github.com/ouraigua/jenkins-library@v0.0.0-20231028010029-fbeaf2f3aa9b/cmd/fortifyExecuteScan.go (about) 1 package cmd 2 3 import ( 4 "context" 5 "encoding/json" 6 "fmt" 7 "math" 8 "net/url" 9 "os" 10 "os/exec" 11 "path/filepath" 12 "regexp" 13 "runtime" 14 "strconv" 15 "strings" 16 "time" 17 18 piperhttp "github.com/SAP/jenkins-library/pkg/http" 19 20 "github.com/bmatcuk/doublestar" 21 22 "github.com/google/go-github/v45/github" 23 "github.com/google/uuid" 24 25 "github.com/piper-validation/fortify-client-go/models" 26 27 "github.com/SAP/jenkins-library/pkg/command" 28 "github.com/SAP/jenkins-library/pkg/fortify" 29 "github.com/SAP/jenkins-library/pkg/gradle" 30 "github.com/SAP/jenkins-library/pkg/log" 31 "github.com/SAP/jenkins-library/pkg/maven" 32 "github.com/SAP/jenkins-library/pkg/piperutils" 33 "github.com/SAP/jenkins-library/pkg/reporting" 34 "github.com/SAP/jenkins-library/pkg/telemetry" 35 "github.com/SAP/jenkins-library/pkg/toolrecord" 36 "github.com/SAP/jenkins-library/pkg/versioning" 37 38 piperGithub "github.com/SAP/jenkins-library/pkg/github" 39 40 "github.com/pkg/errors" 41 ) 42 43 const getClasspathScriptContent = ` 44 gradle.allprojects { 45 task getClasspath { 46 doLast { 47 new File(projectDir, filename).text = sourceSets.main.compileClasspath.asPath 48 } 49 } 50 } 51 ` 52 53 type pullRequestService interface { 54 ListPullRequestsWithCommit(ctx context.Context, owner, repo, sha string, opts *github.PullRequestListOptions) ([]*github.PullRequest, *github.Response, error) 55 } 56 57 type fortifyUtils interface { 58 maven.Utils 59 gradle.Utils 60 piperutils.FileUtils 61 62 SetDir(d string) 63 GetArtifact(buildTool, buildDescriptorFile string, options *versioning.Options) (versioning.Artifact, error) 64 GetIssueService() *github.IssuesService 65 GetSearchService() *github.SearchService 66 } 67 68 type fortifyUtilsBundle struct { 69 *command.Command 70 *piperutils.Files 71 *piperhttp.Client 72 issues *github.IssuesService 73 search *github.SearchService 74 } 75 76 func (f *fortifyUtilsBundle) GetArtifact(buildTool, buildDescriptorFile string, options *versioning.Options) (versioning.Artifact, error) { 77 return versioning.GetArtifact(buildTool, buildDescriptorFile, options, f) 78 } 79 80 func (f *fortifyUtilsBundle) CreateIssue(ghCreateIssueOptions *piperGithub.CreateIssueOptions) error { 81 _, err := piperGithub.CreateIssue(ghCreateIssueOptions) 82 return err 83 } 84 85 func (f *fortifyUtilsBundle) GetIssueService() *github.IssuesService { 86 return f.issues 87 } 88 89 func (f *fortifyUtilsBundle) GetSearchService() *github.SearchService { 90 return f.search 91 } 92 93 func newFortifyUtilsBundle(client *github.Client) fortifyUtils { 94 utils := fortifyUtilsBundle{ 95 Command: &command.Command{}, 96 Files: &piperutils.Files{}, 97 Client: &piperhttp.Client{}, 98 } 99 if client != nil { 100 utils.issues = client.Issues 101 utils.search = client.Search 102 } 103 utils.Stdout(log.Writer()) 104 utils.Stderr(log.Writer()) 105 return &utils 106 } 107 108 const ( 109 checkString = "<---CHECK FORTIFY---" 110 classpathFileName = "fortify-execute-scan-cp.txt" 111 ) 112 113 var execInPath = exec.LookPath 114 115 func fortifyExecuteScan(config fortifyExecuteScanOptions, telemetryData *telemetry.CustomData, influx *fortifyExecuteScanInflux) { 116 // TODO provide parameter for trusted certs 117 ctx, client, err := piperGithub.NewClientBuilder(config.GithubToken, config.GithubAPIURL).Build() 118 if err != nil { 119 log.Entry().WithError(err).Warning("Failed to get GitHub client") 120 } 121 auditStatus := map[string]string{} 122 sys := fortify.NewSystemInstance(config.ServerURL, config.APIEndpoint, config.AuthToken, config.Proxy, time.Minute*15) 123 utils := newFortifyUtilsBundle(client) 124 125 influx.step_data.fields.fortify = false 126 reports, err := runFortifyScan(ctx, config, sys, utils, telemetryData, influx, auditStatus) 127 piperutils.PersistReportsAndLinks("fortifyExecuteScan", config.ModulePath, utils, reports, nil) 128 if err != nil { 129 log.Entry().WithError(err).Fatal("Fortify scan and check failed") 130 } 131 influx.step_data.fields.fortify = true 132 // make sure that no specific error category is set in success case 133 log.SetErrorCategory(log.ErrorUndefined) 134 } 135 136 func determineArtifact(config fortifyExecuteScanOptions, utils fortifyUtils) (versioning.Artifact, error) { 137 versioningOptions := versioning.Options{ 138 M2Path: config.M2Path, 139 GlobalSettingsFile: config.GlobalSettingsFile, 140 ProjectSettingsFile: config.ProjectSettingsFile, 141 Defines: config.AdditionalMvnParameters, 142 } 143 144 artifact, err := utils.GetArtifact(config.BuildTool, config.BuildDescriptorFile, &versioningOptions) 145 if err != nil { 146 return nil, fmt.Errorf("Unable to get artifact from descriptor %v: %w", config.BuildDescriptorFile, err) 147 } 148 return artifact, nil 149 } 150 151 func runFortifyScan(ctx context.Context, config fortifyExecuteScanOptions, sys fortify.System, utils fortifyUtils, telemetryData *telemetry.CustomData, influx *fortifyExecuteScanInflux, auditStatus map[string]string) ([]piperutils.Path, error) { 152 var reports []piperutils.Path 153 log.Entry().Debugf("Running Fortify scan against SSC at %v", config.ServerURL) 154 executableList := []string{"fortifyupdate", "sourceanalyzer"} 155 for _, exec := range executableList { 156 _, err := execInPath(exec) 157 if err != nil { 158 return reports, fmt.Errorf("Command not found: %v. Please configure a supported docker image or install Fortify SCA on the system.", exec) 159 } 160 } 161 162 if config.BuildTool == "maven" && config.InstallArtifacts { 163 err := maven.InstallMavenArtifacts(&maven.EvaluateOptions{ 164 M2Path: config.M2Path, 165 ProjectSettingsFile: config.ProjectSettingsFile, 166 GlobalSettingsFile: config.GlobalSettingsFile, 167 PomPath: config.BuildDescriptorFile, 168 }, utils) 169 if err != nil { 170 return reports, fmt.Errorf("Unable to install artifacts: %w", err) 171 } 172 } 173 174 artifact, err := determineArtifact(config, utils) 175 if err != nil { 176 log.Entry().WithError(err).Fatal() 177 } 178 coordinates, err := artifact.GetCoordinates() 179 if err != nil { 180 log.SetErrorCategory(log.ErrorConfiguration) 181 return reports, fmt.Errorf("unable to get project coordinates from descriptor %v: %w", config.BuildDescriptorFile, err) 182 } 183 log.Entry().Debugf("loaded project coordinates %v from descriptor", coordinates) 184 185 if len(config.Version) > 0 { 186 log.Entry().Infof("Resolving product version from default provided '%s' with versioning '%s'", config.Version, config.VersioningModel) 187 coordinates.Version = config.Version 188 } 189 190 fortifyProjectName, fortifyProjectVersion := versioning.DetermineProjectCoordinatesWithCustomVersion(config.ProjectName, config.VersioningModel, config.CustomScanVersion, coordinates) 191 project, err := sys.GetProjectByName(fortifyProjectName, config.AutoCreate, fortifyProjectVersion) 192 if err != nil { 193 classifyErrorOnLookup(err) 194 return reports, fmt.Errorf("Failed to load project %v: %w", fortifyProjectName, err) 195 } 196 projectVersion, err := sys.GetProjectVersionDetailsByProjectIDAndVersionName(project.ID, fortifyProjectVersion, config.AutoCreate, fortifyProjectName) 197 if err != nil { 198 classifyErrorOnLookup(err) 199 return reports, fmt.Errorf("Failed to load project version %v: %w", fortifyProjectVersion, err) 200 } 201 202 if len(config.PullRequestName) > 0 { 203 fortifyProjectVersion = config.PullRequestName 204 projectVersion, err = sys.LookupOrCreateProjectVersionDetailsForPullRequest(project.ID, projectVersion, fortifyProjectVersion) 205 if err != nil { 206 classifyErrorOnLookup(err) 207 return reports, fmt.Errorf("Failed to lookup / create project version for pull request %v: %w", fortifyProjectVersion, err) 208 } 209 log.Entry().Debugf("Looked up / created project version with ID %v for PR %v", projectVersion.ID, fortifyProjectVersion) 210 } else { 211 prID, prAuthor := determinePullRequestMerge(config) 212 if prID != "0" { 213 log.Entry().Debugf("Determined PR ID '%v' for merge check", prID) 214 if len(prAuthor) > 0 && !piperutils.ContainsString(config.Assignees, prAuthor) { 215 log.Entry().Debugf("Determined PR Author '%v' for result assignment", prAuthor) 216 config.Assignees = append(config.Assignees, prAuthor) 217 } else { 218 log.Entry().Debugf("Unable to determine PR Author, using assignees: %v", config.Assignees) 219 } 220 pullRequestProjectName := fmt.Sprintf("PR-%v", prID) 221 err = sys.MergeProjectVersionStateOfPRIntoMaster(config.FprDownloadEndpoint, config.FprUploadEndpoint, project.ID, projectVersion.ID, pullRequestProjectName) 222 if err != nil { 223 return reports, fmt.Errorf("Failed to merge project version state for pull request %v into project version %v of project %v: %w", pullRequestProjectName, fortifyProjectVersion, project.ID, err) 224 } 225 } 226 } 227 228 filterSet, err := sys.GetFilterSetOfProjectVersionByTitle(projectVersion.ID, config.FilterSetTitle) 229 if filterSet == nil || err != nil { 230 return reports, fmt.Errorf("Failed to load filter set with title %v", config.FilterSetTitle) 231 } 232 233 // create toolrecord file 234 // tbd - how to handle verifyOnly 235 toolRecordFileName, err := createToolRecordFortify(utils, "./", config, project.ID, fortifyProjectName, projectVersion.ID, fortifyProjectVersion) 236 if err != nil { 237 // do not fail until the framework is well established 238 log.Entry().Warning("TR_FORTIFY: Failed to create toolrecord file ...", err) 239 } else { 240 reports = append(reports, piperutils.Path{Target: toolRecordFileName}) 241 } 242 243 if config.VerifyOnly { 244 log.Entry().Infof("Starting audit status check on project %v with version %v and project version ID %v", fortifyProjectName, fortifyProjectVersion, projectVersion.ID) 245 paths, err := verifyFFProjectCompliance(ctx, config, utils, sys, project, projectVersion, filterSet, influx, auditStatus) 246 reports = append(reports, paths...) 247 return reports, err 248 } 249 250 log.Entry().Infof("Scanning and uploading to project %v with version %v and projectVersionId %v", fortifyProjectName, fortifyProjectVersion, projectVersion.ID) 251 buildLabel := fmt.Sprintf("%v/repos/%v/%v/commits/%v", config.GithubAPIURL, config.Owner, config.Repository, config.CommitID) 252 253 // Create sourceanalyzer command based on configuration 254 buildID := uuid.New().String() 255 utils.SetDir(config.ModulePath) 256 if err := os.MkdirAll(fmt.Sprintf("%v/%v", config.ModulePath, "target"), os.ModePerm); err != nil { 257 log.Entry().WithError(err).Error("failed to create directory") 258 } 259 260 if config.UpdateRulePack { 261 262 fortifyUpdateParams := []string{"-acceptKey", "-acceptSSLCertificate", "-url", config.ServerURL} 263 proxyPort, proxyHost := getProxyParams(config.Proxy) 264 if proxyHost != "" && proxyPort != "" { 265 fortifyUpdateParams = append(fortifyUpdateParams, "-proxyhost", proxyHost, "-proxyport", proxyPort) 266 } 267 268 err := utils.RunExecutable("fortifyupdate", fortifyUpdateParams...) 269 if err != nil { 270 return reports, fmt.Errorf("failed to update rule pack, serverUrl: %v", config.ServerURL) 271 } 272 273 err = utils.RunExecutable("fortifyupdate", "-acceptKey", "-acceptSSLCertificate", "-showInstalledRules") 274 if err != nil { 275 return reports, fmt.Errorf("failed to fetch details of installed rule pack, serverUrl: %v", config.ServerURL) 276 } 277 } 278 279 err = triggerFortifyScan(config, utils, buildID, buildLabel, fortifyProjectName) 280 reports = append(reports, piperutils.Path{Target: fmt.Sprintf("%vtarget/fortify-scan.*", config.ModulePath)}) 281 reports = append(reports, piperutils.Path{Target: fmt.Sprintf("%vtarget/*.fpr", config.ModulePath)}) 282 if err != nil { 283 return reports, errors.Wrapf(err, "failed to scan project") 284 } 285 286 var message string 287 if config.UploadResults { 288 log.Entry().Debug("Uploading results") 289 resultFilePath := fmt.Sprintf("%vtarget/result.fpr", config.ModulePath) 290 err = sys.UploadResultFile(config.FprUploadEndpoint, resultFilePath, projectVersion.ID) 291 message = fmt.Sprintf("Failed to upload result file %v to Fortify SSC at %v", resultFilePath, config.ServerURL) 292 } else { 293 log.Entry().Debug("Generating XML report") 294 xmlReportName := "fortify_result.xml" 295 err = utils.RunExecutable("ReportGenerator", "-format", "xml", "-f", xmlReportName, "-source", fmt.Sprintf("%vtarget/result.fpr", config.ModulePath)) 296 message = fmt.Sprintf("Failed to generate XML report %v", xmlReportName) 297 if err != nil { 298 reports = append(reports, piperutils.Path{Target: fmt.Sprintf("%vfortify_result.xml", config.ModulePath)}) 299 } 300 } 301 if err != nil { 302 return reports, fmt.Errorf(message+": %w", err) 303 } 304 305 log.Entry().Infof("Ensuring latest FPR is processed for project %v with version %v and project version ID %v", fortifyProjectName, fortifyProjectVersion, projectVersion.ID) 306 // Ensure latest FPR is processed 307 err = verifyScanResultsFinishedUploading(config, sys, projectVersion.ID, buildLabel, filterSet, 308 10*time.Second, time.Duration(config.PollingMinutes)*time.Minute) 309 if err != nil { 310 return reports, err 311 } 312 313 // SARIF conversion done after latest FPR is processed, but before the compliance is checked 314 if config.ConvertToSarif { 315 resultFilePath := fmt.Sprintf("%vtarget/result.fpr", config.ModulePath) 316 log.Entry().Info("Calling conversion to SARIF function.") 317 sarif, sarifSimplified, err := fortify.ConvertFprToSarif(sys, projectVersion, resultFilePath, filterSet) 318 if err != nil { 319 return reports, fmt.Errorf("failed to generate SARIF") 320 } 321 log.Entry().Debug("Writing simplified sarif file in plain text to disk.") 322 paths, err := fortify.WriteSarif(sarifSimplified, "result.sarif") 323 if err != nil { 324 return reports, fmt.Errorf("failed to write simplified sarif") 325 } 326 reports = append(reports, paths...) 327 328 log.Entry().Debug("Writing full sarif file to disk and gzip it.") 329 paths, err = fortify.WriteGzipSarif(sarif, "result.sarif.gz") 330 if err != nil { 331 return reports, fmt.Errorf("failed to write gzip sarif") 332 } 333 reports = append(reports, paths...) 334 } 335 336 log.Entry().Infof("Starting audit status check on project %v with version %v and project version ID %v", fortifyProjectName, fortifyProjectVersion, projectVersion.ID) 337 paths, err := verifyFFProjectCompliance(ctx, config, utils, sys, project, projectVersion, filterSet, influx, auditStatus) 338 reports = append(reports, paths...) 339 return reports, err 340 } 341 342 func classifyErrorOnLookup(err error) { 343 if strings.Contains(err.Error(), "connect: connection refused") || strings.Contains(err.Error(), "net/http: TLS handshake timeout") { 344 log.SetErrorCategory(log.ErrorService) 345 } 346 } 347 348 func verifyFFProjectCompliance(ctx context.Context, config fortifyExecuteScanOptions, utils fortifyUtils, sys fortify.System, project *models.Project, projectVersion *models.ProjectVersion, filterSet *models.FilterSet, influx *fortifyExecuteScanInflux, auditStatus map[string]string) ([]piperutils.Path, error) { 349 reports := []piperutils.Path{} 350 // Generate report 351 if config.Reporting { 352 resultURL := []byte(fmt.Sprintf("%v/html/ssc/version/%v/fix/null/", config.ServerURL, projectVersion.ID)) 353 if err := os.WriteFile(fmt.Sprintf("%vtarget/%v-%v.%v", config.ModulePath, *project.Name, *projectVersion.Name, "txt"), resultURL, 0o700); err != nil { 354 log.Entry().WithError(err).Error("failed to write file") 355 } 356 357 data, err := generateAndDownloadQGateReport(config, sys, project, projectVersion) 358 if err != nil { 359 return reports, err 360 } 361 if err := os.WriteFile(fmt.Sprintf("%vtarget/%v-%v.%v", config.ModulePath, *project.Name, *projectVersion.Name, config.ReportType), data, 0o700); err != nil { 362 log.Entry().WithError(err).Warning("failed to write file") 363 } 364 } 365 366 // Perform audit compliance checks 367 issueFilterSelectorSet, err := sys.GetIssueFilterSelectorOfProjectVersionByName(projectVersion.ID, []string{"Analysis", "Folder", "Category"}, nil) 368 if err != nil { 369 return reports, errors.Wrapf(err, "failed to fetch project version issue filter selector for project version ID %v", projectVersion.ID) 370 } 371 log.Entry().Debugf("initial filter selector set: %v", issueFilterSelectorSet) 372 373 spotChecksCountByCategory := []fortify.SpotChecksAuditCount{} 374 numberOfViolations, issueGroups, err := analyseUnauditedIssues(config, sys, projectVersion, filterSet, issueFilterSelectorSet, influx, auditStatus, &spotChecksCountByCategory) 375 if err != nil { 376 return reports, errors.Wrap(err, "failed to analyze unaudited issues") 377 } 378 numberOfSuspiciousExploitable, issueGroupsSuspiciousExploitable := analyseSuspiciousExploitable(config, sys, projectVersion, filterSet, issueFilterSelectorSet, influx, auditStatus) 379 numberOfViolations += numberOfSuspiciousExploitable 380 issueGroups = append(issueGroups, issueGroupsSuspiciousExploitable...) 381 382 log.Entry().Infof("Counted %v violations, details: %v", numberOfViolations, auditStatus) 383 384 influx.fortify_data.fields.projectID = project.ID 385 influx.fortify_data.fields.projectName = *project.Name 386 influx.fortify_data.fields.projectVersion = *projectVersion.Name 387 influx.fortify_data.fields.projectVersionID = projectVersion.ID 388 influx.fortify_data.fields.violations = numberOfViolations 389 390 fortifyReportingData := prepareReportData(influx) 391 scanReport := fortify.CreateCustomReport(fortifyReportingData, issueGroups) 392 paths, err := fortify.WriteCustomReports(scanReport) 393 if err != nil { 394 return reports, errors.Wrap(err, "failed to write custom reports") 395 } 396 reports = append(reports, paths...) 397 398 log.Entry().Debug("Checking whether GitHub issue creation/update is active") 399 log.Entry().Debugf("%v, %v, %v, %v, %v, %v", config.CreateResultIssue, numberOfViolations > 0, len(config.GithubToken) > 0, len(config.GithubAPIURL) > 0, len(config.Owner) > 0, len(config.Repository) > 0) 400 if config.CreateResultIssue && numberOfViolations > 0 && len(config.GithubToken) > 0 && len(config.GithubAPIURL) > 0 && len(config.Owner) > 0 && len(config.Repository) > 0 { 401 log.Entry().Debug("Creating/updating GitHub issue with scan results") 402 gh := reporting.GitHub{ 403 Owner: &config.Owner, 404 Repository: &config.Repository, 405 Assignees: &config.Assignees, 406 IssueService: utils.GetIssueService(), 407 SearchService: utils.GetSearchService(), 408 } 409 if err := gh.UploadSingleReport(ctx, scanReport); err != nil { 410 return reports, fmt.Errorf("failed to upload scan results into GitHub: %w", err) 411 } 412 } 413 414 jsonReport := fortify.CreateJSONReport(fortifyReportingData, spotChecksCountByCategory, config.ServerURL) 415 paths, err = fortify.WriteJSONReport(jsonReport) 416 if err != nil { 417 return reports, errors.Wrap(err, "failed to write json report") 418 } 419 reports = append(reports, paths...) 420 421 if numberOfViolations > 0 { 422 log.SetErrorCategory(log.ErrorCompliance) 423 return reports, errors.New("fortify scan failed, the project is not compliant. For details check the archived report") 424 } 425 return reports, nil 426 } 427 428 func prepareReportData(influx *fortifyExecuteScanInflux) fortify.FortifyReportData { 429 input := influx.fortify_data.fields 430 output := fortify.FortifyReportData{} 431 output.ProjectID = input.projectID 432 output.ProjectName = input.projectName 433 output.ProjectVersion = input.projectVersion 434 output.AuditAllAudited = input.auditAllAudited 435 output.AuditAllTotal = input.auditAllTotal 436 output.CorporateAudited = input.corporateAudited 437 output.CorporateTotal = input.corporateTotal 438 output.SpotChecksAudited = input.spotChecksAudited 439 output.SpotChecksGap = input.spotChecksGap 440 output.SpotChecksTotal = input.spotChecksTotal 441 output.Exploitable = input.exploitable 442 output.Suppressed = input.suppressed 443 output.Suspicious = input.suspicious 444 output.ProjectVersionID = input.projectVersionID 445 output.Violations = input.violations 446 return output 447 } 448 449 func analyseUnauditedIssues(config fortifyExecuteScanOptions, sys fortify.System, projectVersion *models.ProjectVersion, filterSet *models.FilterSet, issueFilterSelectorSet *models.IssueFilterSelectorSet, influx *fortifyExecuteScanInflux, auditStatus map[string]string, spotChecksCountByCategory *[]fortify.SpotChecksAuditCount) (int, []*models.ProjectVersionIssueGroup, error) { 450 log.Entry().Info("Analyzing unaudited issues") 451 452 if config.SpotCheckMinimumUnit != "percentage" && config.SpotCheckMinimumUnit != "number" { 453 return 0, nil, fmt.Errorf("Invalid spotCheckMinimumUnit. Please set it as 'percentage' or 'number'.") 454 } 455 456 reducedFilterSelectorSet := sys.ReduceIssueFilterSelectorSet(issueFilterSelectorSet, []string{"Folder"}, nil) 457 fetchedIssueGroups, err := sys.GetProjectIssuesByIDAndFilterSetGroupedBySelector(projectVersion.ID, "", filterSet.GUID, reducedFilterSelectorSet) 458 if err != nil { 459 return 0, fetchedIssueGroups, errors.Wrapf(err, "failed to fetch project version issue groups with filter set %v and selector %v for project version ID %v", filterSet, issueFilterSelectorSet, projectVersion.ID) 460 } 461 overallViolations := 0 462 for _, issueGroup := range fetchedIssueGroups { 463 issueDelta, err := getIssueDeltaFor(config, sys, issueGroup, projectVersion.ID, filterSet, issueFilterSelectorSet, influx, auditStatus, spotChecksCountByCategory) 464 if err != nil { 465 return overallViolations, fetchedIssueGroups, errors.Wrap(err, "failed to get issue delta") 466 } 467 overallViolations += issueDelta 468 } 469 return overallViolations, fetchedIssueGroups, nil 470 } 471 472 func getIssueDeltaFor(config fortifyExecuteScanOptions, sys fortify.System, issueGroup *models.ProjectVersionIssueGroup, projectVersionID int64, filterSet *models.FilterSet, issueFilterSelectorSet *models.IssueFilterSelectorSet, influx *fortifyExecuteScanInflux, auditStatus map[string]string, spotChecksCountByCategory *[]fortify.SpotChecksAuditCount) (int, error) { 473 totalMinusAuditedDelta := 0 474 group := "" 475 total := 0 476 audited := 0 477 if issueGroup != nil { 478 group = *issueGroup.ID 479 total = int(*issueGroup.TotalCount) 480 audited = int(*issueGroup.AuditedCount) 481 } 482 groupTotalMinusAuditedDelta := total - audited 483 if groupTotalMinusAuditedDelta > 0 { 484 reducedFilterSelectorSet := sys.ReduceIssueFilterSelectorSet(issueFilterSelectorSet, []string{"Folder", "Analysis"}, []string{group}) 485 folderSelector := sys.GetFilterSetByDisplayName(reducedFilterSelectorSet, "Folder") 486 if folderSelector == nil { 487 return totalMinusAuditedDelta, fmt.Errorf("folder selector not found") 488 } 489 analysisSelector := sys.GetFilterSetByDisplayName(reducedFilterSelectorSet, "Analysis") 490 491 auditStatus[group] = fmt.Sprintf("%v total : %v audited", total, audited) 492 493 if strings.Contains(config.MustAuditIssueGroups, group) { 494 totalMinusAuditedDelta += groupTotalMinusAuditedDelta 495 if group == "Corporate Security Requirements" { 496 influx.fortify_data.fields.corporateTotal = total 497 influx.fortify_data.fields.corporateAudited = audited 498 } 499 if group == "Audit All" { 500 influx.fortify_data.fields.auditAllTotal = total 501 influx.fortify_data.fields.auditAllAudited = audited 502 } 503 log.Entry().Errorf("[projectVersionId %v]: Unaudited %v detected, count %v", projectVersionID, group, totalMinusAuditedDelta) 504 logIssueURL(config, projectVersionID, folderSelector, analysisSelector) 505 } 506 507 if strings.Contains(config.SpotAuditIssueGroups, group) { 508 log.Entry().Infof("Analyzing %v", config.SpotAuditIssueGroups) 509 filter := fmt.Sprintf("%v:%v", folderSelector.EntityType, folderSelector.SelectorOptions[0].Value) 510 fetchedIssueGroups, err := sys.GetProjectIssuesByIDAndFilterSetGroupedBySelector(projectVersionID, filter, filterSet.GUID, sys.ReduceIssueFilterSelectorSet(issueFilterSelectorSet, []string{"Category"}, nil)) 511 if err != nil { 512 return totalMinusAuditedDelta, errors.Wrapf(err, "failed to fetch project version issue groups with filter %v, filter set %v and selector %v for project version ID %v", filter, filterSet, issueFilterSelectorSet, projectVersionID) 513 } 514 totalMinusAuditedDelta += getSpotIssueCount(config, sys, fetchedIssueGroups, projectVersionID, filterSet, reducedFilterSelectorSet, influx, auditStatus, spotChecksCountByCategory) 515 } 516 } 517 return totalMinusAuditedDelta, nil 518 } 519 520 func getSpotIssueCount(config fortifyExecuteScanOptions, sys fortify.System, spotCheckCategories []*models.ProjectVersionIssueGroup, projectVersionID int64, filterSet *models.FilterSet, issueFilterSelectorSet *models.IssueFilterSelectorSet, influx *fortifyExecuteScanInflux, auditStatus map[string]string, spotChecksCountByCategory *[]fortify.SpotChecksAuditCount) int { 521 overallDelta := 0 522 overallIssues := 0 523 overallIssuesAudited := 0 524 525 for _, issueGroup := range spotCheckCategories { 526 group := "" 527 total := 0 528 audited := 0 529 if issueGroup != nil { 530 group = *issueGroup.ID 531 total = int(*issueGroup.TotalCount) 532 audited = int(*issueGroup.AuditedCount) 533 } 534 flagOutput := "" 535 536 minSpotChecksPerCategory := getMinSpotChecksPerCategory(config, total) 537 log.Entry().Debugf("Minimum spot checks for group %v is %v with audit count %v and total issue count %v", group, minSpotChecksPerCategory, audited, total) 538 539 if ((total <= minSpotChecksPerCategory || minSpotChecksPerCategory < 0) && audited != total) || (total > minSpotChecksPerCategory && audited < minSpotChecksPerCategory) { 540 currentDelta := minSpotChecksPerCategory - audited 541 if minSpotChecksPerCategory < 0 || minSpotChecksPerCategory > total { 542 currentDelta = total - audited 543 } 544 if currentDelta > 0 { 545 filterSelectorFolder := sys.GetFilterSetByDisplayName(issueFilterSelectorSet, "Folder") 546 filterSelectorAnalysis := sys.GetFilterSetByDisplayName(issueFilterSelectorSet, "Analysis") 547 overallDelta += currentDelta 548 log.Entry().Errorf("[projectVersionId %v]: %v unaudited spot check issues detected in group %v", projectVersionID, currentDelta, group) 549 logIssueURL(config, projectVersionID, filterSelectorFolder, filterSelectorAnalysis) 550 flagOutput = checkString 551 } 552 } 553 554 overallIssues += total 555 overallIssuesAudited += audited 556 557 auditStatus[group] = fmt.Sprintf("%v total : %v audited %v", total, audited, flagOutput) 558 *spotChecksCountByCategory = append(*spotChecksCountByCategory, fortify.SpotChecksAuditCount{Audited: audited, Total: total, Type: group}) 559 } 560 561 influx.fortify_data.fields.spotChecksTotal = overallIssues 562 influx.fortify_data.fields.spotChecksAudited = overallIssuesAudited 563 influx.fortify_data.fields.spotChecksGap = overallDelta 564 565 return overallDelta 566 } 567 568 func getMinSpotChecksPerCategory(config fortifyExecuteScanOptions, totalCount int) int { 569 if config.SpotCheckMinimumUnit == "percentage" { 570 spotCheckMinimumPercentageValue := int(math.Ceil(float64(config.SpotCheckMinimum) / 100.0 * float64(totalCount))) 571 return getSpotChecksMinAsPerMaximum(config.SpotCheckMaximum, spotCheckMinimumPercentageValue) 572 } 573 574 return getSpotChecksMinAsPerMaximum(config.SpotCheckMaximum, config.SpotCheckMinimum) 575 } 576 577 func getSpotChecksMinAsPerMaximum(spotCheckMax int, spotCheckMin int) int { 578 if spotCheckMax < 1 { 579 return spotCheckMin 580 } 581 582 if spotCheckMin > spotCheckMax { 583 return spotCheckMax 584 } 585 586 return spotCheckMin 587 } 588 589 func analyseSuspiciousExploitable(config fortifyExecuteScanOptions, sys fortify.System, projectVersion *models.ProjectVersion, filterSet *models.FilterSet, issueFilterSelectorSet *models.IssueFilterSelectorSet, influx *fortifyExecuteScanInflux, auditStatus map[string]string) (int, []*models.ProjectVersionIssueGroup) { 590 log.Entry().Info("Analyzing suspicious and exploitable issues") 591 reducedFilterSelectorSet := sys.ReduceIssueFilterSelectorSet(issueFilterSelectorSet, []string{"Analysis"}, []string{}) 592 fetchedGroups, err := sys.GetProjectIssuesByIDAndFilterSetGroupedBySelector(projectVersion.ID, "", filterSet.GUID, reducedFilterSelectorSet) 593 if err != nil { 594 log.Entry().WithError(err).Errorf("failed to get project issues") 595 } 596 597 suspiciousCount := 0 598 exploitableCount := 0 599 for _, issueGroup := range fetchedGroups { 600 if *issueGroup.ID == "3" { 601 suspiciousCount = int(*issueGroup.TotalCount) 602 } else if *issueGroup.ID == "4" { 603 exploitableCount = int(*issueGroup.TotalCount) 604 } 605 } 606 607 result := 0 608 if (suspiciousCount > 0 && config.ConsiderSuspicious) || exploitableCount > 0 { 609 result = result + suspiciousCount + exploitableCount 610 log.Entry().Errorf("[projectVersionId %v]: %v suspicious and %v exploitable issues detected", projectVersion.ID, suspiciousCount, exploitableCount) 611 log.Entry().Errorf("%v/html/ssc/index.jsp#!/version/%v/fix?issueGrouping=%v_%v&issueFilters=%v_%v", config.ServerURL, projectVersion.ID, reducedFilterSelectorSet.GroupBySet[0].EntityType, reducedFilterSelectorSet.GroupBySet[0].Value, reducedFilterSelectorSet.FilterBySet[0].EntityType, reducedFilterSelectorSet.FilterBySet[0].Value) 612 } 613 issueStatistics, err := sys.GetIssueStatisticsOfProjectVersion(projectVersion.ID) 614 if err != nil { 615 log.Entry().WithError(err).Errorf("Failed to fetch project version statistics for project version ID %v", projectVersion.ID) 616 } 617 auditStatus["Suspicious"] = fmt.Sprintf("%v", suspiciousCount) 618 auditStatus["Exploitable"] = fmt.Sprintf("%v", exploitableCount) 619 suppressedCount := *issueStatistics[0].SuppressedCount 620 if suppressedCount > 0 { 621 auditStatus["Suppressed"] = fmt.Sprintf("WARNING: Detected %v suppressed issues which could violate audit compliance!!!", suppressedCount) 622 } 623 influx.fortify_data.fields.suspicious = suspiciousCount 624 influx.fortify_data.fields.exploitable = exploitableCount 625 influx.fortify_data.fields.suppressed = int(suppressedCount) 626 627 return result, fetchedGroups 628 } 629 630 func logIssueURL(config fortifyExecuteScanOptions, projectVersionID int64, folderSelector, analysisSelector *models.IssueFilterSelector) { 631 url := fmt.Sprintf("%v/html/ssc/index.jsp#!/version/%v/fix", config.ServerURL, projectVersionID) 632 if len(folderSelector.SelectorOptions) > 0 { 633 url += fmt.Sprintf("?issueFilters=%v_%v:%v", 634 folderSelector.EntityType, 635 folderSelector.Value, 636 folderSelector.SelectorOptions[0].Value) 637 } else { 638 log.Entry().Debugf("no 'filter by set' array entries") 639 } 640 if analysisSelector != nil { 641 url += fmt.Sprintf("&issueFilters=%v_%v:", 642 analysisSelector.EntityType, 643 analysisSelector.Value) 644 } else { 645 log.Entry().Debugf("no second entry in 'filter by set' array") 646 } 647 log.Entry().Error(url) 648 } 649 650 func generateAndDownloadQGateReport(config fortifyExecuteScanOptions, sys fortify.System, project *models.Project, projectVersion *models.ProjectVersion) ([]byte, error) { 651 log.Entry().Infof("Generating report with template ID %v", config.ReportTemplateID) 652 report, err := sys.GenerateQGateReport(project.ID, projectVersion.ID, int64(config.ReportTemplateID), *project.Name, *projectVersion.Name, config.ReportType) 653 if err != nil { 654 return []byte{}, errors.Wrap(err, "failed to generate Q-Gate report") 655 } 656 log.Entry().Debugf("Triggered report generation of report ID %v", report.ID) 657 status := report.Status 658 for status == "PROCESSING" || status == "SCHED_PROCESSING" { 659 time.Sleep(10 * time.Second) 660 report, err = sys.GetReportDetails(report.ID) 661 if err != nil { 662 return []byte{}, fmt.Errorf("Failed to fetch Q-Gate report generation status: %w", err) 663 } 664 status = report.Status 665 } 666 data, err := sys.DownloadReportFile(config.ReportDownloadEndpoint, report.ID) 667 if err != nil { 668 return []byte{}, fmt.Errorf("Failed to download Q-Gate Report: %w", err) 669 } 670 return data, nil 671 } 672 673 var errProcessing = errors.New("artifact still processing") 674 675 func checkArtifactStatus(config fortifyExecuteScanOptions, projectVersionID int64, filterSet *models.FilterSet, artifact *models.Artifact, retries int, pollingDelay, timeout time.Duration) error { 676 if "PROCESSING" == artifact.Status || "SCHED_PROCESSING" == artifact.Status { 677 pollingTime := time.Duration(retries) * pollingDelay 678 if pollingTime >= timeout { 679 log.SetErrorCategory(log.ErrorService) 680 return fmt.Errorf("terminating after %v since artifact for Project Version %v is still in status %v", timeout, projectVersionID, artifact.Status) 681 } 682 log.Entry().Infof("Most recent artifact uploaded on %v of Project Version %v is still in status %v...", artifact.UploadDate, projectVersionID, artifact.Status) 683 time.Sleep(pollingDelay) 684 return errProcessing 685 } 686 if "REQUIRE_AUTH" == artifact.Status { 687 // verify no manual issue approval needed 688 log.SetErrorCategory(log.ErrorCompliance) 689 return fmt.Errorf("There are artifacts that require manual approval for Project Version %v, please visit Fortify SSC and approve them for processing\n%v/html/ssc/index.jsp#!/version/%v/artifacts?filterSet=%v", projectVersionID, config.ServerURL, projectVersionID, filterSet.GUID) 690 } 691 if "ERROR_PROCESSING" == artifact.Status { 692 log.SetErrorCategory(log.ErrorService) 693 return fmt.Errorf("There are artifacts that failed processing for Project Version %v\n%v/html/ssc/index.jsp#!/version/%v/artifacts?filterSet=%v", projectVersionID, config.ServerURL, projectVersionID, filterSet.GUID) 694 } 695 return nil 696 } 697 698 func verifyScanResultsFinishedUploading(config fortifyExecuteScanOptions, sys fortify.System, projectVersionID int64, buildLabel string, filterSet *models.FilterSet, pollingDelay, timeout time.Duration) error { 699 log.Entry().Debug("Verifying scan results have finished uploading and processing") 700 var artifacts []*models.Artifact 701 var relatedUpload *models.Artifact 702 var err error 703 retries := 0 704 for relatedUpload == nil { 705 artifacts, err = sys.GetArtifactsOfProjectVersion(projectVersionID) 706 log.Entry().Debugf("Received %v artifacts for project version ID %v", len(artifacts), projectVersionID) 707 if err != nil { 708 return fmt.Errorf("failed to fetch artifacts of project version ID %v: %w", projectVersionID, err) 709 } 710 if len(artifacts) == 0 { 711 return fmt.Errorf("no uploaded artifacts for assessment detected for project version with ID %v", projectVersionID) 712 } 713 latest := artifacts[0] 714 err = checkArtifactStatus(config, projectVersionID, filterSet, latest, retries, pollingDelay, timeout) 715 if err != nil { 716 if err == errProcessing { 717 retries++ 718 continue 719 } 720 return err 721 } 722 relatedUpload = findArtifactByBuildLabel(artifacts, buildLabel) 723 if relatedUpload == nil { 724 log.Entry().Warn("Unable to identify artifact based on the build label, will consider most recent artifact as related to the scan") 725 relatedUpload = artifacts[0] 726 } 727 } 728 729 differenceInSeconds := calculateTimeDifferenceToLastUpload(relatedUpload.UploadDate, projectVersionID) 730 // Use the absolute value for checking the time difference 731 if differenceInSeconds > float64(60*config.DeltaMinutes) { 732 return errors.New("no recent upload detected on Project Version") 733 } 734 for _, upload := range artifacts { 735 if upload.Status == "ERROR_PROCESSING" { 736 log.Entry().Warn("Previous uploads detected that failed processing, please ensure that your scans are properly configured") 737 break 738 } 739 } 740 return nil 741 } 742 743 func findArtifactByBuildLabel(artifacts []*models.Artifact, buildLabel string) *models.Artifact { 744 if len(buildLabel) == 0 { 745 return nil 746 } 747 for _, artifact := range artifacts { 748 if len(buildLabel) > 0 && artifact.Embed != nil && artifact.Embed.Scans != nil && len(artifact.Embed.Scans) > 0 { 749 scan := artifact.Embed.Scans[0] 750 if scan != nil && strings.HasSuffix(scan.BuildLabel, buildLabel) { 751 return artifact 752 } 753 } 754 } 755 return nil 756 } 757 758 func calculateTimeDifferenceToLastUpload(uploadDate models.Iso8601MilliDateTime, projectVersionID int64) float64 { 759 log.Entry().Infof("Last upload on project version %v happened on %v", projectVersionID, uploadDate) 760 uploadDateAsTime := time.Time(uploadDate) 761 duration := time.Since(uploadDateAsTime) 762 log.Entry().Debugf("Difference duration is %v", duration) 763 absoluteSeconds := math.Abs(duration.Seconds()) 764 log.Entry().Infof("Difference since %v in seconds is %v", uploadDateAsTime, absoluteSeconds) 765 return absoluteSeconds 766 } 767 768 func executeTemplatedCommand(utils fortifyUtils, cmdTemplate []string, context map[string]string) error { 769 for index, cmdTemplatePart := range cmdTemplate { 770 result, err := piperutils.ExecuteTemplate(cmdTemplatePart, context) 771 if err != nil { 772 return errors.Wrapf(err, "failed to transform template for command fragment: %v", cmdTemplatePart) 773 } 774 cmdTemplate[index] = result 775 } 776 err := utils.RunExecutable(cmdTemplate[0], cmdTemplate[1:]...) 777 if err != nil { 778 return errors.Wrapf(err, "failed to execute command %v", cmdTemplate) 779 } 780 return nil 781 } 782 783 func autoresolvePipClasspath(executable string, parameters []string, file string, utils fortifyUtils) (string, error) { 784 // redirect stdout and create cp file from command output 785 outfile, err := os.Create(file) 786 if err != nil { 787 return "", errors.Wrapf(err, "failed to create classpath file") 788 } 789 defer outfile.Close() 790 utils.Stdout(outfile) 791 err = utils.RunExecutable(executable, parameters...) 792 if err != nil { 793 return "", errors.Wrapf(err, "failed to run classpath autodetection command %v with parameters %v", executable, parameters) 794 } 795 utils.Stdout(log.Entry().Writer()) 796 return readClasspathFile(file), nil 797 } 798 799 func autoresolveMavenClasspath(config fortifyExecuteScanOptions, file string, utils fortifyUtils) (string, error) { 800 if filepath.IsAbs(file) { 801 log.Entry().Warnf("Passing an absolute path for -Dmdep.outputFile results in the classpath only for the last module in multi-module maven projects.") 802 } 803 defines := generateMavenFortifyDefines(&config, file) 804 executeOptions := maven.ExecuteOptions{ 805 PomPath: config.BuildDescriptorFile, 806 ProjectSettingsFile: config.ProjectSettingsFile, 807 GlobalSettingsFile: config.GlobalSettingsFile, 808 M2Path: config.M2Path, 809 Goals: []string{"dependency:build-classpath", "package"}, 810 Defines: defines, 811 ReturnStdout: false, 812 } 813 _, err := maven.Execute(&executeOptions, utils) 814 if err != nil { 815 log.Entry().WithError(err).Warnf("failed to determine classpath using Maven: %v", err) 816 } 817 return readAllClasspathFiles(file), nil 818 } 819 820 func autoresolveGradleClasspath(config fortifyExecuteScanOptions, file string, utils fortifyUtils) (string, error) { 821 gradleOptions := &gradle.ExecuteOptions{ 822 Task: "getClasspath", 823 UseWrapper: true, 824 InitScriptContent: getClasspathScriptContent, 825 ProjectProperties: map[string]string{"filename": file}, 826 } 827 if _, err := gradle.Execute(gradleOptions, utils); err != nil { 828 log.Entry().WithError(err).Warnf("failed to determine classpath using Gradle: %v", err) 829 } 830 return readAllClasspathFiles(file), nil 831 } 832 833 func generateMavenFortifyDefines(config *fortifyExecuteScanOptions, file string) []string { 834 defines := []string{ 835 fmt.Sprintf("-Dmdep.outputFile=%v", file), 836 // Parameter to indicate to maven build that the fortify step is the trigger, can be used for optimizations 837 "-Dfortify", 838 "-DincludeScope=compile", 839 "-DskipTests", 840 "-Dmaven.javadoc.skip=true", 841 "--fail-at-end", 842 } 843 844 if len(config.BuildDescriptorExcludeList) > 0 { 845 // From the documentation, these are file paths to a module's pom.xml. 846 // For MTA projects, we support pom.xml files here and skip others. 847 for _, exclude := range config.BuildDescriptorExcludeList { 848 if !strings.HasSuffix(exclude, "pom.xml") { 849 continue 850 } 851 exists, _ := piperutils.FileExists(exclude) 852 if !exists { 853 continue 854 } 855 moduleName := filepath.Dir(exclude) 856 if moduleName != "" { 857 defines = append(defines, "-pl", "!"+moduleName) 858 } 859 } 860 } 861 862 return defines 863 } 864 865 // readAllClasspathFiles tests whether the passed file is an absolute path. If not, it will glob for 866 // all files under the current directory with the given file name and concatenate their contents. 867 // Otherwise it will return the contents pointed to by the absolute path. 868 func readAllClasspathFiles(file string) string { 869 var paths []string 870 if filepath.IsAbs(file) { 871 paths = []string{file} 872 } else { 873 paths, _ = doublestar.Glob(filepath.Join("**", file)) 874 log.Entry().Debugf("Concatenating the class paths from %v", paths) 875 } 876 var contents string 877 const separator = ":" 878 for _, path := range paths { 879 contents += separator + readClasspathFile(path) 880 } 881 return removeDuplicates(contents, separator) 882 } 883 884 func readClasspathFile(file string) string { 885 data, err := os.ReadFile(file) 886 if err != nil { 887 log.Entry().WithError(err).Warnf("failed to read classpath from file '%v'", file) 888 } 889 result := strings.TrimSpace(string(data)) 890 if len(result) == 0 { 891 log.Entry().Warnf("classpath from file '%v' was empty", file) 892 } 893 return result 894 } 895 896 func removeDuplicates(contents, separator string) string { 897 if separator == "" || contents == "" { 898 return contents 899 } 900 entries := strings.Split(contents, separator) 901 entrySet := map[string]struct{}{} 902 contents = "" 903 for _, entry := range entries { 904 if entry == "" { 905 continue 906 } 907 _, contained := entrySet[entry] 908 if !contained { 909 entrySet[entry] = struct{}{} 910 contents += entry + separator 911 } 912 } 913 if contents != "" { 914 // Remove trailing "separator" 915 contents = contents[:len(contents)-len(separator)] 916 } 917 return contents 918 } 919 920 func triggerFortifyScan(config fortifyExecuteScanOptions, utils fortifyUtils, buildID, buildLabel, buildProject string) error { 921 var err error 922 // Do special Python related prep 923 pipVersion := "pip3" 924 if config.PythonVersion != "python3" { 925 pipVersion = "pip2" 926 } 927 928 classpath := "" 929 if config.BuildTool == "maven" { 930 if config.AutodetectClasspath { 931 classpath, err = autoresolveMavenClasspath(config, classpathFileName, utils) 932 if err != nil { 933 return err 934 } 935 } 936 config.Translate, err = populateMavenGradleTranslate(&config, classpath) 937 if err != nil { 938 log.Entry().WithError(err).Warnf("failed to apply src ('%s') or exclude ('%s') parameter", config.Src, config.Exclude) 939 } 940 } else if config.BuildTool == "gradle" { 941 if config.AutodetectClasspath { 942 classpath, err = autoresolveGradleClasspath(config, classpathFileName, utils) 943 if err != nil { 944 return err 945 } 946 } 947 config.Translate, err = populateMavenGradleTranslate(&config, classpath) 948 if err != nil { 949 log.Entry().WithError(err).Warnf("failed to apply src ('%s') or exclude ('%s') parameter", config.Src, config.Exclude) 950 } 951 } else if config.BuildTool == "pip" { 952 if config.AutodetectClasspath { 953 separator := getSeparator() 954 script := fmt.Sprintf("import sys;p=sys.path;p.remove('');print('%v'.join(p))", separator) 955 classpath, err = autoresolvePipClasspath(config.PythonVersion, []string{"-c", script}, classpathFileName, utils) 956 if err != nil { 957 return errors.Wrap(err, "failed to autoresolve pip classpath") 958 } 959 } 960 // install the dev dependencies 961 if len(config.PythonRequirementsFile) > 0 { 962 context := map[string]string{} 963 cmdTemplate := []string{pipVersion, "install", "--user", "-r", config.PythonRequirementsFile} 964 cmdTemplate = append(cmdTemplate, tokenize(config.PythonRequirementsInstallSuffix)...) 965 if err := executeTemplatedCommand(utils, cmdTemplate, context); err != nil { 966 log.Entry().WithError(err).Error("failed to execute template command") 967 } 968 } 969 970 if err := executeTemplatedCommand(utils, tokenize(config.PythonInstallCommand), map[string]string{"Pip": pipVersion}); err != nil { 971 log.Entry().WithError(err).Error("failed to execute template command") 972 } 973 974 config.Translate, err = populatePipTranslate(&config, classpath) 975 if err != nil { 976 log.Entry().WithError(err).Warnf("failed to apply pythonAdditionalPath ('%s') or src ('%s') parameter", config.PythonAdditionalPath, config.Src) 977 } 978 979 } else { 980 return fmt.Errorf("buildTool '%s' is not supported by this step", config.BuildTool) 981 } 982 983 err = translateProject(&config, utils, buildID, classpath) 984 if err != nil { 985 return err 986 } 987 988 return scanProject(&config, utils, buildID, buildLabel, buildProject) 989 } 990 991 func appendPythonVersionToTranslate(translateOptions map[string]interface{}, pythonVersion string) error { 992 if pythonVersion == "python2" { 993 translateOptions["pythonVersion"] = "2" 994 } else if pythonVersion == "python3" { 995 translateOptions["pythonVersion"] = "3" 996 } else { 997 return fmt.Errorf("Invalid pythonVersion '%s'. Possible values for pythonVersion are 'python2' and 'python3'. ", pythonVersion) 998 } 999 1000 return nil 1001 } 1002 1003 func populatePipTranslate(config *fortifyExecuteScanOptions, classpath string) (string, error) { 1004 if len(config.Translate) > 0 { 1005 return config.Translate, nil 1006 } 1007 1008 var translateList []map[string]interface{} 1009 translateList = append(translateList, make(map[string]interface{})) 1010 separator := getSeparator() 1011 1012 err := appendPythonVersionToTranslate(translateList[0], config.PythonVersion) 1013 if err != nil { 1014 return "", err 1015 } 1016 1017 translateList[0]["pythonPath"] = classpath + separator + 1018 getSuppliedOrDefaultListAsString(config.PythonAdditionalPath, []string{}, separator) 1019 translateList[0]["src"] = getSuppliedOrDefaultListAsString( 1020 config.Src, []string{"./**/*"}, ":") 1021 translateList[0]["exclude"] = getSuppliedOrDefaultListAsString( 1022 config.Exclude, []string{"./**/tests/**/*", "./**/setup.py"}, separator) 1023 1024 translateJSON, err := json.Marshal(translateList) 1025 1026 return string(translateJSON), err 1027 } 1028 1029 func populateMavenGradleTranslate(config *fortifyExecuteScanOptions, classpath string) (string, error) { 1030 if len(config.Translate) > 0 { 1031 return config.Translate, nil 1032 } 1033 1034 var translateList []map[string]interface{} 1035 translateList = append(translateList, make(map[string]interface{})) 1036 translateList[0]["classpath"] = classpath 1037 1038 setTranslateEntryIfNotEmpty(translateList[0], "src", ":", config.Src, 1039 []string{"**/*.xml", "**/*.html", "**/*.jsp", "**/*.js", "**/src/main/resources/**/*", "**/src/main/java/**/*", "**/src/gen/java/cds/**/*", "**/target/main/java/**/*", "**/target/main/resources/**/*", "**/target/generated-sources/**/*"}) 1040 1041 setTranslateEntryIfNotEmpty(translateList[0], "exclude", getSeparator(), config.Exclude, []string{"**/src/test/**/*"}) 1042 1043 translateJSON, err := json.Marshal(translateList) 1044 1045 return string(translateJSON), err 1046 } 1047 1048 func translateProject(config *fortifyExecuteScanOptions, utils fortifyUtils, buildID, classpath string) error { 1049 var translateList []map[string]string 1050 json.Unmarshal([]byte(config.Translate), &translateList) 1051 log.Entry().Debugf("Translating with options: %v", translateList) 1052 for _, translate := range translateList { 1053 if len(classpath) > 0 { 1054 translate["autoClasspath"] = classpath 1055 } 1056 err := handleSingleTranslate(config, utils, buildID, translate) 1057 if err != nil { 1058 return err 1059 } 1060 } 1061 return nil 1062 } 1063 1064 func handleSingleTranslate(config *fortifyExecuteScanOptions, command fortifyUtils, buildID string, t map[string]string) error { 1065 if t != nil { 1066 log.Entry().Debugf("Handling translate config %v", t) 1067 translateOptions := []string{ 1068 "-verbose", 1069 "-64", 1070 "-b", 1071 buildID, 1072 } 1073 translateOptions = append(translateOptions, tokenize(config.Memory)...) 1074 translateOptions = appendToOptions(config, translateOptions, t) 1075 log.Entry().Debugf("Running sourceanalyzer translate command with options %v", translateOptions) 1076 err := command.RunExecutable("sourceanalyzer", translateOptions...) 1077 if err != nil { 1078 return errors.Wrapf(err, "failed to execute sourceanalyzer translate command with options %v", translateOptions) 1079 } 1080 } else { 1081 log.Entry().Debug("Skipping translate with nil value") 1082 } 1083 return nil 1084 } 1085 1086 func scanProject(config *fortifyExecuteScanOptions, command fortifyUtils, buildID, buildLabel, buildProject string) error { 1087 scanOptions := []string{ 1088 "-verbose", 1089 "-64", 1090 "-b", 1091 buildID, 1092 "-scan", 1093 } 1094 scanOptions = append(scanOptions, tokenize(config.Memory)...) 1095 if config.QuickScan { 1096 scanOptions = append(scanOptions, "-quick") 1097 } 1098 if len(config.AdditionalScanParameters) > 0 { 1099 scanOptions = append(scanOptions, config.AdditionalScanParameters...) 1100 } 1101 if len(buildLabel) > 0 { 1102 scanOptions = append(scanOptions, "-build-label", buildLabel) 1103 } 1104 if len(buildProject) > 0 { 1105 scanOptions = append(scanOptions, "-build-project", buildProject) 1106 } 1107 scanOptions = append(scanOptions, "-logfile", "target/fortify-scan.log", "-f", "target/result.fpr") 1108 1109 err := command.RunExecutable("sourceanalyzer", scanOptions...) 1110 if err != nil { 1111 return errors.Wrapf(err, "failed to execute sourceanalyzer scan command with scanOptions %v", scanOptions) 1112 } 1113 return nil 1114 } 1115 1116 func determinePullRequestMerge(config fortifyExecuteScanOptions) (string, string) { 1117 author := "" 1118 // TODO provide parameter for trusted certs 1119 ctx, client, err := piperGithub.NewClientBuilder(config.GithubToken, config.GithubAPIURL).Build() 1120 if err == nil && ctx != nil && client != nil { 1121 prID, author, err := determinePullRequestMergeGithub(ctx, config, client.PullRequests) 1122 if err != nil { 1123 log.Entry().WithError(err).Warn("Failed to get PR metadata via GitHub client") 1124 } else { 1125 return prID, author 1126 } 1127 } else { 1128 log.Entry().WithError(err).Warn("Failed to instantiate GitHub client to get PR metadata") 1129 } 1130 1131 log.Entry().Infof("Trying to determine PR ID in commit message: %v", config.CommitMessage) 1132 r, _ := regexp.Compile(config.PullRequestMessageRegex) 1133 matches := r.FindSubmatch([]byte(config.CommitMessage)) 1134 if matches != nil && len(matches) > 1 { 1135 return string(matches[config.PullRequestMessageRegexGroup]), author 1136 } 1137 return "0", "" 1138 } 1139 1140 func determinePullRequestMergeGithub(ctx context.Context, config fortifyExecuteScanOptions, pullRequestServiceInstance pullRequestService) (string, string, error) { 1141 number := "0" 1142 author := "" 1143 options := github.PullRequestListOptions{State: "closed", Sort: "updated", Direction: "desc"} 1144 prList, _, err := pullRequestServiceInstance.ListPullRequestsWithCommit(ctx, config.Owner, config.Repository, config.CommitID, &options) 1145 if err == nil && prList != nil && len(prList) > 0 { 1146 number = fmt.Sprintf("%v", prList[0].GetNumber()) 1147 if prList[0].GetUser() != nil { 1148 author = prList[0].GetUser().GetLogin() 1149 } 1150 return number, author, nil 1151 } 1152 1153 log.Entry().Infof("Unable to resolve PR via commit ID: %v", config.CommitID) 1154 return number, author, err 1155 } 1156 1157 func appendToOptions(config *fortifyExecuteScanOptions, options []string, t map[string]string) []string { 1158 switch config.BuildTool { 1159 case "windows": 1160 if len(t["aspnetcore"]) > 0 { 1161 options = append(options, "-aspnetcore") 1162 } 1163 if len(t["dotNetCoreVersion"]) > 0 { 1164 options = append(options, "-dotnet-core-version", t["dotNetCoreVersion"]) 1165 } 1166 if len(t["libDirs"]) > 0 { 1167 options = append(options, "-libdirs", t["libDirs"]) 1168 } 1169 1170 case "maven", "gradle": 1171 if len(t["autoClasspath"]) > 0 { 1172 options = append(options, "-cp", t["autoClasspath"]) 1173 } else if len(t["classpath"]) > 0 { 1174 options = append(options, "-cp", t["classpath"]) 1175 } else { 1176 log.Entry().Debugf("no field 'autoClasspath' or 'classpath' in map or both empty") 1177 } 1178 if len(t["extdirs"]) > 0 { 1179 options = append(options, "-extdirs", t["extdirs"]) 1180 } 1181 if len(t["javaBuildDir"]) > 0 { 1182 options = append(options, "-java-build-dir", t["javaBuildDir"]) 1183 } 1184 if len(t["source"]) > 0 { 1185 options = append(options, "-source", t["source"]) 1186 } 1187 if len(t["jdk"]) > 0 { 1188 options = append(options, "-jdk", t["jdk"]) 1189 } 1190 if len(t["sourcepath"]) > 0 { 1191 options = append(options, "-sourcepath", t["sourcepath"]) 1192 } 1193 1194 case "pip": 1195 if len(t["autoClasspath"]) > 0 { 1196 options = append(options, "-python-path", t["autoClasspath"]) 1197 } else if len(t["pythonPath"]) > 0 { 1198 options = append(options, "-python-path", t["pythonPath"]) 1199 } 1200 if len(t["djangoTemplatDirs"]) > 0 { 1201 options = append(options, "-django-template-dirs", t["djangoTemplatDirs"]) 1202 } 1203 if len(t["pythonVersion"]) > 0 { 1204 options = append(options, "-python-version", t["pythonVersion"]) 1205 } 1206 1207 default: 1208 return options 1209 } 1210 1211 if len(t["exclude"]) > 0 { 1212 options = append(options, "-exclude", t["exclude"]) 1213 } 1214 return append(options, strings.Split(t["src"], ":")...) 1215 } 1216 1217 func getSuppliedOrDefaultList(suppliedList, defaultList []string) []string { 1218 if len(suppliedList) > 0 { 1219 return suppliedList 1220 } 1221 return defaultList 1222 } 1223 1224 func getSuppliedOrDefaultListAsString(suppliedList, defaultList []string, separator string) string { 1225 effectiveList := getSuppliedOrDefaultList(suppliedList, defaultList) 1226 return strings.Join(effectiveList, separator) 1227 } 1228 1229 // setTranslateEntryIfNotEmpty builds a string from either the user-supplied list, or the default list, 1230 // by joining the entries with the given separator. If the resulting string is not empty, it will be 1231 // placed as an entry in the provided map under the given key. 1232 func setTranslateEntryIfNotEmpty(translate map[string]interface{}, key, separator string, suppliedList, defaultList []string) { 1233 value := getSuppliedOrDefaultListAsString(suppliedList, defaultList, separator) 1234 if value != "" { 1235 translate[key] = value 1236 } 1237 } 1238 1239 // getSeparator returns the separator string depending on the host platform. This assumes that 1240 // Piper executes the Fortify command line tools within the same OS platform as it is running on itself. 1241 func getSeparator() string { 1242 if runtime.GOOS == "windows" { 1243 return ";" 1244 } 1245 return ":" 1246 } 1247 1248 func createToolRecordFortify(utils fortifyUtils, workspace string, config fortifyExecuteScanOptions, projectID int64, projectName string, projectVersionID int64, projectVersion string) (string, error) { 1249 record := toolrecord.New(utils, workspace, "fortify", config.ServerURL) 1250 // Project 1251 err := record.AddKeyData("project", 1252 strconv.FormatInt(projectID, 10), 1253 projectName, 1254 "") 1255 if err != nil { 1256 return "", err 1257 } 1258 // projectVersion 1259 projectVersionURL := config.ServerURL + "/html/ssc/version/" + strconv.FormatInt(projectVersionID, 10) 1260 err = record.AddKeyData("projectVersion", 1261 strconv.FormatInt(projectVersionID, 10), 1262 projectVersion, 1263 projectVersionURL) 1264 if err != nil { 1265 return "", err 1266 } 1267 err = record.Persist() 1268 if err != nil { 1269 return "", err 1270 } 1271 return record.GetFileName(), nil 1272 } 1273 1274 func getProxyParams(proxyUrl string) (string, string) { 1275 if proxyUrl == "" { 1276 return "", "" 1277 } 1278 1279 urlParams, err := url.Parse(proxyUrl) 1280 if err != nil { 1281 log.Entry().Warningf("Failed to parse proxy url %s", proxyUrl) 1282 return "", "" 1283 } 1284 return urlParams.Port(), urlParams.Hostname() 1285 }