github.com/SAP/jenkins-library@v1.362.0/cmd/checkmarxExecuteScan.go (about) 1 package cmd 2 3 import ( 4 "archive/zip" 5 "context" 6 "encoding/json" 7 "encoding/xml" 8 "fmt" 9 "io" 10 "math" 11 "os" 12 "path/filepath" 13 "regexp" 14 "sort" 15 "strconv" 16 "strings" 17 "time" 18 19 "github.com/SAP/jenkins-library/pkg/checkmarx" 20 piperGithub "github.com/SAP/jenkins-library/pkg/github" 21 piperHttp "github.com/SAP/jenkins-library/pkg/http" 22 "github.com/SAP/jenkins-library/pkg/log" 23 "github.com/SAP/jenkins-library/pkg/piperutils" 24 "github.com/SAP/jenkins-library/pkg/reporting" 25 "github.com/SAP/jenkins-library/pkg/telemetry" 26 "github.com/SAP/jenkins-library/pkg/toolrecord" 27 "github.com/bmatcuk/doublestar" 28 "github.com/pkg/errors" 29 30 "github.com/google/go-github/v45/github" 31 ) 32 33 type checkmarxExecuteScanUtils interface { 34 FileInfoHeader(fi os.FileInfo) (*zip.FileHeader, error) 35 Stat(name string) (os.FileInfo, error) 36 Open(name string) (*os.File, error) 37 WriteFile(filename string, data []byte, perm os.FileMode) error 38 MkdirAll(path string, perm os.FileMode) error 39 PathMatch(pattern, name string) (bool, error) 40 GetWorkspace() string 41 GetIssueService() *github.IssuesService 42 GetSearchService() *github.SearchService 43 } 44 45 type checkmarxExecuteScanUtilsBundle struct { 46 workspace string 47 issues *github.IssuesService 48 search *github.SearchService 49 } 50 51 func (c *checkmarxExecuteScanUtilsBundle) PathMatch(pattern, name string) (bool, error) { 52 return doublestar.PathMatch(pattern, name) 53 } 54 55 func (c *checkmarxExecuteScanUtilsBundle) GetWorkspace() string { 56 return c.workspace 57 } 58 59 func (c *checkmarxExecuteScanUtilsBundle) WriteFile(filename string, data []byte, perm os.FileMode) error { 60 return os.WriteFile(filename, data, perm) 61 } 62 63 func (c *checkmarxExecuteScanUtilsBundle) MkdirAll(path string, perm os.FileMode) error { 64 return os.MkdirAll(path, perm) 65 } 66 67 func (c *checkmarxExecuteScanUtilsBundle) FileInfoHeader(fi os.FileInfo) (*zip.FileHeader, error) { 68 return zip.FileInfoHeader(fi) 69 } 70 71 func (c *checkmarxExecuteScanUtilsBundle) Stat(name string) (os.FileInfo, error) { 72 return os.Stat(name) 73 } 74 75 func (c *checkmarxExecuteScanUtilsBundle) Open(name string) (*os.File, error) { 76 return os.Open(name) 77 } 78 79 func (c *checkmarxExecuteScanUtilsBundle) CreateIssue(ghCreateIssueOptions *piperGithub.CreateIssueOptions) error { 80 _, err := piperGithub.CreateIssue(ghCreateIssueOptions) 81 return err 82 } 83 84 func (c *checkmarxExecuteScanUtilsBundle) GetIssueService() *github.IssuesService { 85 return c.issues 86 } 87 88 func (c *checkmarxExecuteScanUtilsBundle) GetSearchService() *github.SearchService { 89 return c.search 90 } 91 92 func newCheckmarxExecuteScanUtilsBundle(workspace string, client *github.Client) checkmarxExecuteScanUtils { 93 utils := checkmarxExecuteScanUtilsBundle{ 94 workspace: workspace, 95 } 96 if client != nil { 97 utils.issues = client.Issues 98 utils.search = client.Search 99 } 100 return &utils 101 } 102 103 func checkmarxExecuteScan(config checkmarxExecuteScanOptions, _ *telemetry.CustomData, influx *checkmarxExecuteScanInflux) { 104 client := &piperHttp.Client{} 105 options := piperHttp.ClientOptions{MaxRetries: config.MaxRetries} 106 client.SetOptions(options) 107 // TODO provide parameter for trusted certs 108 ctx, ghClient, err := piperGithub.NewClientBuilder(config.GithubToken, config.GithubAPIURL).Build() 109 if err != nil { 110 log.Entry().WithError(err).Warning("Failed to get GitHub client") 111 } 112 sys, err := checkmarx.NewSystemInstance(client, config.ServerURL, config.Username, config.Password) 113 if err != nil { 114 log.Entry().WithError(err).Fatalf("Failed to create Checkmarx client talking to URL %v", config.ServerURL) 115 } 116 influx.step_data.fields.checkmarx = false 117 utils := newCheckmarxExecuteScanUtilsBundle("./", ghClient) 118 if err := runScan(ctx, config, sys, influx, utils); err != nil { 119 log.Entry().WithError(err).Fatal("Failed to execute Checkmarx scan.") 120 } 121 influx.step_data.fields.checkmarx = true 122 } 123 124 func runScan(ctx context.Context, config checkmarxExecuteScanOptions, sys checkmarx.System, influx *checkmarxExecuteScanInflux, utils checkmarxExecuteScanUtils) error { 125 teamID := config.TeamID 126 if len(teamID) == 0 { 127 readTeamID, err := loadTeamIDByTeamName(config, sys, teamID) 128 if err != nil { 129 return err 130 } 131 teamID = readTeamID 132 } 133 project, projectName, err := loadExistingProject(sys, config.ProjectName, config.PullRequestName, teamID) 134 if err != nil { 135 return errors.Wrap(err, "error when trying to load project") 136 } 137 if strings.EqualFold(project.Name, projectName) { // case insensitive string comparison 138 err = presetExistingProject(config, sys, projectName, project) 139 if err != nil { 140 return err 141 } 142 } else { 143 if len(teamID) == 0 { 144 return errors.Wrap(err, "TeamName or TeamID is required to create a new project") 145 } 146 project, err = createNewProject(config, sys, projectName, teamID) 147 if err != nil { 148 return err 149 } 150 } 151 152 err = uploadAndScan(ctx, config, sys, project, influx, utils) 153 if err != nil { 154 return errors.Wrap(err, "scan, upload, and result validation returned an error") 155 } 156 return nil 157 } 158 159 func loadTeamIDByTeamName(config checkmarxExecuteScanOptions, sys checkmarx.System, teamID string) (string, error) { 160 team, err := loadTeam(sys, config.TeamName) 161 if err != nil { 162 return "", errors.Wrap(err, "failed to load team") 163 } 164 teamIDBytes, _ := team.ID.MarshalJSON() 165 err = json.Unmarshal(teamIDBytes, &teamID) 166 if err != nil { 167 var teamIDInt int 168 err = json.Unmarshal(teamIDBytes, &teamIDInt) 169 if err != nil { 170 return "", errors.Wrap(err, "failed to unmarshall team.ID") 171 } 172 teamID = strconv.Itoa(teamIDInt) 173 } 174 return teamID, nil 175 } 176 177 func createNewProject(config checkmarxExecuteScanOptions, sys checkmarx.System, projectName string, teamID string) (checkmarx.Project, error) { 178 log.Entry().Infof("Project %v does not exist, starting to create it...", projectName) 179 presetID, _ := strconv.Atoi(config.Preset) 180 project, err := createAndConfigureNewProject(sys, projectName, teamID, presetID, config.Preset, config.EngineConfigurationID) 181 if err != nil { 182 return checkmarx.Project{}, errors.Wrapf(err, "failed to create and configure new project %v", projectName) 183 } 184 return project, nil 185 } 186 187 func presetExistingProject(config checkmarxExecuteScanOptions, sys checkmarx.System, projectName string, project checkmarx.Project) error { 188 log.Entry().Infof("Project %v exists...", projectName) 189 if len(config.Preset) > 0 { 190 presetID, _ := strconv.Atoi(config.Preset) 191 err := setPresetForProject(sys, project.ID, presetID, projectName, config.Preset, config.EngineConfigurationID) 192 if err != nil { 193 return errors.Wrapf(err, "failed to set preset %v for project %v", config.Preset, projectName) 194 } 195 } 196 return nil 197 } 198 199 func loadTeam(sys checkmarx.System, teamName string) (checkmarx.Team, error) { 200 teams := sys.GetTeams() 201 team := checkmarx.Team{} 202 var err error 203 if len(teams) > 0 && len(teamName) > 0 { 204 team, err = sys.FilterTeamByName(teams, teamName) 205 } 206 if err != nil { 207 return team, fmt.Errorf("failed to identify team by teamName %v", teamName) 208 } else { 209 return team, nil 210 } 211 } 212 213 func loadExistingProject(sys checkmarx.System, initialProjectName, pullRequestName, teamID string) (checkmarx.Project, string, error) { 214 var project checkmarx.Project 215 projectName := initialProjectName 216 if len(initialProjectName) == 0 { 217 return project, projectName, errors.New("You need to provide the Checkmarx project name, projectName parameter is mandatory") 218 } 219 if len(pullRequestName) > 0 { 220 projectName = fmt.Sprintf("%v_%v", initialProjectName, pullRequestName) 221 projects, err := sys.GetProjectsByNameAndTeam(projectName, teamID) 222 if err != nil || len(projects) == 0 { 223 projects, err = sys.GetProjectsByNameAndTeam(initialProjectName, teamID) 224 if err != nil { 225 return project, projectName, errors.Wrap(err, "failed getting projects") 226 } 227 if len(projects) == 0 { 228 return checkmarx.Project{}, projectName, nil 229 } 230 branchProject, err := sys.GetProjectByID(sys.CreateBranch(projects[0].ID, projectName)) 231 if err != nil { 232 return project, projectName, fmt.Errorf("failed to create branch %v for project %v", projectName, initialProjectName) 233 } 234 project = branchProject 235 } else { 236 project = projects[0] 237 log.Entry().Debugf("Loaded project with name %v", project.Name) 238 } 239 } else { 240 projects, err := sys.GetProjectsByNameAndTeam(projectName, teamID) 241 if err != nil { 242 return project, projectName, errors.Wrap(err, "failed getting projects") 243 } 244 if len(projects) == 0 { 245 return checkmarx.Project{}, projectName, nil 246 } 247 if len(projects) == 1 { 248 project = projects[0] 249 } else { 250 for _, current_project := range projects { 251 if projectName == current_project.Name { 252 project = current_project 253 break 254 } 255 } 256 if len(project.Name) == 0 { 257 return project, projectName, errors.New("Cannot find project " + projectName + ". You need to provide the teamName parameter if you want a new project to be created.") 258 } 259 } 260 log.Entry().Debugf("Loaded project with name %v", project.Name) 261 } 262 return project, projectName, nil 263 } 264 265 func zipWorkspaceFiles(filterPattern string, utils checkmarxExecuteScanUtils) (*os.File, error) { 266 zipFileName := filepath.Join(utils.GetWorkspace(), "workspace.zip") 267 patterns := piperutils.Trim(strings.Split(filterPattern, ",")) 268 sort.Strings(patterns) 269 zipFile, err := os.Create(zipFileName) 270 if err != nil { 271 return zipFile, errors.Wrap(err, "failed to create archive of project sources") 272 } 273 defer zipFile.Close() 274 err = zipFolder(utils.GetWorkspace(), zipFile, patterns, utils) 275 if err != nil { 276 return nil, errors.Wrap(err, "failed to compact folder") 277 } 278 return zipFile, nil 279 } 280 281 func uploadAndScan(ctx context.Context, config checkmarxExecuteScanOptions, sys checkmarx.System, project checkmarx.Project, influx *checkmarxExecuteScanInflux, utils checkmarxExecuteScanUtils) error { 282 previousScans, err := sys.GetScans(project.ID) 283 if err != nil && config.VerifyOnly { 284 log.Entry().Warnf("Cannot load scans for project %v, verification only mode aborted", project.Name) 285 } 286 if len(previousScans) > 0 && config.VerifyOnly { 287 err := verifyCxProjectCompliance(ctx, config, sys, previousScans[0].ID, influx, utils) 288 if err != nil { 289 log.SetErrorCategory(log.ErrorCompliance) 290 return errors.Wrapf(err, "project %v not compliant", project.Name) 291 } 292 } else { 293 zipFile, err := zipWorkspaceFiles(config.FilterPattern, utils) 294 if err != nil { 295 return errors.Wrap(err, "failed to zip workspace files") 296 } 297 err = sys.UploadProjectSourceCode(project.ID, zipFile.Name()) 298 if err != nil { 299 return errors.Wrapf(err, "failed to upload source code for project %v", project.Name) 300 } 301 302 log.Entry().Debugf("Source code uploaded for project %v", project.Name) 303 err = os.Remove(zipFile.Name()) 304 if err != nil { 305 log.Entry().WithError(err).Warnf("Failed to delete zipped source code for project %v", project.Name) 306 } 307 308 incremental := config.Incremental 309 fullScanCycle, err := strconv.Atoi(config.FullScanCycle) 310 if err != nil { 311 log.SetErrorCategory(log.ErrorConfiguration) 312 return errors.Wrapf(err, "invalid configuration value for fullScanCycle %v, must be a positive int", config.FullScanCycle) 313 } 314 315 if config.IsOptimizedAndScheduled { 316 incremental = false 317 } else if incremental && config.FullScansScheduled && fullScanCycle > 0 && (getNumCoherentIncrementalScans(previousScans)+1)%fullScanCycle == 0 { 318 incremental = false 319 } else if incremental && isLastScanFailedIncremental(previousScans) { // if the last incremental scan failed, trigger a full scan instead 320 incremental = false 321 log.Entry().Infof("Last incremental scan for project %v failed, triggering full scan instead", project.Name) 322 } 323 324 return triggerScan(ctx, config, sys, project, incremental, influx, utils) 325 } 326 return nil 327 } 328 329 func isLastScanFailedIncremental(scans []checkmarx.ScanStatus) bool { 330 if len(scans) == 0 { 331 return false 332 } 333 334 scan := scans[0] 335 if scan.IsIncremental && scan.Status.Name == "Failed" { 336 return true 337 } else { 338 return false 339 } 340 } 341 342 func triggerScan(ctx context.Context, config checkmarxExecuteScanOptions, sys checkmarx.System, project checkmarx.Project, incremental bool, influx *checkmarxExecuteScanInflux, utils checkmarxExecuteScanUtils) error { 343 scan, err := sys.ScanProject(project.ID, incremental, true, !config.AvoidDuplicateProjectScans) 344 if err != nil { 345 return errors.Wrapf(err, "cannot scan project %v", project.Name) 346 } 347 348 log.Entry().Debugf("Scanning project %v ", project.Name) 349 err = pollScanStatus(sys, scan) 350 if err != nil { 351 return errors.Wrap(err, "polling scan status failed") 352 } 353 354 log.Entry().Debugln("Scan finished") 355 return verifyCxProjectCompliance(ctx, config, sys, scan.ID, influx, utils) 356 } 357 358 func verifyCxProjectCompliance(ctx context.Context, config checkmarxExecuteScanOptions, sys checkmarx.System, scanID int, influx *checkmarxExecuteScanInflux, utils checkmarxExecuteScanUtils) error { 359 var reports []piperutils.Path 360 if config.GeneratePdfReport { 361 pdfReportName := createReportName(utils.GetWorkspace(), "CxSASTReport_%v.pdf") 362 err := downloadAndSaveReport(sys, pdfReportName, scanID, utils) 363 if err != nil { 364 log.Entry().Warning("Report download failed - continue processing ...") 365 } else { 366 reports = append(reports, piperutils.Path{Target: pdfReportName, Mandatory: true}) 367 } 368 } else { 369 log.Entry().Debug("Report generation is disabled via configuration") 370 } 371 372 xmlReportName := createReportName(utils.GetWorkspace(), "CxSASTResults_%v.xml") 373 results, err := getDetailedResults(config, sys, xmlReportName, scanID, utils) 374 if err != nil { 375 return errors.Wrap(err, "failed to get detailed results") 376 } 377 reports = append(reports, piperutils.Path{Target: xmlReportName}) 378 379 // generate sarif report 380 if config.ConvertToSarif { 381 log.Entry().Info("Calling conversion to SARIF function.") 382 sarif, err := checkmarx.ConvertCxxmlToSarif(sys, xmlReportName, scanID) 383 if err != nil { 384 return fmt.Errorf("failed to generate SARIF") 385 } 386 paths, err := checkmarx.WriteSarif(sarif) 387 if err != nil { 388 return fmt.Errorf("failed to write sarif") 389 } 390 reports = append(reports, paths...) 391 } 392 393 // create toolrecord 394 toolRecordFileName, err := createToolRecordCx(utils, utils.GetWorkspace(), config, results) 395 if err != nil { 396 // do not fail until the framework is well established 397 log.Entry().Warning("TR_CHECKMARX: Failed to create toolrecord file ...", err) 398 } else { 399 reports = append(reports, piperutils.Path{Target: toolRecordFileName}) 400 } 401 402 // create JSON report (regardless vulnerabilityThreshold enabled or not) 403 jsonReport := checkmarx.CreateJSONReport(results) 404 paths, err := checkmarx.WriteJSONReport(jsonReport) 405 if err != nil { 406 log.Entry().Warning("failed to write JSON report...", err) 407 } else { 408 // add JSON report to archiving list 409 reports = append(reports, paths...) 410 } 411 links := []piperutils.Path{{Target: results["DeepLink"].(string), Name: "Checkmarx Web UI"}} 412 413 insecure := false 414 var insecureResults []string 415 var neutralResults []string 416 417 if config.VulnerabilityThresholdEnabled { 418 insecure, insecureResults, neutralResults = enforceThresholds(config, results) 419 scanReport := checkmarx.CreateCustomReport(results, insecureResults, neutralResults) 420 421 if insecure && config.CreateResultIssue && len(config.GithubToken) > 0 && len(config.GithubAPIURL) > 0 && len(config.Owner) > 0 && len(config.Repository) > 0 { 422 log.Entry().Debug("Creating/updating GitHub issue with check results") 423 gh := reporting.GitHub{ 424 Owner: &config.Owner, 425 Repository: &config.Repository, 426 Assignees: &config.Assignees, 427 IssueService: utils.GetIssueService(), 428 SearchService: utils.GetSearchService(), 429 } 430 if err := gh.UploadSingleReport(ctx, scanReport); err != nil { 431 return fmt.Errorf("failed to upload scan results into GitHub: %w", err) 432 } 433 } 434 435 paths, err := checkmarx.WriteCustomReports(scanReport, fmt.Sprint(results["ProjectName"]), fmt.Sprint(results["ProjectID"])) 436 if err != nil { 437 // do not fail until we have a better idea to handle it 438 log.Entry().Warning("failed to write HTML/MarkDown report file ...", err) 439 } else { 440 reports = append(reports, paths...) 441 } 442 } 443 444 piperutils.PersistReportsAndLinks("checkmarxExecuteScan", utils.GetWorkspace(), utils, reports, links) 445 reportToInflux(results, influx) 446 447 if insecure { 448 if config.VulnerabilityThresholdResult == "FAILURE" { 449 log.SetErrorCategory(log.ErrorCompliance) 450 return fmt.Errorf("the project is not compliant - see report for details") 451 } 452 log.Entry().Errorf("Checkmarx scan result set to %v, some results are not meeting defined thresholds. For details see the archived report.", config.VulnerabilityThresholdResult) 453 } else { 454 log.Entry().Infoln("Checkmarx scan finished successfully") 455 } 456 return nil 457 } 458 459 func createReportName(workspace, reportFileNameTemplate string) string { 460 regExpFileName := regexp.MustCompile(`[^\w\d]`) 461 timeStamp, _ := time.Now().Local().MarshalText() 462 return filepath.Join(workspace, fmt.Sprintf(reportFileNameTemplate, regExpFileName.ReplaceAllString(string(timeStamp), "_"))) 463 } 464 465 func pollScanStatus(sys checkmarx.System, scan checkmarx.Scan) error { 466 status := "Scan phase: New" 467 pastStatus := status 468 log.Entry().Info(status) 469 stepDetail := "..." 470 stageDetail := "..." 471 for { 472 var detail checkmarx.ScanStatusDetail 473 status, detail = sys.GetScanStatusAndDetail(scan.ID) 474 if len(detail.Stage) > 0 { 475 stageDetail = detail.Stage 476 } 477 if len(detail.Step) > 0 { 478 stepDetail = detail.Step 479 } 480 if status == "Finished" || status == "Canceled" || status == "Failed" { 481 break 482 } 483 484 status = fmt.Sprintf("Scan phase: %v (%v / %v)", status, stageDetail, stepDetail) 485 if pastStatus != status { 486 log.Entry().Info(status) 487 pastStatus = status 488 } 489 log.Entry().Debug("Polling for status: sleeping...") 490 time.Sleep(10 * time.Second) 491 } 492 if status == "Canceled" { 493 log.SetErrorCategory(log.ErrorCustom) 494 return fmt.Errorf("scan canceled via web interface") 495 } 496 if status == "Failed" { 497 if strings.Contains(stageDetail, "<ErrorCode>17033</ErrorCode>") { // Translate a cryptic XML error into a human-readable message 498 stageDetail = "Failed to start scanning due to one of following reasons: source folder is empty, all source files are of an unsupported language or file format" 499 } 500 return fmt.Errorf("Checkmarx scan failed with the following error: %v", stageDetail) 501 } 502 return nil 503 } 504 505 func reportToInflux(results map[string]interface{}, influx *checkmarxExecuteScanInflux) { 506 influx.checkmarx_data.fields.high_issues = results["High"].(map[string]int)["Issues"] 507 influx.checkmarx_data.fields.high_not_false_positive = results["High"].(map[string]int)["NotFalsePositive"] 508 influx.checkmarx_data.fields.high_not_exploitable = results["High"].(map[string]int)["NotExploitable"] 509 influx.checkmarx_data.fields.high_confirmed = results["High"].(map[string]int)["Confirmed"] 510 influx.checkmarx_data.fields.high_urgent = results["High"].(map[string]int)["Urgent"] 511 influx.checkmarx_data.fields.high_proposed_not_exploitable = results["High"].(map[string]int)["ProposedNotExploitable"] 512 influx.checkmarx_data.fields.high_to_verify = results["High"].(map[string]int)["ToVerify"] 513 influx.checkmarx_data.fields.medium_issues = results["Medium"].(map[string]int)["Issues"] 514 influx.checkmarx_data.fields.medium_not_false_positive = results["Medium"].(map[string]int)["NotFalsePositive"] 515 influx.checkmarx_data.fields.medium_not_exploitable = results["Medium"].(map[string]int)["NotExploitable"] 516 influx.checkmarx_data.fields.medium_confirmed = results["Medium"].(map[string]int)["Confirmed"] 517 influx.checkmarx_data.fields.medium_urgent = results["Medium"].(map[string]int)["Urgent"] 518 influx.checkmarx_data.fields.medium_proposed_not_exploitable = results["Medium"].(map[string]int)["ProposedNotExploitable"] 519 influx.checkmarx_data.fields.medium_to_verify = results["Medium"].(map[string]int)["ToVerify"] 520 influx.checkmarx_data.fields.low_issues = results["Low"].(map[string]int)["Issues"] 521 influx.checkmarx_data.fields.low_not_false_positive = results["Low"].(map[string]int)["NotFalsePositive"] 522 influx.checkmarx_data.fields.low_not_exploitable = results["Low"].(map[string]int)["NotExploitable"] 523 influx.checkmarx_data.fields.low_confirmed = results["Low"].(map[string]int)["Confirmed"] 524 influx.checkmarx_data.fields.low_urgent = results["Low"].(map[string]int)["Urgent"] 525 influx.checkmarx_data.fields.low_proposed_not_exploitable = results["Low"].(map[string]int)["ProposedNotExploitable"] 526 influx.checkmarx_data.fields.low_to_verify = results["Low"].(map[string]int)["ToVerify"] 527 influx.checkmarx_data.fields.information_issues = results["Information"].(map[string]int)["Issues"] 528 influx.checkmarx_data.fields.information_not_false_positive = results["Information"].(map[string]int)["NotFalsePositive"] 529 influx.checkmarx_data.fields.information_not_exploitable = results["Information"].(map[string]int)["NotExploitable"] 530 influx.checkmarx_data.fields.information_confirmed = results["Information"].(map[string]int)["Confirmed"] 531 influx.checkmarx_data.fields.information_urgent = results["Information"].(map[string]int)["Urgent"] 532 influx.checkmarx_data.fields.information_proposed_not_exploitable = results["Information"].(map[string]int)["ProposedNotExploitable"] 533 influx.checkmarx_data.fields.information_to_verify = results["Information"].(map[string]int)["ToVerify"] 534 influx.checkmarx_data.fields.initiator_name = results["InitiatorName"].(string) 535 influx.checkmarx_data.fields.owner = results["Owner"].(string) 536 influx.checkmarx_data.fields.scan_id = results["ScanId"].(string) 537 influx.checkmarx_data.fields.project_id = results["ProjectId"].(string) 538 influx.checkmarx_data.fields.projectName = results["ProjectName"].(string) 539 influx.checkmarx_data.fields.team = results["Team"].(string) 540 influx.checkmarx_data.fields.team_full_path_on_report_date = results["TeamFullPathOnReportDate"].(string) 541 influx.checkmarx_data.fields.scan_start = results["ScanStart"].(string) 542 influx.checkmarx_data.fields.scan_time = results["ScanTime"].(string) 543 influx.checkmarx_data.fields.lines_of_code_scanned = results["LinesOfCodeScanned"].(int) 544 influx.checkmarx_data.fields.files_scanned = results["FilesScanned"].(int) 545 influx.checkmarx_data.fields.checkmarx_version = results["CheckmarxVersion"].(string) 546 influx.checkmarx_data.fields.scan_type = results["ScanType"].(string) 547 influx.checkmarx_data.fields.preset = results["Preset"].(string) 548 influx.checkmarx_data.fields.deep_link = results["DeepLink"].(string) 549 influx.checkmarx_data.fields.report_creation_time = results["ReportCreationTime"].(string) 550 } 551 552 func downloadAndSaveReport(sys checkmarx.System, reportFileName string, scanID int, utils checkmarxExecuteScanUtils) error { 553 report, err := generateAndDownloadReport(sys, scanID, "PDF") 554 if err != nil { 555 return errors.Wrap(err, "failed to download the report") 556 } 557 log.Entry().Debugf("Saving report to file %v...", reportFileName) 558 return utils.WriteFile(reportFileName, report, 0o700) 559 } 560 561 func enforceThresholds(config checkmarxExecuteScanOptions, results map[string]interface{}) (bool, []string, []string) { 562 neutralResults := []string{} 563 insecureResults := []string{} 564 insecure := false 565 cxHighThreshold := config.VulnerabilityThresholdHigh 566 cxMediumThreshold := config.VulnerabilityThresholdMedium 567 cxLowThreshold := config.VulnerabilityThresholdLow 568 cxLowThresholdPerQuery := config.VulnerabilityThresholdLowPerQuery 569 cxLowThresholdPerQueryMax := config.VulnerabilityThresholdLowPerQueryMax 570 highValue := results["High"].(map[string]int)["NotFalsePositive"] 571 mediumValue := results["Medium"].(map[string]int)["NotFalsePositive"] 572 lowValue := results["Low"].(map[string]int)["NotFalsePositive"] 573 var unit string 574 highViolation := "" 575 mediumViolation := "" 576 lowViolation := "" 577 if config.VulnerabilityThresholdUnit == "percentage" { 578 unit = "%" 579 highAudited := results["High"].(map[string]int)["Issues"] - results["High"].(map[string]int)["NotFalsePositive"] 580 highOverall := results["High"].(map[string]int)["Issues"] 581 if highOverall == 0 { 582 highAudited = 1 583 highOverall = 1 584 } 585 mediumAudited := results["Medium"].(map[string]int)["Issues"] - results["Medium"].(map[string]int)["NotFalsePositive"] 586 mediumOverall := results["Medium"].(map[string]int)["Issues"] 587 if mediumOverall == 0 { 588 mediumAudited = 1 589 mediumOverall = 1 590 } 591 lowAudited := results["Low"].(map[string]int)["Confirmed"] + results["Low"].(map[string]int)["NotExploitable"] 592 lowOverall := results["Low"].(map[string]int)["Issues"] 593 if lowOverall == 0 { 594 lowAudited = 1 595 lowOverall = 1 596 } 597 highValue = int(float32(highAudited) / float32(highOverall) * 100.0) 598 mediumValue = int(float32(mediumAudited) / float32(mediumOverall) * 100.0) 599 lowValue = int(float32(lowAudited) / float32(lowOverall) * 100.0) 600 601 if highValue < cxHighThreshold { 602 insecure = true 603 highViolation = fmt.Sprintf("<-- %v %v deviation", cxHighThreshold-highValue, unit) 604 } 605 if mediumValue < cxMediumThreshold { 606 insecure = true 607 mediumViolation = fmt.Sprintf("<-- %v %v deviation", cxMediumThreshold-mediumValue, unit) 608 } 609 // if the flag is switched on, calculate the Low findings threshold per query 610 if cxLowThresholdPerQuery { 611 lowPerQueryMap := results["LowPerQuery"].(map[string]map[string]int) 612 if lowPerQueryMap != nil { 613 for lowQuery, resultsLowQuery := range lowPerQueryMap { 614 lowAuditedPerQuery := resultsLowQuery["Confirmed"] + resultsLowQuery["NotExploitable"] 615 lowOverallPerQuery := resultsLowQuery["Issues"] 616 lowAuditedRequiredPerQuery := int(math.Ceil(float64(lowOverallPerQuery) * float64(cxLowThreshold) / 100.0)) 617 if lowAuditedPerQuery < lowAuditedRequiredPerQuery && lowAuditedPerQuery < cxLowThresholdPerQueryMax { 618 insecure = true 619 msgSeperator := "|" 620 if lowViolation == "" { 621 msgSeperator = "<--" 622 } 623 lowViolation += fmt.Sprintf(" %v query: %v, audited: %v, required: %v ", msgSeperator, lowQuery, lowAuditedPerQuery, lowAuditedRequiredPerQuery) 624 } 625 } 626 } 627 } else { // calculate the Low findings threshold in total 628 if lowValue < cxLowThreshold { 629 insecure = true 630 lowViolation = fmt.Sprintf("<-- %v %v deviation", cxLowThreshold-lowValue, unit) 631 } 632 } 633 634 } 635 if config.VulnerabilityThresholdUnit == "absolute" { 636 unit = " findings" 637 if highValue > cxHighThreshold { 638 insecure = true 639 highViolation = fmt.Sprintf("<-- %v%v deviation", highValue-cxHighThreshold, unit) 640 } 641 if mediumValue > cxMediumThreshold { 642 insecure = true 643 mediumViolation = fmt.Sprintf("<-- %v%v deviation", mediumValue-cxMediumThreshold, unit) 644 } 645 if lowValue > cxLowThreshold { 646 insecure = true 647 lowViolation = fmt.Sprintf("<-- %v%v deviation", lowValue-cxLowThreshold, unit) 648 } 649 } 650 651 highText := fmt.Sprintf("High %v%v %v", highValue, unit, highViolation) 652 mediumText := fmt.Sprintf("Medium %v%v %v", mediumValue, unit, mediumViolation) 653 lowText := fmt.Sprintf("Low %v%v %v", lowValue, unit, lowViolation) 654 if len(highViolation) > 0 { 655 insecureResults = append(insecureResults, highText) 656 log.Entry().Error(highText) 657 } else { 658 neutralResults = append(neutralResults, highText) 659 log.Entry().Info(highText) 660 } 661 if len(mediumViolation) > 0 { 662 insecureResults = append(insecureResults, mediumText) 663 log.Entry().Error(mediumText) 664 } else { 665 neutralResults = append(neutralResults, mediumText) 666 log.Entry().Info(mediumText) 667 } 668 if len(lowViolation) > 0 { 669 insecureResults = append(insecureResults, lowText) 670 log.Entry().Error(lowText) 671 } else { 672 neutralResults = append(neutralResults, lowText) 673 log.Entry().Info(lowText) 674 } 675 676 return insecure, insecureResults, neutralResults 677 } 678 679 func createAndConfigureNewProject(sys checkmarx.System, projectName, teamID string, presetIDValue int, presetValue, engineConfiguration string) (checkmarx.Project, error) { 680 if len(presetValue) == 0 { 681 log.SetErrorCategory(log.ErrorConfiguration) 682 return checkmarx.Project{}, fmt.Errorf("preset not specified, creation of project %v failed", projectName) 683 } 684 685 projectCreateResult, err := sys.CreateProject(projectName, teamID) 686 if err != nil { 687 return checkmarx.Project{}, errors.Wrapf(err, "cannot create project %v", projectName) 688 } 689 690 if err := setPresetForProject(sys, projectCreateResult.ID, presetIDValue, projectName, presetValue, engineConfiguration); err != nil { 691 return checkmarx.Project{}, errors.Wrapf(err, "failed to set preset %v for project", presetValue) 692 } 693 694 projects, err := sys.GetProjectsByNameAndTeam(projectName, teamID) 695 if err != nil || len(projects) == 0 { 696 return checkmarx.Project{}, errors.Wrapf(err, "failed to load newly created project %v", projectName) 697 } 698 log.Entry().Debugf("New Project %v created", projectName) 699 log.Entry().Debugf("Projects: %v", projects) 700 return projects[0], nil 701 } 702 703 // loadPreset finds a checkmarx.Preset that has either the ID or Name given by presetValue. 704 // presetValue is not expected to be empty. 705 func loadPreset(sys checkmarx.System, presetValue string) (checkmarx.Preset, error) { 706 presets := sys.GetPresets() 707 var preset checkmarx.Preset 708 var configuredPresetName string 709 preset = sys.FilterPresetByName(presets, presetValue) 710 configuredPresetName = presetValue 711 if len(configuredPresetName) > 0 && preset.Name == configuredPresetName { 712 log.Entry().Infof("Loaded preset %v", preset.Name) 713 return preset, nil 714 } 715 log.Entry().Infof("Preset '%s' not found. Available presets are:", presetValue) 716 for _, prs := range presets { 717 log.Entry().Infof("preset id: %v, name: '%v'", prs.ID, prs.Name) 718 } 719 return checkmarx.Preset{}, fmt.Errorf("preset %v not found", preset.Name) 720 } 721 722 // setPresetForProject is only called when it has already been established that the preset needs to be set. 723 // It will exit via the logging framework in case the preset could be found, or the project could not be updated. 724 func setPresetForProject(sys checkmarx.System, projectID, presetIDValue int, projectName, presetValue, engineConfiguration string) error { 725 presetID := presetIDValue 726 if presetID <= 0 { 727 preset, err := loadPreset(sys, presetValue) 728 if err != nil { 729 return errors.Wrapf(err, "preset %v not found, configuration of project %v failed", presetValue, projectName) 730 } 731 presetID = preset.ID 732 } 733 err := sys.UpdateProjectConfiguration(projectID, presetID, engineConfiguration) 734 if err != nil { 735 return errors.Wrapf(err, "updating configuration of project %v failed", projectName) 736 } 737 return nil 738 } 739 740 func generateAndDownloadReport(sys checkmarx.System, scanID int, reportType string) ([]byte, error) { 741 report, err := sys.RequestNewReport(scanID, reportType) 742 if err != nil { 743 return []byte{}, errors.Wrap(err, "failed to request new report") 744 } 745 finalStatus := 1 746 for { 747 reportStatus, err := sys.GetReportStatus(report.ReportID) 748 if err != nil { 749 return []byte{}, errors.Wrap(err, "failed to get report status") 750 } 751 finalStatus = reportStatus.Status.ID 752 if finalStatus != 1 { 753 break 754 } 755 time.Sleep(10 * time.Second) 756 } 757 if finalStatus == 2 { 758 return sys.DownloadReport(report.ReportID) 759 } 760 return []byte{}, fmt.Errorf("unexpected status %v recieved", finalStatus) 761 } 762 763 func getNumCoherentIncrementalScans(scans []checkmarx.ScanStatus) int { 764 count := 0 765 for _, scan := range scans { 766 if !scan.IsIncremental { 767 break 768 } 769 count++ 770 } 771 return count 772 } 773 774 func getDetailedResults(config checkmarxExecuteScanOptions, sys checkmarx.System, reportFileName string, scanID int, utils checkmarxExecuteScanUtils) (map[string]interface{}, error) { 775 resultMap := map[string]interface{}{} 776 data, err := generateAndDownloadReport(sys, scanID, "XML") 777 if err != nil { 778 return resultMap, errors.Wrap(err, "failed to download xml report") 779 } 780 if len(data) > 0 { 781 err = utils.WriteFile(reportFileName, data, 0o700) 782 if err != nil { 783 return resultMap, errors.Wrap(err, "failed to write file") 784 } 785 var xmlResult checkmarx.DetailedResult 786 err := xml.Unmarshal(data, &xmlResult) 787 if err != nil { 788 return resultMap, errors.Wrapf(err, "failed to unmarshal XML report for scan %v", scanID) 789 } 790 resultMap["InitiatorName"] = xmlResult.InitiatorName 791 resultMap["Owner"] = xmlResult.Owner 792 resultMap["ScanId"] = xmlResult.ScanID 793 resultMap["ProjectId"] = xmlResult.ProjectID 794 resultMap["ProjectName"] = xmlResult.ProjectName 795 resultMap["Team"] = xmlResult.Team 796 resultMap["TeamFullPathOnReportDate"] = xmlResult.TeamFullPathOnReportDate 797 resultMap["ScanStart"] = xmlResult.ScanStart 798 resultMap["ScanTime"] = xmlResult.ScanTime 799 resultMap["LinesOfCodeScanned"] = xmlResult.LinesOfCodeScanned 800 resultMap["FilesScanned"] = xmlResult.FilesScanned 801 resultMap["CheckmarxVersion"] = xmlResult.CheckmarxVersion 802 resultMap["ScanType"] = xmlResult.ScanType 803 resultMap["Preset"] = xmlResult.Preset 804 resultMap["DeepLink"] = xmlResult.DeepLink 805 resultMap["ReportCreationTime"] = xmlResult.ReportCreationTime 806 resultMap["High"] = map[string]int{} 807 resultMap["Medium"] = map[string]int{} 808 resultMap["Low"] = map[string]int{} 809 resultMap["Information"] = map[string]int{} 810 for _, query := range xmlResult.Queries { 811 for _, result := range query.Results { 812 key := result.Severity 813 var submap map[string]int 814 if resultMap[key] == nil { 815 submap = map[string]int{} 816 resultMap[key] = submap 817 } else { 818 submap = resultMap[key].(map[string]int) 819 } 820 submap["Issues"]++ 821 822 auditState := "ToVerify" 823 switch result.State { 824 case "1": 825 auditState = "NotExploitable" 826 case "2": 827 auditState = "Confirmed" 828 case "3": 829 auditState = "Urgent" 830 case "4": 831 auditState = "ProposedNotExploitable" 832 case "0": 833 default: 834 auditState = "ToVerify" 835 } 836 submap[auditState]++ 837 838 if result.FalsePositive != "True" { 839 submap["NotFalsePositive"]++ 840 } 841 } 842 } 843 844 // if the flag is switched on, build the list of Low findings per query 845 if config.VulnerabilityThresholdLowPerQuery { 846 var lowPerQuery = map[string]map[string]int{} 847 for _, query := range xmlResult.Queries { 848 for _, result := range query.Results { 849 if result.Severity != "Low" { 850 continue 851 } 852 key := query.Name 853 var submap map[string]int 854 if lowPerQuery[key] == nil { 855 submap = map[string]int{} 856 lowPerQuery[key] = submap 857 } else { 858 submap = lowPerQuery[key] 859 } 860 submap["Issues"]++ 861 auditState := "ToVerify" 862 switch result.State { 863 case "1": 864 auditState = "NotExploitable" 865 break 866 case "2": 867 auditState = "Confirmed" 868 break 869 case "3": 870 auditState = "Urgent" 871 break 872 case "4": 873 auditState = "ProposedNotExploitable" 874 break 875 case "0": 876 default: 877 auditState = "ToVerify" 878 break 879 } 880 submap[auditState]++ 881 882 if result.FalsePositive != "True" { 883 submap["NotFalsePositive"]++ 884 } 885 } 886 } 887 resultMap["LowPerQuery"] = lowPerQuery 888 } 889 } 890 return resultMap, nil 891 } 892 893 func zipFolder(source string, zipFile io.Writer, patterns []string, utils checkmarxExecuteScanUtils) error { 894 archive := zip.NewWriter(zipFile) 895 defer archive.Close() 896 897 info, err := utils.Stat(source) 898 if err != nil { 899 return nil 900 } 901 902 var baseDir string 903 if info.IsDir() { 904 baseDir = filepath.Base(source) 905 } 906 907 fileCount := 0 908 err = filepath.Walk(source, func(path string, info os.FileInfo, err error) error { 909 if err != nil { 910 return err 911 } 912 913 noMatch, err := isFileNotMatchingPattern(patterns, path, info, utils) 914 if err != nil || noMatch { 915 return err 916 } 917 918 header, err := utils.FileInfoHeader(info) 919 if err != nil { 920 return err 921 } 922 923 if baseDir != "" { 924 header.Name = filepath.Join(baseDir, strings.TrimPrefix(path, source)) 925 } 926 927 adaptHeader(info, header) 928 929 writer, err := archive.CreateHeader(header) 930 if err != nil || info.IsDir() { 931 return err 932 } 933 934 file, err := utils.Open(path) 935 if err != nil { 936 return err 937 } 938 defer file.Close() 939 _, err = io.Copy(writer, file) 940 fileCount++ 941 return err 942 }) 943 log.Entry().Infof("Zipped %d files", fileCount) 944 err = handleZeroFilesZipped(source, err, fileCount) 945 return err 946 } 947 948 func adaptHeader(info os.FileInfo, header *zip.FileHeader) { 949 if info.IsDir() { 950 header.Name += "/" 951 } else { 952 header.Method = zip.Deflate 953 } 954 } 955 956 func handleZeroFilesZipped(source string, err error, fileCount int) error { 957 if err == nil && fileCount == 0 { 958 log.SetErrorCategory(log.ErrorConfiguration) 959 err = fmt.Errorf("filterPattern matched no files or workspace directory '%s' was empty", source) 960 } 961 return err 962 } 963 964 // isFileNotMatchingPattern checks if file path does not match one of the patterns. 965 // If it matches a negative pattern (starting with '!') then true is returned. 966 // 967 // If it is a directory, false is returned. 968 // If no patterns are provided, false is returned. 969 func isFileNotMatchingPattern(patterns []string, path string, info os.FileInfo, utils checkmarxExecuteScanUtils) (bool, error) { 970 if len(patterns) == 0 || info.IsDir() { 971 return false, nil 972 } 973 974 for _, pattern := range patterns { 975 negative := false 976 if strings.HasPrefix(pattern, "!") { 977 pattern = strings.TrimLeft(pattern, "!") 978 negative = true 979 } 980 match, err := utils.PathMatch(pattern, path) 981 if err != nil { 982 return false, errors.Wrapf(err, "Pattern %v could not get executed", pattern) 983 } 984 985 if match { 986 return negative, nil 987 } 988 } 989 return true, nil 990 } 991 992 func createToolRecordCx(utils checkmarxExecuteScanUtils, workspace string, config checkmarxExecuteScanOptions, results map[string]interface{}) (string, error) { 993 record := toolrecord.New(utils, workspace, "checkmarx", config.ServerURL) 994 // Todo TeamId - see run_scan() 995 // record.AddKeyData("team", XXX, resultMap["Team"], "") 996 // Project 997 err := record.AddKeyData("project", 998 results["ProjectId"].(string), 999 results["ProjectName"].(string), 1000 "") 1001 if err != nil { 1002 return "", err 1003 } 1004 // Scan 1005 err = record.AddKeyData("scanid", 1006 results["ScanId"].(string), 1007 results["ScanId"].(string), 1008 results["DeepLink"].(string)) 1009 if err != nil { 1010 return "", err 1011 } 1012 err = record.Persist() 1013 if err != nil { 1014 return "", err 1015 } 1016 return record.GetFileName(), nil 1017 }