github.com/jaylevin/jenkins-library@v1.230.4/cmd/whitesourceExecuteScan.go (about) 1 package cmd 2 3 import ( 4 "encoding/json" 5 "fmt" 6 "os" 7 "path/filepath" 8 "strconv" 9 "strings" 10 "time" 11 12 piperDocker "github.com/SAP/jenkins-library/pkg/docker" 13 piperGithub "github.com/SAP/jenkins-library/pkg/github" 14 piperhttp "github.com/SAP/jenkins-library/pkg/http" 15 ws "github.com/SAP/jenkins-library/pkg/whitesource" 16 17 "github.com/SAP/jenkins-library/pkg/command" 18 "github.com/SAP/jenkins-library/pkg/log" 19 "github.com/SAP/jenkins-library/pkg/npm" 20 "github.com/SAP/jenkins-library/pkg/piperutils" 21 "github.com/SAP/jenkins-library/pkg/reporting" 22 "github.com/SAP/jenkins-library/pkg/telemetry" 23 "github.com/SAP/jenkins-library/pkg/toolrecord" 24 "github.com/SAP/jenkins-library/pkg/versioning" 25 "github.com/pkg/errors" 26 "github.com/xuri/excelize/v2" 27 ) 28 29 // ScanOptions is just used to make the lines less long 30 type ScanOptions = whitesourceExecuteScanOptions 31 32 // WhiteSource defines the functions that are expected by the step implementation to 33 // be available from the WhiteSource system. 34 type whitesource interface { 35 GetProductByName(productName string) (ws.Product, error) 36 CreateProduct(productName string) (string, error) 37 SetProductAssignments(productToken string, membership, admins, alertReceivers *ws.Assignment) error 38 GetProjectsMetaInfo(productToken string) ([]ws.Project, error) 39 GetProjectToken(productToken, projectName string) (string, error) 40 GetProjectByToken(projectToken string) (ws.Project, error) 41 GetProjectRiskReport(projectToken string) ([]byte, error) 42 GetProjectVulnerabilityReport(projectToken string, format string) ([]byte, error) 43 GetProjectAlerts(projectToken string) ([]ws.Alert, error) 44 GetProjectAlertsByType(projectToken, alertType string) ([]ws.Alert, error) 45 GetProjectLibraryLocations(projectToken string) ([]ws.Library, error) 46 } 47 48 type whitesourceUtils interface { 49 ws.Utils 50 piperutils.FileUtils 51 GetArtifactCoordinates(buildTool, buildDescriptorFile string, 52 options *versioning.Options) (versioning.Coordinates, error) 53 54 CreateIssue(ghCreateIssueOptions *piperGithub.CreateIssueOptions) error 55 56 Now() time.Time 57 } 58 59 type whitesourceUtilsBundle struct { 60 *piperhttp.Client 61 *command.Command 62 *piperutils.Files 63 npmExecutor npm.Executor 64 } 65 66 // CreateIssue supplies capability for GitHub issue creation 67 func (w *whitesourceUtilsBundle) CreateIssue(ghCreateIssueOptions *piperGithub.CreateIssueOptions) error { 68 return piperGithub.CreateIssue(ghCreateIssueOptions) 69 } 70 71 func (w *whitesourceUtilsBundle) FileOpen(name string, flag int, perm os.FileMode) (ws.File, error) { 72 return os.OpenFile(name, flag, perm) 73 } 74 75 func (w *whitesourceUtilsBundle) GetArtifactCoordinates(buildTool, buildDescriptorFile string, options *versioning.Options) (versioning.Coordinates, error) { 76 artifact, err := versioning.GetArtifact(buildTool, buildDescriptorFile, options, w) 77 if err != nil { 78 return versioning.Coordinates{}, err 79 } 80 return artifact.GetCoordinates() 81 } 82 83 func (w *whitesourceUtilsBundle) getNpmExecutor(config *ws.ScanOptions) npm.Executor { 84 if w.npmExecutor == nil { 85 w.npmExecutor = npm.NewExecutor(npm.ExecutorOptions{DefaultNpmRegistry: config.DefaultNpmRegistry}) 86 } 87 return w.npmExecutor 88 } 89 90 func (w *whitesourceUtilsBundle) FindPackageJSONFiles(config *ws.ScanOptions) ([]string, error) { 91 return w.getNpmExecutor(config).FindPackageJSONFilesWithExcludes(config.BuildDescriptorExcludeList) 92 } 93 94 func (w *whitesourceUtilsBundle) InstallAllNPMDependencies(config *ws.ScanOptions, packageJSONFiles []string) error { 95 return w.getNpmExecutor(config).InstallAllDependencies(packageJSONFiles) 96 } 97 98 func (w *whitesourceUtilsBundle) SetOptions(o piperhttp.ClientOptions) { 99 w.Client.SetOptions(o) 100 } 101 102 func (w *whitesourceUtilsBundle) Now() time.Time { 103 return time.Now() 104 } 105 106 func newWhitesourceUtils(config *ScanOptions) *whitesourceUtilsBundle { 107 utils := whitesourceUtilsBundle{ 108 Client: &piperhttp.Client{}, 109 Command: &command.Command{}, 110 Files: &piperutils.Files{}, 111 } 112 // Reroute cmd output to logging framework 113 utils.Stdout(log.Writer()) 114 utils.Stderr(log.Writer()) 115 // Configure HTTP Client 116 utils.SetOptions(piperhttp.ClientOptions{TransportTimeout: time.Duration(config.Timeout) * time.Second}) 117 return &utils 118 } 119 120 func newWhitesourceScan(config *ScanOptions) *ws.Scan { 121 return &ws.Scan{ 122 AggregateProjectName: config.ProjectName, 123 ProductVersion: config.Version, 124 } 125 } 126 127 func whitesourceExecuteScan(config ScanOptions, _ *telemetry.CustomData, commonPipelineEnvironment *whitesourceExecuteScanCommonPipelineEnvironment, influx *whitesourceExecuteScanInflux) { 128 utils := newWhitesourceUtils(&config) 129 scan := newWhitesourceScan(&config) 130 sys := ws.NewSystem(config.ServiceURL, config.OrgToken, config.UserToken, time.Duration(config.Timeout)*time.Second) 131 influx.step_data.fields.whitesource = false 132 err := runWhitesourceExecuteScan(&config, scan, utils, sys, commonPipelineEnvironment, influx) 133 if err != nil { 134 log.Entry().WithError(err).Fatal("step execution failed") 135 } 136 influx.step_data.fields.whitesource = true 137 } 138 139 func runWhitesourceExecuteScan(config *ScanOptions, scan *ws.Scan, utils whitesourceUtils, sys whitesource, commonPipelineEnvironment *whitesourceExecuteScanCommonPipelineEnvironment, influx *whitesourceExecuteScanInflux) error { 140 if err := resolveAggregateProjectName(config, scan, sys); err != nil { 141 return errors.Wrapf(err, "failed to resolve and aggregate project name") 142 } 143 144 if err := resolveProjectIdentifiers(config, scan, utils, sys); err != nil { 145 if strings.Contains(fmt.Sprint(err), "User is not allowed to perform this action") { 146 log.SetErrorCategory(log.ErrorConfiguration) 147 } 148 return errors.Wrapf(err, "failed to resolve project identifiers") 149 } 150 151 if config.AggregateVersionWideReport { 152 // Generate a vulnerability report for all projects with version = config.ProjectVersion 153 // Note that this is not guaranteed that all projects are from the same scan. 154 // For example, if a module was removed from the source code, the project may still 155 // exist in the WhiteSource system. 156 if err := aggregateVersionWideLibraries(config, utils, sys); err != nil { 157 return errors.Wrapf(err, "failed to aggregate version wide libraries") 158 } 159 if err := aggregateVersionWideVulnerabilities(config, utils, sys); err != nil { 160 return errors.Wrapf(err, "failed to aggregate version wide vulnerabilities") 161 } 162 } else { 163 if err := runWhitesourceScan(config, scan, utils, sys, commonPipelineEnvironment, influx); err != nil { 164 return errors.Wrapf(err, "failed to execute WhiteSource scan") 165 } 166 } 167 return nil 168 } 169 170 func runWhitesourceScan(config *ScanOptions, scan *ws.Scan, utils whitesourceUtils, sys whitesource, commonPipelineEnvironment *whitesourceExecuteScanCommonPipelineEnvironment, influx *whitesourceExecuteScanInflux) error { 171 // Download Docker image for container scan 172 // ToDo: move it to improve testability 173 if config.BuildTool == "docker" { 174 saveImageOptions := containerSaveImageOptions{ 175 ContainerImage: config.ScanImage, 176 ContainerRegistryURL: config.ScanImageRegistryURL, 177 ContainerRegistryUser: config.ContainerRegistryUser, 178 ContainerRegistryPassword: config.ContainerRegistryPassword, 179 DockerConfigJSON: config.DockerConfigJSON, 180 FilePath: config.ProjectName, 181 ImageFormat: "legacy", // keep the image format legacy or whitesource is not able to read layers 182 } 183 dClientOptions := piperDocker.ClientOptions{ImageName: saveImageOptions.ContainerImage, RegistryURL: saveImageOptions.ContainerRegistryURL, LocalPath: "", ImageFormat: "legacy"} 184 dClient := &piperDocker.Client{} 185 dClient.SetOptions(dClientOptions) 186 if _, err := runContainerSaveImage(&saveImageOptions, &telemetry.CustomData{}, "./cache", "", dClient, utils); err != nil { 187 if strings.Contains(fmt.Sprint(err), "no image found") { 188 log.SetErrorCategory(log.ErrorConfiguration) 189 } 190 return errors.Wrapf(err, "failed to download Docker image %v", config.ScanImage) 191 } 192 193 } 194 195 // Start the scan 196 if err := executeScan(config, scan, utils); err != nil { 197 return errors.Wrapf(err, "failed to execute Scan") 198 } 199 200 // ToDo: Check this: 201 // Why is this required at all, resolveProjectIdentifiers() is already called before the scan in runWhitesourceExecuteScan() 202 // Could perhaps use scan.updateProjects(sys) directly... have not investigated what could break 203 if err := resolveProjectIdentifiers(config, scan, utils, sys); err != nil { 204 return errors.Wrapf(err, "failed to resolve project identifiers") 205 } 206 207 log.Entry().Info("-----------------------------------------------------") 208 log.Entry().Infof("Product Version: '%s'", config.Version) 209 log.Entry().Info("Scanned projects:") 210 for _, project := range scan.ScannedProjects() { 211 log.Entry().Infof(" Name: '%s', token: %s", project.Name, project.Token) 212 } 213 log.Entry().Info("-----------------------------------------------------") 214 215 paths, err := checkAndReportScanResults(config, scan, utils, sys, influx) 216 piperutils.PersistReportsAndLinks("whitesourceExecuteScan", "", paths, nil) 217 persistScannedProjects(config, scan, commonPipelineEnvironment) 218 if err != nil { 219 return errors.Wrapf(err, "failed to check and report scan results") 220 } 221 return nil 222 } 223 224 func checkAndReportScanResults(config *ScanOptions, scan *ws.Scan, utils whitesourceUtils, sys whitesource, influx *whitesourceExecuteScanInflux) ([]piperutils.Path, error) { 225 reportPaths := []piperutils.Path{} 226 if !config.Reporting && !config.SecurityVulnerabilities { 227 return reportPaths, nil 228 } 229 // Wait for WhiteSource backend to propagate the changes before downloading any reports. 230 if err := scan.BlockUntilReportsAreReady(sys); err != nil { 231 return reportPaths, err 232 } 233 234 if config.Reporting { 235 var err error 236 reportPaths, err = scan.DownloadReports(ws.ReportOptions{ 237 ReportDirectory: ws.ReportsDirectory, 238 VulnerabilityReportFormat: config.VulnerabilityReportFormat, 239 }, utils, sys) 240 if err != nil { 241 return reportPaths, err 242 } 243 } 244 245 checkErrors := []string{} 246 247 rPath, err := checkPolicyViolations(config, scan, sys, utils, reportPaths, influx) 248 if err != nil { 249 checkErrors = append(checkErrors, fmt.Sprint(err)) 250 } 251 reportPaths = append(reportPaths, rPath) 252 253 if config.SecurityVulnerabilities { 254 rPaths, err := checkSecurityViolations(config, scan, sys, utils, influx) 255 reportPaths = append(reportPaths, rPaths...) 256 if err != nil { 257 checkErrors = append(checkErrors, fmt.Sprint(err)) 258 } 259 } 260 261 // create toolrecord file 262 // tbd - how to handle verifyOnly 263 toolRecordFileName, err := createToolRecordWhitesource("./", config, scan) 264 if err != nil { 265 // do not fail until the framework is well established 266 log.Entry().Warning("TR_WHITESOURCE: Failed to create toolrecord file ...", err) 267 } else { 268 reportPaths = append(reportPaths, piperutils.Path{Target: toolRecordFileName}) 269 } 270 271 if len(checkErrors) > 0 { 272 return reportPaths, fmt.Errorf(strings.Join(checkErrors, ": ")) 273 } 274 return reportPaths, nil 275 } 276 277 func createWhiteSourceProduct(config *ScanOptions, sys whitesource) (string, error) { 278 log.Entry().Infof("Attempting to create new WhiteSource product for '%s'..", config.ProductName) 279 productToken, err := sys.CreateProduct(config.ProductName) 280 if err != nil { 281 return "", fmt.Errorf("failed to create WhiteSource product: %w", err) 282 } 283 284 var admins ws.Assignment 285 for _, address := range config.EmailAddressesOfInitialProductAdmins { 286 admins.UserAssignments = append(admins.UserAssignments, ws.UserAssignment{Email: address}) 287 } 288 289 err = sys.SetProductAssignments(productToken, nil, &admins, nil) 290 if err != nil { 291 return "", fmt.Errorf("failed to set admins on new WhiteSource product: %w", err) 292 } 293 294 return productToken, nil 295 } 296 297 func resolveProjectIdentifiers(config *ScanOptions, scan *ws.Scan, utils whitesourceUtils, sys whitesource) error { 298 if len(scan.AggregateProjectName) > 0 && (len(config.Version)+len(config.CustomScanVersion) > 0) { 299 if config.Version == "" { 300 config.Version = config.CustomScanVersion 301 } 302 } else { 303 options := &versioning.Options{ 304 DockerImage: config.ScanImage, 305 ProjectSettingsFile: config.ProjectSettingsFile, 306 GlobalSettingsFile: config.GlobalSettingsFile, 307 M2Path: config.M2Path, 308 } 309 coordinates, err := utils.GetArtifactCoordinates(config.BuildTool, config.BuildDescriptorFile, options) 310 if err != nil { 311 return errors.Wrap(err, "failed to get build artifact description") 312 } 313 314 if len(config.Version) > 0 { 315 log.Entry().Infof("Resolving product version from default provided '%s' with versioning '%s'", config.Version, config.VersioningModel) 316 coordinates.Version = config.Version 317 } 318 319 nameTmpl := `{{list .GroupID .ArtifactID | join "-" | trimAll "-"}}` 320 name, version := versioning.DetermineProjectCoordinatesWithCustomVersion(nameTmpl, config.VersioningModel, config.CustomScanVersion, coordinates) 321 if scan.AggregateProjectName == "" { 322 log.Entry().Infof("Resolved project name '%s' from descriptor file", name) 323 scan.AggregateProjectName = name 324 } 325 326 config.Version = version 327 log.Entry().Infof("Resolved product version '%s'", version) 328 } 329 330 scan.ProductVersion = validateProductVersion(config.Version) 331 332 if err := resolveProductToken(config, sys); err != nil { 333 return errors.Wrap(err, "error resolving product token") 334 } 335 if err := resolveAggregateProjectToken(config, sys); err != nil { 336 return errors.Wrap(err, "error resolving aggregate project token") 337 } 338 339 return scan.UpdateProjects(config.ProductToken, sys) 340 } 341 342 // resolveProductToken resolves the token of the WhiteSource Product specified by config.ProductName, 343 // unless the user provided a token in config.ProductToken already, or it was previously resolved. 344 // If no Product can be found for the given config.ProductName, and the parameter 345 // config.CreatePipelineFromProduct is set, an attempt will be made to create the product and 346 // configure the initial product admins. 347 func resolveProductToken(config *ScanOptions, sys whitesource) error { 348 if config.ProductToken != "" { 349 return nil 350 } 351 log.Entry().Infof("Attempting to resolve product token for product '%s'..", config.ProductName) 352 product, err := sys.GetProductByName(config.ProductName) 353 if err != nil && config.CreateProductFromPipeline { 354 product = ws.Product{} 355 product.Token, err = createWhiteSourceProduct(config, sys) 356 if err != nil { 357 return errors.Wrapf(err, "failed to create whitesource product") 358 } 359 } 360 if err != nil { 361 return errors.Wrapf(err, "failed to get product by name") 362 } 363 log.Entry().Infof("Resolved product token: '%s'..", product.Token) 364 config.ProductToken = product.Token 365 return nil 366 } 367 368 // resolveAggregateProjectName checks if config.ProjectToken is configured, and if so, expects a WhiteSource 369 // project with that token to exist. The AggregateProjectName in the ws.Scan is then configured with that 370 // project's name. 371 func resolveAggregateProjectName(config *ScanOptions, scan *ws.Scan, sys whitesource) error { 372 if config.ProjectToken == "" { 373 return nil 374 } 375 log.Entry().Infof("Attempting to resolve aggregate project name for token '%s'..", config.ProjectToken) 376 // If the user configured the "projectToken" parameter, we expect this project to exist in the backend. 377 project, err := sys.GetProjectByToken(config.ProjectToken) 378 if err != nil { 379 return errors.Wrapf(err, "failed to get project by token") 380 } 381 nameVersion := strings.Split(project.Name, " - ") 382 scan.AggregateProjectName = nameVersion[0] 383 log.Entry().Infof("Resolve aggregate project name '%s'..", scan.AggregateProjectName) 384 return nil 385 } 386 387 // resolveAggregateProjectToken fetches the token of the WhiteSource Project specified by config.ProjectName 388 // and stores it in config.ProjectToken. 389 // The user can configure a projectName or projectToken of the project to be used as for aggregation of scan results. 390 func resolveAggregateProjectToken(config *ScanOptions, sys whitesource) error { 391 if config.ProjectToken != "" || config.ProjectName == "" { 392 return nil 393 } 394 log.Entry().Infof("Attempting to resolve project token for project '%s'..", config.ProjectName) 395 fullProjName := fmt.Sprintf("%s - %s", config.ProjectName, config.Version) 396 projectToken, err := sys.GetProjectToken(config.ProductToken, fullProjName) 397 if err != nil { 398 return errors.Wrapf(err, "failed to get project token") 399 } 400 // A project may not yet exist for this project name-version combo. 401 // It will be created by the scan, we retrieve the token again after scanning. 402 if projectToken != "" { 403 log.Entry().Infof("Resolved project token: '%s'..", projectToken) 404 config.ProjectToken = projectToken 405 } else { 406 log.Entry().Infof("Project '%s' not yet present in WhiteSource", fullProjName) 407 } 408 return nil 409 } 410 411 // validateProductVersion makes sure that the version does not contain a dash "-". 412 func validateProductVersion(version string) string { 413 // TrimLeft() removes all "-" from the beginning, unlike TrimPrefix()! 414 version = strings.TrimLeft(version, "-") 415 if strings.Contains(version, "-") { 416 version = strings.SplitN(version, "-", 1)[0] 417 } 418 return version 419 } 420 421 func wsScanOptions(config *ScanOptions) *ws.ScanOptions { 422 return &ws.ScanOptions{ 423 BuildTool: config.BuildTool, 424 ScanType: "", // no longer provided via config 425 OrgToken: config.OrgToken, 426 UserToken: config.UserToken, 427 ProductName: config.ProductName, 428 ProductToken: config.ProductToken, 429 ProductVersion: config.Version, 430 ProjectName: config.ProjectName, 431 BuildDescriptorFile: config.BuildDescriptorFile, 432 BuildDescriptorExcludeList: config.BuildDescriptorExcludeList, 433 PomPath: config.BuildDescriptorFile, 434 M2Path: config.M2Path, 435 GlobalSettingsFile: config.GlobalSettingsFile, 436 ProjectSettingsFile: config.ProjectSettingsFile, 437 InstallArtifacts: config.InstallArtifacts, 438 DefaultNpmRegistry: config.DefaultNpmRegistry, 439 AgentDownloadURL: config.AgentDownloadURL, 440 AgentFileName: config.AgentFileName, 441 ConfigFilePath: config.ConfigFilePath, 442 Includes: config.Includes, 443 Excludes: config.Excludes, 444 JreDownloadURL: config.JreDownloadURL, 445 AgentURL: config.AgentURL, 446 ServiceURL: config.ServiceURL, 447 ScanPath: config.ScanPath, 448 Verbose: GeneralConfig.Verbose, 449 } 450 } 451 452 // Unified Agent is the only supported option by WhiteSource going forward: 453 // The Unified Agent will be used to perform the scan. 454 func executeScan(config *ScanOptions, scan *ws.Scan, utils whitesourceUtils) error { 455 456 options := wsScanOptions(config) 457 458 // Execute scan with Unified Agent jar file 459 if err := scan.ExecuteUAScan(options, utils); err != nil { 460 return errors.Wrapf(err, "failed to execute Unified Agent scan") 461 } 462 return nil 463 } 464 465 func checkPolicyViolations(config *ScanOptions, scan *ws.Scan, sys whitesource, utils whitesourceUtils, reportPaths []piperutils.Path, influx *whitesourceExecuteScanInflux) (piperutils.Path, error) { 466 467 policyViolationCount := 0 468 for _, project := range scan.ScannedProjects() { 469 alerts, err := sys.GetProjectAlertsByType(project.Token, "REJECTED_BY_POLICY_RESOURCE") 470 if err != nil { 471 return piperutils.Path{}, fmt.Errorf("failed to retrieve project policy alerts from WhiteSource: %w", err) 472 } 473 policyViolationCount += len(alerts) 474 } 475 476 violations := struct { 477 PolicyViolations int `json:"policyViolations"` 478 Reports []string `json:"reports"` 479 }{ 480 PolicyViolations: policyViolationCount, 481 Reports: []string{}, 482 } 483 for _, report := range reportPaths { 484 _, reportFile := filepath.Split(report.Target) 485 violations.Reports = append(violations.Reports, reportFile) 486 } 487 488 violationContent, err := json.Marshal(violations) 489 if err != nil { 490 return piperutils.Path{}, fmt.Errorf("failed to marshal policy violation data: %w", err) 491 } 492 493 jsonViolationReportPath := filepath.Join(ws.ReportsDirectory, "whitesource-ip.json") 494 err = utils.FileWrite(jsonViolationReportPath, violationContent, 0666) 495 if err != nil { 496 return piperutils.Path{}, fmt.Errorf("failed to write policy violation report: %w", err) 497 } 498 499 policyReport := piperutils.Path{Name: "WhiteSource Policy Violation Report", Target: jsonViolationReportPath} 500 501 // create a json report to be used later, e.g. issue creation in GitHub 502 ipReport := reporting.ScanReport{ 503 ReportTitle: "WhiteSource IP Report", 504 Subheaders: []reporting.Subheader{ 505 {Description: "WhiteSource product name", Details: config.ProductName}, 506 {Description: "Filtered project names", Details: strings.Join(scan.ScannedProjectNames(), ", ")}, 507 }, 508 Overview: []reporting.OverviewRow{ 509 {Description: "Total number of licensing vulnerabilities", Details: fmt.Sprint(policyViolationCount)}, 510 }, 511 SuccessfulScan: policyViolationCount == 0, 512 ReportTime: utils.Now(), 513 } 514 515 // JSON reports are used by step pipelineCreateSummary in order to e.g. prepare an issue creation in GitHub 516 // ignore JSON errors since structure is in our hands 517 jsonReport, _ := ipReport.ToJSON() 518 if exists, _ := utils.DirExists(reporting.StepReportDirectory); !exists { 519 err := utils.MkdirAll(reporting.StepReportDirectory, 0777) 520 if err != nil { 521 return policyReport, errors.Wrap(err, "failed to create reporting directory") 522 } 523 } 524 if err := utils.FileWrite(filepath.Join(reporting.StepReportDirectory, fmt.Sprintf("whitesourceExecuteScan_ip_%v.json", ws.ReportSha(config.ProductName, scan))), jsonReport, 0666); err != nil { 525 return policyReport, errors.Wrapf(err, "failed to write json report") 526 } 527 // we do not add the json report to the overall list of reports for now, 528 // since it is just an intermediary report used as input for later 529 // and there does not seem to be real benefit in archiving it. 530 531 if policyViolationCount > 0 { 532 log.SetErrorCategory(log.ErrorCompliance) 533 influx.whitesource_data.fields.policy_violations = policyViolationCount 534 return policyReport, fmt.Errorf("%v policy violation(s) found", policyViolationCount) 535 } 536 537 return policyReport, nil 538 } 539 540 func checkSecurityViolations(config *ScanOptions, scan *ws.Scan, sys whitesource, utils whitesourceUtils, influx *whitesourceExecuteScanInflux) ([]piperutils.Path, error) { 541 var reportPaths []piperutils.Path 542 // Check for security vulnerabilities and fail the build if cvssSeverityLimit threshold is crossed 543 // convert config.CvssSeverityLimit to float64 544 cvssSeverityLimit, err := strconv.ParseFloat(config.CvssSeverityLimit, 64) 545 if err != nil { 546 log.SetErrorCategory(log.ErrorConfiguration) 547 return reportPaths, fmt.Errorf("failed to parse parameter cvssSeverityLimit (%s) "+ 548 "as floating point number: %w", config.CvssSeverityLimit, err) 549 } 550 551 if config.ProjectToken != "" { 552 project := ws.Project{Name: config.ProjectName, Token: config.ProjectToken} 553 // ToDo: see if HTML report generation is really required here 554 // we anyway need to do some refactoring here since config.ProjectToken != "" essentially indicates an aggregated project 555 if _, _, err := checkProjectSecurityViolations(cvssSeverityLimit, project, sys, influx); err != nil { 556 return reportPaths, err 557 } 558 } else { 559 vulnerabilitiesCount := 0 560 var errorsOccured []string 561 allAlerts := []ws.Alert{} 562 for _, project := range scan.ScannedProjects() { 563 // collect errors and aggregate vulnerabilities from all projects 564 if vulCount, alerts, err := checkProjectSecurityViolations(cvssSeverityLimit, project, sys, influx); err != nil { 565 allAlerts = append(allAlerts, alerts...) 566 vulnerabilitiesCount += vulCount 567 errorsOccured = append(errorsOccured, fmt.Sprint(err)) 568 } 569 } 570 log.Entry().Debugf("Aggregated %v alerts for scanned projects", len(allAlerts)) 571 572 if config.CreateResultIssue && vulnerabilitiesCount > 0 && len(config.GithubToken) > 0 && len(config.GithubAPIURL) > 0 && len(config.Owner) > 0 && len(config.Repository) > 0 { 573 log.Entry().Debugf("Creating result issues for %v alert(s)", vulnerabilitiesCount) 574 issueDetails := make([]reporting.IssueDetail, len(allAlerts)) 575 piperutils.CopyAtoB(allAlerts, issueDetails) 576 err = reporting.UploadMultipleReportsToGithub(&issueDetails, config.GithubToken, config.GithubAPIURL, config.Owner, config.Repository, config.Assignees, config.CustomTLSCertificateLinks, utils) 577 if err != nil { 578 errorsOccured = append(errorsOccured, fmt.Sprint(err)) 579 } 580 } 581 582 scanReport := ws.CreateCustomVulnerabilityReport(config.ProductName, scan, &allAlerts, cvssSeverityLimit) 583 paths, err := ws.WriteCustomVulnerabilityReports(config.ProductName, scan, scanReport, utils) 584 if err != nil { 585 errorsOccured = append(errorsOccured, fmt.Sprint(err)) 586 } 587 reportPaths = append(reportPaths, paths...) 588 589 sarif := ws.CreateSarifResultFile(scan, &allAlerts) 590 paths, err = ws.WriteSarifFile(sarif, utils) 591 if err != nil { 592 errorsOccured = append(errorsOccured, fmt.Sprint(err)) 593 } 594 reportPaths = append(reportPaths, paths...) 595 596 if len(errorsOccured) > 0 { 597 if vulnerabilitiesCount > 0 { 598 log.SetErrorCategory(log.ErrorCompliance) 599 } 600 return reportPaths, fmt.Errorf(strings.Join(errorsOccured, ": ")) 601 } 602 } 603 return reportPaths, nil 604 } 605 606 // checkSecurityViolations checks security violations and returns an error if the configured severity limit is crossed. 607 func checkProjectSecurityViolations(cvssSeverityLimit float64, project ws.Project, sys whitesource, influx *whitesourceExecuteScanInflux) (int, []ws.Alert, error) { 608 // get project alerts (vulnerabilities) 609 alerts, err := sys.GetProjectAlertsByType(project.Token, "SECURITY_VULNERABILITY") 610 if err != nil { 611 return 0, alerts, fmt.Errorf("failed to retrieve project alerts from WhiteSource: %w", err) 612 } 613 614 severeVulnerabilities, nonSevereVulnerabilities := ws.CountSecurityVulnerabilities(&alerts, cvssSeverityLimit) 615 influx.whitesource_data.fields.minor_vulnerabilities = nonSevereVulnerabilities 616 influx.whitesource_data.fields.major_vulnerabilities = severeVulnerabilities 617 influx.whitesource_data.fields.vulnerabilities = nonSevereVulnerabilities + severeVulnerabilities 618 if nonSevereVulnerabilities > 0 { 619 log.Entry().Warnf("WARNING: %v Open Source Software Security vulnerabilities with "+ 620 "CVSS score below threshold %.1f detected in project %s.", nonSevereVulnerabilities, 621 cvssSeverityLimit, project.Name) 622 } else if len(alerts) == 0 { 623 log.Entry().Infof("No Open Source Software Security vulnerabilities detected in project %s", 624 project.Name) 625 } 626 // https://github.com/SAP/jenkins-library/blob/master/vars/whitesourceExecuteScan.groovy#L558 627 if severeVulnerabilities > 0 { 628 log.SetErrorCategory(log.ErrorCompliance) 629 return severeVulnerabilities, alerts, fmt.Errorf("%v Open Source Software Security vulnerabilities with CVSS score greater "+ 630 "or equal to %.1f detected in project %s", 631 severeVulnerabilities, cvssSeverityLimit, project.Name) 632 } 633 return 0, alerts, nil 634 } 635 636 func aggregateVersionWideLibraries(config *ScanOptions, utils whitesourceUtils, sys whitesource) error { 637 log.Entry().Infof("Aggregating list of libraries used for all projects with version: %s", config.Version) 638 639 projects, err := sys.GetProjectsMetaInfo(config.ProductToken) 640 if err != nil { 641 return errors.Wrapf(err, "failed to get projects meta info") 642 } 643 644 versionWideLibraries := map[string][]ws.Library{} // maps project name to slice of libraries 645 for _, project := range projects { 646 projectVersion := strings.Split(project.Name, " - ")[1] 647 projectName := strings.Split(project.Name, " - ")[0] 648 if projectVersion == config.Version { 649 libs, err := sys.GetProjectLibraryLocations(project.Token) 650 if err != nil { 651 return errors.Wrapf(err, "failed to get project library locations") 652 } 653 log.Entry().Infof("Found project: %s with %v libraries.", project.Name, len(libs)) 654 versionWideLibraries[projectName] = libs 655 } 656 } 657 if err := newLibraryCSVReport(versionWideLibraries, config, utils); err != nil { 658 return errors.Wrapf(err, "failed toget new libary CSV report") 659 } 660 return nil 661 } 662 663 func aggregateVersionWideVulnerabilities(config *ScanOptions, utils whitesourceUtils, sys whitesource) error { 664 log.Entry().Infof("Aggregating list of vulnerabilities for all projects with version: %s", config.Version) 665 666 projects, err := sys.GetProjectsMetaInfo(config.ProductToken) 667 if err != nil { 668 return errors.Wrapf(err, "failed to get projects meta info") 669 } 670 671 var versionWideAlerts []ws.Alert // all alerts for a given project version 672 projectNames := `` // holds all project tokens considered a part of the report for debugging 673 for _, project := range projects { 674 projectVersion := strings.Split(project.Name, " - ")[1] 675 if projectVersion == config.Version { 676 projectNames += project.Name + "\n" 677 alerts, err := sys.GetProjectAlertsByType(project.Token, "SECURITY_VULNERABILITY") 678 if err != nil { 679 return errors.Wrapf(err, "failed to get project alerts by type") 680 } 681 log.Entry().Infof("Found project: %s with %v vulnerabilities.", project.Name, len(alerts)) 682 versionWideAlerts = append(versionWideAlerts, alerts...) 683 } 684 } 685 686 reportPath := filepath.Join(ws.ReportsDirectory, "project-names-aggregated.txt") 687 if err := utils.FileWrite(reportPath, []byte(projectNames), 0666); err != nil { 688 return errors.Wrapf(err, "failed to write report: %s", reportPath) 689 } 690 if err := newVulnerabilityExcelReport(versionWideAlerts, config, utils); err != nil { 691 return errors.Wrapf(err, "failed to create new vulnerability excel report") 692 } 693 return nil 694 } 695 696 const wsReportTimeStampLayout = "20060102-150405" 697 698 // outputs an slice of alerts to an excel file 699 func newVulnerabilityExcelReport(alerts []ws.Alert, config *ScanOptions, utils whitesourceUtils) error { 700 file := excelize.NewFile() 701 streamWriter, err := file.NewStreamWriter("Sheet1") 702 if err != nil { 703 return err 704 } 705 styleID, err := file.NewStyle(`{"font":{"color":"#777777"}}`) 706 if err != nil { 707 return err 708 } 709 if err := fillVulnerabilityExcelReport(alerts, streamWriter, styleID); err != nil { 710 return err 711 } 712 if err := streamWriter.Flush(); err != nil { 713 return err 714 } 715 716 if err := utils.MkdirAll(ws.ReportsDirectory, 0777); err != nil { 717 return err 718 } 719 720 fileName := filepath.Join(ws.ReportsDirectory, 721 fmt.Sprintf("vulnerabilities-%s.xlsx", utils.Now().Format(wsReportTimeStampLayout))) 722 stream, err := utils.FileOpen(fileName, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0666) 723 if err != nil { 724 return err 725 } 726 if err := file.Write(stream); err != nil { 727 return err 728 } 729 filePath := piperutils.Path{Name: "aggregated-vulnerabilities", Target: fileName} 730 piperutils.PersistReportsAndLinks("whitesourceExecuteScan", "", []piperutils.Path{filePath}, nil) 731 return nil 732 } 733 734 func fillVulnerabilityExcelReport(alerts []ws.Alert, streamWriter *excelize.StreamWriter, styleID int) error { 735 rows := []struct { 736 axis string 737 title string 738 }{ 739 {"A1", "Severity"}, 740 {"B1", "Library"}, 741 {"C1", "Vulnerability Id"}, 742 {"D1", "CVSS 3"}, 743 {"E1", "Project"}, 744 {"F1", "Resolution"}, 745 } 746 for _, row := range rows { 747 err := streamWriter.SetRow(row.axis, []interface{}{excelize.Cell{StyleID: styleID, Value: row.title}}) 748 if err != nil { 749 return err 750 } 751 } 752 753 for i, alert := range alerts { 754 row := make([]interface{}, 6) 755 vuln := alert.Vulnerability 756 row[0] = vuln.CVSS3Severity 757 row[1] = alert.Library.Filename 758 row[2] = vuln.Name 759 row[3] = vuln.CVSS3Score 760 row[4] = alert.Project 761 row[5] = vuln.FixResolutionText 762 cell, _ := excelize.CoordinatesToCellName(1, i+2) 763 if err := streamWriter.SetRow(cell, row); err != nil { 764 log.Entry().Errorf("failed to write alert row: %v", err) 765 } 766 } 767 return nil 768 } 769 770 // outputs an slice of libraries to an excel file based on projects with version == config.Version 771 func newLibraryCSVReport(libraries map[string][]ws.Library, config *ScanOptions, utils whitesourceUtils) error { 772 output := "Library Name, Project Name\n" 773 for projectName, libraries := range libraries { 774 log.Entry().Infof("Writing %v libraries for project %s to excel report..", len(libraries), projectName) 775 for _, library := range libraries { 776 output += library.Name + ", " + projectName + "\n" 777 } 778 } 779 780 // Ensure reporting directory exists 781 if err := utils.MkdirAll(ws.ReportsDirectory, 0777); err != nil { 782 return errors.Wrapf(err, "failed to create directories: %s", ws.ReportsDirectory) 783 } 784 785 // Write result to file 786 fileName := fmt.Sprintf("%s/libraries-%s.csv", ws.ReportsDirectory, 787 utils.Now().Format(wsReportTimeStampLayout)) 788 if err := utils.FileWrite(fileName, []byte(output), 0666); err != nil { 789 return errors.Wrapf(err, "failed to write file: %s", fileName) 790 } 791 filePath := piperutils.Path{Name: "aggregated-libraries", Target: fileName} 792 piperutils.PersistReportsAndLinks("whitesourceExecuteScan", "", []piperutils.Path{filePath}, nil) 793 return nil 794 } 795 796 // persistScannedProjects writes all actually scanned WhiteSource project names as list 797 // into the Common Pipeline Environment, from where it can be used by sub-sequent steps. 798 func persistScannedProjects(config *ScanOptions, scan *ws.Scan, commonPipelineEnvironment *whitesourceExecuteScanCommonPipelineEnvironment) { 799 projectNames := []string{} 800 if config.ProjectName != "" { 801 projectNames = []string{config.ProjectName + " - " + config.Version} 802 } else { 803 projectNames = scan.ScannedProjectNames() 804 } 805 commonPipelineEnvironment.custom.whitesourceProjectNames = projectNames 806 } 807 808 // create toolrecord file for whitesource 809 // 810 func createToolRecordWhitesource(workspace string, config *whitesourceExecuteScanOptions, scan *ws.Scan) (string, error) { 811 record := toolrecord.New(workspace, "whitesource", config.ServiceURL) 812 wsUiRoot := "https://saas.whitesourcesoftware.com" 813 productURL := wsUiRoot + "/Wss/WSS.html#!product;token=" + config.ProductToken 814 err := record.AddKeyData("product", 815 config.ProductToken, 816 config.ProductName, 817 productURL) 818 if err != nil { 819 return "", err 820 } 821 max_idx := 0 822 for idx, project := range scan.ScannedProjects() { 823 max_idx = idx 824 name := project.Name 825 token := project.Token 826 projectURL := "" 827 if token != "" { 828 projectURL = wsUiRoot + "/Wss/WSS.html#!project;token=" + token 829 } else { 830 // token is empty, provide a dummy to have an indication 831 token = "unknown" 832 } 833 err = record.AddKeyData("project", 834 token, 835 name, 836 projectURL) 837 if err != nil { 838 return "", err 839 } 840 } 841 // set overall display data to product if there 842 // is more than one project 843 if max_idx > 1 { 844 record.SetOverallDisplayData(config.ProductName, productURL) 845 } 846 err = record.Persist() 847 if err != nil { 848 return "", err 849 } 850 return record.GetFileName(), nil 851 }