github.com/SAP/jenkins-library@v1.362.0/cmd/whitesourceExecuteScan.go (about) 1 package cmd 2 3 import ( 4 "context" 5 "encoding/json" 6 "fmt" 7 "net/url" 8 "os" 9 "path/filepath" 10 "strconv" 11 "strings" 12 "time" 13 14 piperDocker "github.com/SAP/jenkins-library/pkg/docker" 15 piperGithub "github.com/SAP/jenkins-library/pkg/github" 16 piperhttp "github.com/SAP/jenkins-library/pkg/http" 17 ws "github.com/SAP/jenkins-library/pkg/whitesource" 18 19 "github.com/SAP/jenkins-library/pkg/command" 20 "github.com/SAP/jenkins-library/pkg/format" 21 "github.com/SAP/jenkins-library/pkg/golang" 22 "github.com/SAP/jenkins-library/pkg/log" 23 "github.com/SAP/jenkins-library/pkg/npm" 24 "github.com/SAP/jenkins-library/pkg/piperutils" 25 "github.com/SAP/jenkins-library/pkg/reporting" 26 "github.com/SAP/jenkins-library/pkg/telemetry" 27 "github.com/SAP/jenkins-library/pkg/toolrecord" 28 "github.com/SAP/jenkins-library/pkg/versioning" 29 "github.com/pkg/errors" 30 "github.com/xuri/excelize/v2" 31 32 "github.com/google/go-github/v45/github" 33 ) 34 35 // ScanOptions is just used to make the lines less long 36 type ScanOptions = whitesourceExecuteScanOptions 37 38 // WhiteSource defines the functions that are expected by the step implementation to 39 // be available from the WhiteSource system. 40 type whitesource interface { 41 GetProductByName(productName string) (ws.Product, error) 42 CreateProduct(productName string) (string, error) 43 SetProductAssignments(productToken string, membership, admins, alertReceivers *ws.Assignment) error 44 GetProjectsMetaInfo(productToken string) ([]ws.Project, error) 45 GetProjectToken(productToken, projectName string) (string, error) 46 GetProjectByToken(projectToken string) (ws.Project, error) 47 GetProjectRiskReport(projectToken string) ([]byte, error) 48 GetProjectVulnerabilityReport(projectToken string, format string) ([]byte, error) 49 GetProjectAlerts(projectToken string) ([]ws.Alert, error) 50 GetProjectAlertsByType(projectToken, alertType string) ([]ws.Alert, error) 51 GetProjectIgnoredAlertsByType(projectToken string, alertType string) ([]ws.Alert, error) 52 GetProjectLibraryLocations(projectToken string) ([]ws.Library, error) 53 GetProjectHierarchy(projectToken string, includeInHouse bool) ([]ws.Library, error) 54 } 55 56 type whitesourceUtils interface { 57 ws.Utils 58 piperutils.FileUtils 59 GetArtifactCoordinates(buildTool, buildDescriptorFile string, options *versioning.Options) (versioning.Coordinates, error) 60 Now() time.Time 61 GetIssueService() *github.IssuesService 62 GetSearchService() *github.SearchService 63 } 64 65 type whitesourceUtilsBundle struct { 66 *piperhttp.Client 67 *command.Command 68 *piperutils.Files 69 npmExecutor npm.Executor 70 issues *github.IssuesService 71 search *github.SearchService 72 } 73 74 func (w *whitesourceUtilsBundle) FileOpen(name string, flag int, perm os.FileMode) (ws.File, error) { 75 return os.OpenFile(name, flag, perm) 76 } 77 78 func (w *whitesourceUtilsBundle) GetArtifactCoordinates(buildTool, buildDescriptorFile string, options *versioning.Options) (versioning.Coordinates, error) { 79 artifact, err := versioning.GetArtifact(buildTool, buildDescriptorFile, options, w) 80 if err != nil { 81 return versioning.Coordinates{}, err 82 } 83 return artifact.GetCoordinates() 84 } 85 86 func (w *whitesourceUtilsBundle) getNpmExecutor(config *ws.ScanOptions) npm.Executor { 87 if w.npmExecutor == nil { 88 w.npmExecutor = npm.NewExecutor(npm.ExecutorOptions{DefaultNpmRegistry: config.DefaultNpmRegistry}) 89 } 90 return w.npmExecutor 91 } 92 93 func (w *whitesourceUtilsBundle) FindPackageJSONFiles(config *ws.ScanOptions) ([]string, error) { 94 return w.getNpmExecutor(config).FindPackageJSONFilesWithExcludes(config.BuildDescriptorExcludeList) 95 } 96 97 func (w *whitesourceUtilsBundle) InstallAllNPMDependencies(config *ws.ScanOptions, packageJSONFiles []string) error { 98 return w.getNpmExecutor(config).InstallAllDependencies(packageJSONFiles) 99 } 100 101 func (w *whitesourceUtilsBundle) SetOptions(o piperhttp.ClientOptions) { 102 w.Client.SetOptions(o) 103 } 104 105 func (w *whitesourceUtilsBundle) Now() time.Time { 106 return time.Now() 107 } 108 109 func (w *whitesourceUtilsBundle) GetIssueService() *github.IssuesService { 110 return w.issues 111 } 112 113 func (w *whitesourceUtilsBundle) GetSearchService() *github.SearchService { 114 return w.search 115 } 116 117 func newWhitesourceUtils(config *ScanOptions, client *github.Client) *whitesourceUtilsBundle { 118 utils := whitesourceUtilsBundle{ 119 Client: &piperhttp.Client{}, 120 Command: &command.Command{}, 121 Files: &piperutils.Files{}, 122 } 123 if client != nil { 124 utils.issues = client.Issues 125 utils.search = client.Search 126 } 127 // Reroute cmd output to logging framework 128 utils.Stdout(log.Writer()) 129 utils.Stderr(log.Writer()) 130 // Configure HTTP Client 131 utils.SetOptions(piperhttp.ClientOptions{TransportTimeout: time.Duration(config.Timeout) * time.Second}) 132 return &utils 133 } 134 135 func newWhitesourceScan(config *ScanOptions) *ws.Scan { 136 return &ws.Scan{ 137 AggregateProjectName: config.ProjectName, 138 ProductVersion: config.Version, 139 BuildTool: config.BuildTool, 140 SkipProjectsWithEmptyTokens: config.SkipProjectsWithEmptyTokens, 141 } 142 } 143 144 func whitesourceExecuteScan(config ScanOptions, _ *telemetry.CustomData, commonPipelineEnvironment *whitesourceExecuteScanCommonPipelineEnvironment, influx *whitesourceExecuteScanInflux) { 145 ctx, client, err := piperGithub. 146 NewClientBuilder(config.GithubToken, config.GithubAPIURL). 147 WithTrustedCerts(config.CustomTLSCertificateLinks).Build() 148 if err != nil { 149 log.Entry().WithError(err).Warning("Failed to get GitHub client") 150 } 151 if log.IsVerbose() { 152 logWorkspaceContent() 153 } 154 utils := newWhitesourceUtils(&config, client) 155 scan := newWhitesourceScan(&config) 156 sys := ws.NewSystem(config.ServiceURL, config.OrgToken, config.UserToken, time.Duration(config.Timeout)*time.Second) 157 influx.step_data.fields.whitesource = false 158 if err := runWhitesourceExecuteScan(ctx, &config, scan, utils, sys, commonPipelineEnvironment, influx); err != nil { 159 log.Entry().WithError(err).Fatal("step execution failed") 160 } 161 influx.step_data.fields.whitesource = true 162 } 163 164 func runWhitesourceExecuteScan(ctx context.Context, config *ScanOptions, scan *ws.Scan, utils whitesourceUtils, sys whitesource, commonPipelineEnvironment *whitesourceExecuteScanCommonPipelineEnvironment, influx *whitesourceExecuteScanInflux) error { 165 if config != nil && config.PrivateModules != "" && config.PrivateModulesGitToken != "" { 166 //configuring go private packages 167 if err := golang.PrepareGolangPrivatePackages("WhitesourceExecuteStep", config.PrivateModules, config.PrivateModulesGitToken); err != nil { 168 log.Entry().Warningf("couldn't set private packages for golang, error: %s", err.Error()) 169 } 170 } 171 172 if err := resolveAggregateProjectName(config, scan, sys); err != nil { 173 return errors.Wrapf(err, "failed to resolve and aggregate project name") 174 } 175 176 if err := resolveProjectIdentifiers(config, scan, utils, sys); err != nil { 177 if strings.Contains(fmt.Sprint(err), "User is not allowed to perform this action") { 178 log.SetErrorCategory(log.ErrorConfiguration) 179 } 180 return errors.Wrapf(err, "failed to resolve project identifiers") 181 } 182 183 if config.AggregateVersionWideReport { 184 // Generate a vulnerability report for all projects with version = config.ProjectVersion 185 // Note that this is not guaranteed that all projects are from the same scan. 186 // For example, if a module was removed from the source code, the project may still 187 // exist in the WhiteSource system. 188 if err := aggregateVersionWideLibraries(config, utils, sys); err != nil { 189 return errors.Wrapf(err, "failed to aggregate version wide libraries") 190 } 191 if err := aggregateVersionWideVulnerabilities(config, utils, sys); err != nil { 192 return errors.Wrapf(err, "failed to aggregate version wide vulnerabilities") 193 } 194 } else { 195 if err := runWhitesourceScan(ctx, config, scan, utils, sys, commonPipelineEnvironment, influx); err != nil { 196 return errors.Wrapf(err, "failed to execute WhiteSource scan") 197 } 198 } 199 return nil 200 } 201 202 func runWhitesourceScan(ctx context.Context, config *ScanOptions, scan *ws.Scan, utils whitesourceUtils, sys whitesource, commonPipelineEnvironment *whitesourceExecuteScanCommonPipelineEnvironment, influx *whitesourceExecuteScanInflux) error { 203 204 // Download Docker image for container scan 205 // ToDo: move it to improve testability 206 if config.BuildTool == "docker" { 207 if len(config.ScanImages) != 0 && config.ActivateMultipleImagesScan { 208 for _, image := range config.ScanImages { 209 config.ScanImage = image 210 err := downloadMultipleDockerImageAsTar(config, utils) 211 if err != nil { 212 return errors.Wrapf(err, "failed to download docker image") 213 } 214 } 215 216 } else { 217 err := downloadDockerImageAsTar(config, utils) 218 if err != nil { 219 return errors.Wrapf(err, "failed to download docker image") 220 } 221 } 222 } 223 224 // Start the scan 225 if err := executeScan(config, scan, utils); err != nil { 226 return errors.Wrapf(err, "failed to execute Scan") 227 } 228 229 // ToDo: Check this: 230 // Why is this required at all, resolveProjectIdentifiers() is already called before the scan in runWhitesourceExecuteScan() 231 // Could perhaps use scan.updateProjects(sys) directly... have not investigated what could break 232 if err := resolveProjectIdentifiers(config, scan, utils, sys); err != nil { 233 return errors.Wrapf(err, "failed to resolve project identifiers") 234 } 235 236 log.Entry().Info("-----------------------------------------------------") 237 log.Entry().Infof("Product Version: '%s'", config.Version) 238 log.Entry().Info("Scanned projects:") 239 for _, project := range scan.ScannedProjects() { 240 log.Entry().Infof(" Name: '%s', token: %s", project.Name, project.Token) 241 } 242 log.Entry().Info("-----------------------------------------------------") 243 244 paths, err := checkAndReportScanResults(ctx, config, scan, utils, sys, influx) 245 piperutils.PersistReportsAndLinks("whitesourceExecuteScan", "", utils, paths, nil) 246 persistScannedProjects(config, scan, commonPipelineEnvironment) 247 if err != nil { 248 return errors.Wrapf(err, "failed to check and report scan results") 249 } 250 return nil 251 } 252 253 func checkAndReportScanResults(ctx context.Context, config *ScanOptions, scan *ws.Scan, utils whitesourceUtils, sys whitesource, influx *whitesourceExecuteScanInflux) ([]piperutils.Path, error) { 254 reportPaths := []piperutils.Path{} 255 if !config.Reporting && !config.SecurityVulnerabilities { 256 return reportPaths, nil 257 } 258 // Wait for WhiteSource backend to propagate the changes before downloading any reports. 259 if err := scan.BlockUntilReportsAreReady(sys); err != nil { 260 return reportPaths, err 261 } 262 263 if config.Reporting { 264 var err error 265 reportPaths, err = scan.DownloadReports(ws.ReportOptions{ 266 ReportDirectory: ws.ReportsDirectory, 267 VulnerabilityReportFormat: config.VulnerabilityReportFormat, 268 }, utils, sys) 269 if err != nil { 270 return reportPaths, err 271 } 272 } 273 274 checkErrors := []string{} 275 276 rPath, err := checkPolicyViolations(ctx, config, scan, sys, utils, reportPaths, influx) 277 278 if err != nil { 279 if !config.FailOnSevereVulnerabilities && log.GetErrorCategory() == log.ErrorCompliance { 280 log.Entry().Infof("policy violation(s) found - step will only create data but not fail due to setting failOnSevereVulnerabilities: false") 281 } else { 282 checkErrors = append(checkErrors, fmt.Sprint(err)) 283 } 284 } 285 reportPaths = append(reportPaths, rPath) 286 287 if config.SecurityVulnerabilities { 288 rPaths, err := checkSecurityViolations(ctx, config, scan, sys, utils, influx) 289 reportPaths = append(reportPaths, rPaths...) 290 if err != nil { 291 if !config.FailOnSevereVulnerabilities && log.GetErrorCategory() == log.ErrorCompliance { 292 log.Entry().Infof("policy violation(s) found - step will only create data but not fail due to setting failOnSevereVulnerabilities: false") 293 } else { 294 checkErrors = append(checkErrors, fmt.Sprint(err)) 295 } 296 } 297 } 298 299 // create toolrecord file 300 // tbd - how to handle verifyOnly 301 toolRecordFileName, err := createToolRecordWhitesource(utils, "./", config, scan) 302 if err != nil { 303 // do not fail until the framework is well established 304 log.Entry().Warning("TR_WHITESOURCE: Failed to create toolrecord file ...", err) 305 } else { 306 reportPaths = append(reportPaths, piperutils.Path{Target: toolRecordFileName}) 307 } 308 309 if len(checkErrors) > 0 { 310 return reportPaths, fmt.Errorf(strings.Join(checkErrors, ": ")) 311 } 312 return reportPaths, nil 313 } 314 315 func createWhiteSourceProduct(config *ScanOptions, sys whitesource) (string, error) { 316 log.Entry().Infof("Attempting to create new WhiteSource product for '%s'..", config.ProductName) 317 productToken, err := sys.CreateProduct(config.ProductName) 318 if err != nil { 319 return "", fmt.Errorf("failed to create WhiteSource product: %w", err) 320 } 321 322 var admins ws.Assignment 323 for _, address := range config.EmailAddressesOfInitialProductAdmins { 324 admins.UserAssignments = append(admins.UserAssignments, ws.UserAssignment{Email: address}) 325 } 326 327 err = sys.SetProductAssignments(productToken, nil, &admins, nil) 328 if err != nil { 329 return "", fmt.Errorf("failed to set admins on new WhiteSource product: %w", err) 330 } 331 332 return productToken, nil 333 } 334 335 func resolveProjectIdentifiers(config *ScanOptions, scan *ws.Scan, utils whitesourceUtils, sys whitesource) error { 336 if len(scan.AggregateProjectName) > 0 && (len(config.Version)+len(config.CustomScanVersion) > 0) { 337 if len(config.CustomScanVersion) > 0 { 338 log.Entry().Infof("Using custom version: %v", config.CustomScanVersion) 339 config.Version = config.CustomScanVersion 340 } else if len(config.Version) > 0 { 341 log.Entry().Infof("Resolving product version from default provided '%s' with versioning '%s'", config.Version, config.VersioningModel) 342 config.Version = versioning.ApplyVersioningModel(config.VersioningModel, config.Version) 343 log.Entry().Infof("Resolved product version '%s'", config.Version) 344 } 345 } else { 346 options := &versioning.Options{ 347 DockerImage: config.ScanImage, 348 ProjectSettingsFile: config.ProjectSettingsFile, 349 GlobalSettingsFile: config.GlobalSettingsFile, 350 M2Path: config.M2Path, 351 } 352 coordinates, err := utils.GetArtifactCoordinates(config.BuildTool, config.BuildDescriptorFile, options) 353 if err != nil { 354 return errors.Wrap(err, "failed to get build artifact description") 355 } 356 scan.Coordinates = coordinates 357 358 if len(config.Version) > 0 { 359 log.Entry().Infof("Resolving product version from default provided '%s' with versioning '%s'", config.Version, config.VersioningModel) 360 coordinates.Version = config.Version 361 } 362 363 nameTmpl := `{{list .GroupID .ArtifactID | join "-" | trimAll "-"}}` 364 name, version := versioning.DetermineProjectCoordinatesWithCustomVersion(nameTmpl, config.VersioningModel, config.CustomScanVersion, coordinates) 365 if scan.AggregateProjectName == "" { 366 log.Entry().Infof("Resolved project name '%s' from descriptor file", name) 367 scan.AggregateProjectName = name 368 } 369 370 config.Version = version 371 log.Entry().Infof("Resolved product version '%s'", version) 372 } 373 374 scan.ProductVersion = validateProductVersion(config.Version) 375 376 if err := resolveProductToken(config, sys); err != nil { 377 return errors.Wrap(err, "error resolving product token") 378 } 379 380 if !config.SkipParentProjectResolution { 381 if err := resolveAggregateProjectToken(config, sys); err != nil { 382 return errors.Wrap(err, "error resolving aggregate project token") 383 } 384 } 385 386 scan.ProductToken = config.ProductToken 387 388 return scan.UpdateProjects(config.ProductToken, sys) 389 } 390 391 // resolveProductToken resolves the token of the WhiteSource Product specified by config.ProductName, 392 // unless the user provided a token in config.ProductToken already, or it was previously resolved. 393 // If no Product can be found for the given config.ProductName, and the parameter 394 // config.CreatePipelineFromProduct is set, an attempt will be made to create the product and 395 // configure the initial product admins. 396 func resolveProductToken(config *ScanOptions, sys whitesource) error { 397 if config.ProductToken != "" { 398 return nil 399 } 400 log.Entry().Infof("Attempting to resolve product token for product '%s'..", config.ProductName) 401 product, err := sys.GetProductByName(config.ProductName) 402 if err != nil && config.CreateProductFromPipeline { 403 product = ws.Product{} 404 product.Token, err = createWhiteSourceProduct(config, sys) 405 if err != nil { 406 return errors.Wrapf(err, "failed to create whitesource product") 407 } 408 } 409 if err != nil { 410 return errors.Wrapf(err, "failed to get product by name") 411 } 412 log.Entry().Infof("Resolved product token: '%s'..", product.Token) 413 config.ProductToken = product.Token 414 return nil 415 } 416 417 // resolveAggregateProjectName checks if config.ProjectToken is configured, and if so, expects a WhiteSource 418 // project with that token to exist. The AggregateProjectName in the ws.Scan is then configured with that 419 // project's name. 420 func resolveAggregateProjectName(config *ScanOptions, scan *ws.Scan, sys whitesource) error { 421 if config.ProjectToken == "" { 422 return nil 423 } 424 log.Entry().Infof("Attempting to resolve aggregate project name for token '%s'..", config.ProjectToken) 425 // If the user configured the "projectToken" parameter, we expect this project to exist in the backend. 426 project, err := sys.GetProjectByToken(config.ProjectToken) 427 if err != nil { 428 return errors.Wrapf(err, "failed to get project by token") 429 } 430 nameVersion := strings.Split(project.Name, " - ") 431 scan.AggregateProjectName = nameVersion[0] 432 log.Entry().Infof("Resolve aggregate project name '%s'..", scan.AggregateProjectName) 433 return nil 434 } 435 436 // resolveAggregateProjectToken fetches the token of the WhiteSource Project specified by config.ProjectName 437 // and stores it in config.ProjectToken. 438 // The user can configure a projectName or projectToken of the project to be used as for aggregation of scan results. 439 func resolveAggregateProjectToken(config *ScanOptions, sys whitesource) error { 440 if config.ProjectToken != "" || config.ProjectName == "" { 441 return nil 442 } 443 log.Entry().Infof("Attempting to resolve project token for project '%s'..", config.ProjectName) 444 fullProjName := fmt.Sprintf("%s - %s", config.ProjectName, config.Version) 445 projectToken, err := sys.GetProjectToken(config.ProductToken, fullProjName) 446 if err != nil { 447 return errors.Wrapf(err, "failed to get project token") 448 } 449 // A project may not yet exist for this project name-version combo. 450 // It will be created by the scan, we retrieve the token again after scanning. 451 if projectToken != "" { 452 log.Entry().Infof("Resolved project token: '%s'..", projectToken) 453 config.ProjectToken = projectToken 454 } else { 455 log.Entry().Infof("Project '%s' not yet present in WhiteSource", fullProjName) 456 } 457 return nil 458 } 459 460 // validateProductVersion makes sure that the version does not contain a dash "-". 461 func validateProductVersion(version string) string { 462 // TrimLeft() removes all "-" from the beginning, unlike TrimPrefix()! 463 version = strings.TrimLeft(version, "-") 464 if strings.Contains(version, "-") { 465 version = strings.SplitN(version, "-", 1)[0] 466 } 467 return version 468 } 469 470 func wsScanOptions(config *ScanOptions) *ws.ScanOptions { 471 return &ws.ScanOptions{ 472 BuildTool: config.BuildTool, 473 ScanType: "", // no longer provided via config 474 OrgToken: config.OrgToken, 475 UserToken: config.UserToken, 476 ProductName: config.ProductName, 477 ProductToken: config.ProductToken, 478 ProductVersion: config.Version, 479 ProjectName: config.ProjectName, 480 BuildDescriptorFile: config.BuildDescriptorFile, 481 BuildDescriptorExcludeList: config.BuildDescriptorExcludeList, 482 PomPath: config.BuildDescriptorFile, 483 M2Path: config.M2Path, 484 GlobalSettingsFile: config.GlobalSettingsFile, 485 ProjectSettingsFile: config.ProjectSettingsFile, 486 InstallArtifacts: config.InstallArtifacts, 487 DefaultNpmRegistry: config.DefaultNpmRegistry, 488 AgentDownloadURL: config.AgentDownloadURL, 489 AgentFileName: config.AgentFileName, 490 ConfigFilePath: config.ConfigFilePath, 491 Includes: config.Includes, 492 Excludes: config.Excludes, 493 JreDownloadURL: config.JreDownloadURL, 494 AgentURL: config.AgentURL, 495 ServiceURL: config.ServiceURL, 496 ScanPath: config.ScanPath, 497 InstallCommand: config.InstallCommand, 498 Verbose: GeneralConfig.Verbose, 499 SkipParentProjectResolution: config.SkipParentProjectResolution, 500 } 501 } 502 503 // Unified Agent is the only supported option by WhiteSource going forward: 504 // The Unified Agent will be used to perform the scan. 505 func executeScan(config *ScanOptions, scan *ws.Scan, utils whitesourceUtils) error { 506 options := wsScanOptions(config) 507 508 if options.InstallCommand != "" { 509 installCommandTokens := strings.Split(config.InstallCommand, " ") 510 if err := utils.RunExecutable(installCommandTokens[0], installCommandTokens[1:]...); err != nil { 511 log.SetErrorCategory(log.ErrorCustom) 512 return errors.Wrapf(err, "failed to execute install command: %v", config.InstallCommand) 513 } 514 } 515 516 // Execute scan with Unified Agent jar file 517 if err := scan.ExecuteUAScan(options, utils); err != nil { 518 return errors.Wrapf(err, "failed to execute Unified Agent scan") 519 } 520 return nil 521 } 522 523 func checkPolicyViolations(ctx context.Context, config *ScanOptions, scan *ws.Scan, sys whitesource, utils whitesourceUtils, reportPaths []piperutils.Path, influx *whitesourceExecuteScanInflux) (piperutils.Path, error) { 524 policyViolationCount := 0 525 allAlerts := []ws.Alert{} 526 for _, project := range scan.ScannedProjects() { 527 alerts, err := sys.GetProjectAlertsByType(project.Token, "REJECTED_BY_POLICY_RESOURCE") 528 if err != nil { 529 return piperutils.Path{}, fmt.Errorf("failed to retrieve project policy alerts from WhiteSource: %w", err) 530 } 531 532 policyViolationCount += len(alerts) 533 allAlerts = append(allAlerts, alerts...) 534 } 535 536 violations := struct { 537 PolicyViolations int `json:"policyViolations"` 538 Reports []string `json:"reports"` 539 }{ 540 PolicyViolations: policyViolationCount, 541 Reports: []string{}, 542 } 543 for _, report := range reportPaths { 544 _, reportFile := filepath.Split(report.Target) 545 violations.Reports = append(violations.Reports, reportFile) 546 } 547 548 violationContent, err := json.Marshal(violations) 549 if err != nil { 550 return piperutils.Path{}, fmt.Errorf("failed to marshal policy violation data: %w", err) 551 } 552 553 jsonViolationReportPath := filepath.Join(ws.ReportsDirectory, "whitesource-ip.json") 554 err = utils.FileWrite(jsonViolationReportPath, violationContent, 0o666) 555 if err != nil { 556 return piperutils.Path{}, fmt.Errorf("failed to write policy violation report: %w", err) 557 } 558 559 policyReport := piperutils.Path{Name: "WhiteSource Policy Violation Report", Target: jsonViolationReportPath} 560 561 // create a json report to be used later, e.g. issue creation in GitHub 562 ipReport := reporting.ScanReport{ 563 ReportTitle: "WhiteSource IP Report", 564 Subheaders: []reporting.Subheader{ 565 {Description: "WhiteSource product name", Details: config.ProductName}, 566 {Description: "Filtered project names", Details: strings.Join(scan.ScannedProjectNames(), ", ")}, 567 }, 568 Overview: []reporting.OverviewRow{ 569 {Description: "Total number of licensing vulnerabilities", Details: fmt.Sprint(policyViolationCount)}, 570 }, 571 SuccessfulScan: policyViolationCount == 0, 572 ReportTime: utils.Now(), 573 } 574 575 // JSON reports are used by step pipelineCreateSummary in order to e.g. prepare an issue creation in GitHub 576 // ignore JSON errors since structure is in our hands 577 jsonReport, _ := ipReport.ToJSON() 578 if exists, _ := utils.DirExists(reporting.StepReportDirectory); !exists { 579 err := utils.MkdirAll(reporting.StepReportDirectory, 0o777) 580 if err != nil { 581 return policyReport, errors.Wrap(err, "failed to create reporting directory") 582 } 583 } 584 if err := utils.FileWrite(filepath.Join(reporting.StepReportDirectory, fmt.Sprintf("whitesourceExecuteScan_ip_%v.json", ws.ReportSha(config.ProductName, scan))), jsonReport, 0o666); err != nil { 585 return policyReport, errors.Wrapf(err, "failed to write json report") 586 } 587 // we do not add the json report to the overall list of reports for now, 588 // since it is just an intermediary report used as input for later 589 // and there does not seem to be real benefit in archiving it. 590 591 if policyViolationCount > 0 { 592 influx.whitesource_data.fields.policy_violations = policyViolationCount 593 log.SetErrorCategory(log.ErrorCompliance) 594 595 if config.CreateResultIssue && policyViolationCount > 0 && len(config.GithubToken) > 0 && len(config.GithubAPIURL) > 0 && len(config.Owner) > 0 && len(config.Repository) > 0 { 596 log.Entry().Debugf("Creating result issues for %v alert(s)", policyViolationCount) 597 issueDetails := make([]reporting.IssueDetail, len(allAlerts)) 598 piperutils.CopyAtoB(allAlerts, issueDetails) 599 gh := reporting.GitHub{ 600 Owner: &config.Owner, 601 Repository: &config.Repository, 602 Assignees: &config.Assignees, 603 IssueService: utils.GetIssueService(), 604 SearchService: utils.GetSearchService(), 605 } 606 if err := gh.UploadMultipleReports(ctx, &issueDetails); err != nil { 607 return policyReport, fmt.Errorf("failed to upload reports to GitHub for %v policy violations: %w", policyViolationCount, err) 608 } 609 } 610 return policyReport, fmt.Errorf("%v policy violation(s) found", policyViolationCount) 611 } 612 613 return policyReport, nil 614 } 615 616 func checkSecurityViolations(ctx context.Context, config *ScanOptions, scan *ws.Scan, sys whitesource, utils whitesourceUtils, influx *whitesourceExecuteScanInflux) ([]piperutils.Path, error) { 617 // Check for security vulnerabilities and fail the build if cvssSeverityLimit threshold is crossed 618 // convert config.CvssSeverityLimit to float64 619 cvssSeverityLimit, err := strconv.ParseFloat(config.CvssSeverityLimit, 64) 620 if err != nil { 621 log.SetErrorCategory(log.ErrorConfiguration) 622 return []piperutils.Path{}, fmt.Errorf("failed to parse parameter cvssSeverityLimit (%s) "+ 623 "as floating point number: %w", config.CvssSeverityLimit, err) 624 } 625 626 // inhale assessments from file system 627 assessments := readAssessmentsFromFile(config.AssessmentFile, utils) 628 629 vulnerabilitiesCount := 0 630 var allOccurredErrors []string 631 allAlerts := []ws.Alert{} 632 allAssessedAlerts := []ws.Alert{} 633 allLibraries := []ws.Library{} 634 635 if config.ProjectToken != "" { 636 project := ws.Project{Name: config.ProjectName, Token: config.ProjectToken} 637 // ToDo: see if HTML report generation is really required here 638 // we anyway need to do some refactoring here since config.ProjectToken != "" essentially indicates an aggregated project 639 640 vulnerabilitiesCount, allAlerts, allAssessedAlerts, allLibraries, allOccurredErrors = collectVulnsAndLibsForProject( 641 config, 642 cvssSeverityLimit, 643 project, 644 sys, 645 assessments, 646 influx, 647 ) 648 649 log.Entry().Debugf("Collected %v libraries for project %v", len(allLibraries), project.Name) 650 651 } else { 652 for _, project := range scan.ScannedProjects() { 653 // collect errors and aggregate vulnerabilities from all projects 654 vulCount, alerts, assessedAlerts, libraries, occurredErrors := collectVulnsAndLibsForProject( 655 config, 656 cvssSeverityLimit, 657 project, 658 sys, 659 assessments, 660 influx, 661 ) 662 if len(occurredErrors) != 0 { 663 allOccurredErrors = append(allOccurredErrors, occurredErrors...) 664 } 665 666 allAlerts = append(allAlerts, alerts...) 667 allAssessedAlerts = append(allAssessedAlerts, assessedAlerts...) 668 vulnerabilitiesCount += vulCount 669 allLibraries = append(allLibraries, libraries...) 670 } 671 log.Entry().Debugf("Aggregated %v alerts for scanned projects", len(allAlerts)) 672 } 673 674 reportPaths, errors := reportGitHubIssuesAndCreateReports( 675 ctx, 676 config, 677 utils, 678 scan, 679 allAlerts, 680 allLibraries, 681 allAssessedAlerts, 682 cvssSeverityLimit, 683 vulnerabilitiesCount, 684 ) 685 686 allOccurredErrors = append(allOccurredErrors, errors...) 687 688 if len(allOccurredErrors) > 0 { 689 if vulnerabilitiesCount > 0 { 690 log.SetErrorCategory(log.ErrorCompliance) 691 } 692 return reportPaths, fmt.Errorf(strings.Join(allOccurredErrors, ": ")) 693 } 694 695 return reportPaths, nil 696 } 697 698 func collectVulnsAndLibsForProject( 699 config *ScanOptions, 700 cvssSeverityLimit float64, 701 project ws.Project, 702 sys whitesource, 703 assessments *[]format.Assessment, 704 influx *whitesourceExecuteScanInflux, 705 ) ( 706 int, 707 []ws.Alert, 708 []ws.Alert, 709 []ws.Library, 710 []string, 711 ) { 712 var errorsOccurred []string 713 vulCount, alerts, assessedAlerts, err := checkProjectSecurityViolations(config, cvssSeverityLimit, project, sys, assessments, influx) 714 if err != nil { 715 errorsOccurred = append(errorsOccurred, fmt.Sprint(err)) 716 } 717 718 // collect all libraries detected in all related projects and errors 719 libraries, err := sys.GetProjectHierarchy(project.Token, true) 720 if err != nil { 721 errorsOccurred = append(errorsOccurred, fmt.Sprint(err)) 722 } 723 log.Entry().Debugf("Collected %v libraries for project %v", len(libraries), project.Name) 724 725 return vulCount, alerts, assessedAlerts, libraries, errorsOccurred 726 } 727 728 func reportGitHubIssuesAndCreateReports( 729 ctx context.Context, 730 config *ScanOptions, 731 utils whitesourceUtils, 732 scan *ws.Scan, 733 allAlerts []ws.Alert, 734 allLibraries []ws.Library, 735 allAssessedAlerts []ws.Alert, 736 cvssSeverityLimit float64, 737 vulnerabilitiesCount int, 738 ) ([]piperutils.Path, []string) { 739 errorsOccured := make([]string, 0) 740 reportPaths := make([]piperutils.Path, 0) 741 742 if config.CreateResultIssue && vulnerabilitiesCount > 0 && len(config.GithubToken) > 0 && len(config.GithubAPIURL) > 0 && len(config.Owner) > 0 && len(config.Repository) > 0 { 743 log.Entry().Debugf("Creating result issues for %v alert(s)", vulnerabilitiesCount) 744 issueDetails := make([]reporting.IssueDetail, len(allAlerts)) 745 piperutils.CopyAtoB(allAlerts, issueDetails) 746 gh := reporting.GitHub{ 747 Owner: &config.Owner, 748 Repository: &config.Repository, 749 Assignees: &config.Assignees, 750 IssueService: utils.GetIssueService(), 751 SearchService: utils.GetSearchService(), 752 } 753 754 if err := gh.UploadMultipleReports(ctx, &issueDetails); err != nil { 755 errorsOccured = append(errorsOccured, fmt.Sprint(err)) 756 } 757 } 758 759 scanReport := ws.CreateCustomVulnerabilityReport(config.ProductName, scan, &allAlerts, cvssSeverityLimit) 760 paths, err := ws.WriteCustomVulnerabilityReports(config.ProductName, scan, scanReport, utils) 761 if err != nil { 762 errorsOccured = append(errorsOccured, fmt.Sprint(err)) 763 } 764 765 reportPaths = append(reportPaths, paths...) 766 767 combinedAlerts := make([]ws.Alert, 0, len(allAlerts)+len(allAssessedAlerts)) 768 combinedAlerts = append(combinedAlerts, allAlerts...) 769 combinedAlerts = append(combinedAlerts, allAssessedAlerts...) 770 771 sarif := ws.CreateSarifResultFile(scan, &combinedAlerts) 772 paths, err = ws.WriteSarifFile(sarif, utils) 773 if err != nil { 774 errorsOccured = append(errorsOccured, fmt.Sprint(err)) 775 } 776 777 reportPaths = append(reportPaths, paths...) 778 779 sbom, err := ws.CreateCycloneSBOM(scan, &allLibraries, &allAlerts, &allAssessedAlerts) 780 if err != nil { 781 errorsOccured = append(errorsOccured, fmt.Sprint(err)) 782 } 783 784 paths, err = ws.WriteCycloneSBOM(sbom, utils) 785 if err != nil { 786 errorsOccured = append(errorsOccured, fmt.Sprint(err)) 787 } 788 789 reportPaths = append(reportPaths, paths...) 790 791 return reportPaths, errorsOccured 792 } 793 794 // read assessments from file and expose them to match alerts and filter them before processing 795 func readAssessmentsFromFile(assessmentFilePath string, utils whitesourceUtils) *[]format.Assessment { 796 exists, err := utils.FileExists(assessmentFilePath) 797 if err != nil { 798 log.SetErrorCategory(log.ErrorConfiguration) 799 log.Entry().WithError(err).Errorf("unable to check existence of assessment file at '%s'", assessmentFilePath) 800 } 801 assessmentFile, err := utils.Open(assessmentFilePath) 802 if exists && err != nil { 803 log.SetErrorCategory(log.ErrorConfiguration) 804 log.Entry().WithError(err).Errorf("unable to open assessment file at '%s'", assessmentFilePath) 805 } 806 assessments := &[]format.Assessment{} 807 if exists { 808 defer assessmentFile.Close() 809 assessments, err = format.ReadAssessments(assessmentFile) 810 if err != nil { 811 log.SetErrorCategory(log.ErrorConfiguration) 812 log.Entry().WithError(err).Errorf("unable to parse assessment file at '%s'", assessmentFilePath) 813 } 814 } 815 return assessments 816 } 817 818 // checkSecurityViolations checks security violations and returns an error if the configured severity limit is crossed. Besides the potential error the list of unassessed and assessed alerts are being returned to allow generating reports and issues from the data. 819 func checkProjectSecurityViolations(config *ScanOptions, cvssSeverityLimit float64, project ws.Project, sys whitesource, assessments *[]format.Assessment, influx *whitesourceExecuteScanInflux) (int, []ws.Alert, []ws.Alert, error) { 820 // get project alerts (vulnerabilities) 821 alerts, err := sys.GetProjectAlertsByType(project.Token, "SECURITY_VULNERABILITY") 822 if err != nil { 823 return 0, alerts, []ws.Alert{}, fmt.Errorf("failed to retrieve project alerts from WhiteSource: %w", err) 824 } 825 826 assessedAlerts, err := sys.GetProjectIgnoredAlertsByType(project.Token, "SECURITY_VULNERABILITY") 827 if err != nil { 828 return 0, alerts, []ws.Alert{}, fmt.Errorf("failed to retrieve project ignored alerts from WhiteSource: %w", err) 829 } 830 831 // filter alerts related to existing assessments 832 filteredAlerts := []ws.Alert{} 833 if assessments != nil && len(*assessments) > 0 { 834 for _, alert := range alerts { 835 if result, err := alert.ContainedIn(assessments); err == nil && !result { 836 filteredAlerts = append(filteredAlerts, alert) 837 } else if alert.Assessment != nil { 838 log.Entry().Debugf("Matched assessment with status %v and analysis %v to vulnerability %v affecting packages %v", alert.Assessment.Status, alert.Assessment.Analysis, alert.Assessment.Vulnerability, alert.Assessment.Purls) 839 assessedAlerts = append(assessedAlerts, alert) 840 } 841 } 842 // intentionally overwriting original list of alerts with those remaining unassessed after processing of assessments 843 alerts = filteredAlerts 844 } 845 846 severeVulnerabilities, nonSevereVulnerabilities := ws.CountSecurityVulnerabilities(&alerts, cvssSeverityLimit) 847 influx.whitesource_data.fields.minor_vulnerabilities = nonSevereVulnerabilities 848 influx.whitesource_data.fields.major_vulnerabilities = severeVulnerabilities 849 influx.whitesource_data.fields.vulnerabilities = nonSevereVulnerabilities + severeVulnerabilities 850 if nonSevereVulnerabilities > 0 { 851 log.Entry().Warnf("WARNING: %v Open Source Software Security vulnerabilities with "+ 852 "CVSS score below threshold %.1f detected in project %s.", nonSevereVulnerabilities, 853 cvssSeverityLimit, project.Name) 854 } else if len(alerts) == 0 { 855 log.Entry().Infof("No Open Source Software Security vulnerabilities detected in project %s", 856 project.Name) 857 } 858 // https://github.com/SAP/jenkins-library/blob/master/vars/whitesourceExecuteScan.groovy#L558 859 if severeVulnerabilities > 0 { 860 if config.FailOnSevereVulnerabilities { 861 log.SetErrorCategory(log.ErrorCompliance) 862 return severeVulnerabilities, alerts, assessedAlerts, fmt.Errorf("%v Open Source Software Security vulnerabilities with CVSS score greater or equal to %.1f detected in project %s", severeVulnerabilities, cvssSeverityLimit, project.Name) 863 } 864 log.Entry().Infof("%v Open Source Software Security vulnerabilities with CVSS score greater or equal to %.1f detected in project %s", severeVulnerabilities, cvssSeverityLimit, project.Name) 865 log.Entry().Info("Step will only create data but not fail due to setting failOnSevereVulnerabilities: false") 866 return severeVulnerabilities, alerts, assessedAlerts, nil 867 } 868 return 0, alerts, assessedAlerts, nil 869 } 870 871 func aggregateVersionWideLibraries(config *ScanOptions, utils whitesourceUtils, sys whitesource) error { 872 log.Entry().Infof("Aggregating list of libraries used for all projects with version: %s", config.Version) 873 874 projects, err := sys.GetProjectsMetaInfo(config.ProductToken) 875 if err != nil { 876 return errors.Wrapf(err, "failed to get projects meta info") 877 } 878 879 versionWideLibraries := map[string][]ws.Library{} // maps project name to slice of libraries 880 for _, project := range projects { 881 projectVersion := strings.Split(project.Name, " - ")[1] 882 projectName := strings.Split(project.Name, " - ")[0] 883 if projectVersion == config.Version { 884 libs, err := sys.GetProjectLibraryLocations(project.Token) 885 if err != nil { 886 return errors.Wrapf(err, "failed to get project library locations") 887 } 888 log.Entry().Infof("Found project: %s with %v libraries.", project.Name, len(libs)) 889 versionWideLibraries[projectName] = libs 890 } 891 } 892 if err := newLibraryCSVReport(versionWideLibraries, config, utils); err != nil { 893 return errors.Wrapf(err, "failed toget new libary CSV report") 894 } 895 return nil 896 } 897 898 func aggregateVersionWideVulnerabilities(config *ScanOptions, utils whitesourceUtils, sys whitesource) error { 899 log.Entry().Infof("Aggregating list of vulnerabilities for all projects with version: %s", config.Version) 900 901 projects, err := sys.GetProjectsMetaInfo(config.ProductToken) 902 if err != nil { 903 return errors.Wrapf(err, "failed to get projects meta info") 904 } 905 906 var versionWideAlerts []ws.Alert // all alerts for a given project version 907 projectNames := `` // holds all project tokens considered a part of the report for debugging 908 for _, project := range projects { 909 projectVersion := strings.Split(project.Name, " - ")[1] 910 if projectVersion == config.Version { 911 projectNames += project.Name + "\n" 912 alerts, err := sys.GetProjectAlertsByType(project.Token, "SECURITY_VULNERABILITY") 913 if err != nil { 914 return errors.Wrapf(err, "failed to get project alerts by type") 915 } 916 917 log.Entry().Infof("Found project: %s with %v vulnerabilities.", project.Name, len(alerts)) 918 versionWideAlerts = append(versionWideAlerts, alerts...) 919 } 920 } 921 922 reportPath := filepath.Join(ws.ReportsDirectory, "project-names-aggregated.txt") 923 if err := utils.FileWrite(reportPath, []byte(projectNames), 0o666); err != nil { 924 return errors.Wrapf(err, "failed to write report: %s", reportPath) 925 } 926 if err := newVulnerabilityExcelReport(versionWideAlerts, config, utils); err != nil { 927 return errors.Wrapf(err, "failed to create new vulnerability excel report") 928 } 929 return nil 930 } 931 932 const wsReportTimeStampLayout = "20060102-150405" 933 934 // outputs an slice of alerts to an excel file 935 func newVulnerabilityExcelReport(alerts []ws.Alert, config *ScanOptions, utils whitesourceUtils) error { 936 file := excelize.NewFile() 937 streamWriter, err := file.NewStreamWriter("Sheet1") 938 if err != nil { 939 return err 940 } 941 styleID, err := file.NewStyle(`{"font":{"color":"#777777"}}`) 942 if err != nil { 943 return err 944 } 945 if err := fillVulnerabilityExcelReport(alerts, streamWriter, styleID); err != nil { 946 return err 947 } 948 if err := streamWriter.Flush(); err != nil { 949 return err 950 } 951 952 if err := utils.MkdirAll(ws.ReportsDirectory, 0o777); err != nil { 953 return err 954 } 955 956 fileName := filepath.Join(ws.ReportsDirectory, 957 fmt.Sprintf("vulnerabilities-%s.xlsx", utils.Now().Format(wsReportTimeStampLayout))) 958 stream, err := utils.FileOpen(fileName, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0o666) 959 if err != nil { 960 return err 961 } 962 if err := file.Write(stream); err != nil { 963 return err 964 } 965 filePath := piperutils.Path{Name: "aggregated-vulnerabilities", Target: fileName} 966 piperutils.PersistReportsAndLinks("whitesourceExecuteScan", "", utils, []piperutils.Path{filePath}, nil) 967 return nil 968 } 969 970 func fillVulnerabilityExcelReport(alerts []ws.Alert, streamWriter *excelize.StreamWriter, styleID int) error { 971 rows := []struct { 972 axis string 973 title string 974 }{ 975 {"A1", "Severity"}, 976 {"B1", "Library"}, 977 {"C1", "Vulnerability Id"}, 978 {"D1", "CVSS 3"}, 979 {"E1", "Project"}, 980 {"F1", "Resolution"}, 981 } 982 for _, row := range rows { 983 err := streamWriter.SetRow(row.axis, []interface{}{excelize.Cell{StyleID: styleID, Value: row.title}}) 984 if err != nil { 985 return err 986 } 987 } 988 989 for i, alert := range alerts { 990 row := make([]interface{}, 6) 991 vuln := alert.Vulnerability 992 row[0] = vuln.CVSS3Severity 993 row[1] = alert.Library.Filename 994 row[2] = vuln.Name 995 row[3] = vuln.CVSS3Score 996 row[4] = alert.Project 997 row[5] = vuln.FixResolutionText 998 cell, _ := excelize.CoordinatesToCellName(1, i+2) 999 if err := streamWriter.SetRow(cell, row); err != nil { 1000 log.Entry().Errorf("failed to write alert row: %v", err) 1001 } 1002 } 1003 return nil 1004 } 1005 1006 // outputs an slice of libraries to an excel file based on projects with version == config.Version 1007 func newLibraryCSVReport(libraries map[string][]ws.Library, config *ScanOptions, utils whitesourceUtils) error { 1008 output := "Library Name, Project Name\n" 1009 for projectName, libraries := range libraries { 1010 log.Entry().Infof("Writing %v libraries for project %s to excel report..", len(libraries), projectName) 1011 for _, library := range libraries { 1012 output += library.Name + ", " + projectName + "\n" 1013 } 1014 } 1015 1016 // Ensure reporting directory exists 1017 if err := utils.MkdirAll(ws.ReportsDirectory, 0o777); err != nil { 1018 return errors.Wrapf(err, "failed to create directories: %s", ws.ReportsDirectory) 1019 } 1020 1021 // Write result to file 1022 fileName := fmt.Sprintf("%s/libraries-%s.csv", ws.ReportsDirectory, 1023 utils.Now().Format(wsReportTimeStampLayout)) 1024 if err := utils.FileWrite(fileName, []byte(output), 0o666); err != nil { 1025 return errors.Wrapf(err, "failed to write file: %s", fileName) 1026 } 1027 filePath := piperutils.Path{Name: "aggregated-libraries", Target: fileName} 1028 piperutils.PersistReportsAndLinks("whitesourceExecuteScan", "", utils, []piperutils.Path{filePath}, nil) 1029 return nil 1030 } 1031 1032 // persistScannedProjects writes all actually scanned WhiteSource project names as list 1033 // into the Common Pipeline Environment, from where it can be used by sub-sequent steps. 1034 func persistScannedProjects(config *ScanOptions, scan *ws.Scan, commonPipelineEnvironment *whitesourceExecuteScanCommonPipelineEnvironment) { 1035 var projectNames []string 1036 if config.ProjectName != "" { 1037 projectNames = []string{config.ProjectName + " - " + config.Version} 1038 } else { 1039 projectNames = scan.ScannedProjectNames() 1040 } 1041 commonPipelineEnvironment.custom.whitesourceProjectNames = projectNames 1042 } 1043 1044 // create toolrecord file for whitesource 1045 func createToolRecordWhitesource(utils whitesourceUtils, workspace string, config *whitesourceExecuteScanOptions, scan *ws.Scan) (string, error) { 1046 record := toolrecord.New(utils, workspace, "whitesource", config.ServiceURL) 1047 // rest api url https://.../api/v1.x 1048 apiUrl, err := url.Parse(config.ServiceURL) 1049 if err != nil { 1050 return "", err 1051 } 1052 wsUiRoot := "https://" + apiUrl.Hostname() 1053 productURL := wsUiRoot + "/Wss/WSS.html#!product;token=" + config.ProductToken 1054 err = record.AddKeyData("product", 1055 config.ProductToken, 1056 config.ProductName, 1057 productURL) 1058 if err != nil { 1059 return "", err 1060 } 1061 max_idx := 0 1062 for idx, project := range scan.ScannedProjects() { 1063 max_idx = idx 1064 name := project.Name 1065 projectId := strconv.FormatInt(project.ID, 10) 1066 token := project.Token 1067 projectURL := "" 1068 if projectId != "" { 1069 projectURL = wsUiRoot + "/Wss/WSS.html#!project;id=" + projectId 1070 } 1071 if token == "" { 1072 // token is empty, provide a dummy to have an indication 1073 token = "unknown" 1074 } 1075 err = record.AddKeyData("project", 1076 token, 1077 name, 1078 projectURL) 1079 if err != nil { 1080 return "", err 1081 } 1082 } 1083 // set overall display data to product if there 1084 // is more than one project 1085 if max_idx > 1 { 1086 record.SetOverallDisplayData(config.ProductName, productURL) 1087 } 1088 err = record.Persist() 1089 if err != nil { 1090 return "", err 1091 } 1092 return record.GetFileName(), nil 1093 } 1094 1095 func downloadMultipleDockerImageAsTar(config *ScanOptions, utils whitesourceUtils) error { 1096 1097 imageNameToSave := strings.Replace(config.ScanImage, "/", "-", -1) 1098 1099 saveImageOptions := containerSaveImageOptions{ 1100 ContainerImage: config.ScanImage, 1101 ContainerRegistryURL: config.ScanImageRegistryURL, 1102 ContainerRegistryUser: config.ContainerRegistryUser, 1103 ContainerRegistryPassword: config.ContainerRegistryPassword, 1104 DockerConfigJSON: config.DockerConfigJSON, 1105 FilePath: config.ScanPath + "/" + imageNameToSave, // previously was config.ProjectName 1106 ImageFormat: "legacy", // keep the image format legacy or whitesource is not able to read layers 1107 } 1108 dClientOptions := piperDocker.ClientOptions{ImageName: saveImageOptions.ContainerImage, RegistryURL: saveImageOptions.ContainerRegistryURL, LocalPath: "", ImageFormat: "legacy"} 1109 dClient := &piperDocker.Client{} 1110 dClient.SetOptions(dClientOptions) 1111 tarFilePath, err := runContainerSaveImage(&saveImageOptions, &telemetry.CustomData{}, "./cache", "", dClient, utils) 1112 if err != nil { 1113 if strings.Contains(fmt.Sprint(err), "no image found") { 1114 log.SetErrorCategory(log.ErrorConfiguration) 1115 } 1116 return errors.Wrapf(err, "failed to download Docker image %v", config.ScanImage) 1117 } 1118 // remove contents after : in the image name 1119 if err := renameTarfilePath(tarFilePath); err != nil { 1120 return errors.Wrapf(err, "failed to rename image %v", err) 1121 } 1122 1123 return nil 1124 } 1125 1126 func downloadDockerImageAsTar(config *ScanOptions, utils whitesourceUtils) error { 1127 1128 saveImageOptions := containerSaveImageOptions{ 1129 ContainerImage: config.ScanImage, 1130 ContainerRegistryURL: config.ScanImageRegistryURL, 1131 ContainerRegistryUser: config.ContainerRegistryUser, 1132 ContainerRegistryPassword: config.ContainerRegistryPassword, 1133 DockerConfigJSON: config.DockerConfigJSON, 1134 FilePath: config.ProjectName, // consider changing this to config.ScanPath + "/" + config.ProjectName 1135 ImageFormat: "legacy", // keep the image format legacy or whitesource is not able to read layers 1136 } 1137 dClientOptions := piperDocker.ClientOptions{ImageName: saveImageOptions.ContainerImage, RegistryURL: saveImageOptions.ContainerRegistryURL, LocalPath: "", ImageFormat: "legacy"} 1138 dClient := &piperDocker.Client{} 1139 dClient.SetOptions(dClientOptions) 1140 if _, err := runContainerSaveImage(&saveImageOptions, &telemetry.CustomData{}, "./cache", "", dClient, utils); err != nil { 1141 if strings.Contains(fmt.Sprint(err), "no image found") { 1142 log.SetErrorCategory(log.ErrorConfiguration) 1143 } 1144 return errors.Wrapf(err, "failed to download Docker image %v", config.ScanImage) 1145 } 1146 1147 return nil 1148 } 1149 1150 // rename tarFilepath to remove all contents after : 1151 func renameTarfilePath(tarFilepath string) error { 1152 if _, err := os.Stat(tarFilepath); os.IsNotExist(err) { 1153 return fmt.Errorf("file %s does not exist", tarFilepath) 1154 } 1155 newFileName := "" 1156 if index := strings.Index(tarFilepath, ":"); index != -1 { 1157 newFileName = tarFilepath[:index] 1158 newFileName += ".tar" 1159 } 1160 if err := os.Rename(tarFilepath, newFileName); err != nil { 1161 return fmt.Errorf("error renaming file %s to %s: %v", tarFilepath, newFileName, err) 1162 } 1163 return nil 1164 }