github.com/jaylevin/jenkins-library@v1.230.4/pkg/whitesource/scanUA.go (about) 1 package whitesource 2 3 import ( 4 "bufio" 5 "bytes" 6 "encoding/json" 7 "fmt" 8 "io" 9 "os" 10 "path/filepath" 11 "regexp" 12 "strings" 13 "sync" 14 15 "github.com/SAP/jenkins-library/pkg/log" 16 "github.com/pkg/errors" 17 ) 18 19 const jvmTarGz = "jvm.tar.gz" 20 const jvmDir = "./jvm" 21 const projectRegEx = `Project name: ([^,]*), URL: (.*)` 22 23 // ExecuteUAScan executes a scan with the Whitesource Unified Agent. 24 func (s *Scan) ExecuteUAScan(config *ScanOptions, utils Utils) error { 25 s.AgentName = "WhiteSource Unified Agent" 26 if config.BuildTool != "mta" { 27 return s.ExecuteUAScanInPath(config, utils, config.ScanPath) 28 } 29 30 log.Entry().Infof("Executing WhiteSource UA scan for MTA project") 31 pomExists, _ := utils.FileExists("pom.xml") 32 if pomExists { 33 mavenConfig := *config 34 mavenConfig.BuildTool = "maven" 35 if err := s.ExecuteUAScanInPath(&mavenConfig, utils, config.ScanPath); err != nil { 36 return errors.Wrap(err, "failed to run scan for maven modules of mta") 37 } 38 } else { 39 if pomFiles, _ := utils.Glob("**/pom.xml"); len(pomFiles) > 0 { 40 log.SetErrorCategory(log.ErrorCustom) 41 return fmt.Errorf("mta project with java modules does not contain an aggregator pom.xml in the root - this is mandatory") 42 } 43 } 44 45 packageJSONFiles, err := utils.FindPackageJSONFiles(config) 46 if err != nil { 47 return errors.Wrap(err, "failed to find package.json files") 48 } 49 if len(packageJSONFiles) > 0 { 50 npmConfig := *config 51 npmConfig.BuildTool = "npm" 52 for _, packageJSONFile := range packageJSONFiles { 53 // we only need the path here 54 modulePath, _ := filepath.Split(packageJSONFile) 55 projectName, err := getProjectNameFromPackageJSON(packageJSONFile, utils) 56 if err != nil { 57 return errors.Wrapf(err, "failed retrieve project name") 58 } 59 npmConfig.ProjectName = projectName 60 // ToDo: likely needs to be refactored, AggregateProjectName should only be available if we want to force aggregation? 61 s.AggregateProjectName = projectName 62 if err := s.ExecuteUAScanInPath(&npmConfig, utils, modulePath); err != nil { 63 return errors.Wrapf(err, "failed to run scan for npm module %v", modulePath) 64 } 65 } 66 } 67 68 _ = removeJre(filepath.Join(jvmDir, "bin", "java"), utils) 69 70 return nil 71 } 72 73 // ExecuteUAScanInPath executes a scan with the Whitesource Unified Agent in a dedicated scanPath. 74 func (s *Scan) ExecuteUAScanInPath(config *ScanOptions, utils Utils, scanPath string) error { 75 // Download the unified agent jar file if one does not exist 76 err := downloadAgent(config, utils) 77 if err != nil { 78 return err 79 } 80 81 // Download JRE in case none is available 82 javaPath, err := downloadJre(config, utils) 83 if err != nil { 84 return err 85 } 86 87 // Fetch version of UA 88 versionBuffer := bytes.Buffer{} 89 utils.Stdout(&versionBuffer) 90 err = utils.RunExecutable(javaPath, "-jar", config.AgentFileName, "-v") 91 if err != nil { 92 return errors.Wrap(err, "Failed to determine UA version") 93 } 94 s.AgentVersion = strings.TrimSpace(versionBuffer.String()) 95 log.Entry().Debugf("Read UA version %v from Stdout", s.AgentVersion) 96 utils.Stdout(log.Writer()) 97 98 // ToDo: Check if Download of Docker/container image should be done here instead of in cmd/whitesourceExecuteScan.go 99 100 // ToDo: check if this is required 101 if err := s.AppendScannedProject(s.AggregateProjectName); err != nil { 102 return err 103 } 104 105 configPath, err := config.RewriteUAConfigurationFile(utils, s.AggregateProjectName) 106 if err != nil { 107 return err 108 } 109 110 if len(scanPath) == 0 { 111 scanPath = "." 112 } 113 114 // log parsing in order to identify the projects WhiteSource really scanned 115 // we may refactor this in case there is a safer way to identify the projects e.g. via REST API 116 117 //ToDO: we only need stdOut or stdErr, let's see where UA writes to ... 118 prOut, stdOut := io.Pipe() 119 trOut := io.TeeReader(prOut, os.Stderr) 120 utils.Stdout(stdOut) 121 122 prErr, stdErr := io.Pipe() 123 trErr := io.TeeReader(prErr, os.Stderr) 124 utils.Stdout(stdErr) 125 126 var wg sync.WaitGroup 127 wg.Add(2) 128 129 go func() { 130 defer wg.Done() 131 scanLog(trOut, s) 132 }() 133 134 go func() { 135 defer wg.Done() 136 scanLog(trErr, s) 137 }() 138 err = utils.RunExecutable(javaPath, "-jar", config.AgentFileName, "-d", scanPath, "-c", configPath, "-wss.url", config.AgentURL) 139 140 if err := removeJre(javaPath, utils); err != nil { 141 log.Entry().Warning(err) 142 } 143 144 if err != nil { 145 if err := removeJre(javaPath, utils); err != nil { 146 log.Entry().Warning(err) 147 } 148 exitCode := utils.GetExitCode() 149 log.Entry().Infof("WhiteSource scan failed with exit code %v", exitCode) 150 evaluateExitCode(exitCode) 151 return errors.Wrapf(err, "failed to execute WhiteSource scan with exit code %v", exitCode) 152 } 153 return nil 154 } 155 156 func evaluateExitCode(exitCode int) { 157 switch exitCode { 158 case 255: 159 log.Entry().Info("General error has occurred.") 160 log.SetErrorCategory(log.ErrorUndefined) 161 case 254: 162 log.Entry().Info("Whitesource found one or multiple policy violations.") 163 log.SetErrorCategory(log.ErrorCompliance) 164 case 253: 165 log.Entry().Info("The local scan client failed to execute the scan.") 166 log.SetErrorCategory(log.ErrorUndefined) 167 case 252: 168 log.Entry().Info("There was a failure in the connection to the WhiteSource servers.") 169 log.SetErrorCategory(log.ErrorInfrastructure) 170 case 251: 171 log.Entry().Info("The server failed to analyze the scan.") 172 log.SetErrorCategory(log.ErrorService) 173 case 250: 174 log.Entry().Info("One of the package manager's prerequisite steps (e.g. npm install) failed.") 175 log.SetErrorCategory(log.ErrorCustom) 176 default: 177 log.Entry().Info("Whitesource scan failed with unknown error code") 178 log.SetErrorCategory(log.ErrorUndefined) 179 } 180 } 181 182 // downloadAgent downloads the unified agent jar file if one does not exist 183 func downloadAgent(config *ScanOptions, utils Utils) error { 184 agentFile := config.AgentFileName 185 exists, err := utils.FileExists(agentFile) 186 if err != nil { 187 return errors.Wrapf(err, "failed to check if file '%s' exists", agentFile) 188 } 189 if !exists { 190 err := utils.DownloadFile(config.AgentDownloadURL, agentFile, nil, nil) 191 if err != nil { 192 // we check if the copy and the unauthorized error occurs and retry the download 193 // if the copy error did not happen, we rerun the whole download mechanism once 194 if strings.Contains(err.Error(), "unable to copy content from url to file") || strings.Contains(err.Error(), "returned with response 404 Not Found") || strings.Contains(err.Error(), "returned with response 403 Forbidden") { 195 // retry the download once again 196 log.Entry().Warnf("[Retry] Previous download failed due to %v", err) 197 err = nil // reset error to nil 198 err = utils.DownloadFile(config.AgentDownloadURL, agentFile, nil, nil) 199 } 200 } 201 202 if err != nil { 203 return errors.Wrapf(err, "failed to download unified agent from URL '%s' to file '%s'", config.AgentDownloadURL, agentFile) 204 } 205 } 206 return nil 207 } 208 209 // downloadJre downloads the a JRE in case no java command can be executed 210 func downloadJre(config *ScanOptions, utils Utils) (string, error) { 211 // cater for multiple executions 212 if exists, _ := utils.FileExists(filepath.Join(jvmDir, "bin", "java")); exists { 213 return filepath.Join(jvmDir, "bin", "java"), nil 214 } 215 err := utils.RunExecutable("java", "-version") 216 javaPath := "java" 217 if err != nil { 218 log.Entry().Infof("No Java installation found, downloading JVM from %v", config.JreDownloadURL) 219 err = utils.DownloadFile(config.JreDownloadURL, jvmTarGz, nil, nil) 220 if err != nil { 221 // we check if the copy error occurs and retry the download 222 // if the copy error did not happen, we rerun the whole download mechanism once 223 if strings.Contains(err.Error(), "unable to copy content from url to file") { 224 // retry the download once again 225 log.Entry().Warnf("Previous Download failed due to %v", err) 226 err = nil 227 err = utils.DownloadFile(config.JreDownloadURL, jvmTarGz, nil, nil) 228 } 229 } 230 231 if err != nil { 232 return "", errors.Wrapf(err, "failed to download jre from URL '%s'", config.JreDownloadURL) 233 } 234 235 // ToDo: replace tar call with go library call 236 err = utils.MkdirAll(jvmDir, 0755) 237 238 err = utils.RunExecutable("tar", fmt.Sprintf("--directory=%v", jvmDir), "--strip-components=1", "-xzf", jvmTarGz) 239 if err != nil { 240 return "", errors.Wrapf(err, "failed to extract %v", jvmTarGz) 241 } 242 log.Entry().Info("Java successfully installed") 243 javaPath = filepath.Join(jvmDir, "bin", "java") 244 } 245 return javaPath, nil 246 } 247 248 func removeJre(javaPath string, utils Utils) error { 249 if javaPath == "java" { 250 return nil 251 } 252 if err := utils.RemoveAll(jvmDir); err != nil { 253 return fmt.Errorf("failed to remove downloaded and extracted jvm from %v", jvmDir) 254 } 255 log.Entry().Debugf("Java successfully removed from %v", jvmDir) 256 if err := utils.FileRemove(jvmTarGz); err != nil { 257 return fmt.Errorf("failed to remove downloaded %v", jvmTarGz) 258 } 259 log.Entry().Debugf("%v successfully removed", jvmTarGz) 260 return nil 261 } 262 263 func getProjectNameFromPackageJSON(packageJSONPath string, utils Utils) (string, error) { 264 fileContents, err := utils.FileRead(packageJSONPath) 265 if err != nil { 266 return "", errors.Wrapf(err, "failed to read file %v", packageJSONPath) 267 } 268 var packageJSON = make(map[string]interface{}) 269 if err := json.Unmarshal(fileContents, &packageJSON); err != nil { 270 return "", errors.Wrapf(err, "failed to read file content of %v", packageJSONPath) 271 } 272 273 projectNameEntry, exists := packageJSON["name"] 274 if !exists { 275 return "", fmt.Errorf("the file '%s' must configure a name", packageJSONPath) 276 } 277 278 projectName, isString := projectNameEntry.(string) 279 if !isString { 280 return "", fmt.Errorf("the file '%s' must configure a name as string", packageJSONPath) 281 } 282 283 return projectName, nil 284 } 285 286 func scanLog(in io.Reader, scan *Scan) { 287 scanner := bufio.NewScanner(in) 288 scanner.Split(scanShortLines) 289 for scanner.Scan() { 290 line := scanner.Text() 291 parseForProjects(line, scan) 292 } 293 if err := scanner.Err(); err != nil { 294 log.Entry().WithError(err).Info("failed to scan log file") 295 } 296 } 297 298 func parseForProjects(logLine string, scan *Scan) { 299 compile := regexp.MustCompile(projectRegEx) 300 values := compile.FindStringSubmatch(logLine) 301 302 if len(values) > 0 && scan.scannedProjects != nil && len(scan.scannedProjects[values[1]].Name) == 0 { 303 scan.scannedProjects[values[1]] = Project{Name: values[1]} 304 } 305 306 } 307 308 func scanShortLines(data []byte, atEOF bool) (advance int, token []byte, err error) { 309 lenData := len(data) 310 if atEOF && lenData == 0 { 311 return 0, nil, nil 312 } 313 if lenData > 32767 && !bytes.Contains(data[0:lenData], []byte("\n")) { 314 // we will neglect long output 315 // no use cases known where this would be relevant 316 return lenData, nil, nil 317 } 318 if i := bytes.IndexByte(data, '\n'); i >= 0 && i < 32767 { 319 // We have a full newline-terminated line with a size limit 320 // Size limit is required since otherwise scanner would stall 321 return i + 1, data[0:i], nil 322 } 323 // If we're at EOF, we have a final, non-terminated line. Return it. 324 if atEOF { 325 return len(data), data, nil 326 } 327 // Request more data. 328 return 0, nil, nil 329 }