github.com/jaylevin/jenkins-library@v1.230.4/pkg/whitesource/scanNPM.go (about) 1 package whitesource 2 3 import ( 4 "encoding/json" 5 "fmt" 6 "io/ioutil" 7 "os" 8 "path/filepath" 9 10 "github.com/SAP/jenkins-library/pkg/log" 11 ) 12 13 const whiteSourceConfig = "whitesource.config.json" 14 15 func setValueAndLogChange(config map[string]interface{}, key string, value interface{}) { 16 oldValue, exists := config[key] 17 if exists && oldValue != value { 18 log.Entry().Infof("overwriting '%s' in %s: %v -> %v", key, whiteSourceConfig, oldValue, value) 19 } 20 config[key] = value 21 } 22 23 func setValueOmitIfPresent(config map[string]interface{}, key, omitIfPresent string, value interface{}) { 24 _, exists := config[omitIfPresent] 25 if exists { 26 return 27 } 28 setValueAndLogChange(config, key, value) 29 } 30 31 // writeWhitesourceConfigJSON creates or merges the file whitesource.config.json in the current 32 // directory from the given NPMScanOptions. 33 func (s *Scan) writeWhitesourceConfigJSON(config *ScanOptions, utils Utils, devDep, ignoreLsErrors bool) error { 34 var npmConfig = make(map[string]interface{}) 35 36 exists, _ := utils.FileExists(whiteSourceConfig) 37 if exists { 38 fileContents, err := utils.FileRead(whiteSourceConfig) 39 if err != nil { 40 return fmt.Errorf("file '%s' already exists, but could not be read: %w", whiteSourceConfig, err) 41 } 42 err = json.Unmarshal(fileContents, &npmConfig) 43 if err != nil { 44 return fmt.Errorf("file '%s' already exists, but could not be parsed: %w", whiteSourceConfig, err) 45 } 46 log.Entry().Infof("The file '%s' already exists in the project. Changed config details will be logged.", 47 whiteSourceConfig) 48 } 49 50 npmConfig["apiKey"] = config.OrgToken 51 npmConfig["userKey"] = config.UserToken 52 setValueAndLogChange(npmConfig, "checkPolicies", true) 53 // When checkPolicies detects any violations, it will by default not update the WS project in the backend. 54 // Therefore we also need "forceUpdate". 55 setValueAndLogChange(npmConfig, "forceUpdate", true) 56 setValueAndLogChange(npmConfig, "productName", config.ProductName) 57 setValueAndLogChange(npmConfig, "productVer", s.ProductVersion) 58 setValueOmitIfPresent(npmConfig, "productToken", "projectToken", config.ProductToken) 59 if config.ProjectName != "" { 60 // In case there are other modules (i.e. maven modules in MTA projects), 61 // or more than one NPM module, setting the project name will lead to 62 // overwriting any previous scan results with the one from this module! 63 // If this is not provided, the WhiteSource project name will be generated 64 // from "name" in package.json plus " - " plus productVersion. 65 setValueAndLogChange(npmConfig, "projectName", config.ProjectName) 66 } 67 setValueAndLogChange(npmConfig, "devDep", devDep) 68 setValueAndLogChange(npmConfig, "ignoreNpmLsErrors", ignoreLsErrors) 69 70 jsonBuffer, err := json.Marshal(npmConfig) 71 if err != nil { 72 return fmt.Errorf("failed to generate '%s': %w", whiteSourceConfig, err) 73 } 74 75 err = utils.FileWrite(whiteSourceConfig, jsonBuffer, 0644) 76 if err != nil { 77 return fmt.Errorf("failed to write '%s': %w", whiteSourceConfig, err) 78 } 79 return nil 80 } 81 82 // ExecuteNpmScan iterates over all found npm modules and performs a scan in each one. 83 func (s *Scan) ExecuteNpmScan(config *ScanOptions, utils Utils) error { 84 s.AgentName = "WhiteSource NPM Plugin" 85 s.AgentVersion = "unknown" 86 modules, err := utils.FindPackageJSONFiles(config) 87 if err != nil { 88 return fmt.Errorf("failed to find package.json files with excludes: %w", err) 89 } 90 if len(modules) == 0 { 91 return fmt.Errorf("found no NPM modules to scan. Configured excludes: %v", 92 config.BuildDescriptorExcludeList) 93 } 94 for _, module := range modules { 95 err := s.executeNpmScanForModule(module, config, utils) 96 if err != nil { 97 return fmt.Errorf("failed to scan NPM module '%s': %w", module, err) 98 } 99 } 100 return nil 101 } 102 103 // executeNpmScanForModule generates a configuration file whitesource.config.json with appropriate values from config, 104 // installs all dependencies if necessary, and executes the scan via "npx whitesource run". 105 func (s *Scan) executeNpmScanForModule(modulePath string, config *ScanOptions, utils Utils) error { 106 log.Entry().Infof("Executing Whitesource scan for NPM module '%s'", modulePath) 107 108 resetDir, err := utils.Getwd() 109 if err != nil { 110 return fmt.Errorf("failed to obtain current directory: %w", err) 111 } 112 113 dir := filepath.Dir(modulePath) 114 if err := utils.Chdir(dir); err != nil { 115 return fmt.Errorf("failed to change into directory '%s': %w", dir, err) 116 } 117 defer func() { 118 err = utils.Chdir(resetDir) 119 if err != nil { 120 log.Entry().Errorf("Failed to reset into directory '%s': %v", resetDir, err) 121 } 122 }() 123 124 if err := s.writeWhitesourceConfigJSON(config, utils, false, true); err != nil { 125 return err 126 } 127 defer func() { _ = utils.FileRemove(whiteSourceConfig) }() 128 129 projectName, err := getNpmProjectName(modulePath, utils) 130 if err != nil { 131 return err 132 } 133 134 if err := reinstallNodeModulesIfLsFails(config, utils); err != nil { 135 return err 136 } 137 138 if err := s.AppendScannedProject(projectName); err != nil { 139 return err 140 } 141 142 return utils.RunExecutable("npx", "whitesource", "run") 143 } 144 145 // getNpmProjectName tries to read a property "name" of type string from the 146 // package.json file in the current directory and returns an error, if this is not possible. 147 func getNpmProjectName(modulePath string, utils Utils) (string, error) { 148 fileContents, err := utils.FileRead("package.json") 149 if err != nil { 150 return "", fmt.Errorf("could not read %s: %w", modulePath, err) 151 } 152 var packageJSON = make(map[string]interface{}) 153 err = json.Unmarshal(fileContents, &packageJSON) 154 155 projectNameEntry, exists := packageJSON["name"] 156 if !exists { 157 return "", fmt.Errorf("the file '%s' must configure a name", modulePath) 158 } 159 160 projectName, isString := projectNameEntry.(string) 161 if !isString { 162 return "", fmt.Errorf("the file '%s' must configure a name", modulePath) 163 } 164 165 return projectName, nil 166 } 167 168 // reinstallNodeModulesIfLsFails tests running of "npm ls". 169 // If that fails, the node_modules directory is cleared and the file "package-lock.json" is removed. 170 // Then "npm install" is performed. Without this, the npm whitesource plugin will consistently hang, 171 // when encountering npm ls errors, even with "ignoreNpmLsErrors:true" in the configuration. 172 // The consequence is that what was scanned is not guaranteed to be identical to what was built & deployed. 173 // This hack/work-around that should be removed once scanning it consistently performed using the Unified Agent. 174 // A possible reason for encountering "npm ls" errors in the first place is that a different node version 175 // is used for whitesourceExecuteScan due to a different docker image being used compared to the build stage. 176 func reinstallNodeModulesIfLsFails(config *ScanOptions, utils Utils) error { 177 // No need to have output from "npm ls" in the log 178 utils.Stdout(ioutil.Discard) 179 defer utils.Stdout(log.Writer()) 180 181 err := utils.RunExecutable("npm", "ls") 182 if err == nil { 183 return nil 184 } 185 log.Entry().Warnf("'npm ls' failed. Re-installing NPM Node Modules") 186 err = utils.RemoveAll("node_modules") 187 if err != nil { 188 return fmt.Errorf("failed to remove node_modules directory: %w", err) 189 } 190 err = utils.MkdirAll("node_modules", os.ModePerm) 191 if err != nil { 192 return fmt.Errorf("failed to recreate node_modules directory: %w", err) 193 } 194 exists, _ := utils.FileExists("package-lock.json") 195 if exists { 196 err = utils.FileRemove("package-lock.json") 197 if err != nil { 198 return fmt.Errorf("failed to remove package-lock.json: %w", err) 199 } 200 } 201 // Passing only "package.json", because we are already inside the module's directory. 202 return utils.InstallAllNPMDependencies(config, []string{"package.json"}) 203 } 204 205 // ExecuteYarnScan generates a configuration file whitesource.config.json with appropriate values from config, 206 // installs whitesource yarn plugin and executes the scan. 207 func (s *Scan) ExecuteYarnScan(config *ScanOptions, utils Utils) error { 208 // To stay compatible with what the step was doing before, trigger aggregation, although 209 // there is a great chance that it doesn't work with yarn the same way it doesn't with npm. 210 // Maybe the yarn code-path should be removed, and only npm stays. 211 config.ProjectName = s.AggregateProjectName 212 if err := s.writeWhitesourceConfigJSON(config, utils, true, false); err != nil { 213 return err 214 } 215 defer func() { _ = utils.FileRemove(whiteSourceConfig) }() 216 if err := utils.RunExecutable("yarn", "global", "add", "whitesource"); err != nil { 217 return err 218 } 219 if err := utils.RunExecutable("yarn", "install"); err != nil { 220 return err 221 } 222 if err := utils.RunExecutable("whitesource", "yarn"); err != nil { 223 return err 224 } 225 return nil 226 }