github.com/ouraigua/jenkins-library@v0.0.0-20231028010029-fbeaf2f3aa9b/cmd/codeqlExecuteScan.go (about) 1 package cmd 2 3 import ( 4 "bytes" 5 "fmt" 6 "os" 7 "path/filepath" 8 "regexp" 9 "strings" 10 "time" 11 12 "github.com/SAP/jenkins-library/pkg/codeql" 13 "github.com/SAP/jenkins-library/pkg/command" 14 "github.com/SAP/jenkins-library/pkg/log" 15 "github.com/SAP/jenkins-library/pkg/orchestrator" 16 "github.com/SAP/jenkins-library/pkg/piperutils" 17 "github.com/SAP/jenkins-library/pkg/telemetry" 18 "github.com/SAP/jenkins-library/pkg/toolrecord" 19 "github.com/pkg/errors" 20 ) 21 22 type codeqlExecuteScanUtils interface { 23 command.ExecRunner 24 25 piperutils.FileUtils 26 } 27 28 type RepoInfo struct { 29 serverUrl string 30 repo string 31 commitId string 32 ref string 33 owner string 34 } 35 36 type codeqlExecuteScanUtilsBundle struct { 37 *command.Command 38 *piperutils.Files 39 } 40 41 const sarifUploadComplete = "complete" 42 const sarifUploadFailed = "failed" 43 44 func newCodeqlExecuteScanUtils() codeqlExecuteScanUtils { 45 utils := codeqlExecuteScanUtilsBundle{ 46 Command: &command.Command{}, 47 Files: &piperutils.Files{}, 48 } 49 50 utils.Stdout(log.Writer()) 51 utils.Stderr(log.Writer()) 52 return &utils 53 } 54 55 func codeqlExecuteScan(config codeqlExecuteScanOptions, telemetryData *telemetry.CustomData) { 56 57 utils := newCodeqlExecuteScanUtils() 58 59 reports, err := runCodeqlExecuteScan(&config, telemetryData, utils) 60 piperutils.PersistReportsAndLinks("codeqlExecuteScan", "./", utils, reports, nil) 61 62 if err != nil { 63 log.Entry().WithError(err).Fatal("Codeql scan failed") 64 } 65 } 66 67 func codeqlQuery(cmd []string, codeqlQuery string) []string { 68 if len(codeqlQuery) > 0 { 69 cmd = append(cmd, codeqlQuery) 70 } 71 72 return cmd 73 } 74 75 func execute(utils codeqlExecuteScanUtils, cmd []string, isVerbose bool) error { 76 if isVerbose { 77 cmd = append(cmd, "-v") 78 } 79 80 return utils.RunExecutable("codeql", cmd...) 81 } 82 83 func getLangFromBuildTool(buildTool string) string { 84 switch buildTool { 85 case "maven": 86 return "java" 87 case "pip": 88 return "python" 89 case "npm": 90 return "javascript" 91 case "yarn": 92 return "javascript" 93 case "golang": 94 return "go" 95 default: 96 return "" 97 } 98 } 99 100 func getGitRepoInfo(repoUri string, repoInfo *RepoInfo) error { 101 if repoUri == "" { 102 return errors.New("repository param is not set or it cannot be auto populated") 103 } 104 105 pat := regexp.MustCompile(`^(https:\/\/|git@)([\S]+:[\S]+@)?([^\/:]+)[\/:]([^\/:]+\/[\S]+)$`) 106 matches := pat.FindAllStringSubmatch(repoUri, -1) 107 if len(matches) > 0 { 108 match := matches[0] 109 repoInfo.serverUrl = "https://" + match[3] 110 repoData := strings.Split(strings.TrimSuffix(match[4], ".git"), "/") 111 if len(repoData) != 2 { 112 return fmt.Errorf("Invalid repository %s", repoUri) 113 } 114 115 repoInfo.owner = repoData[0] 116 repoInfo.repo = repoData[1] 117 return nil 118 } 119 120 return fmt.Errorf("Invalid repository %s", repoUri) 121 } 122 123 func initGitInfo(config *codeqlExecuteScanOptions) (RepoInfo, error) { 124 var repoInfo RepoInfo 125 err := getGitRepoInfo(config.Repository, &repoInfo) 126 if err != nil { 127 log.Entry().Error(err) 128 } 129 130 repoInfo.ref = config.AnalyzedRef 131 repoInfo.commitId = config.CommitID 132 133 provider, err := orchestrator.NewOrchestratorSpecificConfigProvider() 134 if err != nil { 135 log.Entry().Warn("No orchestrator found. We assume piper is running locally.") 136 } else { 137 if repoInfo.ref == "" { 138 repoInfo.ref = provider.GetReference() 139 } 140 141 if repoInfo.commitId == "" || repoInfo.commitId == "NA" { 142 repoInfo.commitId = provider.GetCommit() 143 } 144 145 if repoInfo.serverUrl == "" { 146 err = getGitRepoInfo(provider.GetRepoURL(), &repoInfo) 147 if err != nil { 148 log.Entry().Error(err) 149 } 150 } 151 } 152 if len(config.TargetGithubRepoURL) > 0 { 153 if strings.Contains(repoInfo.serverUrl, "github") { 154 log.Entry().Errorf("TargetGithubRepoURL should not be set as the source repo is on github.") 155 return repoInfo, errors.New("TargetGithubRepoURL should not be set as the source repo is on github.") 156 } 157 err := getGitRepoInfo(config.TargetGithubRepoURL, &repoInfo) 158 if err != nil { 159 log.Entry().Error(err) 160 return repoInfo, err 161 } 162 if len(config.TargetGithubBranchName) > 0 { 163 repoInfo.ref = config.TargetGithubBranchName 164 if len(strings.Split(config.TargetGithubBranchName, "/")) < 3 { 165 repoInfo.ref = "refs/heads/" + config.TargetGithubBranchName 166 } 167 } 168 } 169 170 return repoInfo, nil 171 } 172 173 func getToken(config *codeqlExecuteScanOptions) (bool, string) { 174 if len(config.GithubToken) > 0 { 175 return true, config.GithubToken 176 } 177 178 envVal, isEnvGithubToken := os.LookupEnv("GITHUB_TOKEN") 179 if isEnvGithubToken { 180 return true, envVal 181 } 182 183 return false, "" 184 } 185 186 func uploadResults(config *codeqlExecuteScanOptions, repoInfo RepoInfo, token string, utils codeqlExecuteScanUtils) (string, error) { 187 cmd := []string{"github", "upload-results", "--sarif=" + filepath.Join(config.ModulePath, "target", "codeqlReport.sarif")} 188 189 if config.GithubToken != "" { 190 cmd = append(cmd, "-a="+token) 191 } 192 193 if repoInfo.commitId != "" { 194 cmd = append(cmd, "--commit="+repoInfo.commitId) 195 } 196 197 if repoInfo.serverUrl != "" { 198 cmd = append(cmd, "--github-url="+repoInfo.serverUrl) 199 } 200 201 if repoInfo.repo != "" { 202 cmd = append(cmd, "--repository="+(repoInfo.owner+"/"+repoInfo.repo)) 203 } 204 205 if repoInfo.ref != "" { 206 cmd = append(cmd, "--ref="+repoInfo.ref) 207 } 208 209 //if no git params are passed(commitId, reference, serverUrl, repository), then codeql tries to auto populate it based on git information of the checkout repository. 210 //It also depends on the orchestrator. Some orchestrator keep git information and some not. 211 212 var bufferOut, bufferErr bytes.Buffer 213 utils.Stdout(&bufferOut) 214 defer utils.Stdout(log.Writer()) 215 utils.Stderr(&bufferErr) 216 defer utils.Stderr(log.Writer()) 217 218 err := execute(utils, cmd, GeneralConfig.Verbose) 219 if err != nil { 220 e := bufferErr.String() 221 log.Entry().Error(e) 222 if strings.Contains(e, "Unauthorized") { 223 log.Entry().Error("Either your Github Token is invalid or you use both Vault and Jenkins credentials where your Vault credentials are invalid, to use your Jenkins credentials try setting 'skipVault:true'") 224 } 225 log.Entry().Error("failed to upload sarif results") 226 return "", err 227 } 228 229 url := bufferOut.String() 230 return strings.TrimSpace(url), nil 231 } 232 233 func waitSarifUploaded(config *codeqlExecuteScanOptions, codeqlSarifUploader codeql.CodeqlSarifUploader) error { 234 maxRetries := config.SarifCheckMaxRetries 235 retryInterval := time.Duration(config.SarifCheckRetryInterval) * time.Second 236 237 log.Entry().Info("waiting for the SARIF to upload") 238 i := 1 239 for { 240 sarifStatus, err := codeqlSarifUploader.GetSarifStatus() 241 if err != nil { 242 return err 243 } 244 log.Entry().Infof("the SARIF processing status: %s", sarifStatus.ProcessingStatus) 245 if sarifStatus.ProcessingStatus == sarifUploadComplete { 246 return nil 247 } 248 if sarifStatus.ProcessingStatus == sarifUploadFailed { 249 for e := range sarifStatus.Errors { 250 log.Entry().Error(e) 251 } 252 return errors.New("failed to upload sarif file") 253 } 254 if i <= maxRetries { 255 log.Entry().Infof("still waiting for the SARIF to upload: retrying in %d seconds... (retry %d/%d)", config.SarifCheckRetryInterval, i, maxRetries) 256 time.Sleep(retryInterval) 257 i++ 258 continue 259 } 260 return errors.New("failed to check sarif uploading status: max retries reached") 261 } 262 } 263 264 func runCodeqlExecuteScan(config *codeqlExecuteScanOptions, telemetryData *telemetry.CustomData, utils codeqlExecuteScanUtils) ([]piperutils.Path, error) { 265 codeqlVersion, err := os.ReadFile("/etc/image-version") 266 if err != nil { 267 log.Entry().Infof("CodeQL image version: unknown") 268 } else { 269 log.Entry().Infof("CodeQL image version: %s", string(codeqlVersion)) 270 } 271 272 var reports []piperutils.Path 273 cmd := []string{"database", "create", config.Database, "--overwrite", "--source-root", ".", "--working-dir", config.ModulePath} 274 275 language := getLangFromBuildTool(config.BuildTool) 276 277 if len(language) == 0 && len(config.Language) == 0 { 278 if config.BuildTool == "custom" { 279 return reports, fmt.Errorf("as the buildTool is custom. please specify the language parameter") 280 } else { 281 return reports, fmt.Errorf("the step could not recognize the specified buildTool %s. please specify valid buildtool", config.BuildTool) 282 } 283 } 284 if len(language) > 0 { 285 cmd = append(cmd, "--language="+language) 286 } else { 287 cmd = append(cmd, "--language="+config.Language) 288 } 289 290 cmd = append(cmd, getRamAndThreadsFromConfig(config)...) 291 292 //codeql has an autobuilder which tries to build the project based on specified programming language 293 if len(config.BuildCommand) > 0 { 294 cmd = append(cmd, "--command="+config.BuildCommand) 295 } 296 297 err = execute(utils, cmd, GeneralConfig.Verbose) 298 if err != nil { 299 log.Entry().Error("failed running command codeql database create") 300 return reports, err 301 } 302 303 err = os.MkdirAll(filepath.Join(config.ModulePath, "target"), os.ModePerm) 304 if err != nil { 305 return reports, fmt.Errorf("failed to create directory: %w", err) 306 } 307 308 cmd = nil 309 cmd = append(cmd, "database", "analyze", "--format=sarif-latest", fmt.Sprintf("--output=%v", filepath.Join(config.ModulePath, "target", "codeqlReport.sarif")), config.Database) 310 cmd = append(cmd, getRamAndThreadsFromConfig(config)...) 311 cmd = codeqlQuery(cmd, config.QuerySuite) 312 err = execute(utils, cmd, GeneralConfig.Verbose) 313 if err != nil { 314 log.Entry().Error("failed running command codeql database analyze for sarif generation") 315 return reports, err 316 } 317 318 reports = append(reports, piperutils.Path{Target: filepath.Join(config.ModulePath, "target", "codeqlReport.sarif")}) 319 320 cmd = nil 321 cmd = append(cmd, "database", "analyze", "--format=csv", fmt.Sprintf("--output=%v", filepath.Join(config.ModulePath, "target", "codeqlReport.csv")), config.Database) 322 cmd = append(cmd, getRamAndThreadsFromConfig(config)...) 323 cmd = codeqlQuery(cmd, config.QuerySuite) 324 err = execute(utils, cmd, GeneralConfig.Verbose) 325 if err != nil { 326 log.Entry().Error("failed running command codeql database analyze for csv generation") 327 return reports, err 328 } 329 330 reports = append(reports, piperutils.Path{Target: filepath.Join(config.ModulePath, "target", "codeqlReport.csv")}) 331 332 repoInfo, err := initGitInfo(config) 333 if err != nil { 334 return reports, err 335 } 336 repoUrl := fmt.Sprintf("%s/%s/%s", repoInfo.serverUrl, repoInfo.owner, repoInfo.repo) 337 repoReference, err := buildRepoReference(repoUrl, repoInfo.ref) 338 repoCodeqlScanUrl := fmt.Sprintf("%s/security/code-scanning?query=is:open+ref:%s", repoUrl, repoInfo.ref) 339 340 if len(config.TargetGithubRepoURL) > 0 { 341 hasToken, token := getToken(config) 342 if !hasToken { 343 return reports, errors.New("failed running upload db sources to GitHub as githubToken was not specified") 344 } 345 repoUploader, err := codeql.NewGitUploaderInstance( 346 token, 347 repoInfo.ref, 348 config.Database, 349 repoInfo.commitId, 350 config.Repository, 351 config.TargetGithubRepoURL, 352 ) 353 if err != nil { 354 return reports, err 355 } 356 targetCommitId, err := repoUploader.UploadProjectToGithub() 357 if err != nil { 358 return reports, errors.Wrap(err, "failed uploading db sources from non-GitHub SCM to GitHub") 359 } 360 repoInfo.commitId = targetCommitId 361 } 362 363 if !config.UploadResults { 364 log.Entry().Warn("The sarif results will not be uploaded to the repository and compliance report will not be generated as uploadResults is set to false.") 365 } else { 366 hasToken, token := getToken(config) 367 if !hasToken { 368 return reports, errors.New("failed running upload-results as githubToken was not specified") 369 } 370 371 sarifUrl, err := uploadResults(config, repoInfo, token, utils) 372 if err != nil { 373 return reports, err 374 } 375 codeqlSarifUploader := codeql.NewCodeqlSarifUploaderInstance(sarifUrl, token) 376 err = waitSarifUploaded(config, &codeqlSarifUploader) 377 if err != nil { 378 return reports, errors.Wrap(err, "failed to upload sarif") 379 } 380 381 codeqlScanAuditInstance := codeql.NewCodeqlScanAuditInstance(repoInfo.serverUrl, repoInfo.owner, repoInfo.repo, token, []string{}) 382 scanResults, err := codeqlScanAuditInstance.GetVulnerabilities(repoInfo.ref) 383 if err != nil { 384 return reports, errors.Wrap(err, "failed to get scan results") 385 } 386 387 codeqlAudit := codeql.CodeqlAudit{ToolName: "codeql", RepositoryUrl: repoUrl, CodeScanningLink: repoCodeqlScanUrl, RepositoryReferenceUrl: repoReference, QuerySuite: config.QuerySuite, ScanResults: scanResults} 388 paths, err := codeql.WriteJSONReport(codeqlAudit, config.ModulePath) 389 if err != nil { 390 return reports, errors.Wrap(err, "failed to write json compliance report") 391 } 392 reports = append(reports, paths...) 393 394 if config.CheckForCompliance { 395 for _, scanResult := range scanResults { 396 unaudited := scanResult.Total - scanResult.Audited 397 if unaudited > config.VulnerabilityThresholdTotal { 398 msg := fmt.Sprintf("Your repository %v with ref %v is not compliant. Total unaudited issues are %v which is greater than the VulnerabilityThresholdTotal count %v", repoUrl, repoInfo.ref, unaudited, config.VulnerabilityThresholdTotal) 399 return reports, errors.Errorf(msg) 400 } 401 } 402 } 403 } 404 405 toolRecordFileName, err := createAndPersistToolRecord(utils, repoInfo, repoReference, repoUrl, repoCodeqlScanUrl) 406 if err != nil { 407 log.Entry().Warning("TR_CODEQL: Failed to create toolrecord file ...", err) 408 } else { 409 reports = append(reports, piperutils.Path{Target: toolRecordFileName}) 410 } 411 412 return reports, nil 413 } 414 415 func createAndPersistToolRecord(utils codeqlExecuteScanUtils, repoInfo RepoInfo, repoReference string, repoUrl string, repoCodeqlScanUrl string) (string, error) { 416 toolRecord, err := createToolRecordCodeql(utils, repoInfo, repoReference, repoUrl, repoCodeqlScanUrl) 417 if err != nil { 418 return "", err 419 } 420 421 toolRecordFileName, err := persistToolRecord(toolRecord) 422 if err != nil { 423 return "", err 424 } 425 426 return toolRecordFileName, nil 427 } 428 429 func createToolRecordCodeql(utils codeqlExecuteScanUtils, repoInfo RepoInfo, repoUrl string, repoReference string, repoCodeqlScanUrl string) (*toolrecord.Toolrecord, error) { 430 record := toolrecord.New(utils, "./", "codeql", repoInfo.serverUrl) 431 432 if repoInfo.serverUrl == "" { 433 return record, errors.New("Repository not set") 434 } 435 436 if repoInfo.commitId == "" || repoInfo.commitId == "NA" { 437 return record, errors.New("CommitId not set") 438 } 439 440 if repoInfo.ref == "" { 441 return record, errors.New("Analyzed Reference not set") 442 } 443 444 record.DisplayName = fmt.Sprintf("%s %s - %s %s", repoInfo.owner, repoInfo.repo, repoInfo.ref, repoInfo.commitId) 445 record.DisplayURL = fmt.Sprintf("%s/security/code-scanning?query=is:open+ref:%s", repoUrl, repoInfo.ref) 446 447 err := record.AddKeyData("repository", 448 fmt.Sprintf("%s/%s", repoInfo.owner, repoInfo.repo), 449 fmt.Sprintf("%s %s", repoInfo.owner, repoInfo.repo), 450 repoUrl) 451 if err != nil { 452 return record, err 453 } 454 455 err = record.AddKeyData("repositoryReference", 456 repoInfo.ref, 457 fmt.Sprintf("%s - %s", repoInfo.repo, repoInfo.ref), 458 repoReference) 459 if err != nil { 460 return record, err 461 } 462 463 err = record.AddKeyData("scanResult", 464 fmt.Sprintf("%s/%s", repoInfo.ref, repoInfo.commitId), 465 fmt.Sprintf("%s %s - %s %s", repoInfo.owner, repoInfo.repo, repoInfo.ref, repoInfo.commitId), 466 fmt.Sprintf("%s/security/code-scanning?query=is:open+ref:%s", repoUrl, repoInfo.ref)) 467 if err != nil { 468 return record, err 469 } 470 471 return record, nil 472 } 473 474 func buildRepoReference(repository, analyzedRef string) (string, error) { 475 ref := strings.Split(analyzedRef, "/") 476 if len(ref) < 3 { 477 return "", errors.New(fmt.Sprintf("Wrong analyzedRef format: %s", analyzedRef)) 478 } 479 if strings.Contains(analyzedRef, "pull") { 480 if len(ref) < 4 { 481 return "", errors.New(fmt.Sprintf("Wrong analyzedRef format: %s", analyzedRef)) 482 } 483 return fmt.Sprintf("%s/pull/%s", repository, ref[2]), nil 484 } 485 return fmt.Sprintf("%s/tree/%s", repository, ref[2]), nil 486 } 487 488 func persistToolRecord(toolRecord *toolrecord.Toolrecord) (string, error) { 489 err := toolRecord.Persist() 490 if err != nil { 491 return "", err 492 } 493 return toolRecord.GetFileName(), nil 494 } 495 496 func getRamAndThreadsFromConfig(config *codeqlExecuteScanOptions) []string { 497 params := make([]string, 0, 2) 498 if len(config.Threads) > 0 { 499 params = append(params, "--threads="+config.Threads) 500 } 501 if len(config.Ram) > 0 { 502 params = append(params, "--ram="+config.Ram) 503 } 504 return params 505 }