github.com/jfrog/jfrog-cli-core/v2@v2.52.0/utils/coreutils/techutils.go (about) 1 package coreutils 2 3 import ( 4 "fmt" 5 "os" 6 "path/filepath" 7 "strings" 8 9 "github.com/jfrog/gofrog/datastructures" 10 "github.com/jfrog/jfrog-client-go/artifactory/services/fspatterns" 11 "github.com/jfrog/jfrog-client-go/utils/errorutils" 12 "github.com/jfrog/jfrog-client-go/utils/log" 13 14 "golang.org/x/exp/maps" 15 "golang.org/x/text/cases" 16 "golang.org/x/text/language" 17 18 "github.com/jfrog/jfrog-client-go/utils/io/fileutils" 19 ) 20 21 type Technology string 22 23 const ( 24 Maven Technology = "maven" 25 Gradle Technology = "gradle" 26 Npm Technology = "npm" 27 Pnpm Technology = "pnpm" 28 Yarn Technology = "yarn" 29 Go Technology = "go" 30 Pip Technology = "pip" 31 Pipenv Technology = "pipenv" 32 Poetry Technology = "poetry" 33 Nuget Technology = "nuget" 34 Dotnet Technology = "dotnet" 35 Docker Technology = "docker" 36 Oci Technology = "oci" 37 ) 38 39 const Pypi = "pypi" 40 41 type TechData struct { 42 // The name of the package type used in this technology. 43 packageType string 44 // Suffixes of file/directory names that indicate if a project uses this technology. 45 // The name of at least one of the files/directories in the project's directory must end with one of these suffixes. 46 indicators []string 47 // Suffixes of file/directory names that indicate if a project does not use this technology. 48 // The names of all the files/directories in the project's directory must NOT end with any of these suffixes. 49 exclude []string 50 // Whether this technology is supported by the 'jf ci-setup' command. 51 ciSetupSupport bool 52 // Whether Contextual Analysis supported in this technology. 53 applicabilityScannable bool 54 // The files that handle the project's dependencies. 55 packageDescriptors []string 56 // Formal name of the technology 57 formal string 58 // The executable name of the technology 59 execCommand string 60 // The operator for package versioning 61 packageVersionOperator string 62 // The package installation command of a package 63 packageInstallationCommand string 64 } 65 66 var technologiesData = map[Technology]TechData{ 67 Maven: { 68 indicators: []string{"pom.xml"}, 69 ciSetupSupport: true, 70 packageDescriptors: []string{"pom.xml"}, 71 execCommand: "mvn", 72 applicabilityScannable: true, 73 }, 74 Gradle: { 75 indicators: []string{"build.gradle", "build.gradle.kts"}, 76 ciSetupSupport: true, 77 packageDescriptors: []string{"build.gradle", "build.gradle.kts"}, 78 applicabilityScannable: true, 79 }, 80 Npm: { 81 indicators: []string{"package.json", "package-lock.json", "npm-shrinkwrap.json"}, 82 exclude: []string{"pnpm-lock.yaml", ".yarnrc.yml", "yarn.lock", ".yarn"}, 83 ciSetupSupport: true, 84 packageDescriptors: []string{"package.json"}, 85 formal: string(Npm), 86 packageVersionOperator: "@", 87 packageInstallationCommand: "install", 88 applicabilityScannable: true, 89 }, 90 Pnpm: { 91 indicators: []string{"pnpm-lock.yaml"}, 92 exclude: []string{".yarnrc.yml", "yarn.lock", ".yarn"}, 93 packageDescriptors: []string{"package.json"}, 94 packageVersionOperator: "@", 95 packageInstallationCommand: "update", 96 applicabilityScannable: true, 97 }, 98 Yarn: { 99 indicators: []string{".yarnrc.yml", "yarn.lock", ".yarn", ".yarnrc"}, 100 exclude: []string{"pnpm-lock.yaml"}, 101 packageDescriptors: []string{"package.json"}, 102 packageVersionOperator: "@", 103 applicabilityScannable: true, 104 }, 105 Go: { 106 indicators: []string{"go.mod"}, 107 packageDescriptors: []string{"go.mod"}, 108 packageVersionOperator: "@v", 109 packageInstallationCommand: "get", 110 }, 111 Pip: { 112 packageType: Pypi, 113 indicators: []string{"setup.py", "requirements.txt"}, 114 packageDescriptors: []string{"setup.py", "requirements.txt"}, 115 exclude: []string{"Pipfile", "Pipfile.lock", "pyproject.toml", "poetry.lock"}, 116 applicabilityScannable: true, 117 }, 118 Pipenv: { 119 packageType: Pypi, 120 indicators: []string{"Pipfile", "Pipfile.lock"}, 121 packageDescriptors: []string{"Pipfile"}, 122 packageVersionOperator: "==", 123 packageInstallationCommand: "install", 124 applicabilityScannable: true, 125 }, 126 Poetry: { 127 packageType: Pypi, 128 indicators: []string{"pyproject.toml", "poetry.lock"}, 129 packageDescriptors: []string{"pyproject.toml"}, 130 packageInstallationCommand: "add", 131 packageVersionOperator: "==", 132 applicabilityScannable: true, 133 }, 134 Nuget: { 135 indicators: []string{".sln", ".csproj"}, 136 packageDescriptors: []string{".sln", ".csproj"}, 137 formal: "NuGet", 138 // .NET CLI is used for NuGet projects 139 execCommand: "dotnet", 140 packageInstallationCommand: "add", 141 // packageName -v packageVersion 142 packageVersionOperator: " -v ", 143 }, 144 Dotnet: { 145 indicators: []string{".sln", ".csproj"}, 146 packageDescriptors: []string{".sln", ".csproj"}, 147 formal: ".NET", 148 }, 149 Docker: { 150 applicabilityScannable: true, 151 }, 152 Oci: { 153 applicabilityScannable: true, 154 }, 155 } 156 157 func (tech Technology) ToFormal() string { 158 if technologiesData[tech].formal == "" { 159 return cases.Title(language.Und).String(tech.String()) 160 } 161 return technologiesData[tech].formal 162 } 163 164 func (tech Technology) String() string { 165 return string(tech) 166 } 167 168 func (tech Technology) GetExecCommandName() string { 169 if technologiesData[tech].execCommand == "" { 170 return tech.String() 171 } 172 return technologiesData[tech].execCommand 173 } 174 175 func (tech Technology) GetPackageType() string { 176 if technologiesData[tech].packageType == "" { 177 return tech.String() 178 } 179 return technologiesData[tech].packageType 180 } 181 182 func (tech Technology) GetPackageDescriptor() []string { 183 return technologiesData[tech].packageDescriptors 184 } 185 186 func (tech Technology) IsCiSetup() bool { 187 return technologiesData[tech].ciSetupSupport 188 } 189 190 func (tech Technology) GetPackageVersionOperator() string { 191 return technologiesData[tech].packageVersionOperator 192 } 193 194 func (tech Technology) GetPackageInstallationCommand() string { 195 return technologiesData[tech].packageInstallationCommand 196 } 197 198 func (tech Technology) ApplicabilityScannable() bool { 199 return technologiesData[tech].applicabilityScannable 200 } 201 202 func DetectedTechnologiesList() (technologies []string) { 203 wd, err := os.Getwd() 204 if errorutils.CheckError(err) != nil { 205 return 206 } 207 return detectedTechnologiesListInPath(wd, false) 208 } 209 210 func detectedTechnologiesListInPath(path string, recursive bool) (technologies []string) { 211 detectedTechnologies, err := DetectTechnologies(path, false, recursive) 212 if err != nil { 213 return 214 } 215 if len(detectedTechnologies) == 0 { 216 return 217 } 218 techStringsList := DetectedTechnologiesToSlice(detectedTechnologies) 219 log.Info(fmt.Sprintf("Detected: %s.", strings.Join(techStringsList, ", "))) 220 return techStringsList 221 } 222 223 // If recursive is true, the search will not be limited to files in the root path. 224 // If requestedTechs is empty, all technologies will be checked. 225 // If excludePathPattern is not empty, files/directories that match the wildcard pattern will be excluded from the search. 226 func DetectTechnologiesDescriptors(path string, recursive bool, requestedTechs []string, requestedDescriptors map[Technology][]string, excludePathPattern string) (technologiesDetected map[Technology]map[string][]string, err error) { 227 filesList, err := fspatterns.ListFiles(path, recursive, false, true, true, excludePathPattern) 228 if err != nil { 229 return 230 } 231 workingDirectoryToIndicators, excludedTechAtWorkingDir := mapFilesToRelevantWorkingDirectories(filesList, requestedDescriptors) 232 var strJson string 233 if strJson, err = GetJsonIndent(workingDirectoryToIndicators); err != nil { 234 return 235 } else if len(workingDirectoryToIndicators) > 0 { 236 log.Debug(fmt.Sprintf("mapped %d working directories with indicators/descriptors:\n%s", len(workingDirectoryToIndicators), strJson)) 237 } 238 technologiesDetected = mapWorkingDirectoriesToTechnologies(workingDirectoryToIndicators, excludedTechAtWorkingDir, ToTechnologies(requestedTechs), requestedDescriptors) 239 if len(technologiesDetected) > 0 { 240 log.Debug(fmt.Sprintf("Detected %d technologies at %s: %s.", len(technologiesDetected), path, maps.Keys(technologiesDetected))) 241 } 242 return 243 } 244 245 // Map files to relevant working directories according to the technologies' indicators/descriptors and requested descriptors. 246 // files: The file paths to map. 247 // requestedDescriptors: Special requested descriptors (for example in Pip requirement.txt can have different path) for each technology. 248 // Returns: 249 // 1. workingDirectoryToIndicators: A map of working directories to the files that are relevant to the technologies. 250 // wd1: [wd1/indicator, wd1/descriptor] 251 // wd/wd2: [wd/wd2/indicator] 252 // 2. excludedTechAtWorkingDir: A map of working directories to the technologies that are excluded from the working directory. 253 // wd1: [tech1, tech2] 254 // wd/wd2: [tech1] 255 func mapFilesToRelevantWorkingDirectories(files []string, requestedDescriptors map[Technology][]string) (workingDirectoryToIndicators map[string][]string, excludedTechAtWorkingDir map[string][]Technology) { 256 workingDirectoryToIndicatorsSet := make(map[string]*datastructures.Set[string]) 257 excludedTechAtWorkingDir = make(map[string][]Technology) 258 for _, path := range files { 259 directory := filepath.Dir(path) 260 261 for tech, techData := range technologiesData { 262 // Check if the working directory contains indicators/descriptors for the technology 263 relevant := isIndicator(path, techData) || isDescriptor(path, techData) || isRequestedDescriptor(path, requestedDescriptors[tech]) 264 if relevant { 265 if _, exist := workingDirectoryToIndicatorsSet[directory]; !exist { 266 workingDirectoryToIndicatorsSet[directory] = datastructures.MakeSet[string]() 267 } 268 workingDirectoryToIndicatorsSet[directory].Add(path) 269 } 270 // Check if the working directory contains a file/directory with a name that ends with an excluded suffix 271 if isExclude(path, techData) { 272 excludedTechAtWorkingDir[directory] = append(excludedTechAtWorkingDir[directory], tech) 273 } 274 } 275 } 276 workingDirectoryToIndicators = make(map[string][]string) 277 for wd, indicators := range workingDirectoryToIndicatorsSet { 278 workingDirectoryToIndicators[wd] = indicators.ToSlice() 279 } 280 return 281 } 282 283 func isDescriptor(path string, techData TechData) bool { 284 for _, descriptor := range techData.packageDescriptors { 285 if strings.HasSuffix(path, descriptor) { 286 return true 287 } 288 } 289 return false 290 } 291 292 func isRequestedDescriptor(path string, requestedDescriptors []string) bool { 293 for _, requestedDescriptor := range requestedDescriptors { 294 if strings.HasSuffix(path, requestedDescriptor) { 295 return true 296 } 297 } 298 return false 299 } 300 301 func isIndicator(path string, techData TechData) bool { 302 for _, indicator := range techData.indicators { 303 if strings.HasSuffix(path, indicator) { 304 return true 305 } 306 } 307 return false 308 } 309 310 func isExclude(path string, techData TechData) bool { 311 for _, exclude := range techData.exclude { 312 if strings.HasSuffix(path, exclude) { 313 return true 314 } 315 } 316 return false 317 } 318 319 // Map working directories to technologies according to the given workingDirectoryToIndicators map files. 320 // workingDirectoryToIndicators: A map of working directories to the files inside the directory that are relevant to the technologies. 321 // excludedTechAtWorkingDir: A map of working directories to the technologies that are excluded from the working directory. 322 // requestedTechs: The technologies to check, if empty all technologies will be checked. 323 // requestedDescriptors: Special requested descriptors (for example in Pip requirement.txt can have different path) for each technology to detect. 324 func mapWorkingDirectoriesToTechnologies(workingDirectoryToIndicators map[string][]string, excludedTechAtWorkingDir map[string][]Technology, requestedTechs []Technology, requestedDescriptors map[Technology][]string) (technologiesDetected map[Technology]map[string][]string) { 325 // Get the relevant technologies to check 326 technologies := requestedTechs 327 if len(technologies) == 0 { 328 technologies = GetAllTechnologiesList() 329 } 330 technologiesDetected = make(map[Technology]map[string][]string) 331 // Map working directories to technologies 332 for _, tech := range technologies { 333 techWorkingDirs := getTechInformationFromWorkingDir(tech, workingDirectoryToIndicators, excludedTechAtWorkingDir, requestedDescriptors) 334 if len(techWorkingDirs) > 0 { 335 // Found indicators of the technology, add to detected. 336 technologiesDetected[tech] = techWorkingDirs 337 } 338 } 339 for _, tech := range requestedTechs { 340 if _, exist := technologiesDetected[tech]; !exist { 341 // Requested (forced with flag) technology and not found any indicators/descriptors in detection, add as detected. 342 log.Warn(fmt.Sprintf("Requested technology %s but not found any indicators/descriptors in detection.", tech)) 343 technologiesDetected[tech] = map[string][]string{} 344 } 345 } 346 return 347 } 348 349 func getTechInformationFromWorkingDir(tech Technology, workingDirectoryToIndicators map[string][]string, excludedTechAtWorkingDir map[string][]Technology, requestedDescriptors map[Technology][]string) (techWorkingDirs map[string][]string) { 350 techWorkingDirs = make(map[string][]string) 351 for wd, indicators := range workingDirectoryToIndicators { 352 descriptorsAtWd := []string{} 353 foundIndicator := false 354 if isTechExcludedInWorkingDir(tech, wd, excludedTechAtWorkingDir) { 355 // Exclude this technology from this working directory 356 continue 357 } 358 // Check if the working directory contains indicators/descriptors for the technology 359 for _, path := range indicators { 360 if isDescriptor(path, technologiesData[tech]) || isRequestedDescriptor(path, requestedDescriptors[tech]) { 361 descriptorsAtWd = append(descriptorsAtWd, path) 362 } 363 if isIndicator(path, technologiesData[tech]) || isRequestedDescriptor(path, requestedDescriptors[tech]) { 364 foundIndicator = true 365 } 366 } 367 if foundIndicator { 368 // Found indicators of the technology in the current working directory, add to detected. 369 techWorkingDirs[wd] = descriptorsAtWd 370 } 371 } 372 // Don't allow working directory if sub directory already exists as key for the same technology 373 techWorkingDirs = cleanSubDirectories(techWorkingDirs) 374 return 375 } 376 377 func isTechExcludedInWorkingDir(tech Technology, wd string, excludedTechAtWorkingDir map[string][]Technology) bool { 378 if excludedTechs, exist := excludedTechAtWorkingDir[wd]; exist { 379 for _, excludedTech := range excludedTechs { 380 if excludedTech == tech { 381 return true 382 } 383 } 384 } 385 return false 386 } 387 388 // Remove sub directories keys from the given workingDirectoryToFiles map. 389 // Keys: [dir/dir, dir/directory] -> [dir/dir, dir/directory] 390 // Keys: [dir, directory] -> [dir, directory] 391 // Keys: [dir/dir2, dir/dir2/dir3, dir/dir2/dir3/dir4] -> [dir/dir2] 392 // Values of removed sub directories will be added to the root directory. 393 func cleanSubDirectories(workingDirectoryToFiles map[string][]string) (result map[string][]string) { 394 result = make(map[string][]string) 395 for wd, files := range workingDirectoryToFiles { 396 root := getExistingRootDir(wd, workingDirectoryToFiles) 397 result[root] = append(result[root], files...) 398 } 399 return 400 } 401 402 // Get the root directory of the given path according to the given workingDirectoryToIndicators map. 403 func getExistingRootDir(path string, workingDirectoryToIndicators map[string][]string) (root string) { 404 root = path 405 for wd := range workingDirectoryToIndicators { 406 parentWd := filepath.Dir(wd) 407 parentRoot := filepath.Dir(root) 408 if parentRoot != parentWd && strings.HasPrefix(root, wd) { 409 root = wd 410 } 411 } 412 return 413 } 414 415 // DetectTechnologies tries to detect all technologies types according to the files in the given path. 416 // 'isCiSetup' will limit the search of possible techs to Maven, Gradle, and npm. 417 // 'recursive' will determine if the search will be limited to files in the root path or not. 418 func DetectTechnologies(path string, isCiSetup, recursive bool) (map[Technology]bool, error) { 419 var filesList []string 420 var err error 421 if recursive { 422 filesList, err = fileutils.ListFilesRecursiveWalkIntoDirSymlink(path, false) 423 } else { 424 filesList, err = fileutils.ListFiles(path, true) 425 } 426 if err != nil { 427 return nil, err 428 } 429 log.Info(fmt.Sprintf("Scanning %d file(s):%s", len(filesList), filesList)) 430 detectedTechnologies := detectTechnologiesByFilePaths(filesList, isCiSetup) 431 return detectedTechnologies, nil 432 } 433 434 func detectTechnologiesByFilePaths(paths []string, isCiSetup bool) (detected map[Technology]bool) { 435 detected = make(map[Technology]bool) 436 exclude := make(map[Technology]bool) 437 for _, path := range paths { 438 for techName, techData := range technologiesData { 439 // If the detection is in a 'jf ci-setup' command, then the checked technology must be supported. 440 if !isCiSetup || (isCiSetup && techData.ciSetupSupport) { 441 // If the project contains a file/directory with a name that ends with an excluded suffix, then this technology is excluded. 442 for _, excludeFile := range techData.exclude { 443 if strings.HasSuffix(path, excludeFile) { 444 exclude[techName] = true 445 } 446 } 447 // If this technology was already excluded, there's no need to look for indicator files/directories. 448 if _, exist := exclude[techName]; !exist { 449 // If the project contains a file/directory with a name that ends with the indicator suffix, then the project probably uses this technology. 450 for _, indicator := range techData.indicators { 451 if strings.HasSuffix(path, indicator) { 452 detected[techName] = true 453 } 454 } 455 } 456 } 457 } 458 } 459 // Remove excluded technologies. 460 for excludeTech := range exclude { 461 delete(detected, excludeTech) 462 } 463 return detected 464 } 465 466 // DetectedTechnologiesToSlice returns a string slice that includes all the names of the detected technologies. 467 func DetectedTechnologiesToSlice(detected map[Technology]bool) []string { 468 keys := make([]string, 0, len(detected)) 469 for tech := range detected { 470 keys = append(keys, string(tech)) 471 } 472 return keys 473 } 474 475 func ToTechnologies(args []string) (technologies []Technology) { 476 for _, argument := range args { 477 technologies = append(technologies, Technology(argument)) 478 } 479 return 480 } 481 482 func GetAllTechnologiesList() (technologies []Technology) { 483 for tech := range technologiesData { 484 technologies = append(technologies, tech) 485 } 486 return 487 } 488 489 func ContainsApplicabilityScannableTech(technologies []Technology) bool { 490 for _, technology := range technologies { 491 if technology.ApplicabilityScannable() { 492 return true 493 } 494 } 495 return false 496 }