github.com/castai/kvisor@v1.7.1-0.20240516114728-b3572a2607b5/cmd/linter/kubebench/util.go (about) 1 package kubebench 2 3 import ( 4 "encoding/json" 5 "fmt" 6 "os" 7 "os/exec" 8 "path/filepath" 9 "regexp" 10 "strconv" 11 "strings" 12 13 "github.com/castai/kvisor/cmd/linter/kubebench/check" 14 "github.com/fatih/color" 15 "github.com/golang/glog" 16 "github.com/spf13/viper" 17 ) 18 19 // Print colors 20 var colors = map[check.State]*color.Color{ 21 check.PASS: color.New(color.FgGreen), 22 check.FAIL: color.New(color.FgRed), 23 check.WARN: color.New(color.FgYellow), 24 check.INFO: color.New(color.FgBlue), 25 } 26 27 var ( 28 psFunc func(string) string 29 statFunc func(string) (os.FileInfo, error) 30 getBinariesFunc func(*viper.Viper, check.NodeType) (map[string]string, error) 31 TypeMap = map[string][]string{ 32 "ca": {"cafile", "defaultcafile"}, 33 "kubeconfig": {"kubeconfig", "defaultkubeconfig"}, 34 "service": {"svc", "defaultsvc"}, 35 "config": {"confs", "defaultconf"}, 36 "datadir": {"datadirs", "defaultdatadir"}, 37 } 38 ) 39 40 func init() { 41 psFunc = ps 42 statFunc = os.Stat 43 getBinariesFunc = getBinaries 44 } 45 46 type Platform struct { 47 Name string 48 Version string 49 } 50 51 func (p Platform) String() string { 52 return fmt.Sprintf("Platform{ Name: %s Version: %s }", p.Name, p.Version) 53 } 54 55 func exitWithError(err error) { 56 fmt.Fprintf(os.Stderr, "\n%v\n", err) 57 // flush before exit non-zero 58 glog.Flush() 59 os.Exit(1) 60 } 61 62 func cleanIDs(list string) map[string]bool { 63 list = strings.Trim(list, ",") 64 ids := strings.Split(list, ",") 65 66 set := make(map[string]bool) 67 68 for _, id := range ids { 69 id = strings.Trim(id, " ") 70 set[id] = true 71 } 72 73 return set 74 } 75 76 // ps execs out to the ps command; it's separated into a function so we can write tests 77 func ps(proc string) string { 78 // TODO: truncate proc to 15 chars 79 // See https://github.com/aquasecurity/kube-bench/issues/328#issuecomment-506813344 80 glog.V(2).Info(fmt.Sprintf("ps - proc: %q", proc)) 81 cmd := exec.Command("/bin/ps", "-C", proc, "-o", "cmd", "--no-headers") 82 out, err := cmd.Output() 83 if err != nil { 84 glog.V(2).Info(fmt.Errorf("%s: %s", cmd.Args, err)) 85 } 86 87 glog.V(2).Info(fmt.Sprintf("ps - returning: %q", string(out))) 88 return string(out) 89 } 90 91 // getBinaries finds which of the set of candidate executables are running. 92 // It returns an error if one mandatory executable is not running. 93 func getBinaries(v *viper.Viper, nodetype check.NodeType) (map[string]string, error) { 94 binmap := make(map[string]string) 95 96 for _, component := range v.GetStringSlice("components") { 97 s := v.Sub(component) 98 if s == nil { 99 continue 100 } 101 102 optional := s.GetBool("optional") 103 bins := s.GetStringSlice("bins") 104 if len(bins) > 0 { 105 bin, err := findExecutable(bins) 106 if err != nil && !optional { 107 glog.V(1).Info(buildComponentMissingErrorMessage(nodetype, component, bins)) 108 return nil, fmt.Errorf("unable to detect running programs for component %q", component) 109 } 110 111 // Default the executable name that we'll substitute to the name of the component 112 if bin == "" { 113 bin = component 114 glog.V(2).Info(fmt.Sprintf("Component %s not running", component)) 115 } else { 116 glog.V(2).Info(fmt.Sprintf("Component %s uses running binary %s", component, bin)) 117 } 118 binmap[component] = bin 119 } 120 } 121 122 return binmap, nil 123 } 124 125 // getConfigFilePath locates the config files we should be using for CIS version 126 func getConfigFilePath(benchmarkVersion string, filename string) (path string, err error) { 127 glog.V(2).Info(fmt.Sprintf("Looking for config specific CIS version %q", benchmarkVersion)) 128 129 path = filepath.Join(cfgDir, benchmarkVersion) 130 file := filepath.Join(path, filename) 131 glog.V(2).Info(fmt.Sprintf("Looking for file: %s", file)) 132 133 if _, err := os.Stat(file); err != nil { 134 glog.V(2).Infof("error accessing config file: %q error: %v\n", file, err) 135 return "", fmt.Errorf("no test files found <= benchmark version: %s", benchmarkVersion) 136 } 137 138 return path, nil 139 } 140 141 // getYamlFilesFromDir returns a list of yaml files in the specified directory, ignoring config.yaml 142 func getYamlFilesFromDir(path string) (names []string, err error) { 143 err = filepath.Walk(path, func(path string, info os.FileInfo, err error) error { 144 if err != nil { 145 return err 146 } 147 148 _, name := filepath.Split(path) 149 if name != "" && name != "config.yaml" && filepath.Ext(name) == ".yaml" { 150 names = append(names, path) 151 } 152 153 return nil 154 }) 155 return names, err 156 } 157 158 // decrementVersion decrements the version number 159 // We want to decrement individually even through versions where we don't supply test files 160 // just in case someone wants to specify their own test files for that version 161 func decrementVersion(version string) string { 162 split := strings.Split(version, ".") 163 if len(split) < 2 { 164 return "" 165 } 166 minor, err := strconv.Atoi(split[1]) 167 if err != nil { 168 return "" 169 } 170 if minor <= 1 { 171 return "" 172 } 173 split[1] = strconv.Itoa(minor - 1) 174 return strings.Join(split, ".") 175 } 176 177 // getFiles finds which of the set of candidate files exist 178 func getFiles(v *viper.Viper, fileType string) map[string]string { 179 filemap := make(map[string]string) 180 mainOpt := TypeMap[fileType][0] 181 defaultOpt := TypeMap[fileType][1] 182 183 for _, component := range v.GetStringSlice("components") { 184 s := v.Sub(component) 185 if s == nil { 186 continue 187 } 188 189 // See if any of the candidate files exist 190 file := findConfigFile(s.GetStringSlice(mainOpt)) 191 if file == "" { 192 if s.IsSet(defaultOpt) { 193 file = s.GetString(defaultOpt) 194 glog.V(2).Info(fmt.Sprintf("Using default %s file name '%s' for component %s", fileType, file, component)) 195 } else { 196 // Default the file name that we'll substitute to the name of the component 197 glog.V(2).Info(fmt.Sprintf("Missing %s file for %s", fileType, component)) 198 file = component 199 } 200 } else { 201 glog.V(2).Info(fmt.Sprintf("Component %s uses %s file '%s'", component, fileType, file)) 202 } 203 204 filemap[component] = file 205 } 206 207 return filemap 208 } 209 210 // verifyBin checks that the binary specified is running 211 func verifyBin(bin string) bool { 212 // Strip any quotes 213 bin = strings.Trim(bin, "'\"") 214 215 // bin could consist of more than one word 216 // We'll search for running processes with the first word, and then check the whole 217 // proc as supplied is included in the results 218 proc := strings.Fields(bin)[0] 219 out := psFunc(proc) 220 221 // There could be multiple lines in the ps output 222 // The binary needs to be the first word in the ps output, except that it could be preceded by a path 223 // e.g. /usr/bin/kubelet is a match for kubelet 224 // but apiserver is not a match for kube-apiserver 225 reFirstWord := regexp.MustCompile(`^(\S*\/)*` + bin) 226 lines := strings.Split(out, "\n") 227 for _, l := range lines { 228 glog.V(3).Info(fmt.Sprintf("reFirstWord.Match(%s)", l)) 229 if reFirstWord.Match([]byte(l)) { 230 return true 231 } 232 } 233 234 return false 235 } 236 237 // fundConfigFile looks through a list of possible config files and finds the first one that exists 238 func findConfigFile(candidates []string) string { 239 for _, c := range candidates { 240 _, err := statFunc(c) 241 if err == nil { 242 return c 243 } 244 if !os.IsNotExist(err) && !strings.HasSuffix(err.Error(), "not a directory") { 245 exitWithError(fmt.Errorf("error looking for file %s: %v", c, err)) 246 } 247 } 248 249 return "" 250 } 251 252 // findExecutable looks through a list of possible executable names and finds the first one that's running 253 func findExecutable(candidates []string) (string, error) { 254 for _, c := range candidates { 255 if verifyBin(c) { 256 return c, nil 257 } 258 glog.V(1).Info(fmt.Sprintf("executable '%s' not running", c)) 259 } 260 261 return "", fmt.Errorf("no candidates running") 262 } 263 264 func multiWordReplace(s string, subname string, sub string) string { 265 f := strings.Fields(sub) 266 if len(f) > 1 { 267 sub = "'" + sub + "'" 268 } 269 270 return strings.Replace(s, subname, sub, -1) 271 } 272 273 const missingKubectlKubeletMessage = ` 274 Unable to find the programs kubectl or kubelet in the PATH. 275 These programs are used to determine which version of Kubernetes is running. 276 Make sure the /usr/local/mount-from-host/bin directory is mapped to the container, 277 either in the job.yaml file, or Docker command. 278 279 For job.yaml: 280 ... 281 - name: usr-bin 282 mountPath: /usr/local/mount-from-host/bin 283 ... 284 285 For docker command: 286 docker -v $(which kubectl):/usr/local/mount-from-host/bin/kubectl .... 287 288 Alternatively, you can specify the version with --version 289 kube-bench --version <VERSION> ... 290 ` 291 292 func getKubeVersion() (*KubeVersion, error) { 293 if k8sVer, err := getKubeVersionFromRESTAPI(); err == nil { 294 glog.V(2).Info(fmt.Sprintf("Kubernetes REST API Reported version: %s", k8sVer)) 295 return k8sVer, nil 296 } 297 298 // These executables might not be on the user's path. 299 _, err := exec.LookPath("kubectl") 300 if err != nil { 301 glog.V(3).Infof("Error locating kubectl: %s", err) 302 _, err = exec.LookPath("kubelet") 303 if err != nil { 304 glog.V(3).Infof("Error locating kubelet: %s", err) 305 // Search for the kubelet binary all over the filesystem and run the first match to get the kubernetes version 306 cmd := exec.Command("/bin/sh", "-c", "`find / -type f -executable -name kubelet 2>/dev/null | grep -m1 .` --version") 307 out, err := cmd.CombinedOutput() 308 if err == nil { 309 glog.V(3).Infof("Found kubelet and query kubernetes version is: %s", string(out)) 310 return getVersionFromKubeletOutput(string(out)), nil 311 } 312 313 glog.Warning(missingKubectlKubeletMessage) 314 glog.V(1).Info("unable to find the programs kubectl or kubelet in the PATH") 315 glog.V(1).Infof("Cant detect version, assuming default %s", defaultKubeVersion) 316 return &KubeVersion{baseVersion: defaultKubeVersion}, nil 317 } 318 return getKubeVersionFromKubelet(), nil 319 } 320 321 return getKubeVersionFromKubectl(), nil 322 } 323 324 func getKubeVersionFromKubectl() *KubeVersion { 325 cmd := exec.Command("kubectl", "version", "-o", "json") 326 out, err := cmd.CombinedOutput() 327 if err != nil { 328 glog.V(2).Infof("Failed to query kubectl: %s", err) 329 glog.V(2).Info(err) 330 } 331 332 return getVersionFromKubectlOutput(string(out)) 333 } 334 335 func getKubeVersionFromKubelet() *KubeVersion { 336 cmd := exec.Command("kubelet", "--version") 337 out, err := cmd.CombinedOutput() 338 if err != nil { 339 glog.V(2).Infof("Failed to query kubelet: %s", err) 340 glog.V(2).Info(err) 341 } 342 343 return getVersionFromKubeletOutput(string(out)) 344 } 345 346 func getVersionFromKubectlOutput(s string) *KubeVersion { 347 glog.V(2).Infof("Kubectl output: %s", s) 348 type versionResult struct { 349 ServerVersion VersionResponse 350 } 351 vrObj := &versionResult{} 352 if err := json.Unmarshal([]byte(s), vrObj); err != nil { 353 glog.V(2).Info(err) 354 if strings.Contains(s, "The connection to the server") { 355 msg := fmt.Sprintf(`Warning: Kubernetes version was not auto-detected because kubectl could not connect to the Kubernetes server. This may be because the kubeconfig information is missing or has credentials that do not match the server. Assuming default version %s`, defaultKubeVersion) 356 fmt.Fprintln(os.Stderr, msg) 357 } 358 glog.V(1).Info(fmt.Sprintf("Unable to get Kubernetes version from kubectl, using default version: %s", defaultKubeVersion)) 359 return &KubeVersion{baseVersion: defaultKubeVersion} 360 } 361 sv := vrObj.ServerVersion 362 return &KubeVersion{ 363 Major: sv.Major, 364 Minor: sv.Minor, 365 GitVersion: sv.GitVersion, 366 } 367 } 368 369 func getVersionFromKubeletOutput(s string) *KubeVersion { 370 glog.V(2).Infof("Kubelet output: %s", s) 371 serverVersionRe := regexp.MustCompile(`Kubernetes v(\d+.\d+)`) 372 subs := serverVersionRe.FindStringSubmatch(s) 373 if len(subs) < 2 { 374 glog.V(1).Info(fmt.Sprintf("Unable to get Kubernetes version from kubelet, using default version: %s", defaultKubeVersion)) 375 return &KubeVersion{baseVersion: defaultKubeVersion} 376 } 377 return &KubeVersion{baseVersion: subs[1]} 378 } 379 380 func makeSubstitutions(s string, ext string, m map[string]string) (string, []string) { 381 substitutions := make([]string, 0) 382 for k, v := range m { 383 subst := "$" + k + ext 384 if v == "" { 385 glog.V(2).Info(fmt.Sprintf("No substitution for '%s'\n", subst)) 386 continue 387 } 388 glog.V(2).Info(fmt.Sprintf("Substituting %s with '%s'\n", subst, v)) 389 beforeS := s 390 s = multiWordReplace(s, subst, v) 391 if beforeS != s { 392 substitutions = append(substitutions, v) 393 } 394 } 395 396 return s, substitutions 397 } 398 399 func isEmpty(str string) bool { 400 return strings.TrimSpace(str) == "" 401 } 402 403 func buildComponentMissingErrorMessage(nodetype check.NodeType, component string, bins []string) string { 404 errMessageTemplate := ` 405 Unable to detect running programs for component %q 406 The following %q programs have been searched, but none of them have been found: 407 %s 408 409 These program names are provided in the config.yaml, section '%s.%s.bins' 410 ` 411 412 var componentRoleName, componentType string 413 switch nodetype { 414 415 case check.NODE: 416 componentRoleName = "worker node" 417 componentType = "node" 418 case check.ETCD: 419 componentRoleName = "etcd node" 420 componentType = "etcd" 421 default: 422 componentRoleName = "master node" 423 componentType = "master" 424 } 425 426 binList := "" 427 for _, bin := range bins { 428 binList = fmt.Sprintf("%s\t- %s\n", binList, bin) 429 } 430 431 return fmt.Sprintf(errMessageTemplate, component, componentRoleName, binList, componentType, component) 432 } 433 434 func getPlatformInfo() Platform { 435 436 openShiftInfo := getOpenShiftInfo() 437 if openShiftInfo.Name != "" && openShiftInfo.Version != "" { 438 return openShiftInfo 439 } 440 441 kv, err := getKubeVersion() 442 if err != nil { 443 glog.V(2).Info(err) 444 return Platform{} 445 } 446 return getPlatformInfoFromVersion(kv.GitVersion) 447 } 448 449 func getPlatformInfoFromVersion(s string) Platform { 450 versionRe := regexp.MustCompile(`v(\d+\.\d+)\.\d+[-+](\w+)(?:[.\-])\w+`) 451 subs := versionRe.FindStringSubmatch(s) 452 if len(subs) < 3 { 453 return Platform{} 454 } 455 return Platform{ 456 Name: subs[2], 457 Version: subs[1], 458 } 459 } 460 461 func getPlatformBenchmarkVersion(platform Platform) string { 462 glog.V(3).Infof("getPlatformBenchmarkVersion platform: %s", platform) 463 switch platform.Name { 464 case "eks": 465 return "eks-1.2.0" 466 case "gke": 467 switch platform.Version { 468 case "1.15", "1.16", "1.17", "1.18", "1.19": 469 return "gke-1.0" 470 default: 471 return "gke-1.2.0" 472 } 473 case "aliyun": 474 return "ack-1.0" 475 case "ocp": 476 switch platform.Version { 477 case "3.10": 478 return "rh-0.7" 479 case "4.1": 480 return "rh-1.0" 481 } 482 case "vmware": 483 return "tkgi-1.2.53" 484 } 485 return "" 486 } 487 488 func getOpenShiftInfo() Platform { 489 glog.V(1).Info("Checking for oc") 490 _, err := exec.LookPath("oc") 491 492 if err == nil { 493 cmd := exec.Command("oc", "version") 494 out, err := cmd.CombinedOutput() 495 496 if err == nil { 497 versionRe := regexp.MustCompile(`oc v(\d+\.\d+)`) 498 subs := versionRe.FindStringSubmatch(string(out)) 499 if len(subs) < 1 { 500 versionRe = regexp.MustCompile(`Client Version:\s*(\d+\.\d+)`) 501 subs = versionRe.FindStringSubmatch(string(out)) 502 } 503 if len(subs) > 1 { 504 glog.V(2).Infof("OCP output '%s' \nplatform is %s \nocp %v", string(out), getPlatformInfoFromVersion(string(out)), subs[1]) 505 ocpBenchmarkVersion, err := getOcpValidVersion(subs[1]) 506 if err == nil { 507 return Platform{Name: "ocp", Version: ocpBenchmarkVersion} 508 } else { 509 glog.V(1).Infof("Can't get getOcpValidVersion: %v", err) 510 } 511 } else { 512 glog.V(1).Infof("Can't parse version output: %v", subs) 513 } 514 } else { 515 glog.V(1).Infof("Can't use oc command: %v", err) 516 } 517 } else { 518 glog.V(1).Infof("Can't find oc command: %v", err) 519 } 520 return Platform{} 521 } 522 523 func getOcpValidVersion(ocpVer string) (string, error) { 524 ocpOriginal := ocpVer 525 526 for !isEmpty(ocpVer) { 527 glog.V(3).Info(fmt.Sprintf("getOcpBenchmarkVersion check for ocp: %q \n", ocpVer)) 528 if ocpVer == "3.10" || ocpVer == "4.1" { 529 glog.V(1).Info(fmt.Sprintf("getOcpBenchmarkVersion found valid version for ocp: %q \n", ocpVer)) 530 return ocpVer, nil 531 } 532 ocpVer = decrementVersion(ocpVer) 533 } 534 535 glog.V(1).Info(fmt.Sprintf("getOcpBenchmarkVersion unable to find a match for: %q", ocpOriginal)) 536 return "", fmt.Errorf("unable to find a matching Benchmark Version match for ocp version: %s", ocpOriginal) 537 }