github.com/khulnasoft-lab/kube-bench@v0.2.1-0.20240330183753-9df52345ae58/cmd/util.go (about) 1 package cmd 2 3 import ( 4 "context" 5 "encoding/json" 6 "fmt" 7 "os" 8 "os/exec" 9 "path/filepath" 10 "regexp" 11 "strconv" 12 "strings" 13 14 "github.com/fatih/color" 15 "github.com/golang/glog" 16 "github.com/khulnasoft-lab/kube-bench/check" 17 "github.com/spf13/viper" 18 19 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 20 "k8s.io/client-go/kubernetes" 21 "k8s.io/client-go/rest" 22 ) 23 24 // Print colors 25 var colors = map[check.State]*color.Color{ 26 check.PASS: color.New(color.FgGreen), 27 check.FAIL: color.New(color.FgRed), 28 check.WARN: color.New(color.FgYellow), 29 check.INFO: color.New(color.FgBlue), 30 } 31 32 var ( 33 psFunc func(string) string 34 statFunc func(string) (os.FileInfo, error) 35 getBinariesFunc func(*viper.Viper, check.NodeType) (map[string]string, error) 36 TypeMap = map[string][]string{ 37 "ca": {"cafile", "defaultcafile"}, 38 "kubeconfig": {"kubeconfig", "defaultkubeconfig"}, 39 "service": {"svc", "defaultsvc"}, 40 "config": {"confs", "defaultconf"}, 41 "datadir": {"datadirs", "defaultdatadir"}, 42 } 43 ) 44 45 func init() { 46 psFunc = ps 47 statFunc = os.Stat 48 getBinariesFunc = getBinaries 49 } 50 51 type Platform struct { 52 Name string 53 Version string 54 } 55 56 func (p Platform) String() string { 57 return fmt.Sprintf("Platform{ Name: %s Version: %s }", p.Name, p.Version) 58 } 59 60 func exitWithError(err error) { 61 fmt.Fprintf(os.Stderr, "\n%v\n", err) 62 // flush before exit non-zero 63 glog.Flush() 64 os.Exit(1) 65 } 66 67 func cleanIDs(list string) map[string]bool { 68 list = strings.Trim(list, ",") 69 ids := strings.Split(list, ",") 70 71 set := make(map[string]bool) 72 73 for _, id := range ids { 74 id = strings.Trim(id, " ") 75 set[id] = true 76 } 77 78 return set 79 } 80 81 // ps execs out to the ps command; it's separated into a function so we can write tests 82 func ps(proc string) string { 83 // TODO: truncate proc to 15 chars 84 // See https://github.com/khulnasoft-lab/kube-bench/issues/328#issuecomment-506813344 85 glog.V(2).Info(fmt.Sprintf("ps - proc: %q", proc)) 86 cmd := exec.Command("/bin/ps", "-C", proc, "-o", "cmd", "--no-headers") 87 out, err := cmd.Output() 88 if err != nil { 89 glog.V(2).Info(fmt.Errorf("%s: %s", cmd.Args, err)) 90 } 91 92 glog.V(2).Info(fmt.Sprintf("ps - returning: %q", string(out))) 93 return string(out) 94 } 95 96 // getBinaries finds which of the set of candidate executables are running. 97 // It returns an error if one mandatory executable is not running. 98 func getBinaries(v *viper.Viper, nodetype check.NodeType) (map[string]string, error) { 99 binmap := make(map[string]string) 100 101 for _, component := range v.GetStringSlice("components") { 102 s := v.Sub(component) 103 if s == nil { 104 continue 105 } 106 107 optional := s.GetBool("optional") 108 bins := s.GetStringSlice("bins") 109 if len(bins) > 0 { 110 bin, err := findExecutable(bins) 111 if err != nil && !optional { 112 glog.V(1).Info(buildComponentMissingErrorMessage(nodetype, component, bins)) 113 return nil, fmt.Errorf("unable to detect running programs for component %q", component) 114 } 115 116 // Default the executable name that we'll substitute to the name of the component 117 if bin == "" { 118 bin = component 119 glog.V(2).Info(fmt.Sprintf("Component %s not running", component)) 120 } else { 121 glog.V(2).Info(fmt.Sprintf("Component %s uses running binary %s", component, bin)) 122 } 123 binmap[component] = bin 124 } 125 } 126 127 return binmap, nil 128 } 129 130 // getConfigFilePath locates the config files we should be using for CIS version 131 func getConfigFilePath(benchmarkVersion string, filename string) (path string, err error) { 132 glog.V(2).Info(fmt.Sprintf("Looking for config specific CIS version %q", benchmarkVersion)) 133 134 path = filepath.Join(cfgDir, benchmarkVersion) 135 file := filepath.Join(path, filename) 136 glog.V(2).Info(fmt.Sprintf("Looking for file: %s", file)) 137 138 if _, err := os.Stat(file); err != nil { 139 glog.V(2).Infof("error accessing config file: %q error: %v\n", file, err) 140 return "", fmt.Errorf("no test files found <= benchmark version: %s", benchmarkVersion) 141 } 142 143 return path, nil 144 } 145 146 // getYamlFilesFromDir returns a list of yaml files in the specified directory, ignoring config.yaml 147 func getYamlFilesFromDir(path string) (names []string, err error) { 148 err = filepath.Walk(path, func(path string, info os.FileInfo, err error) error { 149 if err != nil { 150 return err 151 } 152 153 _, name := filepath.Split(path) 154 if name != "" && name != "config.yaml" && filepath.Ext(name) == ".yaml" { 155 names = append(names, path) 156 } 157 158 return nil 159 }) 160 return names, err 161 } 162 163 // decrementVersion decrements the version number 164 // We want to decrement individually even through versions where we don't supply test files 165 // just in case someone wants to specify their own test files for that version 166 func decrementVersion(version string) string { 167 split := strings.Split(version, ".") 168 if len(split) < 2 { 169 return "" 170 } 171 minor, err := strconv.Atoi(split[1]) 172 if err != nil { 173 return "" 174 } 175 if minor <= 1 { 176 return "" 177 } 178 split[1] = strconv.Itoa(minor - 1) 179 return strings.Join(split, ".") 180 } 181 182 // getFiles finds which of the set of candidate files exist 183 func getFiles(v *viper.Viper, fileType string) map[string]string { 184 filemap := make(map[string]string) 185 mainOpt := TypeMap[fileType][0] 186 defaultOpt := TypeMap[fileType][1] 187 188 for _, component := range v.GetStringSlice("components") { 189 s := v.Sub(component) 190 if s == nil { 191 continue 192 } 193 194 // See if any of the candidate files exist 195 file := findConfigFile(s.GetStringSlice(mainOpt)) 196 if file == "" { 197 if s.IsSet(defaultOpt) { 198 file = s.GetString(defaultOpt) 199 glog.V(2).Info(fmt.Sprintf("Using default %s file name '%s' for component %s", fileType, file, component)) 200 } else { 201 // Default the file name that we'll substitute to the name of the component 202 glog.V(2).Info(fmt.Sprintf("Missing %s file for %s", fileType, component)) 203 file = component 204 } 205 } else { 206 glog.V(2).Info(fmt.Sprintf("Component %s uses %s file '%s'", component, fileType, file)) 207 } 208 209 filemap[component] = file 210 } 211 212 return filemap 213 } 214 215 // verifyBin checks that the binary specified is running 216 func verifyBin(bin string) bool { 217 // Strip any quotes 218 bin = strings.Trim(bin, "'\"") 219 220 // bin could consist of more than one word 221 // We'll search for running processes with the first word, and then check the whole 222 // proc as supplied is included in the results 223 proc := strings.Fields(bin)[0] 224 out := psFunc(proc) 225 226 // There could be multiple lines in the ps output 227 // The binary needs to be the first word in the ps output, except that it could be preceded by a path 228 // e.g. /usr/bin/kubelet is a match for kubelet 229 // but apiserver is not a match for kube-apiserver 230 reFirstWord := regexp.MustCompile(`^(\S*\/)*` + bin) 231 lines := strings.Split(out, "\n") 232 for _, l := range lines { 233 glog.V(3).Info(fmt.Sprintf("reFirstWord.Match(%s)", l)) 234 if reFirstWord.Match([]byte(l)) { 235 return true 236 } 237 } 238 239 return false 240 } 241 242 // fundConfigFile looks through a list of possible config files and finds the first one that exists 243 func findConfigFile(candidates []string) string { 244 for _, c := range candidates { 245 _, err := statFunc(c) 246 if err == nil { 247 return c 248 } 249 if !os.IsNotExist(err) && !strings.HasSuffix(err.Error(), "not a directory") { 250 exitWithError(fmt.Errorf("error looking for file %s: %v", c, err)) 251 } 252 } 253 254 return "" 255 } 256 257 // findExecutable looks through a list of possible executable names and finds the first one that's running 258 func findExecutable(candidates []string) (string, error) { 259 for _, c := range candidates { 260 if verifyBin(c) { 261 return c, nil 262 } 263 glog.V(1).Info(fmt.Sprintf("executable '%s' not running", c)) 264 } 265 266 return "", fmt.Errorf("no candidates running") 267 } 268 269 func multiWordReplace(s string, subname string, sub string) string { 270 f := strings.Fields(sub) 271 if len(f) > 1 { 272 sub = "'" + sub + "'" 273 } 274 275 return strings.Replace(s, subname, sub, -1) 276 } 277 278 const missingKubectlKubeletMessage = ` 279 Unable to find the programs kubectl or kubelet in the PATH. 280 These programs are used to determine which version of Kubernetes is running. 281 Make sure the /usr/local/mount-from-host/bin directory is mapped to the container, 282 either in the job.yaml file, or Docker command. 283 284 For job.yaml: 285 ... 286 - name: usr-bin 287 mountPath: /usr/local/mount-from-host/bin 288 ... 289 290 For docker command: 291 docker -v $(which kubectl):/usr/local/mount-from-host/bin/kubectl .... 292 293 Alternatively, you can specify the version with --version 294 kube-bench --version <VERSION> ... 295 ` 296 297 func getKubeVersion() (*KubeVersion, error) { 298 kubeConfig, err := rest.InClusterConfig() 299 if err != nil { 300 glog.V(3).Infof("Error fetching cluster config: %s", err) 301 } 302 isRKE := false 303 if err == nil && kubeConfig != nil { 304 k8sClient, err := kubernetes.NewForConfig(kubeConfig) 305 if err != nil { 306 glog.V(3).Infof("Failed to fetch k8sClient object from kube config : %s", err) 307 } 308 309 if err == nil { 310 isRKE, err = IsRKE(context.Background(), k8sClient) 311 if err != nil { 312 glog.V(3).Infof("Error detecting RKE cluster: %s", err) 313 } 314 } 315 } 316 317 if k8sVer, err := getKubeVersionFromRESTAPI(); err == nil { 318 glog.V(2).Info(fmt.Sprintf("Kubernetes REST API Reported version: %s", k8sVer)) 319 if isRKE { 320 k8sVer.GitVersion = k8sVer.GitVersion + "-rancher1" 321 } 322 return k8sVer, nil 323 } 324 325 // These executables might not be on the user's path. 326 _, err = exec.LookPath("kubectl") 327 if err != nil { 328 glog.V(3).Infof("Error locating kubectl: %s", err) 329 _, err = exec.LookPath("kubelet") 330 if err != nil { 331 glog.V(3).Infof("Error locating kubelet: %s", err) 332 // Search for the kubelet binary all over the filesystem and run the first match to get the kubernetes version 333 cmd := exec.Command("/bin/sh", "-c", "`find / -type f -executable -name kubelet 2>/dev/null | grep -m1 .` --version") 334 out, err := cmd.CombinedOutput() 335 if err == nil { 336 glog.V(3).Infof("Found kubelet and query kubernetes version is: %s", string(out)) 337 return getVersionFromKubeletOutput(string(out)), nil 338 } 339 340 glog.Warning(missingKubectlKubeletMessage) 341 glog.V(1).Info("unable to find the programs kubectl or kubelet in the PATH") 342 glog.V(1).Infof("Cant detect version, assuming default %s", defaultKubeVersion) 343 return &KubeVersion{baseVersion: defaultKubeVersion}, nil 344 } 345 return getKubeVersionFromKubelet(), nil 346 } 347 348 return getKubeVersionFromKubectl(), nil 349 } 350 351 func getKubeVersionFromKubectl() *KubeVersion { 352 cmd := exec.Command("kubectl", "version", "-o", "json") 353 out, err := cmd.CombinedOutput() 354 if err != nil { 355 glog.V(2).Infof("Failed to query kubectl: %s", err) 356 glog.V(2).Info(err) 357 } 358 359 return getVersionFromKubectlOutput(string(out)) 360 } 361 362 func getKubeVersionFromKubelet() *KubeVersion { 363 cmd := exec.Command("kubelet", "--version") 364 out, err := cmd.CombinedOutput() 365 if err != nil { 366 glog.V(2).Infof("Failed to query kubelet: %s", err) 367 glog.V(2).Info(err) 368 } 369 370 return getVersionFromKubeletOutput(string(out)) 371 } 372 373 func getVersionFromKubectlOutput(s string) *KubeVersion { 374 glog.V(2).Infof("Kubectl output: %s", s) 375 type versionResult struct { 376 ServerVersion VersionResponse 377 } 378 vrObj := &versionResult{} 379 if err := json.Unmarshal([]byte(s), vrObj); err != nil { 380 glog.V(2).Info(err) 381 if strings.Contains(s, "The connection to the server") { 382 msg := fmt.Sprintf(`Warning: Kubernetes version was not auto-detected because kubectl could not connect to the Kubernetes server. This may be because the kubeconfig information is missing or has credentials that do not match the server. Assuming default version %s`, defaultKubeVersion) 383 fmt.Fprintln(os.Stderr, msg) 384 } 385 glog.V(1).Info(fmt.Sprintf("Unable to get Kubernetes version from kubectl, using default version: %s", defaultKubeVersion)) 386 return &KubeVersion{baseVersion: defaultKubeVersion} 387 } 388 sv := vrObj.ServerVersion 389 return &KubeVersion{ 390 Major: sv.Major, 391 Minor: sv.Minor, 392 GitVersion: sv.GitVersion, 393 } 394 } 395 396 func getVersionFromKubeletOutput(s string) *KubeVersion { 397 glog.V(2).Infof("Kubelet output: %s", s) 398 serverVersionRe := regexp.MustCompile(`Kubernetes v(\d+.\d+)`) 399 subs := serverVersionRe.FindStringSubmatch(s) 400 if len(subs) < 2 { 401 glog.V(1).Info(fmt.Sprintf("Unable to get Kubernetes version from kubelet, using default version: %s", defaultKubeVersion)) 402 return &KubeVersion{baseVersion: defaultKubeVersion} 403 } 404 return &KubeVersion{baseVersion: subs[1]} 405 } 406 407 func makeSubstitutions(s string, ext string, m map[string]string) (string, []string) { 408 substitutions := make([]string, 0) 409 for k, v := range m { 410 subst := "$" + k + ext 411 if v == "" { 412 glog.V(2).Info(fmt.Sprintf("No substitution for '%s'\n", subst)) 413 continue 414 } 415 glog.V(2).Info(fmt.Sprintf("Substituting %s with '%s'\n", subst, v)) 416 beforeS := s 417 s = multiWordReplace(s, subst, v) 418 if beforeS != s { 419 substitutions = append(substitutions, v) 420 } 421 } 422 423 return s, substitutions 424 } 425 426 func isEmpty(str string) bool { 427 return strings.TrimSpace(str) == "" 428 } 429 430 func buildComponentMissingErrorMessage(nodetype check.NodeType, component string, bins []string) string { 431 errMessageTemplate := ` 432 Unable to detect running programs for component %q 433 The following %q programs have been searched, but none of them have been found: 434 %s 435 436 These program names are provided in the config.yaml, section '%s.%s.bins' 437 ` 438 439 var componentRoleName, componentType string 440 switch nodetype { 441 442 case check.NODE: 443 componentRoleName = "worker node" 444 componentType = "node" 445 case check.ETCD: 446 componentRoleName = "etcd node" 447 componentType = "etcd" 448 default: 449 componentRoleName = "master node" 450 componentType = "master" 451 } 452 453 binList := "" 454 for _, bin := range bins { 455 binList = fmt.Sprintf("%s\t- %s\n", binList, bin) 456 } 457 458 return fmt.Sprintf(errMessageTemplate, component, componentRoleName, binList, componentType, component) 459 } 460 461 func getPlatformInfo() Platform { 462 463 openShiftInfo := getOpenShiftInfo() 464 if openShiftInfo.Name != "" && openShiftInfo.Version != "" { 465 return openShiftInfo 466 } 467 468 kv, err := getKubeVersion() 469 if err != nil { 470 glog.V(2).Info(err) 471 return Platform{} 472 } 473 return getPlatformInfoFromVersion(kv.GitVersion) 474 } 475 476 func getPlatformInfoFromVersion(s string) Platform { 477 versionRe := regexp.MustCompile(`v(\d+\.\d+)\.\d+[-+](\w+)(?:[.\-+]*)\w+`) 478 subs := versionRe.FindStringSubmatch(s) 479 if len(subs) < 3 { 480 return Platform{} 481 } 482 return Platform{ 483 Name: subs[2], 484 Version: subs[1], 485 } 486 } 487 488 func getPlatformBenchmarkVersion(platform Platform) string { 489 glog.V(3).Infof("getPlatformBenchmarkVersion platform: %s", platform) 490 switch platform.Name { 491 case "eks": 492 return "eks-1.2.0" 493 case "gke": 494 switch platform.Version { 495 case "1.15", "1.16", "1.17", "1.18", "1.19": 496 return "gke-1.0" 497 default: 498 return "gke-1.2.0" 499 } 500 case "aliyun": 501 return "ack-1.0" 502 case "ocp": 503 switch platform.Version { 504 case "3.10": 505 return "rh-0.7" 506 case "4.1": 507 return "rh-1.0" 508 } 509 case "vmware": 510 return "tkgi-1.2.53" 511 case "k3s": 512 switch platform.Version { 513 case "1.23": 514 return "k3s-cis-1.23" 515 case "1.24": 516 return "k3s-cis-1.24" 517 case "1.25", "1.26", "1.27": 518 return "k3s-cis-1.7" 519 } 520 case "rancher": 521 switch platform.Version { 522 case "1.23": 523 return "rke-cis-1.23" 524 case "1.24": 525 return "rke-cis-1.24" 526 case "1.25", "1.26", "1.27": 527 return "rke-cis-1.7" 528 } 529 case "rke2r": 530 switch platform.Version { 531 case "1.23": 532 return "rke2-cis-1.23" 533 case "1.24": 534 return "rke2-cis-1.24" 535 case "1.25", "1.26", "1.27": 536 return "rke2-cis-1.7" 537 } 538 } 539 return "" 540 } 541 542 func getOpenShiftInfo() Platform { 543 glog.V(1).Info("Checking for oc") 544 _, err := exec.LookPath("oc") 545 546 if err == nil { 547 cmd := exec.Command("oc", "version") 548 out, err := cmd.CombinedOutput() 549 550 if err == nil { 551 versionRe := regexp.MustCompile(`oc v(\d+\.\d+)`) 552 subs := versionRe.FindStringSubmatch(string(out)) 553 if len(subs) < 1 { 554 versionRe = regexp.MustCompile(`Client Version:\s*(\d+\.\d+)`) 555 subs = versionRe.FindStringSubmatch(string(out)) 556 } 557 if len(subs) > 1 { 558 glog.V(2).Infof("OCP output '%s' \nplatform is %s \nocp %v", string(out), getPlatformInfoFromVersion(string(out)), subs[1]) 559 ocpBenchmarkVersion, err := getOcpValidVersion(subs[1]) 560 if err == nil { 561 return Platform{Name: "ocp", Version: ocpBenchmarkVersion} 562 } else { 563 glog.V(1).Infof("Can't get getOcpValidVersion: %v", err) 564 } 565 } else { 566 glog.V(1).Infof("Can't parse version output: %v", subs) 567 } 568 } else { 569 glog.V(1).Infof("Can't use oc command: %v", err) 570 } 571 } else { 572 glog.V(1).Infof("Can't find oc command: %v", err) 573 } 574 return Platform{} 575 } 576 577 func getOcpValidVersion(ocpVer string) (string, error) { 578 ocpOriginal := ocpVer 579 580 for !isEmpty(ocpVer) { 581 glog.V(3).Info(fmt.Sprintf("getOcpBenchmarkVersion check for ocp: %q \n", ocpVer)) 582 if ocpVer == "3.10" || ocpVer == "4.1" { 583 glog.V(1).Info(fmt.Sprintf("getOcpBenchmarkVersion found valid version for ocp: %q \n", ocpVer)) 584 return ocpVer, nil 585 } 586 ocpVer = decrementVersion(ocpVer) 587 } 588 589 glog.V(1).Info(fmt.Sprintf("getOcpBenchmarkVersion unable to find a match for: %q", ocpOriginal)) 590 return "", fmt.Errorf("unable to find a matching Benchmark Version match for ocp version: %s", ocpOriginal) 591 } 592 593 // IsRKE Identifies if the cluster belongs to Rancher Distribution RKE 594 func IsRKE(ctx context.Context, k8sClient kubernetes.Interface) (bool, error) { 595 // if there are windows nodes then this should not be counted as rke.linux 596 windowsNodes, err := k8sClient.CoreV1().Nodes().List(ctx, metav1.ListOptions{ 597 Limit: 1, 598 LabelSelector: "kubernetes.io/os=windows", 599 }) 600 if err != nil { 601 return false, err 602 } 603 if len(windowsNodes.Items) != 0 { 604 return false, nil 605 } 606 607 // Any node created by RKE should have the annotation, so just grab 1 608 nodes, err := k8sClient.CoreV1().Nodes().List(ctx, metav1.ListOptions{Limit: 1}) 609 if err != nil { 610 return false, err 611 } 612 613 if len(nodes.Items) == 0 { 614 return false, nil 615 } 616 617 annos := nodes.Items[0].Annotations 618 if _, ok := annos["rke.cattle.io/external-ip"]; ok { 619 return true, nil 620 } 621 if _, ok := annos["rke.cattle.io/internal-ip"]; ok { 622 return true, nil 623 } 624 return false, nil 625 }