k8s.io/kubernetes@v1.29.3/test/e2e_node/remote/node_conformance.go (about) 1 /* 2 Copyright 2016 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package remote 18 19 import ( 20 "fmt" 21 "os" 22 "os/exec" 23 "path/filepath" 24 "runtime" 25 "strings" 26 "time" 27 28 "k8s.io/klog/v2" 29 30 "k8s.io/kubernetes/test/e2e/framework" 31 "k8s.io/kubernetes/test/e2e_node/builder" 32 "k8s.io/kubernetes/test/utils" 33 ) 34 35 // ConformanceRemote contains the specific functions in the node conformance test suite. 36 type ConformanceRemote struct{} 37 38 func init() { 39 RegisterTestSuite("conformance", &ConformanceRemote{}) 40 } 41 42 // getConformanceDirectory gets node conformance test build directory. 43 func getConformanceDirectory() (string, error) { 44 k8sRoot, err := utils.GetK8sRootDir() 45 if err != nil { 46 return "", err 47 } 48 return filepath.Join(k8sRoot, "test", "e2e_node", "conformance", "build"), nil 49 } 50 51 // commandToString is a helper function which formats command to string. 52 func commandToString(c *exec.Cmd) string { 53 return strings.Join(append([]string{c.Path}, c.Args[1:]...), " ") 54 } 55 56 // Image path constants. 57 const ( 58 conformanceRegistry = "registry.k8s.io" 59 conformanceArch = runtime.GOARCH 60 conformanceTarfile = "node_conformance.tar" 61 conformanceTestBinary = "e2e_node.test" 62 conformanceImageLoadTimeout = time.Duration(30) * time.Second 63 ) 64 65 // timestamp is used as an unique id of current test. 66 var timestamp = getTimestamp() 67 68 // getConformanceTestImageName returns name of the conformance test image given the system spec name. 69 func getConformanceTestImageName(systemSpecName string) string { 70 if systemSpecName == "" { 71 return fmt.Sprintf("%s/node-test-%s:%s", conformanceRegistry, conformanceArch, timestamp) 72 } 73 return fmt.Sprintf("%s/node-test-%s-%s:%s", conformanceRegistry, systemSpecName, conformanceArch, timestamp) 74 } 75 76 // buildConformanceTest builds node conformance test image tarball into binDir. 77 func buildConformanceTest(binDir, systemSpecName string) error { 78 // Get node conformance directory. 79 conformancePath, err := getConformanceDirectory() 80 if err != nil { 81 return fmt.Errorf("failed to get node conformance directory: %w", err) 82 } 83 // Build docker image. 84 cmd := exec.Command("make", "-C", conformancePath, "BIN_DIR="+binDir, 85 "REGISTRY="+conformanceRegistry, 86 "ARCH="+conformanceArch, 87 "VERSION="+timestamp, 88 "SYSTEM_SPEC_NAME="+systemSpecName) 89 if output, err := cmd.CombinedOutput(); err != nil { 90 return fmt.Errorf("failed to build node conformance docker image: command - %q, error - %v, output - %q", 91 commandToString(cmd), err, output) 92 } 93 // Save docker image into tar file. 94 cmd = exec.Command("docker", "save", "-o", filepath.Join(binDir, conformanceTarfile), getConformanceTestImageName(systemSpecName)) 95 if output, err := cmd.CombinedOutput(); err != nil { 96 return fmt.Errorf("failed to save node conformance docker image into tar file: command - %q, error - %v, output - %q", 97 commandToString(cmd), err, output) 98 } 99 return nil 100 } 101 102 // SetupTestPackage sets up the test package with binaries k8s required for node conformance test 103 func (c *ConformanceRemote) SetupTestPackage(tardir, systemSpecName string) error { 104 // Build the executables 105 if err := builder.BuildGo(); err != nil { 106 return fmt.Errorf("failed to build the dependencies: %w", err) 107 } 108 109 // Make sure we can find the newly built binaries 110 buildOutputDir, err := utils.GetK8sBuildOutputDir(builder.IsDockerizedBuild(), builder.GetTargetBuildArch()) 111 if err != nil { 112 return fmt.Errorf("failed to locate kubernetes build output directory %v", err) 113 } 114 115 // Build node conformance tarball. 116 if err := buildConformanceTest(buildOutputDir, systemSpecName); err != nil { 117 return fmt.Errorf("failed to build node conformance test: %w", err) 118 } 119 120 // Copy files 121 requiredFiles := []string{"kubelet", conformanceTestBinary, conformanceTarfile} 122 for _, file := range requiredFiles { 123 source := filepath.Join(buildOutputDir, file) 124 if _, err := os.Stat(source); err != nil { 125 return fmt.Errorf("failed to locate test file %s: %w", file, err) 126 } 127 output, err := exec.Command("cp", source, filepath.Join(tardir, file)).CombinedOutput() 128 if err != nil { 129 return fmt.Errorf("failed to copy %q: error - %v output - %q", file, err, output) 130 } 131 } 132 133 return nil 134 } 135 136 // loadConformanceImage loads node conformance image from tar file. 137 func loadConformanceImage(host, workspace string) error { 138 klog.Info("Loading conformance image from tarfile") 139 tarfile := filepath.Join(workspace, conformanceTarfile) 140 if output, err := SSH(host, "timeout", conformanceImageLoadTimeout.String(), 141 "docker", "load", "-i", tarfile); err != nil { 142 return fmt.Errorf("failed to load node conformance image from tar file %q: error - %v output - %q", 143 tarfile, err, output) 144 } 145 return nil 146 } 147 148 // kubeletLauncherLog is the log of kubelet launcher. 149 const kubeletLauncherLog = "kubelet-launcher.log" 150 151 // kubeletPodPath is a fixed known pod specification path. We can not use the random pod 152 // manifest directory generated in e2e_node.test because we need to mount the directory into 153 // the conformance test container, it's easier if it's a known directory. 154 // TODO(random-liu): Get rid of this once we switch to cluster e2e node bootstrap script. 155 var kubeletPodPath = "conformance-pod-manifest-" + timestamp 156 157 // getPodPath returns pod manifest full path. 158 func getPodPath(workspace string) string { 159 return filepath.Join(workspace, kubeletPodPath) 160 } 161 162 // isSystemd returns whether the node is a systemd node. 163 func isSystemd(host string) (bool, error) { 164 // Returns "systemd" if /run/systemd/system is found, empty string otherwise. 165 output, err := SSH(host, "test", "-e", "/run/systemd/system", "&&", "echo", "systemd", "||", "true") 166 if err != nil { 167 return false, fmt.Errorf("failed to check systemd: error - %v output - %q", err, output) 168 } 169 return strings.TrimSpace(output) != "", nil 170 } 171 172 // launchKubelet launches kubelet by running e2e_node.test binary in run-kubelet-mode. 173 // This is a temporary solution, we should change node e2e to use the same node bootstrap 174 // with cluster e2e and launch kubelet outside of the test for both regular node e2e and 175 // node conformance test. 176 // TODO(random-liu): Switch to use standard node bootstrap script. 177 func launchKubelet(host, workspace, results, testArgs, bearerToken string) error { 178 podManifestPath := getPodPath(workspace) 179 if output, err := SSH(host, "mkdir", podManifestPath); err != nil { 180 return fmt.Errorf("failed to create kubelet pod manifest path %q: error - %v output - %q", 181 podManifestPath, err, output) 182 } 183 startKubeletCmd := fmt.Sprintf("./%s --run-kubelet-mode --node-name=%s"+ 184 " --bearer-token=%s"+ 185 " --report-dir=%s %s --kubelet-flags=--pod-manifest-path=%s > %s 2>&1", 186 conformanceTestBinary, host, bearerToken, results, testArgs, podManifestPath, filepath.Join(results, kubeletLauncherLog)) 187 var cmd []string 188 systemd, err := isSystemd(host) 189 if err != nil { 190 return fmt.Errorf("failed to check systemd: %w", err) 191 } 192 if systemd { 193 cmd = []string{ 194 "systemd-run", "sh", "-c", getSSHCommand(" && ", 195 // Switch to workspace. 196 fmt.Sprintf("cd %s", workspace), 197 // Launch kubelet by running e2e_node.test in run-kubelet-mode. 198 startKubeletCmd, 199 ), 200 } 201 } else { 202 cmd = []string{ 203 "sh", "-c", getSSHCommand(" && ", 204 // Switch to workspace. 205 fmt.Sprintf("cd %s", workspace), 206 // Launch kubelet by running e2e_node.test in run-kubelet-mode with nohup. 207 fmt.Sprintf("(nohup %s &)", startKubeletCmd), 208 ), 209 } 210 } 211 klog.V(2).Infof("Launch kubelet with command: %v", cmd) 212 output, err := SSH(host, cmd...) 213 if err != nil { 214 return fmt.Errorf("failed to launch kubelet with command %v: error - %v output - %q", 215 cmd, err, output) 216 } 217 klog.Info("Successfully launch kubelet") 218 return nil 219 } 220 221 // kubeletStopGracePeriod is the grace period to wait before forcibly killing kubelet. 222 const kubeletStopGracePeriod = 10 * time.Second 223 224 // stopKubelet stops kubelet launcher and kubelet gracefully. 225 func stopKubelet(host, workspace string) error { 226 klog.Info("Gracefully stop kubelet launcher") 227 if output, err := SSH(host, "pkill", conformanceTestBinary); err != nil { 228 return fmt.Errorf("failed to gracefully stop kubelet launcher: error - %v output - %q", 229 err, output) 230 } 231 klog.Info("Wait for kubelet launcher to stop") 232 stopped := false 233 for start := time.Now(); time.Since(start) < kubeletStopGracePeriod; time.Sleep(time.Second) { 234 // Check whether the process is still running. 235 output, err := SSH(host, "pidof", conformanceTestBinary, "||", "true") 236 if err != nil { 237 return fmt.Errorf("failed to check kubelet stopping: error - %v output -%q", 238 err, output) 239 } 240 // Kubelet is stopped 241 if strings.TrimSpace(output) == "" { 242 stopped = true 243 break 244 } 245 } 246 if !stopped { 247 klog.Info("Forcibly stop kubelet") 248 if output, err := SSH(host, "pkill", "-SIGKILL", conformanceTestBinary); err != nil { 249 return fmt.Errorf("failed to forcibly stop kubelet: error - %v output - %q", 250 err, output) 251 } 252 } 253 klog.Info("Successfully stop kubelet") 254 // Clean up the pod manifest path 255 podManifestPath := getPodPath(workspace) 256 if output, err := SSH(host, "rm", "-f", filepath.Join(workspace, podManifestPath)); err != nil { 257 return fmt.Errorf("failed to cleanup pod manifest directory %q: error - %v, output - %q", 258 podManifestPath, err, output) 259 } 260 return nil 261 } 262 263 // RunTest runs test on the node. 264 func (c *ConformanceRemote) RunTest(host, workspace, results, imageDesc, junitFilePrefix, testArgs, _, systemSpecName, extraEnvs, _ string, timeout time.Duration) (string, error) { 265 // Install the cni plugins and add a basic CNI configuration. 266 if err := setupCNI(host, workspace); err != nil { 267 return "", err 268 } 269 270 // Configure iptables firewall rules. 271 if err := configureFirewall(host); err != nil { 272 return "", err 273 } 274 275 // Kill any running node processes. 276 cleanupNodeProcesses(host) 277 278 // Load node conformance image. 279 if err := loadConformanceImage(host, workspace); err != nil { 280 return "", err 281 } 282 283 bearerToken, err := framework.GenerateSecureToken(16) 284 if err != nil { 285 return "", err 286 } 287 288 // Launch kubelet. 289 if err := launchKubelet(host, workspace, results, testArgs, bearerToken); err != nil { 290 return "", err 291 } 292 // Stop kubelet. 293 defer func() { 294 if err := stopKubelet(host, workspace); err != nil { 295 // Only log an error if failed to stop kubelet because it is not critical. 296 klog.Errorf("failed to stop kubelet: %v", err) 297 } 298 }() 299 300 // Run the tests 301 klog.V(2).Infof("Starting tests on %q", host) 302 podManifestPath := getPodPath(workspace) 303 cmd := fmt.Sprintf("'timeout -k 30s %fs docker run --rm --privileged=true --net=host -v /:/rootfs -v %s:%s -v %s:/var/result -e TEST_ARGS=--report-prefix=%s -e EXTRA_ENVS=%s -e TEST_ARGS=--bearer-token=%s %s'", 304 timeout.Seconds(), podManifestPath, podManifestPath, results, junitFilePrefix, extraEnvs, bearerToken, getConformanceTestImageName(systemSpecName)) 305 return SSH(host, "sh", "-c", cmd) 306 }