k8s.io/kubernetes@v1.29.3/test/e2e_node/swap_test.go (about) 1 /* 2 Copyright 2023 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package e2enode 18 19 import ( 20 "context" 21 "fmt" 22 "path/filepath" 23 "strconv" 24 25 "github.com/onsi/ginkgo/v2" 26 "github.com/onsi/gomega" 27 v1 "k8s.io/api/core/v1" 28 "k8s.io/apimachinery/pkg/api/resource" 29 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 30 "k8s.io/apimachinery/pkg/util/rand" 31 utilfeature "k8s.io/apiserver/pkg/util/feature" 32 "k8s.io/kubernetes/pkg/features" 33 "k8s.io/kubernetes/pkg/kubelet/types" 34 "k8s.io/kubernetes/test/e2e/framework" 35 e2epod "k8s.io/kubernetes/test/e2e/framework/pod" 36 testutils "k8s.io/kubernetes/test/utils" 37 admissionapi "k8s.io/pod-security-admission/api" 38 ) 39 40 const ( 41 cgroupBasePath = "/sys/fs/cgroup/" 42 cgroupV1SwapLimitFile = "/memory/memory.memsw.limit_in_bytes" 43 cgroupV2SwapLimitFile = "memory.swap.max" 44 cgroupV1MemLimitFile = "/memory/memory.limit_in_bytes" 45 ) 46 47 var _ = SIGDescribe("Swap", framework.WithNodeConformance(), "[LinuxOnly]", func() { 48 f := framework.NewDefaultFramework("swap-test") 49 f.NamespacePodSecurityEnforceLevel = admissionapi.LevelBaseline 50 51 ginkgo.DescribeTable("with configuration", func(qosClass v1.PodQOSClass, memoryRequestEqualLimit bool) { 52 ginkgo.By(fmt.Sprintf("Creating a pod of QOS class %s. memoryRequestEqualLimit: %t", qosClass, memoryRequestEqualLimit)) 53 pod := getSwapTestPod(f, qosClass, memoryRequestEqualLimit) 54 pod = runPodAndWaitUntilScheduled(f, pod) 55 56 isCgroupV2 := isPodCgroupV2(f, pod) 57 isLimitedSwap := isLimitedSwap(f, pod) 58 59 if !isSwapFeatureGateEnabled() || !isCgroupV2 || (isLimitedSwap && (qosClass != v1.PodQOSBurstable || memoryRequestEqualLimit)) { 60 ginkgo.By(fmt.Sprintf("Expecting no swap. feature gate on? %t isCgroupV2? %t is QoS burstable? %t", isSwapFeatureGateEnabled(), isCgroupV2, qosClass == v1.PodQOSBurstable)) 61 expectNoSwap(f, pod, isCgroupV2) 62 return 63 } 64 65 if !isLimitedSwap { 66 ginkgo.By("expecting unlimited swap") 67 expectUnlimitedSwap(f, pod, isCgroupV2) 68 return 69 } 70 71 ginkgo.By("expecting limited swap") 72 expectedSwapLimit := calcSwapForBurstablePod(f, pod) 73 expectLimitedSwap(f, pod, expectedSwapLimit) 74 }, 75 ginkgo.Entry("QOS Best-effort", v1.PodQOSBestEffort, false), 76 ginkgo.Entry("QOS Burstable", v1.PodQOSBurstable, false), 77 ginkgo.Entry("QOS Burstable with memory request equals to limit", v1.PodQOSBurstable, true), 78 ginkgo.Entry("QOS Guaranteed", v1.PodQOSGuaranteed, false), 79 ) 80 }) 81 82 // Note that memoryRequestEqualLimit is effective only when qosClass is PodQOSBestEffort. 83 func getSwapTestPod(f *framework.Framework, qosClass v1.PodQOSClass, memoryRequestEqualLimit bool) *v1.Pod { 84 podMemoryAmount := resource.MustParse("128Mi") 85 86 var resources v1.ResourceRequirements 87 switch qosClass { 88 case v1.PodQOSBestEffort: 89 // nothing to do in this case 90 case v1.PodQOSBurstable: 91 resources = v1.ResourceRequirements{ 92 Requests: v1.ResourceList{ 93 v1.ResourceMemory: podMemoryAmount, 94 }, 95 } 96 97 if memoryRequestEqualLimit { 98 resources.Limits = resources.Requests 99 } 100 case v1.PodQOSGuaranteed: 101 resources = v1.ResourceRequirements{ 102 Limits: v1.ResourceList{ 103 v1.ResourceCPU: resource.MustParse("200m"), 104 v1.ResourceMemory: podMemoryAmount, 105 }, 106 } 107 resources.Requests = resources.Limits 108 } 109 110 pod := &v1.Pod{ 111 ObjectMeta: metav1.ObjectMeta{ 112 Name: "test-pod-swap-" + rand.String(5), 113 Namespace: f.Namespace.Name, 114 }, 115 Spec: v1.PodSpec{ 116 RestartPolicy: v1.RestartPolicyAlways, 117 Containers: []v1.Container{ 118 { 119 Name: "busybox-container", 120 Image: busyboxImage, 121 Command: []string{"sleep", "600"}, 122 Resources: resources, 123 }, 124 }, 125 }, 126 } 127 128 return pod 129 } 130 131 func runPodAndWaitUntilScheduled(f *framework.Framework, pod *v1.Pod) *v1.Pod { 132 ginkgo.By("running swap test pod") 133 podClient := e2epod.NewPodClient(f) 134 135 pod = podClient.CreateSync(context.Background(), pod) 136 pod, err := podClient.Get(context.Background(), pod.Name, metav1.GetOptions{}) 137 138 framework.ExpectNoError(err) 139 isReady, err := testutils.PodRunningReady(pod) 140 framework.ExpectNoError(err) 141 gomega.ExpectWithOffset(1, isReady).To(gomega.BeTrue(), "pod should be ready") 142 143 return pod 144 } 145 146 func isSwapFeatureGateEnabled() bool { 147 ginkgo.By("figuring if NodeSwap feature gate is turned on") 148 return utilfeature.DefaultFeatureGate.Enabled(features.NodeSwap) 149 } 150 151 func readCgroupFile(f *framework.Framework, pod *v1.Pod, filename string) string { 152 filePath := filepath.Join(cgroupBasePath, filename) 153 154 ginkgo.By("reading cgroup file " + filePath) 155 output := e2epod.ExecCommandInContainer(f, pod.Name, pod.Spec.Containers[0].Name, "/bin/sh", "-ec", "cat "+filePath) 156 157 return output 158 } 159 160 func isPodCgroupV2(f *framework.Framework, pod *v1.Pod) bool { 161 ginkgo.By("figuring is test pod runs cgroup v2") 162 output := e2epod.ExecCommandInContainer(f, pod.Name, pod.Spec.Containers[0].Name, "/bin/sh", "-ec", `if test -f "/sys/fs/cgroup/cgroup.controllers"; then echo "true"; else echo "false"; fi`) 163 164 return output == "true" 165 } 166 167 func expectNoSwap(f *framework.Framework, pod *v1.Pod, isCgroupV2 bool) { 168 if isCgroupV2 { 169 swapLimit := readCgroupFile(f, pod, cgroupV2SwapLimitFile) 170 gomega.ExpectWithOffset(1, swapLimit).To(gomega.Equal("0"), "max swap allowed should be zero") 171 } else { 172 swapPlusMemLimit := readCgroupFile(f, pod, cgroupV1SwapLimitFile) 173 memLimit := readCgroupFile(f, pod, cgroupV1MemLimitFile) 174 gomega.ExpectWithOffset(1, swapPlusMemLimit).ToNot(gomega.BeEmpty()) 175 gomega.ExpectWithOffset(1, swapPlusMemLimit).To(gomega.Equal(memLimit)) 176 } 177 } 178 179 func expectUnlimitedSwap(f *framework.Framework, pod *v1.Pod, isCgroupV2 bool) { 180 if isCgroupV2 { 181 swapLimit := readCgroupFile(f, pod, cgroupV2SwapLimitFile) 182 gomega.ExpectWithOffset(1, swapLimit).To(gomega.Equal("max"), "max swap allowed should be \"max\"") 183 } else { 184 swapPlusMemLimit := readCgroupFile(f, pod, cgroupV1SwapLimitFile) 185 gomega.ExpectWithOffset(1, swapPlusMemLimit).To(gomega.Equal("-1")) 186 } 187 } 188 189 // supports v2 only as v1 shouldn't support LimitedSwap 190 func expectLimitedSwap(f *framework.Framework, pod *v1.Pod, expectedSwapLimit int64) { 191 swapLimitStr := readCgroupFile(f, pod, cgroupV2SwapLimitFile) 192 193 swapLimit, err := strconv.Atoi(swapLimitStr) 194 framework.ExpectNoError(err, "cannot convert swap limit to int") 195 196 // cgroup values are always aligned w.r.t. the page size, which is usually 4Ki 197 const cgroupAlignment int64 = 4 * 1024 // 4Ki 198 const errMsg = "swap limitation is not as expected" 199 200 gomega.ExpectWithOffset(1, int64(swapLimit)).To( 201 gomega.Or( 202 gomega.BeNumerically(">=", expectedSwapLimit-cgroupAlignment), 203 gomega.BeNumerically("<=", expectedSwapLimit+cgroupAlignment), 204 ), 205 errMsg, 206 ) 207 } 208 209 func getSwapCapacity(f *framework.Framework, pod *v1.Pod) int64 { 210 output := e2epod.ExecCommandInContainer(f, pod.Name, pod.Spec.Containers[0].Name, "/bin/sh", "-ec", "free -b | grep Swap | xargs | cut -d\" \" -f2") 211 212 swapCapacity, err := strconv.Atoi(output) 213 framework.ExpectNoError(err, "cannot convert swap size to int") 214 215 ginkgo.By(fmt.Sprintf("providing swap capacity: %d", swapCapacity)) 216 217 return int64(swapCapacity) 218 } 219 220 func getMemoryCapacity(f *framework.Framework, pod *v1.Pod) int64 { 221 nodes, err := f.ClientSet.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{}) 222 framework.ExpectNoError(err, "failed listing nodes") 223 224 for _, node := range nodes.Items { 225 if node.Name != pod.Spec.NodeName { 226 continue 227 } 228 229 memCapacity := node.Status.Capacity[v1.ResourceMemory] 230 return memCapacity.Value() 231 } 232 233 framework.ExpectNoError(fmt.Errorf("node %s wasn't found", pod.Spec.NodeName)) 234 return 0 235 } 236 237 func calcSwapForBurstablePod(f *framework.Framework, pod *v1.Pod) int64 { 238 nodeMemoryCapacity := getMemoryCapacity(f, pod) 239 nodeSwapCapacity := getSwapCapacity(f, pod) 240 containerMemoryRequest := pod.Spec.Containers[0].Resources.Requests.Memory().Value() 241 242 containerMemoryProportion := float64(containerMemoryRequest) / float64(nodeMemoryCapacity) 243 swapAllocation := containerMemoryProportion * float64(nodeSwapCapacity) 244 ginkgo.By(fmt.Sprintf("Calculating swap for burstable pods: nodeMemoryCapacity: %d, nodeSwapCapacity: %d, containerMemoryRequest: %d, swapAllocation: %d", 245 nodeMemoryCapacity, nodeSwapCapacity, containerMemoryRequest, int64(swapAllocation))) 246 247 return int64(swapAllocation) 248 } 249 250 func isLimitedSwap(f *framework.Framework, pod *v1.Pod) bool { 251 kubeletCfg, err := getCurrentKubeletConfig(context.Background()) 252 framework.ExpectNoError(err, "cannot get kubelet config") 253 254 return kubeletCfg.MemorySwap.SwapBehavior == types.LimitedSwap 255 }