k8s.io/kubernetes@v1.29.3/test/e2e/apimachinery/resource_quota.go (about) 1 /* 2 Copyright 2015 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package apimachinery 18 19 import ( 20 "context" 21 "encoding/json" 22 "fmt" 23 "strconv" 24 "time" 25 26 appsv1 "k8s.io/api/apps/v1" 27 v1 "k8s.io/api/core/v1" 28 schedulingv1 "k8s.io/api/scheduling/v1" 29 apiequality "k8s.io/apimachinery/pkg/api/equality" 30 apierrors "k8s.io/apimachinery/pkg/api/errors" 31 "k8s.io/apimachinery/pkg/api/resource" 32 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 33 "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" 34 "k8s.io/apimachinery/pkg/labels" 35 "k8s.io/apimachinery/pkg/runtime" 36 "k8s.io/apimachinery/pkg/runtime/schema" 37 "k8s.io/apimachinery/pkg/types" 38 "k8s.io/apimachinery/pkg/util/intstr" 39 utilrand "k8s.io/apimachinery/pkg/util/rand" 40 "k8s.io/apimachinery/pkg/util/wait" 41 watch "k8s.io/apimachinery/pkg/watch" 42 quota "k8s.io/apiserver/pkg/quota/v1" 43 clientset "k8s.io/client-go/kubernetes" 44 clientscheme "k8s.io/client-go/kubernetes/scheme" 45 "k8s.io/client-go/tools/cache" 46 watchtools "k8s.io/client-go/tools/watch" 47 "k8s.io/client-go/util/retry" 48 "k8s.io/kubernetes/pkg/quota/v1/evaluator/core" 49 "k8s.io/kubernetes/test/e2e/feature" 50 "k8s.io/kubernetes/test/e2e/framework" 51 "k8s.io/kubernetes/test/utils/crd" 52 imageutils "k8s.io/kubernetes/test/utils/image" 53 admissionapi "k8s.io/pod-security-admission/api" 54 "k8s.io/utils/pointer" 55 56 "github.com/onsi/ginkgo/v2" 57 "github.com/onsi/gomega" 58 ) 59 60 const ( 61 // how long to wait for a resource quota update to occur 62 resourceQuotaTimeout = time.Minute 63 podName = "pfpod" 64 ) 65 66 var classGold = "gold" 67 var extendedResourceName = "example.com/dongle" 68 69 var _ = SIGDescribe("ResourceQuota", func() { 70 f := framework.NewDefaultFramework("resourcequota") 71 f.NamespacePodSecurityLevel = admissionapi.LevelBaseline 72 73 /* 74 Release: v1.16 75 Testname: ResourceQuota, object count quota, resourcequotas 76 Description: Create a ResourceQuota. Creation MUST be successful and its ResourceQuotaStatus MUST match to expected used and total allowed resource quota count within namespace. 77 */ 78 framework.ConformanceIt("should create a ResourceQuota and ensure its status is promptly calculated.", func(ctx context.Context) { 79 ginkgo.By("Counting existing ResourceQuota") 80 c, err := countResourceQuota(ctx, f.ClientSet, f.Namespace.Name) 81 framework.ExpectNoError(err) 82 83 ginkgo.By("Creating a ResourceQuota") 84 quotaName := "test-quota" 85 resourceQuota := newTestResourceQuota(quotaName) 86 _, err = createResourceQuota(ctx, f.ClientSet, f.Namespace.Name, resourceQuota) 87 framework.ExpectNoError(err) 88 89 ginkgo.By("Ensuring resource quota status is calculated") 90 usedResources := v1.ResourceList{} 91 usedResources[v1.ResourceQuotas] = resource.MustParse(strconv.Itoa(c + 1)) 92 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, quotaName, usedResources) 93 framework.ExpectNoError(err) 94 }) 95 96 /* 97 Release: v1.16 98 Testname: ResourceQuota, object count quota, service 99 Description: Create a ResourceQuota. Creation MUST be successful and its ResourceQuotaStatus MUST match to expected used and total allowed resource quota count within namespace. 100 Create a Service. Its creation MUST be successful and resource usage count against the Service object and resourceQuota object MUST be captured in ResourceQuotaStatus of the ResourceQuota. 101 Delete the Service. Deletion MUST succeed and resource usage count against the Service object MUST be released from ResourceQuotaStatus of the ResourceQuota. 102 */ 103 framework.ConformanceIt("should create a ResourceQuota and capture the life of a service.", func(ctx context.Context) { 104 ginkgo.By("Counting existing ResourceQuota") 105 c, err := countResourceQuota(ctx, f.ClientSet, f.Namespace.Name) 106 framework.ExpectNoError(err) 107 108 ginkgo.By("Creating a ResourceQuota") 109 quotaName := "test-quota" 110 resourceQuota := newTestResourceQuota(quotaName) 111 _, err = createResourceQuota(ctx, f.ClientSet, f.Namespace.Name, resourceQuota) 112 framework.ExpectNoError(err) 113 114 ginkgo.By("Ensuring resource quota status is calculated") 115 usedResources := v1.ResourceList{} 116 usedResources[v1.ResourceQuotas] = resource.MustParse(strconv.Itoa(c + 1)) 117 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, quotaName, usedResources) 118 framework.ExpectNoError(err) 119 120 ginkgo.By("Creating a Service") 121 service := newTestServiceForQuota("test-service", v1.ServiceTypeClusterIP, false) 122 service, err = f.ClientSet.CoreV1().Services(f.Namespace.Name).Create(ctx, service, metav1.CreateOptions{}) 123 framework.ExpectNoError(err) 124 125 ginkgo.By("Creating a NodePort Service") 126 nodeport := newTestServiceForQuota("test-service-np", v1.ServiceTypeNodePort, false) 127 nodeport, err = f.ClientSet.CoreV1().Services(f.Namespace.Name).Create(ctx, nodeport, metav1.CreateOptions{}) 128 framework.ExpectNoError(err) 129 130 ginkgo.By("Not allowing a LoadBalancer Service with NodePort to be created that exceeds remaining quota") 131 loadbalancer := newTestServiceForQuota("test-service-lb", v1.ServiceTypeLoadBalancer, true) 132 _, err = f.ClientSet.CoreV1().Services(f.Namespace.Name).Create(ctx, loadbalancer, metav1.CreateOptions{}) 133 gomega.Expect(err).To(gomega.HaveOccurred()) 134 135 ginkgo.By("Ensuring resource quota status captures service creation") 136 usedResources = v1.ResourceList{} 137 usedResources[v1.ResourceQuotas] = resource.MustParse(strconv.Itoa(c + 1)) 138 usedResources[v1.ResourceServices] = resource.MustParse("2") 139 usedResources[v1.ResourceServicesNodePorts] = resource.MustParse("1") 140 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, quotaName, usedResources) 141 framework.ExpectNoError(err) 142 143 ginkgo.By("Deleting Services") 144 err = f.ClientSet.CoreV1().Services(f.Namespace.Name).Delete(ctx, service.Name, metav1.DeleteOptions{}) 145 framework.ExpectNoError(err) 146 err = f.ClientSet.CoreV1().Services(f.Namespace.Name).Delete(ctx, nodeport.Name, metav1.DeleteOptions{}) 147 framework.ExpectNoError(err) 148 149 ginkgo.By("Ensuring resource quota status released usage") 150 usedResources[v1.ResourceServices] = resource.MustParse("0") 151 usedResources[v1.ResourceServicesNodePorts] = resource.MustParse("0") 152 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, quotaName, usedResources) 153 framework.ExpectNoError(err) 154 }) 155 156 /* 157 Release: v1.16 158 Testname: ResourceQuota, object count quota, secret 159 Description: Create a ResourceQuota. Creation MUST be successful and its ResourceQuotaStatus MUST match to expected used and total allowed resource quota count within namespace. 160 Create a Secret. Its creation MUST be successful and resource usage count against the Secret object and resourceQuota object MUST be captured in ResourceQuotaStatus of the ResourceQuota. 161 Delete the Secret. Deletion MUST succeed and resource usage count against the Secret object MUST be released from ResourceQuotaStatus of the ResourceQuota. 162 */ 163 framework.ConformanceIt("should create a ResourceQuota and capture the life of a secret.", func(ctx context.Context) { 164 ginkgo.By("Discovering how many secrets are in namespace by default") 165 found, unchanged := 0, 0 166 // On contended servers the service account controller can slow down, leading to the count changing during a run. 167 // Wait up to 5s for the count to stabilize, assuming that updates come at a consistent rate, and are not held indefinitely. 168 err := wait.PollWithContext(ctx, 1*time.Second, 30*time.Second, func(ctx context.Context) (bool, error) { 169 secrets, err := f.ClientSet.CoreV1().Secrets(f.Namespace.Name).List(ctx, metav1.ListOptions{}) 170 framework.ExpectNoError(err) 171 if len(secrets.Items) == found { 172 // loop until the number of secrets has stabilized for 5 seconds 173 unchanged++ 174 return unchanged > 4, nil 175 } 176 unchanged = 0 177 found = len(secrets.Items) 178 return false, nil 179 }) 180 framework.ExpectNoError(err) 181 defaultSecrets := fmt.Sprintf("%d", found) 182 hardSecrets := fmt.Sprintf("%d", found+1) 183 184 ginkgo.By("Counting existing ResourceQuota") 185 c, err := countResourceQuota(ctx, f.ClientSet, f.Namespace.Name) 186 framework.ExpectNoError(err) 187 188 ginkgo.By("Creating a ResourceQuota") 189 quotaName := "test-quota" 190 resourceQuota := newTestResourceQuota(quotaName) 191 resourceQuota.Spec.Hard[v1.ResourceSecrets] = resource.MustParse(hardSecrets) 192 _, err = createResourceQuota(ctx, f.ClientSet, f.Namespace.Name, resourceQuota) 193 framework.ExpectNoError(err) 194 195 ginkgo.By("Ensuring resource quota status is calculated") 196 usedResources := v1.ResourceList{} 197 usedResources[v1.ResourceQuotas] = resource.MustParse(strconv.Itoa(c + 1)) 198 usedResources[v1.ResourceSecrets] = resource.MustParse(defaultSecrets) 199 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, quotaName, usedResources) 200 framework.ExpectNoError(err) 201 202 ginkgo.By("Creating a Secret") 203 secret := newTestSecretForQuota("test-secret") 204 secret, err = f.ClientSet.CoreV1().Secrets(f.Namespace.Name).Create(ctx, secret, metav1.CreateOptions{}) 205 framework.ExpectNoError(err) 206 207 ginkgo.By("Ensuring resource quota status captures secret creation") 208 usedResources = v1.ResourceList{} 209 usedResources[v1.ResourceSecrets] = resource.MustParse(hardSecrets) 210 // we expect there to be two secrets because each namespace will receive 211 // a service account token secret by default 212 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, quotaName, usedResources) 213 framework.ExpectNoError(err) 214 215 ginkgo.By("Deleting a secret") 216 err = f.ClientSet.CoreV1().Secrets(f.Namespace.Name).Delete(ctx, secret.Name, metav1.DeleteOptions{}) 217 framework.ExpectNoError(err) 218 219 ginkgo.By("Ensuring resource quota status released usage") 220 usedResources[v1.ResourceSecrets] = resource.MustParse(defaultSecrets) 221 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, quotaName, usedResources) 222 framework.ExpectNoError(err) 223 }) 224 225 /* 226 Release: v1.16 227 Testname: ResourceQuota, object count quota, pod 228 Description: Create a ResourceQuota. Creation MUST be successful and its ResourceQuotaStatus MUST match to expected used and total allowed resource quota count within namespace. 229 Create a Pod with resource request count for CPU, Memory, EphemeralStorage and ExtendedResourceName. Pod creation MUST be successful and respective resource usage count MUST be captured in ResourceQuotaStatus of the ResourceQuota. 230 Create another Pod with resource request exceeding remaining quota. Pod creation MUST fail as the request exceeds ResourceQuota limits. 231 Update the successfully created pod's resource requests. Updation MUST fail as a Pod can not dynamically update its resource requirements. 232 Delete the successfully created Pod. Pod Deletion MUST be scuccessful and it MUST release the allocated resource counts from ResourceQuotaStatus of the ResourceQuota. 233 */ 234 framework.ConformanceIt("should create a ResourceQuota and capture the life of a pod.", func(ctx context.Context) { 235 ginkgo.By("Counting existing ResourceQuota") 236 c, err := countResourceQuota(ctx, f.ClientSet, f.Namespace.Name) 237 framework.ExpectNoError(err) 238 239 ginkgo.By("Creating a ResourceQuota") 240 quotaName := "test-quota" 241 resourceQuota := newTestResourceQuota(quotaName) 242 _, err = createResourceQuota(ctx, f.ClientSet, f.Namespace.Name, resourceQuota) 243 framework.ExpectNoError(err) 244 245 ginkgo.By("Ensuring resource quota status is calculated") 246 usedResources := v1.ResourceList{} 247 usedResources[v1.ResourceQuotas] = resource.MustParse(strconv.Itoa(c + 1)) 248 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, quotaName, usedResources) 249 framework.ExpectNoError(err) 250 251 ginkgo.By("Creating a Pod that fits quota") 252 podName := "test-pod" 253 requests := v1.ResourceList{} 254 limits := v1.ResourceList{} 255 requests[v1.ResourceCPU] = resource.MustParse("500m") 256 requests[v1.ResourceMemory] = resource.MustParse("252Mi") 257 requests[v1.ResourceEphemeralStorage] = resource.MustParse("30Gi") 258 requests[v1.ResourceName(extendedResourceName)] = resource.MustParse("2") 259 limits[v1.ResourceName(extendedResourceName)] = resource.MustParse("2") 260 pod := newTestPodForQuota(f, podName, requests, limits) 261 pod, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(ctx, pod, metav1.CreateOptions{}) 262 framework.ExpectNoError(err) 263 podToUpdate := pod 264 265 ginkgo.By("Ensuring ResourceQuota status captures the pod usage") 266 usedResources[v1.ResourceQuotas] = resource.MustParse(strconv.Itoa(c + 1)) 267 usedResources[v1.ResourcePods] = resource.MustParse("1") 268 usedResources[v1.ResourceCPU] = requests[v1.ResourceCPU] 269 usedResources[v1.ResourceMemory] = requests[v1.ResourceMemory] 270 usedResources[v1.ResourceEphemeralStorage] = requests[v1.ResourceEphemeralStorage] 271 usedResources[v1.ResourceName(v1.DefaultResourceRequestsPrefix+extendedResourceName)] = requests[v1.ResourceName(extendedResourceName)] 272 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, quotaName, usedResources) 273 framework.ExpectNoError(err) 274 275 ginkgo.By("Not allowing a pod to be created that exceeds remaining quota") 276 requests = v1.ResourceList{} 277 requests[v1.ResourceCPU] = resource.MustParse("600m") 278 requests[v1.ResourceMemory] = resource.MustParse("100Mi") 279 pod = newTestPodForQuota(f, "fail-pod", requests, v1.ResourceList{}) 280 _, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(ctx, pod, metav1.CreateOptions{}) 281 gomega.Expect(err).To(gomega.HaveOccurred()) 282 283 ginkgo.By("Not allowing a pod to be created that exceeds remaining quota(validation on extended resources)") 284 requests = v1.ResourceList{} 285 limits = v1.ResourceList{} 286 requests[v1.ResourceCPU] = resource.MustParse("500m") 287 requests[v1.ResourceMemory] = resource.MustParse("100Mi") 288 requests[v1.ResourceEphemeralStorage] = resource.MustParse("30Gi") 289 requests[v1.ResourceName(extendedResourceName)] = resource.MustParse("2") 290 limits[v1.ResourceName(extendedResourceName)] = resource.MustParse("2") 291 pod = newTestPodForQuota(f, "fail-pod-for-extended-resource", requests, limits) 292 _, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(ctx, pod, metav1.CreateOptions{}) 293 gomega.Expect(err).To(gomega.HaveOccurred()) 294 295 ginkgo.By("Ensuring a pod cannot update its resource requirements") 296 // a pod cannot dynamically update its resource requirements. 297 requests = v1.ResourceList{} 298 requests[v1.ResourceCPU] = resource.MustParse("100m") 299 requests[v1.ResourceMemory] = resource.MustParse("100Mi") 300 requests[v1.ResourceEphemeralStorage] = resource.MustParse("10Gi") 301 podToUpdate.Spec.Containers[0].Resources.Requests = requests 302 _, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Update(ctx, podToUpdate, metav1.UpdateOptions{}) 303 gomega.Expect(err).To(gomega.HaveOccurred()) 304 305 ginkgo.By("Ensuring attempts to update pod resource requirements did not change quota usage") 306 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, quotaName, usedResources) 307 framework.ExpectNoError(err) 308 309 ginkgo.By("Deleting the pod") 310 err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Delete(ctx, podName, *metav1.NewDeleteOptions(0)) 311 framework.ExpectNoError(err) 312 313 ginkgo.By("Ensuring resource quota status released the pod usage") 314 usedResources[v1.ResourceQuotas] = resource.MustParse(strconv.Itoa(c + 1)) 315 usedResources[v1.ResourcePods] = resource.MustParse("0") 316 usedResources[v1.ResourceCPU] = resource.MustParse("0") 317 usedResources[v1.ResourceMemory] = resource.MustParse("0") 318 usedResources[v1.ResourceEphemeralStorage] = resource.MustParse("0") 319 usedResources[v1.ResourceName(v1.DefaultResourceRequestsPrefix+extendedResourceName)] = resource.MustParse("0") 320 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, quotaName, usedResources) 321 framework.ExpectNoError(err) 322 }) 323 /* 324 Release: v1.16 325 Testname: ResourceQuota, object count quota, configmap 326 Description: Create a ResourceQuota. Creation MUST be successful and its ResourceQuotaStatus MUST match to expected used and total allowed resource quota count within namespace. 327 Create a ConfigMap. Its creation MUST be successful and resource usage count against the ConfigMap object MUST be captured in ResourceQuotaStatus of the ResourceQuota. 328 Delete the ConfigMap. Deletion MUST succeed and resource usage count against the ConfigMap object MUST be released from ResourceQuotaStatus of the ResourceQuota. 329 */ 330 framework.ConformanceIt("should create a ResourceQuota and capture the life of a configMap.", func(ctx context.Context) { 331 found, unchanged := 0, 0 332 // On contended servers the service account controller can slow down, leading to the count changing during a run. 333 // Wait up to 15s for the count to stabilize, assuming that updates come at a consistent rate, and are not held indefinitely. 334 err := wait.PollWithContext(ctx, 1*time.Second, time.Minute, func(ctx context.Context) (bool, error) { 335 configmaps, err := f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).List(ctx, metav1.ListOptions{}) 336 framework.ExpectNoError(err) 337 if len(configmaps.Items) == found { 338 // loop until the number of configmaps has stabilized for 15 seconds 339 unchanged++ 340 return unchanged > 15, nil 341 } 342 unchanged = 0 343 found = len(configmaps.Items) 344 return false, nil 345 }) 346 framework.ExpectNoError(err) 347 defaultConfigMaps := fmt.Sprintf("%d", found) 348 hardConfigMaps := fmt.Sprintf("%d", found+1) 349 350 ginkgo.By("Counting existing ResourceQuota") 351 c, err := countResourceQuota(ctx, f.ClientSet, f.Namespace.Name) 352 framework.ExpectNoError(err) 353 354 ginkgo.By("Creating a ResourceQuota") 355 quotaName := "test-quota" 356 resourceQuota := newTestResourceQuota(quotaName) 357 resourceQuota.Spec.Hard[v1.ResourceConfigMaps] = resource.MustParse(hardConfigMaps) 358 _, err = createResourceQuota(ctx, f.ClientSet, f.Namespace.Name, resourceQuota) 359 framework.ExpectNoError(err) 360 361 ginkgo.By("Ensuring resource quota status is calculated") 362 usedResources := v1.ResourceList{} 363 usedResources[v1.ResourceQuotas] = resource.MustParse(strconv.Itoa(c + 1)) 364 usedResources[v1.ResourceConfigMaps] = resource.MustParse(defaultConfigMaps) 365 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, quotaName, usedResources) 366 framework.ExpectNoError(err) 367 368 ginkgo.By("Creating a ConfigMap") 369 configMap := newTestConfigMapForQuota("test-configmap") 370 configMap, err = f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Create(ctx, configMap, metav1.CreateOptions{}) 371 framework.ExpectNoError(err) 372 373 ginkgo.By("Ensuring resource quota status captures configMap creation") 374 usedResources = v1.ResourceList{} 375 usedResources[v1.ResourceQuotas] = resource.MustParse(strconv.Itoa(c + 1)) 376 usedResources[v1.ResourceConfigMaps] = resource.MustParse(hardConfigMaps) 377 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, quotaName, usedResources) 378 framework.ExpectNoError(err) 379 380 ginkgo.By("Deleting a ConfigMap") 381 err = f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Delete(ctx, configMap.Name, metav1.DeleteOptions{}) 382 framework.ExpectNoError(err) 383 384 ginkgo.By("Ensuring resource quota status released usage") 385 usedResources[v1.ResourceConfigMaps] = resource.MustParse(defaultConfigMaps) 386 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, quotaName, usedResources) 387 framework.ExpectNoError(err) 388 }) 389 390 /* 391 Release: v1.16 392 Testname: ResourceQuota, object count quota, replicationController 393 Description: Create a ResourceQuota. Creation MUST be successful and its ResourceQuotaStatus MUST match to expected used and total allowed resource quota count within namespace. 394 Create a ReplicationController. Its creation MUST be successful and resource usage count against the ReplicationController object MUST be captured in ResourceQuotaStatus of the ResourceQuota. 395 Delete the ReplicationController. Deletion MUST succeed and resource usage count against the ReplicationController object MUST be released from ResourceQuotaStatus of the ResourceQuota. 396 */ 397 framework.ConformanceIt("should create a ResourceQuota and capture the life of a replication controller.", func(ctx context.Context) { 398 ginkgo.By("Counting existing ResourceQuota") 399 c, err := countResourceQuota(ctx, f.ClientSet, f.Namespace.Name) 400 framework.ExpectNoError(err) 401 402 ginkgo.By("Creating a ResourceQuota") 403 quotaName := "test-quota" 404 resourceQuota := newTestResourceQuota(quotaName) 405 _, err = createResourceQuota(ctx, f.ClientSet, f.Namespace.Name, resourceQuota) 406 framework.ExpectNoError(err) 407 408 ginkgo.By("Ensuring resource quota status is calculated") 409 usedResources := v1.ResourceList{} 410 usedResources[v1.ResourceQuotas] = resource.MustParse(strconv.Itoa(c + 1)) 411 usedResources[v1.ResourceReplicationControllers] = resource.MustParse("0") 412 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, quotaName, usedResources) 413 framework.ExpectNoError(err) 414 415 ginkgo.By("Creating a ReplicationController") 416 replicationController := newTestReplicationControllerForQuota("test-rc", "nginx", 0) 417 replicationController, err = f.ClientSet.CoreV1().ReplicationControllers(f.Namespace.Name).Create(ctx, replicationController, metav1.CreateOptions{}) 418 framework.ExpectNoError(err) 419 420 ginkgo.By("Ensuring resource quota status captures replication controller creation") 421 usedResources = v1.ResourceList{} 422 usedResources[v1.ResourceReplicationControllers] = resource.MustParse("1") 423 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, quotaName, usedResources) 424 framework.ExpectNoError(err) 425 426 ginkgo.By("Deleting a ReplicationController") 427 // Without the delete options, the object isn't actually 428 // removed until the GC verifies that all children have been 429 // detached. ReplicationControllers default to "orphan", which 430 // is different from most resources. (Why? To preserve a common 431 // workflow from prior to the GC's introduction.) 432 err = f.ClientSet.CoreV1().ReplicationControllers(f.Namespace.Name).Delete(ctx, replicationController.Name, metav1.DeleteOptions{ 433 PropagationPolicy: func() *metav1.DeletionPropagation { 434 p := metav1.DeletePropagationBackground 435 return &p 436 }(), 437 }) 438 framework.ExpectNoError(err) 439 440 ginkgo.By("Ensuring resource quota status released usage") 441 usedResources[v1.ResourceReplicationControllers] = resource.MustParse("0") 442 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, quotaName, usedResources) 443 framework.ExpectNoError(err) 444 }) 445 446 /* 447 Release: v1.16 448 Testname: ResourceQuota, object count quota, replicaSet 449 Description: Create a ResourceQuota. Creation MUST be successful and its ResourceQuotaStatus MUST match to expected used and total allowed resource quota count within namespace. 450 Create a ReplicaSet. Its creation MUST be successful and resource usage count against the ReplicaSet object MUST be captured in ResourceQuotaStatus of the ResourceQuota. 451 Delete the ReplicaSet. Deletion MUST succeed and resource usage count against the ReplicaSet object MUST be released from ResourceQuotaStatus of the ResourceQuota. 452 */ 453 framework.ConformanceIt("should create a ResourceQuota and capture the life of a replica set.", func(ctx context.Context) { 454 ginkgo.By("Counting existing ResourceQuota") 455 c, err := countResourceQuota(ctx, f.ClientSet, f.Namespace.Name) 456 framework.ExpectNoError(err) 457 458 ginkgo.By("Creating a ResourceQuota") 459 quotaName := "test-quota" 460 resourceQuota := newTestResourceQuota(quotaName) 461 _, err = createResourceQuota(ctx, f.ClientSet, f.Namespace.Name, resourceQuota) 462 framework.ExpectNoError(err) 463 464 ginkgo.By("Ensuring resource quota status is calculated") 465 usedResources := v1.ResourceList{} 466 usedResources[v1.ResourceQuotas] = resource.MustParse(strconv.Itoa(c + 1)) 467 usedResources[v1.ResourceName("count/replicasets.apps")] = resource.MustParse("0") 468 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, quotaName, usedResources) 469 framework.ExpectNoError(err) 470 471 ginkgo.By("Creating a ReplicaSet") 472 replicaSet := newTestReplicaSetForQuota("test-rs", "nginx", 0) 473 replicaSet, err = f.ClientSet.AppsV1().ReplicaSets(f.Namespace.Name).Create(ctx, replicaSet, metav1.CreateOptions{}) 474 framework.ExpectNoError(err) 475 476 ginkgo.By("Ensuring resource quota status captures replicaset creation") 477 usedResources = v1.ResourceList{} 478 usedResources[v1.ResourceName("count/replicasets.apps")] = resource.MustParse("1") 479 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, quotaName, usedResources) 480 framework.ExpectNoError(err) 481 482 ginkgo.By("Deleting a ReplicaSet") 483 err = f.ClientSet.AppsV1().ReplicaSets(f.Namespace.Name).Delete(ctx, replicaSet.Name, metav1.DeleteOptions{}) 484 framework.ExpectNoError(err) 485 486 ginkgo.By("Ensuring resource quota status released usage") 487 usedResources[v1.ResourceName("count/replicasets.apps")] = resource.MustParse("0") 488 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, quotaName, usedResources) 489 framework.ExpectNoError(err) 490 }) 491 492 /* 493 Release: v1.16 494 Testname: ResourceQuota, object count quota, pvc 495 Description: Create a ResourceQuota. Creation MUST be successful and its ResourceQuotaStatus MUST match to expected used and total allowed resource quota count within namespace. 496 Create PersistentVolumeClaim (PVC) to request storage capacity of 1G. PVC creation MUST be successful and resource usage count against the PVC and storage object MUST be captured in ResourceQuotaStatus of the ResourceQuota. 497 Delete the PVC. Deletion MUST succeed and resource usage count against its PVC and storage object MUST be released from ResourceQuotaStatus of the ResourceQuota. 498 [NotConformancePromotable] as test suite do not have any e2e at this moment which are explicitly verifying PV and PVC behaviour. 499 */ 500 ginkgo.It("should create a ResourceQuota and capture the life of a persistent volume claim", func(ctx context.Context) { 501 ginkgo.By("Counting existing ResourceQuota") 502 c, err := countResourceQuota(ctx, f.ClientSet, f.Namespace.Name) 503 framework.ExpectNoError(err) 504 505 ginkgo.By("Creating a ResourceQuota") 506 quotaName := "test-quota" 507 resourceQuota := newTestResourceQuota(quotaName) 508 _, err = createResourceQuota(ctx, f.ClientSet, f.Namespace.Name, resourceQuota) 509 framework.ExpectNoError(err) 510 511 ginkgo.By("Ensuring resource quota status is calculated") 512 usedResources := v1.ResourceList{} 513 usedResources[v1.ResourceQuotas] = resource.MustParse(strconv.Itoa(c + 1)) 514 usedResources[v1.ResourcePersistentVolumeClaims] = resource.MustParse("0") 515 usedResources[v1.ResourceRequestsStorage] = resource.MustParse("0") 516 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, quotaName, usedResources) 517 framework.ExpectNoError(err) 518 519 ginkgo.By("Creating a PersistentVolumeClaim") 520 pvc := newTestPersistentVolumeClaimForQuota("test-claim") 521 pvc, err = f.ClientSet.CoreV1().PersistentVolumeClaims(f.Namespace.Name).Create(ctx, pvc, metav1.CreateOptions{}) 522 framework.ExpectNoError(err) 523 524 ginkgo.By("Ensuring resource quota status captures persistent volume claim creation") 525 usedResources = v1.ResourceList{} 526 usedResources[v1.ResourcePersistentVolumeClaims] = resource.MustParse("1") 527 usedResources[v1.ResourceRequestsStorage] = resource.MustParse("1Gi") 528 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, quotaName, usedResources) 529 framework.ExpectNoError(err) 530 531 ginkgo.By("Deleting a PersistentVolumeClaim") 532 err = f.ClientSet.CoreV1().PersistentVolumeClaims(f.Namespace.Name).Delete(ctx, pvc.Name, metav1.DeleteOptions{}) 533 framework.ExpectNoError(err) 534 535 ginkgo.By("Ensuring resource quota status released usage") 536 usedResources[v1.ResourcePersistentVolumeClaims] = resource.MustParse("0") 537 usedResources[v1.ResourceRequestsStorage] = resource.MustParse("0") 538 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, quotaName, usedResources) 539 framework.ExpectNoError(err) 540 }) 541 542 /* 543 Release: v1.16 544 Testname: ResourceQuota, object count quota, storageClass 545 Description: Create a ResourceQuota. Creation MUST be successful and its ResourceQuotaStatus MUST match to expected used and total allowed resource quota count within namespace. 546 Create PersistentVolumeClaim (PVC) with specified storageClass to request storage capacity of 1G. PVC creation MUST be successful and resource usage count against PVC, storageClass and storage object MUST be captured in ResourceQuotaStatus of the ResourceQuota. 547 Delete the PVC. Deletion MUST succeed and resource usage count against PVC, storageClass and storage object MUST be released from ResourceQuotaStatus of the ResourceQuota. 548 [NotConformancePromotable] as test suite do not have any e2e at this moment which are explicitly verifying PV and PVC behaviour. 549 */ 550 ginkgo.It("should create a ResourceQuota and capture the life of a persistent volume claim with a storage class", func(ctx context.Context) { 551 ginkgo.By("Counting existing ResourceQuota") 552 c, err := countResourceQuota(ctx, f.ClientSet, f.Namespace.Name) 553 framework.ExpectNoError(err) 554 555 ginkgo.By("Creating a ResourceQuota") 556 quotaName := "test-quota" 557 resourceQuota := newTestResourceQuota(quotaName) 558 _, err = createResourceQuota(ctx, f.ClientSet, f.Namespace.Name, resourceQuota) 559 framework.ExpectNoError(err) 560 561 ginkgo.By("Ensuring resource quota status is calculated") 562 usedResources := v1.ResourceList{} 563 usedResources[v1.ResourceQuotas] = resource.MustParse(strconv.Itoa(c + 1)) 564 usedResources[v1.ResourcePersistentVolumeClaims] = resource.MustParse("0") 565 usedResources[v1.ResourceRequestsStorage] = resource.MustParse("0") 566 usedResources[core.V1ResourceByStorageClass(classGold, v1.ResourcePersistentVolumeClaims)] = resource.MustParse("0") 567 usedResources[core.V1ResourceByStorageClass(classGold, v1.ResourceRequestsStorage)] = resource.MustParse("0") 568 569 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, quotaName, usedResources) 570 framework.ExpectNoError(err) 571 572 ginkgo.By("Creating a PersistentVolumeClaim with storage class") 573 pvc := newTestPersistentVolumeClaimForQuota("test-claim") 574 pvc.Spec.StorageClassName = &classGold 575 pvc, err = f.ClientSet.CoreV1().PersistentVolumeClaims(f.Namespace.Name).Create(ctx, pvc, metav1.CreateOptions{}) 576 framework.ExpectNoError(err) 577 578 ginkgo.By("Ensuring resource quota status captures persistent volume claim creation") 579 usedResources = v1.ResourceList{} 580 usedResources[v1.ResourcePersistentVolumeClaims] = resource.MustParse("1") 581 usedResources[v1.ResourceRequestsStorage] = resource.MustParse("1Gi") 582 usedResources[core.V1ResourceByStorageClass(classGold, v1.ResourcePersistentVolumeClaims)] = resource.MustParse("1") 583 usedResources[core.V1ResourceByStorageClass(classGold, v1.ResourceRequestsStorage)] = resource.MustParse("1Gi") 584 585 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, quotaName, usedResources) 586 framework.ExpectNoError(err) 587 588 ginkgo.By("Deleting a PersistentVolumeClaim") 589 err = f.ClientSet.CoreV1().PersistentVolumeClaims(f.Namespace.Name).Delete(ctx, pvc.Name, metav1.DeleteOptions{}) 590 framework.ExpectNoError(err) 591 592 ginkgo.By("Ensuring resource quota status released usage") 593 usedResources[v1.ResourcePersistentVolumeClaims] = resource.MustParse("0") 594 usedResources[v1.ResourceRequestsStorage] = resource.MustParse("0") 595 usedResources[core.V1ResourceByStorageClass(classGold, v1.ResourcePersistentVolumeClaims)] = resource.MustParse("0") 596 usedResources[core.V1ResourceByStorageClass(classGold, v1.ResourceRequestsStorage)] = resource.MustParse("0") 597 598 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, quotaName, usedResources) 599 framework.ExpectNoError(err) 600 }) 601 602 ginkgo.It("should create a ResourceQuota and capture the life of a custom resource.", func(ctx context.Context) { 603 ginkgo.By("Creating a Custom Resource Definition") 604 testcrd, err := crd.CreateTestCRD(f) 605 framework.ExpectNoError(err) 606 ginkgo.DeferCleanup(testcrd.CleanUp) 607 countResourceName := "count/" + testcrd.Crd.Spec.Names.Plural + "." + testcrd.Crd.Spec.Group 608 // resourcequota controller needs to take 30 seconds at most to detect the new custom resource. 609 // in order to make sure the resourcequota controller knows this resource, we create one test 610 // resourcequota object, and triggering updates on it until the status is updated. 611 quotaName := "quota-for-" + testcrd.Crd.Spec.Names.Plural 612 _, err = createResourceQuota(ctx, f.ClientSet, f.Namespace.Name, &v1.ResourceQuota{ 613 ObjectMeta: metav1.ObjectMeta{Name: quotaName}, 614 Spec: v1.ResourceQuotaSpec{ 615 Hard: v1.ResourceList{ 616 v1.ResourceName(countResourceName): resource.MustParse("0"), 617 }, 618 }, 619 }) 620 framework.ExpectNoError(err) 621 err = updateResourceQuotaUntilUsageAppears(ctx, f.ClientSet, f.Namespace.Name, quotaName, v1.ResourceName(countResourceName)) 622 framework.ExpectNoError(err) 623 err = f.ClientSet.CoreV1().ResourceQuotas(f.Namespace.Name).Delete(ctx, quotaName, metav1.DeleteOptions{}) 624 framework.ExpectNoError(err) 625 626 ginkgo.By("Counting existing ResourceQuota") 627 c, err := countResourceQuota(ctx, f.ClientSet, f.Namespace.Name) 628 framework.ExpectNoError(err) 629 630 ginkgo.By("Creating a ResourceQuota") 631 quotaName = "test-quota" 632 resourceQuota := newTestResourceQuota(quotaName) 633 resourceQuota.Spec.Hard[v1.ResourceName(countResourceName)] = resource.MustParse("1") 634 _, err = createResourceQuota(ctx, f.ClientSet, f.Namespace.Name, resourceQuota) 635 framework.ExpectNoError(err) 636 637 ginkgo.By("Ensuring resource quota status is calculated") 638 usedResources := v1.ResourceList{} 639 usedResources[v1.ResourceQuotas] = resource.MustParse(strconv.Itoa(c + 1)) 640 usedResources[v1.ResourceName(countResourceName)] = resource.MustParse("0") 641 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, quotaName, usedResources) 642 framework.ExpectNoError(err) 643 644 ginkgo.By("Creating a custom resource") 645 resourceClient := testcrd.DynamicClients["v1"] 646 testcr, err := instantiateCustomResource(ctx, &unstructured.Unstructured{ 647 Object: map[string]interface{}{ 648 "apiVersion": testcrd.Crd.Spec.Group + "/" + testcrd.Crd.Spec.Versions[0].Name, 649 "kind": testcrd.Crd.Spec.Names.Kind, 650 "metadata": map[string]interface{}{ 651 "name": "test-cr-1", 652 }, 653 }, 654 }, resourceClient, testcrd.Crd) 655 framework.ExpectNoError(err) 656 657 ginkgo.By("Ensuring resource quota status captures custom resource creation") 658 usedResources = v1.ResourceList{} 659 usedResources[v1.ResourceName(countResourceName)] = resource.MustParse("1") 660 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, quotaName, usedResources) 661 framework.ExpectNoError(err) 662 663 ginkgo.By("Creating a second custom resource") 664 _, err = instantiateCustomResource(ctx, &unstructured.Unstructured{ 665 Object: map[string]interface{}{ 666 "apiVersion": testcrd.Crd.Spec.Group + "/" + testcrd.Crd.Spec.Versions[0].Name, 667 "kind": testcrd.Crd.Spec.Names.Kind, 668 "metadata": map[string]interface{}{ 669 "name": "test-cr-2", 670 }, 671 }, 672 }, resourceClient, testcrd.Crd) 673 // since we only give one quota, this creation should fail. 674 gomega.Expect(err).To(gomega.HaveOccurred()) 675 676 ginkgo.By("Deleting a custom resource") 677 err = deleteCustomResource(ctx, resourceClient, testcr.GetName()) 678 framework.ExpectNoError(err) 679 680 ginkgo.By("Ensuring resource quota status released usage") 681 usedResources[v1.ResourceName(countResourceName)] = resource.MustParse("0") 682 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, quotaName, usedResources) 683 framework.ExpectNoError(err) 684 }) 685 686 /* 687 Release: v1.16 688 Testname: ResourceQuota, quota scope, Terminating and NotTerminating scope 689 Description: Create two ResourceQuotas, one with 'Terminating' scope and another 'NotTerminating' scope. Request and the limit counts for CPU and Memory resources are set for the ResourceQuota. Creation MUST be successful and their ResourceQuotaStatus MUST match to expected used and total allowed resource quota count within namespace. 690 Create a Pod with specified CPU and Memory ResourceRequirements fall within quota limits. Pod creation MUST be successful and usage count MUST be captured in ResourceQuotaStatus of 'NotTerminating' scoped ResourceQuota but MUST NOT in 'Terminating' scoped ResourceQuota. 691 Delete the Pod. Pod deletion MUST succeed and Pod resource usage count MUST be released from ResourceQuotaStatus of 'NotTerminating' scoped ResourceQuota. 692 Create a pod with specified activeDeadlineSeconds and resourceRequirements for CPU and Memory fall within quota limits. Pod creation MUST be successful and usage count MUST be captured in ResourceQuotaStatus of 'Terminating' scoped ResourceQuota but MUST NOT in 'NotTerminating' scoped ResourceQuota. 693 Delete the Pod. Pod deletion MUST succeed and Pod resource usage count MUST be released from ResourceQuotaStatus of 'Terminating' scoped ResourceQuota. 694 */ 695 framework.ConformanceIt("should verify ResourceQuota with terminating scopes.", func(ctx context.Context) { 696 ginkgo.By("Creating a ResourceQuota with terminating scope") 697 quotaTerminatingName := "quota-terminating" 698 resourceQuotaTerminating, err := createResourceQuota(ctx, f.ClientSet, f.Namespace.Name, newTestResourceQuotaWithScope(quotaTerminatingName, v1.ResourceQuotaScopeTerminating)) 699 framework.ExpectNoError(err) 700 701 ginkgo.By("Ensuring ResourceQuota status is calculated") 702 usedResources := v1.ResourceList{} 703 usedResources[v1.ResourcePods] = resource.MustParse("0") 704 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, resourceQuotaTerminating.Name, usedResources) 705 framework.ExpectNoError(err) 706 707 ginkgo.By("Creating a ResourceQuota with not terminating scope") 708 quotaNotTerminatingName := "quota-not-terminating" 709 resourceQuotaNotTerminating, err := createResourceQuota(ctx, f.ClientSet, f.Namespace.Name, newTestResourceQuotaWithScope(quotaNotTerminatingName, v1.ResourceQuotaScopeNotTerminating)) 710 framework.ExpectNoError(err) 711 712 ginkgo.By("Ensuring ResourceQuota status is calculated") 713 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, resourceQuotaNotTerminating.Name, usedResources) 714 framework.ExpectNoError(err) 715 716 ginkgo.By("Creating a long running pod") 717 podName := "test-pod" 718 requests := v1.ResourceList{} 719 requests[v1.ResourceCPU] = resource.MustParse("500m") 720 requests[v1.ResourceMemory] = resource.MustParse("200Mi") 721 limits := v1.ResourceList{} 722 limits[v1.ResourceCPU] = resource.MustParse("1") 723 limits[v1.ResourceMemory] = resource.MustParse("400Mi") 724 pod := newTestPodForQuota(f, podName, requests, limits) 725 _, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(ctx, pod, metav1.CreateOptions{}) 726 framework.ExpectNoError(err) 727 728 ginkgo.By("Ensuring resource quota with not terminating scope captures the pod usage") 729 usedResources[v1.ResourcePods] = resource.MustParse("1") 730 usedResources[v1.ResourceRequestsCPU] = requests[v1.ResourceCPU] 731 usedResources[v1.ResourceRequestsMemory] = requests[v1.ResourceMemory] 732 usedResources[v1.ResourceLimitsCPU] = limits[v1.ResourceCPU] 733 usedResources[v1.ResourceLimitsMemory] = limits[v1.ResourceMemory] 734 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, resourceQuotaNotTerminating.Name, usedResources) 735 framework.ExpectNoError(err) 736 737 ginkgo.By("Ensuring resource quota with terminating scope ignored the pod usage") 738 usedResources[v1.ResourcePods] = resource.MustParse("0") 739 usedResources[v1.ResourceRequestsCPU] = resource.MustParse("0") 740 usedResources[v1.ResourceRequestsMemory] = resource.MustParse("0") 741 usedResources[v1.ResourceLimitsCPU] = resource.MustParse("0") 742 usedResources[v1.ResourceLimitsMemory] = resource.MustParse("0") 743 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, resourceQuotaTerminating.Name, usedResources) 744 framework.ExpectNoError(err) 745 746 ginkgo.By("Deleting the pod") 747 err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Delete(ctx, podName, *metav1.NewDeleteOptions(0)) 748 framework.ExpectNoError(err) 749 750 ginkgo.By("Ensuring resource quota status released the pod usage") 751 usedResources[v1.ResourcePods] = resource.MustParse("0") 752 usedResources[v1.ResourceRequestsCPU] = resource.MustParse("0") 753 usedResources[v1.ResourceRequestsMemory] = resource.MustParse("0") 754 usedResources[v1.ResourceLimitsCPU] = resource.MustParse("0") 755 usedResources[v1.ResourceLimitsMemory] = resource.MustParse("0") 756 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, resourceQuotaNotTerminating.Name, usedResources) 757 framework.ExpectNoError(err) 758 759 ginkgo.By("Creating a terminating pod") 760 podName = "terminating-pod" 761 pod = newTestPodForQuota(f, podName, requests, limits) 762 activeDeadlineSeconds := int64(3600) 763 pod.Spec.ActiveDeadlineSeconds = &activeDeadlineSeconds 764 _, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(ctx, pod, metav1.CreateOptions{}) 765 framework.ExpectNoError(err) 766 767 ginkgo.By("Ensuring resource quota with terminating scope captures the pod usage") 768 usedResources[v1.ResourcePods] = resource.MustParse("1") 769 usedResources[v1.ResourceRequestsCPU] = requests[v1.ResourceCPU] 770 usedResources[v1.ResourceRequestsMemory] = requests[v1.ResourceMemory] 771 usedResources[v1.ResourceLimitsCPU] = limits[v1.ResourceCPU] 772 usedResources[v1.ResourceLimitsMemory] = limits[v1.ResourceMemory] 773 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, resourceQuotaTerminating.Name, usedResources) 774 framework.ExpectNoError(err) 775 776 ginkgo.By("Ensuring resource quota with not terminating scope ignored the pod usage") 777 usedResources[v1.ResourcePods] = resource.MustParse("0") 778 usedResources[v1.ResourceRequestsCPU] = resource.MustParse("0") 779 usedResources[v1.ResourceRequestsMemory] = resource.MustParse("0") 780 usedResources[v1.ResourceLimitsCPU] = resource.MustParse("0") 781 usedResources[v1.ResourceLimitsMemory] = resource.MustParse("0") 782 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, resourceQuotaNotTerminating.Name, usedResources) 783 framework.ExpectNoError(err) 784 785 ginkgo.By("Deleting the pod") 786 err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Delete(ctx, podName, *metav1.NewDeleteOptions(0)) 787 framework.ExpectNoError(err) 788 789 ginkgo.By("Ensuring resource quota status released the pod usage") 790 usedResources[v1.ResourcePods] = resource.MustParse("0") 791 usedResources[v1.ResourceRequestsCPU] = resource.MustParse("0") 792 usedResources[v1.ResourceRequestsMemory] = resource.MustParse("0") 793 usedResources[v1.ResourceLimitsCPU] = resource.MustParse("0") 794 usedResources[v1.ResourceLimitsMemory] = resource.MustParse("0") 795 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, resourceQuotaTerminating.Name, usedResources) 796 framework.ExpectNoError(err) 797 }) 798 799 /* 800 Release: v1.16 801 Testname: ResourceQuota, quota scope, BestEffort and NotBestEffort scope 802 Description: Create two ResourceQuotas, one with 'BestEffort' scope and another with 'NotBestEffort' scope. Creation MUST be successful and their ResourceQuotaStatus MUST match to expected used and total allowed resource quota count within namespace. 803 Create a 'BestEffort' Pod by not explicitly specifying resource limits and requests. Pod creation MUST be successful and usage count MUST be captured in ResourceQuotaStatus of 'BestEffort' scoped ResourceQuota but MUST NOT in 'NotBestEffort' scoped ResourceQuota. 804 Delete the Pod. Pod deletion MUST succeed and Pod resource usage count MUST be released from ResourceQuotaStatus of 'BestEffort' scoped ResourceQuota. 805 Create a 'NotBestEffort' Pod by explicitly specifying resource limits and requests. Pod creation MUST be successful and usage count MUST be captured in ResourceQuotaStatus of 'NotBestEffort' scoped ResourceQuota but MUST NOT in 'BestEffort' scoped ResourceQuota. 806 Delete the Pod. Pod deletion MUST succeed and Pod resource usage count MUST be released from ResourceQuotaStatus of 'NotBestEffort' scoped ResourceQuota. 807 */ 808 framework.ConformanceIt("should verify ResourceQuota with best effort scope.", func(ctx context.Context) { 809 ginkgo.By("Creating a ResourceQuota with best effort scope") 810 resourceQuotaBestEffort, err := createResourceQuota(ctx, f.ClientSet, f.Namespace.Name, newTestResourceQuotaWithScope("quota-besteffort", v1.ResourceQuotaScopeBestEffort)) 811 framework.ExpectNoError(err) 812 813 ginkgo.By("Ensuring ResourceQuota status is calculated") 814 usedResources := v1.ResourceList{} 815 usedResources[v1.ResourcePods] = resource.MustParse("0") 816 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, resourceQuotaBestEffort.Name, usedResources) 817 framework.ExpectNoError(err) 818 819 ginkgo.By("Creating a ResourceQuota with not best effort scope") 820 resourceQuotaNotBestEffort, err := createResourceQuota(ctx, f.ClientSet, f.Namespace.Name, newTestResourceQuotaWithScope("quota-not-besteffort", v1.ResourceQuotaScopeNotBestEffort)) 821 framework.ExpectNoError(err) 822 823 ginkgo.By("Ensuring ResourceQuota status is calculated") 824 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, resourceQuotaNotBestEffort.Name, usedResources) 825 framework.ExpectNoError(err) 826 827 ginkgo.By("Creating a best-effort pod") 828 pod := newTestPodForQuota(f, podName, v1.ResourceList{}, v1.ResourceList{}) 829 pod, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(ctx, pod, metav1.CreateOptions{}) 830 framework.ExpectNoError(err) 831 832 ginkgo.By("Ensuring resource quota with best effort scope captures the pod usage") 833 usedResources[v1.ResourcePods] = resource.MustParse("1") 834 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, resourceQuotaBestEffort.Name, usedResources) 835 framework.ExpectNoError(err) 836 837 ginkgo.By("Ensuring resource quota with not best effort ignored the pod usage") 838 usedResources[v1.ResourcePods] = resource.MustParse("0") 839 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, resourceQuotaNotBestEffort.Name, usedResources) 840 framework.ExpectNoError(err) 841 842 ginkgo.By("Deleting the pod") 843 err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Delete(ctx, pod.Name, *metav1.NewDeleteOptions(0)) 844 framework.ExpectNoError(err) 845 846 ginkgo.By("Ensuring resource quota status released the pod usage") 847 usedResources[v1.ResourcePods] = resource.MustParse("0") 848 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, resourceQuotaBestEffort.Name, usedResources) 849 framework.ExpectNoError(err) 850 851 ginkgo.By("Creating a not best-effort pod") 852 requests := v1.ResourceList{} 853 requests[v1.ResourceCPU] = resource.MustParse("500m") 854 requests[v1.ResourceMemory] = resource.MustParse("200Mi") 855 limits := v1.ResourceList{} 856 limits[v1.ResourceCPU] = resource.MustParse("1") 857 limits[v1.ResourceMemory] = resource.MustParse("400Mi") 858 pod = newTestPodForQuota(f, "burstable-pod", requests, limits) 859 pod, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(ctx, pod, metav1.CreateOptions{}) 860 framework.ExpectNoError(err) 861 862 ginkgo.By("Ensuring resource quota with not best effort scope captures the pod usage") 863 usedResources[v1.ResourcePods] = resource.MustParse("1") 864 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, resourceQuotaNotBestEffort.Name, usedResources) 865 framework.ExpectNoError(err) 866 867 ginkgo.By("Ensuring resource quota with best effort scope ignored the pod usage") 868 usedResources[v1.ResourcePods] = resource.MustParse("0") 869 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, resourceQuotaBestEffort.Name, usedResources) 870 framework.ExpectNoError(err) 871 872 ginkgo.By("Deleting the pod") 873 err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Delete(ctx, pod.Name, *metav1.NewDeleteOptions(0)) 874 framework.ExpectNoError(err) 875 876 ginkgo.By("Ensuring resource quota status released the pod usage") 877 usedResources[v1.ResourcePods] = resource.MustParse("0") 878 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, resourceQuotaNotBestEffort.Name, usedResources) 879 framework.ExpectNoError(err) 880 }) 881 882 /* 883 Release: v1.16 884 Testname: ResourceQuota, update and delete 885 Description: Create a ResourceQuota for CPU and Memory quota limits. Creation MUST be successful. 886 When ResourceQuota is updated to modify CPU and Memory quota limits, update MUST succeed with updated values for CPU and Memory limits. 887 When ResourceQuota is deleted, it MUST not be available in the namespace. 888 */ 889 framework.ConformanceIt("should be able to update and delete ResourceQuota.", func(ctx context.Context) { 890 client := f.ClientSet 891 ns := f.Namespace.Name 892 893 ginkgo.By("Creating a ResourceQuota") 894 quotaName := "test-quota" 895 resourceQuota := &v1.ResourceQuota{ 896 Spec: v1.ResourceQuotaSpec{ 897 Hard: v1.ResourceList{}, 898 }, 899 } 900 resourceQuota.ObjectMeta.Name = quotaName 901 resourceQuota.Spec.Hard[v1.ResourceCPU] = resource.MustParse("1") 902 resourceQuota.Spec.Hard[v1.ResourceMemory] = resource.MustParse("500Mi") 903 _, err := createResourceQuota(ctx, client, ns, resourceQuota) 904 framework.ExpectNoError(err) 905 906 ginkgo.By("Getting a ResourceQuota") 907 resourceQuotaResult, err := client.CoreV1().ResourceQuotas(ns).Get(ctx, quotaName, metav1.GetOptions{}) 908 framework.ExpectNoError(err) 909 gomega.Expect(resourceQuotaResult.Spec.Hard).To(gomega.HaveKeyWithValue(v1.ResourceCPU, resource.MustParse("1"))) 910 gomega.Expect(resourceQuotaResult.Spec.Hard).To(gomega.HaveKeyWithValue(v1.ResourceMemory, resource.MustParse("500Mi"))) 911 912 ginkgo.By("Updating a ResourceQuota") 913 resourceQuota.Spec.Hard[v1.ResourceCPU] = resource.MustParse("2") 914 resourceQuota.Spec.Hard[v1.ResourceMemory] = resource.MustParse("1Gi") 915 resourceQuotaResult, err = client.CoreV1().ResourceQuotas(ns).Update(ctx, resourceQuota, metav1.UpdateOptions{}) 916 framework.ExpectNoError(err) 917 gomega.Expect(resourceQuotaResult.Spec.Hard).To(gomega.HaveKeyWithValue(v1.ResourceCPU, resource.MustParse("2"))) 918 gomega.Expect(resourceQuotaResult.Spec.Hard).To(gomega.HaveKeyWithValue(v1.ResourceMemory, resource.MustParse("1Gi"))) 919 920 ginkgo.By("Verifying a ResourceQuota was modified") 921 resourceQuotaResult, err = client.CoreV1().ResourceQuotas(ns).Get(ctx, quotaName, metav1.GetOptions{}) 922 framework.ExpectNoError(err) 923 gomega.Expect(resourceQuotaResult.Spec.Hard).To(gomega.HaveKeyWithValue(v1.ResourceCPU, resource.MustParse("2"))) 924 gomega.Expect(resourceQuotaResult.Spec.Hard).To(gomega.HaveKeyWithValue(v1.ResourceMemory, resource.MustParse("1Gi"))) 925 926 ginkgo.By("Deleting a ResourceQuota") 927 err = deleteResourceQuota(ctx, client, ns, quotaName) 928 framework.ExpectNoError(err) 929 930 ginkgo.By("Verifying the deleted ResourceQuota") 931 _, err = client.CoreV1().ResourceQuotas(ns).Get(ctx, quotaName, metav1.GetOptions{}) 932 if !apierrors.IsNotFound(err) { 933 framework.Failf("Expected `not found` error, got: %v", err) 934 } 935 }) 936 937 /* 938 Release: v1.25 939 Testname: ResourceQuota, manage lifecycle of a ResourceQuota 940 Description: Attempt to create a ResourceQuota for CPU and Memory 941 quota limits. Creation MUST be successful. Attempt to list all 942 namespaces with a label selector which MUST succeed. One list 943 MUST be found. The ResourceQuota when patched MUST succeed. 944 Given the patching of the ResourceQuota, the fields MUST equal 945 the new values. It MUST succeed at deleting a collection of 946 ResourceQuota via a label selector. 947 */ 948 framework.ConformanceIt("should manage the lifecycle of a ResourceQuota", func(ctx context.Context) { 949 client := f.ClientSet 950 ns := f.Namespace.Name 951 952 rqName := "e2e-quota-" + utilrand.String(5) 953 label := map[string]string{"e2e-rq-label": rqName} 954 labelSelector := labels.SelectorFromSet(label).String() 955 956 ginkgo.By("Creating a ResourceQuota") 957 resourceQuota := &v1.ResourceQuota{ 958 ObjectMeta: metav1.ObjectMeta{ 959 Name: rqName, 960 Labels: label, 961 }, 962 Spec: v1.ResourceQuotaSpec{ 963 Hard: v1.ResourceList{}, 964 }, 965 } 966 resourceQuota.Spec.Hard[v1.ResourceCPU] = resource.MustParse("1") 967 resourceQuota.Spec.Hard[v1.ResourceMemory] = resource.MustParse("500Mi") 968 _, err := createResourceQuota(ctx, client, ns, resourceQuota) 969 framework.ExpectNoError(err) 970 971 ginkgo.By("Getting a ResourceQuota") 972 resourceQuotaResult, err := client.CoreV1().ResourceQuotas(ns).Get(ctx, rqName, metav1.GetOptions{}) 973 framework.ExpectNoError(err) 974 gomega.Expect(resourceQuotaResult.Spec.Hard[v1.ResourceCPU]).To(gomega.Equal(resource.MustParse("1"))) 975 gomega.Expect(resourceQuotaResult.Spec.Hard[v1.ResourceMemory]).To(gomega.Equal(resource.MustParse("500Mi"))) 976 977 ginkgo.By("Listing all ResourceQuotas with LabelSelector") 978 rq, err := client.CoreV1().ResourceQuotas("").List(ctx, metav1.ListOptions{LabelSelector: labelSelector}) 979 framework.ExpectNoError(err, "Failed to list job. %v", err) 980 gomega.Expect(rq.Items).To(gomega.HaveLen(1), "Failed to find ResourceQuotes %v", rqName) 981 982 ginkgo.By("Patching the ResourceQuota") 983 payload := "{\"metadata\":{\"labels\":{\"" + rqName + "\":\"patched\"}},\"spec\":{\"hard\":{ \"memory\":\"750Mi\"}}}" 984 patchedResourceQuota, err := client.CoreV1().ResourceQuotas(ns).Patch(ctx, rqName, types.StrategicMergePatchType, []byte(payload), metav1.PatchOptions{}) 985 framework.ExpectNoError(err, "failed to patch ResourceQuota %s in namespace %s", rqName, ns) 986 gomega.Expect(patchedResourceQuota.Labels[rqName]).To(gomega.Equal("patched"), "Failed to find the label for this ResourceQuota. Current labels: %v", patchedResourceQuota.Labels) 987 gomega.Expect(*patchedResourceQuota.Spec.Hard.Memory()).To(gomega.Equal(resource.MustParse("750Mi")), "Hard memory value for ResourceQuota %q is %s not 750Mi.", patchedResourceQuota.ObjectMeta.Name, patchedResourceQuota.Spec.Hard.Memory().String()) 988 989 ginkgo.By("Deleting a Collection of ResourceQuotas") 990 err = client.CoreV1().ResourceQuotas(ns).DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{LabelSelector: labelSelector}) 991 framework.ExpectNoError(err) 992 993 ginkgo.By("Verifying the deleted ResourceQuota") 994 _, err = client.CoreV1().ResourceQuotas(ns).Get(ctx, rqName, metav1.GetOptions{}) 995 if !apierrors.IsNotFound(err) { 996 framework.Failf("Expected `not found` error, got: %v", err) 997 } 998 }) 999 1000 /* 1001 Release: v1.26 1002 Testname: ResourceQuota, apply changes to a ResourceQuota status 1003 Description: Attempt to create a ResourceQuota for CPU and Memory 1004 quota limits. Creation MUST be successful. Updating the hard 1005 status values MUST succeed and the new values MUST be found. The 1006 reported hard status values MUST equal the spec hard values. 1007 Patching the spec hard values MUST succeed and the new values MUST 1008 be found. Patching the hard status values MUST succeed. The 1009 reported hard status values MUST equal the new spec hard values. 1010 Getting the /status MUST succeed and the reported hard status 1011 values MUST equal the spec hard values. Repatching the hard status 1012 values MUST succeed. The spec MUST NOT be changed when 1013 patching /status. 1014 */ 1015 framework.ConformanceIt("should apply changes to a resourcequota status", func(ctx context.Context) { 1016 ns := f.Namespace.Name 1017 rqClient := f.ClientSet.CoreV1().ResourceQuotas(ns) 1018 rqName := "e2e-rq-status-" + utilrand.String(5) 1019 label := map[string]string{"e2e-rq-label": rqName} 1020 labelSelector := labels.SelectorFromSet(label).String() 1021 1022 w := &cache.ListWatch{ 1023 WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { 1024 options.LabelSelector = labelSelector 1025 return rqClient.Watch(ctx, options) 1026 }, 1027 } 1028 1029 rqList, err := f.ClientSet.CoreV1().ResourceQuotas("").List(ctx, metav1.ListOptions{LabelSelector: labelSelector}) 1030 framework.ExpectNoError(err, "failed to list Services") 1031 1032 ginkgo.By(fmt.Sprintf("Creating resourceQuota %q", rqName)) 1033 resourceQuota := &v1.ResourceQuota{ 1034 ObjectMeta: metav1.ObjectMeta{ 1035 Name: rqName, 1036 Labels: label, 1037 }, 1038 Spec: v1.ResourceQuotaSpec{ 1039 Hard: v1.ResourceList{ 1040 v1.ResourceCPU: resource.MustParse("500m"), 1041 v1.ResourceMemory: resource.MustParse("500Mi"), 1042 }, 1043 }, 1044 } 1045 _, err = createResourceQuota(ctx, f.ClientSet, ns, resourceQuota) 1046 framework.ExpectNoError(err) 1047 1048 initialResourceQuota, err := rqClient.Get(ctx, rqName, metav1.GetOptions{}) 1049 framework.ExpectNoError(err) 1050 gomega.Expect(*initialResourceQuota.Spec.Hard.Cpu()).To(gomega.Equal(resource.MustParse("500m")), "Hard cpu value for ResourceQuota %q is %s not 500m.", initialResourceQuota.Name, initialResourceQuota.Spec.Hard.Cpu().String()) 1051 framework.Logf("Resource quota %q reports spec: hard cpu limit of %s", rqName, initialResourceQuota.Spec.Hard.Cpu()) 1052 gomega.Expect(*initialResourceQuota.Spec.Hard.Memory()).To(gomega.Equal(resource.MustParse("500Mi")), "Hard memory value for ResourceQuota %q is %s not 500Mi.", initialResourceQuota.Name, initialResourceQuota.Spec.Hard.Memory().String()) 1053 framework.Logf("Resource quota %q reports spec: hard memory limit of %s", rqName, initialResourceQuota.Spec.Hard.Memory()) 1054 1055 ginkgo.By(fmt.Sprintf("Updating resourceQuota %q /status", rqName)) 1056 var updatedResourceQuota *v1.ResourceQuota 1057 hardLimits := quota.Add(v1.ResourceList{}, initialResourceQuota.Spec.Hard) 1058 1059 err = retry.RetryOnConflict(retry.DefaultRetry, func() error { 1060 updateStatus, err := rqClient.Get(ctx, rqName, metav1.GetOptions{}) 1061 framework.ExpectNoError(err, "Unable to get ResourceQuota %q", rqName) 1062 updateStatus.Status = v1.ResourceQuotaStatus{ 1063 Hard: hardLimits, 1064 } 1065 updatedResourceQuota, err = rqClient.UpdateStatus(ctx, updateStatus, metav1.UpdateOptions{}) 1066 return err 1067 }) 1068 framework.ExpectNoError(err, "Failed to update resourceQuota") 1069 1070 ginkgo.By(fmt.Sprintf("Confirm /status for %q resourceQuota via watch", rqName)) 1071 ctxUntil, cancel := context.WithTimeout(ctx, f.Timeouts.PodStartShort) 1072 defer cancel() 1073 1074 _, err = watchtools.Until(ctxUntil, rqList.ResourceVersion, w, func(event watch.Event) (bool, error) { 1075 if rq, ok := event.Object.(*v1.ResourceQuota); ok { 1076 found := rq.Name == updatedResourceQuota.Name && 1077 rq.Namespace == ns && 1078 apiequality.Semantic.DeepEqual(rq.Status.Hard, updatedResourceQuota.Spec.Hard) 1079 if !found { 1080 framework.Logf("observed resourceQuota %q in namespace %q with hard status: %#v", rq.Name, rq.Namespace, rq.Status.Hard) 1081 return false, nil 1082 } 1083 framework.Logf("Found resourceQuota %q in namespace %q with hard status: %#v", rq.Name, rq.Namespace, rq.Status.Hard) 1084 return found, nil 1085 } 1086 framework.Logf("Observed event: %+v", event.Object) 1087 return false, nil 1088 }) 1089 framework.ExpectNoError(err, "failed to locate ResourceQuota %q in namespace %q", updatedResourceQuota.Name, ns) 1090 framework.Logf("ResourceQuota %q /status was updated", updatedResourceQuota.Name) 1091 1092 // Sync resourceQuota list before patching /status 1093 rqList, err = f.ClientSet.CoreV1().ResourceQuotas("").List(ctx, metav1.ListOptions{LabelSelector: labelSelector}) 1094 framework.ExpectNoError(err, "failed to list Services") 1095 1096 ginkgo.By("Patching hard spec values for cpu & memory") 1097 xResourceQuota, err := rqClient.Patch(ctx, updatedResourceQuota.Name, types.StrategicMergePatchType, 1098 []byte(`{"spec":{"hard":{"cpu":"1","memory":"1Gi"}}}`), 1099 metav1.PatchOptions{}) 1100 framework.ExpectNoError(err, "Could not patch resourcequota %q. Error: %v", xResourceQuota.Name, err) 1101 framework.Logf("Resource quota %q reports spec: hard cpu limit of %s", rqName, xResourceQuota.Spec.Hard.Cpu()) 1102 framework.Logf("Resource quota %q reports spec: hard memory limit of %s", rqName, xResourceQuota.Spec.Hard.Memory()) 1103 1104 ginkgo.By(fmt.Sprintf("Patching %q /status", rqName)) 1105 hardLimits = quota.Add(v1.ResourceList{}, xResourceQuota.Spec.Hard) 1106 1107 rqStatusJSON, err := json.Marshal(hardLimits) 1108 framework.ExpectNoError(err) 1109 patchedResourceQuota, err := rqClient.Patch(ctx, rqName, types.StrategicMergePatchType, 1110 []byte(`{"status": {"hard": `+string(rqStatusJSON)+`}}`), 1111 metav1.PatchOptions{}, "status") 1112 framework.ExpectNoError(err) 1113 1114 ginkgo.By(fmt.Sprintf("Confirm /status for %q resourceQuota via watch", rqName)) 1115 ctxUntil, cancel = context.WithTimeout(ctx, f.Timeouts.PodStartShort) 1116 defer cancel() 1117 1118 _, err = watchtools.Until(ctxUntil, rqList.ResourceVersion, w, func(event watch.Event) (bool, error) { 1119 if rq, ok := event.Object.(*v1.ResourceQuota); ok { 1120 found := rq.Name == patchedResourceQuota.Name && 1121 rq.Namespace == ns && 1122 apiequality.Semantic.DeepEqual(rq.Status.Hard, patchedResourceQuota.Spec.Hard) 1123 if !found { 1124 framework.Logf("observed resourceQuota %q in namespace %q with hard status: %#v", rq.Name, rq.Namespace, rq.Status.Hard) 1125 return false, nil 1126 } 1127 framework.Logf("Found resourceQuota %q in namespace %q with hard status: %#v", rq.Name, rq.Namespace, rq.Status.Hard) 1128 return found, nil 1129 } 1130 framework.Logf("Observed event: %+v", event.Object) 1131 return false, nil 1132 }) 1133 framework.ExpectNoError(err, "failed to locate ResourceQuota %q in namespace %q", patchedResourceQuota.Name, ns) 1134 framework.Logf("ResourceQuota %q /status was patched", patchedResourceQuota.Name) 1135 1136 ginkgo.By(fmt.Sprintf("Get %q /status", rqName)) 1137 rqResource := schema.GroupVersionResource{Group: "", Version: "v1", Resource: "resourcequotas"} 1138 unstruct, err := f.DynamicClient.Resource(rqResource).Namespace(ns).Get(ctx, resourceQuota.Name, metav1.GetOptions{}, "status") 1139 framework.ExpectNoError(err) 1140 1141 rq, err := unstructuredToResourceQuota(unstruct) 1142 framework.ExpectNoError(err, "Getting the status of the resource quota %q", rq.Name) 1143 1144 gomega.Expect(*rq.Status.Hard.Cpu()).To(gomega.Equal(resource.MustParse("1")), "Hard cpu value for ResourceQuota %q is %s not 1.", rq.Name, rq.Status.Hard.Cpu().String()) 1145 framework.Logf("Resourcequota %q reports status: hard cpu of %s", rqName, rq.Status.Hard.Cpu()) 1146 gomega.Expect(*rq.Status.Hard.Memory()).To(gomega.Equal(resource.MustParse("1Gi")), "Hard memory value for ResourceQuota %q is %s not 1Gi.", rq.Name, rq.Status.Hard.Memory().String()) 1147 framework.Logf("Resourcequota %q reports status: hard memory of %s", rqName, rq.Status.Hard.Memory()) 1148 1149 // Sync resourceQuota list before repatching /status 1150 rqList, err = f.ClientSet.CoreV1().ResourceQuotas("").List(ctx, metav1.ListOptions{LabelSelector: labelSelector}) 1151 framework.ExpectNoError(err, "failed to list Services") 1152 1153 ginkgo.By(fmt.Sprintf("Repatching %q /status before checking Spec is unchanged", rqName)) 1154 newHardLimits := v1.ResourceList{ 1155 v1.ResourceCPU: resource.MustParse("2"), 1156 v1.ResourceMemory: resource.MustParse("2Gi"), 1157 } 1158 rqStatusJSON, err = json.Marshal(newHardLimits) 1159 framework.ExpectNoError(err) 1160 1161 repatchedResourceQuota, err := rqClient.Patch(ctx, rqName, types.StrategicMergePatchType, 1162 []byte(`{"status": {"hard": `+string(rqStatusJSON)+`}}`), 1163 metav1.PatchOptions{}, "status") 1164 framework.ExpectNoError(err) 1165 1166 gomega.Expect(*repatchedResourceQuota.Status.Hard.Cpu()).To(gomega.Equal(resource.MustParse("2")), "Hard cpu value for ResourceQuota %q is %s not 2.", repatchedResourceQuota.Name, repatchedResourceQuota.Status.Hard.Cpu().String()) 1167 framework.Logf("Resourcequota %q reports status: hard cpu of %s", repatchedResourceQuota.Name, repatchedResourceQuota.Status.Hard.Cpu()) 1168 gomega.Expect(*repatchedResourceQuota.Status.Hard.Memory()).To(gomega.Equal(resource.MustParse("2Gi")), "Hard memory value for ResourceQuota %q is %s not 2Gi.", repatchedResourceQuota.Name, repatchedResourceQuota.Status.Hard.Memory().String()) 1169 framework.Logf("Resourcequota %q reports status: hard memory of %s", repatchedResourceQuota.Name, repatchedResourceQuota.Status.Hard.Memory()) 1170 1171 _, err = watchtools.Until(ctxUntil, rqList.ResourceVersion, w, func(event watch.Event) (bool, error) { 1172 if rq, ok := event.Object.(*v1.ResourceQuota); ok { 1173 found := rq.Name == patchedResourceQuota.Name && 1174 rq.Namespace == ns && 1175 *rq.Status.Hard.Cpu() == resource.MustParse("2") && 1176 *rq.Status.Hard.Memory() == resource.MustParse("2Gi") 1177 if !found { 1178 framework.Logf("observed resourceQuota %q in namespace %q with hard status: %#v", rq.Name, rq.Namespace, rq.Status.Hard) 1179 return false, nil 1180 } 1181 framework.Logf("Found resourceQuota %q in namespace %q with hard status: %#v", rq.Name, rq.Namespace, rq.Status.Hard) 1182 return found, nil 1183 } 1184 framework.Logf("Observed event: %+v", event.Object) 1185 return false, nil 1186 }) 1187 framework.ExpectNoError(err, "failed to locate ResourceQuota %q in namespace %q", patchedResourceQuota.Name, ns) 1188 1189 // the resource_quota_controller ignores changes to the status so we have to wait for a full resync of the controller 1190 // to reconcile the status again, this full resync is set every 5 minutes by default so we need to poll at least one 1191 // minute more just in case we we start to poll just after the full resync has happened and he have to wait until 1192 // next full resync. 1193 // Ref: https://issues.k8s.io/121911 1194 err = wait.PollUntilContextTimeout(ctx, 5*time.Second, 6*time.Minute, true, func(ctx context.Context) (bool, error) { 1195 resourceQuotaResult, err := rqClient.Get(ctx, rqName, metav1.GetOptions{}) 1196 if err != nil { 1197 return false, nil 1198 } 1199 1200 if *resourceQuotaResult.Spec.Hard.Cpu() == *resourceQuotaResult.Status.Hard.Cpu() { 1201 if *resourceQuotaResult.Status.Hard.Cpu() != resource.MustParse("1") { 1202 framework.Logf("Hard cpu status value for ResourceQuota %q is %s not 1.", repatchedResourceQuota.Name, resourceQuotaResult.Status.Hard.Cpu().String()) 1203 return false, nil 1204 } 1205 if *resourceQuotaResult.Status.Hard.Memory() != resource.MustParse("1Gi") { 1206 framework.Logf("Hard memory status value for ResourceQuota %q is %s not 1Gi.", repatchedResourceQuota.Name, resourceQuotaResult.Status.Hard.Memory().String()) 1207 return false, nil 1208 } 1209 framework.Logf("ResourceQuota %q Spec was unchanged and /status reset", resourceQuotaResult.Name) 1210 return true, nil 1211 } 1212 framework.Logf("ResourceQuota %q Spec and Status does not match: %#v", resourceQuotaResult.Name, resourceQuotaResult) 1213 return false, nil 1214 }) 1215 if err != nil { 1216 framework.Failf("Error waiting for ResourceQuota %q to reset its Status: %v", patchedResourceQuota.Name, err) 1217 } 1218 1219 }) 1220 }) 1221 1222 var _ = SIGDescribe("ResourceQuota", feature.ScopeSelectors, func() { 1223 f := framework.NewDefaultFramework("scope-selectors") 1224 f.NamespacePodSecurityLevel = admissionapi.LevelBaseline 1225 ginkgo.It("should verify ResourceQuota with best effort scope using scope-selectors.", func(ctx context.Context) { 1226 ginkgo.By("Creating a ResourceQuota with best effort scope") 1227 resourceQuotaBestEffort, err := createResourceQuota(ctx, f.ClientSet, f.Namespace.Name, newTestResourceQuotaWithScopeSelector("quota-besteffort", v1.ResourceQuotaScopeBestEffort)) 1228 framework.ExpectNoError(err) 1229 1230 ginkgo.By("Ensuring ResourceQuota status is calculated") 1231 usedResources := v1.ResourceList{} 1232 usedResources[v1.ResourcePods] = resource.MustParse("0") 1233 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, resourceQuotaBestEffort.Name, usedResources) 1234 framework.ExpectNoError(err) 1235 1236 ginkgo.By("Creating a ResourceQuota with not best effort scope") 1237 resourceQuotaNotBestEffort, err := createResourceQuota(ctx, f.ClientSet, f.Namespace.Name, newTestResourceQuotaWithScopeSelector("quota-not-besteffort", v1.ResourceQuotaScopeNotBestEffort)) 1238 framework.ExpectNoError(err) 1239 1240 ginkgo.By("Ensuring ResourceQuota status is calculated") 1241 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, resourceQuotaNotBestEffort.Name, usedResources) 1242 framework.ExpectNoError(err) 1243 1244 ginkgo.By("Creating a best-effort pod") 1245 pod := newTestPodForQuota(f, podName, v1.ResourceList{}, v1.ResourceList{}) 1246 pod, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(ctx, pod, metav1.CreateOptions{}) 1247 framework.ExpectNoError(err) 1248 1249 ginkgo.By("Ensuring resource quota with best effort scope captures the pod usage") 1250 usedResources[v1.ResourcePods] = resource.MustParse("1") 1251 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, resourceQuotaBestEffort.Name, usedResources) 1252 framework.ExpectNoError(err) 1253 1254 ginkgo.By("Ensuring resource quota with not best effort ignored the pod usage") 1255 usedResources[v1.ResourcePods] = resource.MustParse("0") 1256 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, resourceQuotaNotBestEffort.Name, usedResources) 1257 framework.ExpectNoError(err) 1258 1259 ginkgo.By("Deleting the pod") 1260 err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Delete(ctx, pod.Name, *metav1.NewDeleteOptions(0)) 1261 framework.ExpectNoError(err) 1262 1263 ginkgo.By("Ensuring resource quota status released the pod usage") 1264 usedResources[v1.ResourcePods] = resource.MustParse("0") 1265 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, resourceQuotaBestEffort.Name, usedResources) 1266 framework.ExpectNoError(err) 1267 1268 ginkgo.By("Creating a not best-effort pod") 1269 requests := v1.ResourceList{} 1270 requests[v1.ResourceCPU] = resource.MustParse("500m") 1271 requests[v1.ResourceMemory] = resource.MustParse("200Mi") 1272 limits := v1.ResourceList{} 1273 limits[v1.ResourceCPU] = resource.MustParse("1") 1274 limits[v1.ResourceMemory] = resource.MustParse("400Mi") 1275 pod = newTestPodForQuota(f, "burstable-pod", requests, limits) 1276 pod, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(ctx, pod, metav1.CreateOptions{}) 1277 framework.ExpectNoError(err) 1278 1279 ginkgo.By("Ensuring resource quota with not best effort scope captures the pod usage") 1280 usedResources[v1.ResourcePods] = resource.MustParse("1") 1281 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, resourceQuotaNotBestEffort.Name, usedResources) 1282 framework.ExpectNoError(err) 1283 1284 ginkgo.By("Ensuring resource quota with best effort scope ignored the pod usage") 1285 usedResources[v1.ResourcePods] = resource.MustParse("0") 1286 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, resourceQuotaBestEffort.Name, usedResources) 1287 framework.ExpectNoError(err) 1288 1289 ginkgo.By("Deleting the pod") 1290 err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Delete(ctx, pod.Name, *metav1.NewDeleteOptions(0)) 1291 framework.ExpectNoError(err) 1292 1293 ginkgo.By("Ensuring resource quota status released the pod usage") 1294 usedResources[v1.ResourcePods] = resource.MustParse("0") 1295 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, resourceQuotaNotBestEffort.Name, usedResources) 1296 framework.ExpectNoError(err) 1297 }) 1298 ginkgo.It("should verify ResourceQuota with terminating scopes through scope selectors.", func(ctx context.Context) { 1299 ginkgo.By("Creating a ResourceQuota with terminating scope") 1300 quotaTerminatingName := "quota-terminating" 1301 resourceQuotaTerminating, err := createResourceQuota(ctx, f.ClientSet, f.Namespace.Name, newTestResourceQuotaWithScopeSelector(quotaTerminatingName, v1.ResourceQuotaScopeTerminating)) 1302 framework.ExpectNoError(err) 1303 1304 ginkgo.By("Ensuring ResourceQuota status is calculated") 1305 usedResources := v1.ResourceList{} 1306 usedResources[v1.ResourcePods] = resource.MustParse("0") 1307 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, resourceQuotaTerminating.Name, usedResources) 1308 framework.ExpectNoError(err) 1309 1310 ginkgo.By("Creating a ResourceQuota with not terminating scope") 1311 quotaNotTerminatingName := "quota-not-terminating" 1312 resourceQuotaNotTerminating, err := createResourceQuota(ctx, f.ClientSet, f.Namespace.Name, newTestResourceQuotaWithScopeSelector(quotaNotTerminatingName, v1.ResourceQuotaScopeNotTerminating)) 1313 framework.ExpectNoError(err) 1314 1315 ginkgo.By("Ensuring ResourceQuota status is calculated") 1316 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, resourceQuotaNotTerminating.Name, usedResources) 1317 framework.ExpectNoError(err) 1318 1319 ginkgo.By("Creating a long running pod") 1320 podName := "test-pod" 1321 requests := v1.ResourceList{} 1322 requests[v1.ResourceCPU] = resource.MustParse("500m") 1323 requests[v1.ResourceMemory] = resource.MustParse("200Mi") 1324 limits := v1.ResourceList{} 1325 limits[v1.ResourceCPU] = resource.MustParse("1") 1326 limits[v1.ResourceMemory] = resource.MustParse("400Mi") 1327 pod := newTestPodForQuota(f, podName, requests, limits) 1328 _, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(ctx, pod, metav1.CreateOptions{}) 1329 framework.ExpectNoError(err) 1330 1331 ginkgo.By("Ensuring resource quota with not terminating scope captures the pod usage") 1332 usedResources[v1.ResourcePods] = resource.MustParse("1") 1333 usedResources[v1.ResourceRequestsCPU] = requests[v1.ResourceCPU] 1334 usedResources[v1.ResourceRequestsMemory] = requests[v1.ResourceMemory] 1335 usedResources[v1.ResourceLimitsCPU] = limits[v1.ResourceCPU] 1336 usedResources[v1.ResourceLimitsMemory] = limits[v1.ResourceMemory] 1337 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, resourceQuotaNotTerminating.Name, usedResources) 1338 framework.ExpectNoError(err) 1339 1340 ginkgo.By("Ensuring resource quota with terminating scope ignored the pod usage") 1341 usedResources[v1.ResourcePods] = resource.MustParse("0") 1342 usedResources[v1.ResourceRequestsCPU] = resource.MustParse("0") 1343 usedResources[v1.ResourceRequestsMemory] = resource.MustParse("0") 1344 usedResources[v1.ResourceLimitsCPU] = resource.MustParse("0") 1345 usedResources[v1.ResourceLimitsMemory] = resource.MustParse("0") 1346 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, resourceQuotaTerminating.Name, usedResources) 1347 framework.ExpectNoError(err) 1348 1349 ginkgo.By("Deleting the pod") 1350 err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Delete(ctx, podName, *metav1.NewDeleteOptions(0)) 1351 framework.ExpectNoError(err) 1352 1353 ginkgo.By("Ensuring resource quota status released the pod usage") 1354 usedResources[v1.ResourcePods] = resource.MustParse("0") 1355 usedResources[v1.ResourceRequestsCPU] = resource.MustParse("0") 1356 usedResources[v1.ResourceRequestsMemory] = resource.MustParse("0") 1357 usedResources[v1.ResourceLimitsCPU] = resource.MustParse("0") 1358 usedResources[v1.ResourceLimitsMemory] = resource.MustParse("0") 1359 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, resourceQuotaNotTerminating.Name, usedResources) 1360 framework.ExpectNoError(err) 1361 1362 ginkgo.By("Creating a terminating pod") 1363 podName = "terminating-pod" 1364 pod = newTestPodForQuota(f, podName, requests, limits) 1365 activeDeadlineSeconds := int64(3600) 1366 pod.Spec.ActiveDeadlineSeconds = &activeDeadlineSeconds 1367 _, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(ctx, pod, metav1.CreateOptions{}) 1368 framework.ExpectNoError(err) 1369 1370 ginkgo.By("Ensuring resource quota with terminating scope captures the pod usage") 1371 usedResources[v1.ResourcePods] = resource.MustParse("1") 1372 usedResources[v1.ResourceRequestsCPU] = requests[v1.ResourceCPU] 1373 usedResources[v1.ResourceRequestsMemory] = requests[v1.ResourceMemory] 1374 usedResources[v1.ResourceLimitsCPU] = limits[v1.ResourceCPU] 1375 usedResources[v1.ResourceLimitsMemory] = limits[v1.ResourceMemory] 1376 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, resourceQuotaTerminating.Name, usedResources) 1377 framework.ExpectNoError(err) 1378 1379 ginkgo.By("Ensuring resource quota with not terminating scope ignored the pod usage") 1380 usedResources[v1.ResourcePods] = resource.MustParse("0") 1381 usedResources[v1.ResourceRequestsCPU] = resource.MustParse("0") 1382 usedResources[v1.ResourceRequestsMemory] = resource.MustParse("0") 1383 usedResources[v1.ResourceLimitsCPU] = resource.MustParse("0") 1384 usedResources[v1.ResourceLimitsMemory] = resource.MustParse("0") 1385 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, resourceQuotaNotTerminating.Name, usedResources) 1386 framework.ExpectNoError(err) 1387 1388 ginkgo.By("Deleting the pod") 1389 err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Delete(ctx, podName, *metav1.NewDeleteOptions(0)) 1390 framework.ExpectNoError(err) 1391 1392 ginkgo.By("Ensuring resource quota status released the pod usage") 1393 usedResources[v1.ResourcePods] = resource.MustParse("0") 1394 usedResources[v1.ResourceRequestsCPU] = resource.MustParse("0") 1395 usedResources[v1.ResourceRequestsMemory] = resource.MustParse("0") 1396 usedResources[v1.ResourceLimitsCPU] = resource.MustParse("0") 1397 usedResources[v1.ResourceLimitsMemory] = resource.MustParse("0") 1398 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, resourceQuotaTerminating.Name, usedResources) 1399 framework.ExpectNoError(err) 1400 }) 1401 }) 1402 1403 var _ = SIGDescribe("ResourceQuota", feature.PodPriority, func() { 1404 f := framework.NewDefaultFramework("resourcequota-priorityclass") 1405 f.NamespacePodSecurityLevel = admissionapi.LevelBaseline 1406 1407 ginkgo.It("should verify ResourceQuota's priority class scope (quota set to pod count: 1) against a pod with same priority class.", func(ctx context.Context) { 1408 1409 _, err := f.ClientSet.SchedulingV1().PriorityClasses().Create(ctx, &schedulingv1.PriorityClass{ObjectMeta: metav1.ObjectMeta{Name: "pclass1"}, Value: int32(1000)}, metav1.CreateOptions{}) 1410 if err != nil && !apierrors.IsAlreadyExists(err) { 1411 framework.Failf("unexpected error while creating priority class: %v", err) 1412 } 1413 1414 hard := v1.ResourceList{} 1415 hard[v1.ResourcePods] = resource.MustParse("1") 1416 1417 ginkgo.By("Creating a ResourceQuota with priority class scope") 1418 resourceQuotaPriorityClass, err := createResourceQuota(ctx, f.ClientSet, f.Namespace.Name, newTestResourceQuotaWithScopeForPriorityClass("quota-priorityclass", hard, v1.ScopeSelectorOpIn, []string{"pclass1"})) 1419 framework.ExpectNoError(err) 1420 1421 ginkgo.By("Ensuring ResourceQuota status is calculated") 1422 usedResources := v1.ResourceList{} 1423 usedResources[v1.ResourcePods] = resource.MustParse("0") 1424 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, resourceQuotaPriorityClass.Name, usedResources) 1425 framework.ExpectNoError(err) 1426 1427 ginkgo.By("Creating a pod with priority class") 1428 podName := "testpod-pclass1" 1429 pod := newTestPodForQuotaWithPriority(f, podName, v1.ResourceList{}, v1.ResourceList{}, "pclass1") 1430 pod, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(ctx, pod, metav1.CreateOptions{}) 1431 framework.ExpectNoError(err) 1432 1433 ginkgo.By("Ensuring resource quota with priority class scope captures the pod usage") 1434 usedResources[v1.ResourcePods] = resource.MustParse("1") 1435 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, resourceQuotaPriorityClass.Name, usedResources) 1436 framework.ExpectNoError(err) 1437 1438 ginkgo.By("Deleting the pod") 1439 err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Delete(ctx, pod.Name, *metav1.NewDeleteOptions(0)) 1440 framework.ExpectNoError(err) 1441 1442 ginkgo.By("Ensuring resource quota status released the pod usage") 1443 usedResources[v1.ResourcePods] = resource.MustParse("0") 1444 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, resourceQuotaPriorityClass.Name, usedResources) 1445 framework.ExpectNoError(err) 1446 }) 1447 1448 ginkgo.It("should verify ResourceQuota's priority class scope (quota set to pod count: 1) against 2 pods with same priority class.", func(ctx context.Context) { 1449 1450 _, err := f.ClientSet.SchedulingV1().PriorityClasses().Create(ctx, &schedulingv1.PriorityClass{ObjectMeta: metav1.ObjectMeta{Name: "pclass2"}, Value: int32(1000)}, metav1.CreateOptions{}) 1451 if err != nil && !apierrors.IsAlreadyExists(err) { 1452 framework.Failf("unexpected error while creating priority class: %v", err) 1453 } 1454 1455 hard := v1.ResourceList{} 1456 hard[v1.ResourcePods] = resource.MustParse("1") 1457 1458 ginkgo.By("Creating a ResourceQuota with priority class scope") 1459 resourceQuotaPriorityClass, err := createResourceQuota(ctx, f.ClientSet, f.Namespace.Name, newTestResourceQuotaWithScopeForPriorityClass("quota-priorityclass", hard, v1.ScopeSelectorOpIn, []string{"pclass2"})) 1460 framework.ExpectNoError(err) 1461 1462 ginkgo.By("Ensuring ResourceQuota status is calculated") 1463 usedResources := v1.ResourceList{} 1464 usedResources[v1.ResourcePods] = resource.MustParse("0") 1465 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, resourceQuotaPriorityClass.Name, usedResources) 1466 framework.ExpectNoError(err) 1467 1468 ginkgo.By("Creating first pod with priority class should pass") 1469 podName := "testpod-pclass2-1" 1470 pod := newTestPodForQuotaWithPriority(f, podName, v1.ResourceList{}, v1.ResourceList{}, "pclass2") 1471 pod, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(ctx, pod, metav1.CreateOptions{}) 1472 framework.ExpectNoError(err) 1473 1474 ginkgo.By("Ensuring resource quota with priority class scope captures the pod usage") 1475 usedResources[v1.ResourcePods] = resource.MustParse("1") 1476 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, resourceQuotaPriorityClass.Name, usedResources) 1477 framework.ExpectNoError(err) 1478 1479 ginkgo.By("Creating 2nd pod with priority class should fail") 1480 podName2 := "testpod-pclass2-2" 1481 pod2 := newTestPodForQuotaWithPriority(f, podName2, v1.ResourceList{}, v1.ResourceList{}, "pclass2") 1482 _, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(ctx, pod2, metav1.CreateOptions{}) 1483 gomega.Expect(err).To(gomega.HaveOccurred()) 1484 1485 ginkgo.By("Deleting first pod") 1486 err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Delete(ctx, pod.Name, *metav1.NewDeleteOptions(0)) 1487 framework.ExpectNoError(err) 1488 1489 ginkgo.By("Ensuring resource quota status released the pod usage") 1490 usedResources[v1.ResourcePods] = resource.MustParse("0") 1491 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, resourceQuotaPriorityClass.Name, usedResources) 1492 framework.ExpectNoError(err) 1493 }) 1494 1495 ginkgo.It("should verify ResourceQuota's priority class scope (quota set to pod count: 1) against 2 pods with different priority class.", func(ctx context.Context) { 1496 1497 _, err := f.ClientSet.SchedulingV1().PriorityClasses().Create(ctx, &schedulingv1.PriorityClass{ObjectMeta: metav1.ObjectMeta{Name: "pclass3"}, Value: int32(1000)}, metav1.CreateOptions{}) 1498 if err != nil && !apierrors.IsAlreadyExists(err) { 1499 framework.Failf("unexpected error while creating priority class: %v", err) 1500 } 1501 1502 hard := v1.ResourceList{} 1503 hard[v1.ResourcePods] = resource.MustParse("1") 1504 1505 ginkgo.By("Creating a ResourceQuota with priority class scope") 1506 resourceQuotaPriorityClass, err := createResourceQuota(ctx, f.ClientSet, f.Namespace.Name, newTestResourceQuotaWithScopeForPriorityClass("quota-priorityclass", hard, v1.ScopeSelectorOpIn, []string{"pclass4"})) 1507 framework.ExpectNoError(err) 1508 1509 ginkgo.By("Ensuring ResourceQuota status is calculated") 1510 usedResources := v1.ResourceList{} 1511 usedResources[v1.ResourcePods] = resource.MustParse("0") 1512 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, resourceQuotaPriorityClass.Name, usedResources) 1513 framework.ExpectNoError(err) 1514 1515 ginkgo.By("Creating a pod with priority class with pclass3") 1516 podName := "testpod-pclass3-1" 1517 pod := newTestPodForQuotaWithPriority(f, podName, v1.ResourceList{}, v1.ResourceList{}, "pclass3") 1518 pod, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(ctx, pod, metav1.CreateOptions{}) 1519 framework.ExpectNoError(err) 1520 1521 ginkgo.By("Ensuring resource quota with priority class scope remains same") 1522 usedResources[v1.ResourcePods] = resource.MustParse("0") 1523 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, resourceQuotaPriorityClass.Name, usedResources) 1524 framework.ExpectNoError(err) 1525 1526 ginkgo.By("Creating a 2nd pod with priority class pclass3") 1527 podName2 := "testpod-pclass2-2" 1528 pod2 := newTestPodForQuotaWithPriority(f, podName2, v1.ResourceList{}, v1.ResourceList{}, "pclass3") 1529 pod2, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(ctx, pod2, metav1.CreateOptions{}) 1530 framework.ExpectNoError(err) 1531 1532 ginkgo.By("Ensuring resource quota with priority class scope remains same") 1533 usedResources[v1.ResourcePods] = resource.MustParse("0") 1534 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, resourceQuotaPriorityClass.Name, usedResources) 1535 framework.ExpectNoError(err) 1536 1537 ginkgo.By("Deleting both pods") 1538 err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Delete(ctx, pod.Name, *metav1.NewDeleteOptions(0)) 1539 framework.ExpectNoError(err) 1540 err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Delete(ctx, pod2.Name, *metav1.NewDeleteOptions(0)) 1541 framework.ExpectNoError(err) 1542 }) 1543 1544 ginkgo.It("should verify ResourceQuota's multiple priority class scope (quota set to pod count: 2) against 2 pods with same priority classes.", func(ctx context.Context) { 1545 _, err := f.ClientSet.SchedulingV1().PriorityClasses().Create(ctx, &schedulingv1.PriorityClass{ObjectMeta: metav1.ObjectMeta{Name: "pclass5"}, Value: int32(1000)}, metav1.CreateOptions{}) 1546 if err != nil && !apierrors.IsAlreadyExists(err) { 1547 framework.Failf("unexpected error while creating priority class: %v", err) 1548 } 1549 1550 _, err = f.ClientSet.SchedulingV1().PriorityClasses().Create(ctx, &schedulingv1.PriorityClass{ObjectMeta: metav1.ObjectMeta{Name: "pclass6"}, Value: int32(1000)}, metav1.CreateOptions{}) 1551 if err != nil && !apierrors.IsAlreadyExists(err) { 1552 framework.Failf("unexpected error while creating priority class: %v", err) 1553 } 1554 1555 hard := v1.ResourceList{} 1556 hard[v1.ResourcePods] = resource.MustParse("2") 1557 1558 ginkgo.By("Creating a ResourceQuota with priority class scope") 1559 resourceQuotaPriorityClass, err := createResourceQuota(ctx, f.ClientSet, f.Namespace.Name, newTestResourceQuotaWithScopeForPriorityClass("quota-priorityclass", hard, v1.ScopeSelectorOpIn, []string{"pclass5", "pclass6"})) 1560 framework.ExpectNoError(err) 1561 1562 ginkgo.By("Ensuring ResourceQuota status is calculated") 1563 usedResources := v1.ResourceList{} 1564 usedResources[v1.ResourcePods] = resource.MustParse("0") 1565 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, resourceQuotaPriorityClass.Name, usedResources) 1566 framework.ExpectNoError(err) 1567 1568 ginkgo.By("Creating a pod with priority class pclass5") 1569 podName := "testpod-pclass5" 1570 pod := newTestPodForQuotaWithPriority(f, podName, v1.ResourceList{}, v1.ResourceList{}, "pclass5") 1571 pod, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(ctx, pod, metav1.CreateOptions{}) 1572 framework.ExpectNoError(err) 1573 1574 ginkgo.By("Ensuring resource quota with priority class is updated with the pod usage") 1575 usedResources[v1.ResourcePods] = resource.MustParse("1") 1576 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, resourceQuotaPriorityClass.Name, usedResources) 1577 framework.ExpectNoError(err) 1578 1579 ginkgo.By("Creating 2nd pod with priority class pclass6") 1580 podName2 := "testpod-pclass6" 1581 pod2 := newTestPodForQuotaWithPriority(f, podName2, v1.ResourceList{}, v1.ResourceList{}, "pclass6") 1582 pod2, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(ctx, pod2, metav1.CreateOptions{}) 1583 framework.ExpectNoError(err) 1584 1585 ginkgo.By("Ensuring resource quota with priority class scope is updated with the pod usage") 1586 usedResources[v1.ResourcePods] = resource.MustParse("2") 1587 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, resourceQuotaPriorityClass.Name, usedResources) 1588 framework.ExpectNoError(err) 1589 1590 ginkgo.By("Deleting both pods") 1591 err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Delete(ctx, pod.Name, *metav1.NewDeleteOptions(0)) 1592 framework.ExpectNoError(err) 1593 err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Delete(ctx, pod2.Name, *metav1.NewDeleteOptions(0)) 1594 framework.ExpectNoError(err) 1595 1596 ginkgo.By("Ensuring resource quota status released the pod usage") 1597 usedResources[v1.ResourcePods] = resource.MustParse("0") 1598 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, resourceQuotaPriorityClass.Name, usedResources) 1599 framework.ExpectNoError(err) 1600 }) 1601 1602 ginkgo.It("should verify ResourceQuota's priority class scope (quota set to pod count: 1) against a pod with different priority class (ScopeSelectorOpNotIn).", func(ctx context.Context) { 1603 1604 _, err := f.ClientSet.SchedulingV1().PriorityClasses().Create(ctx, &schedulingv1.PriorityClass{ObjectMeta: metav1.ObjectMeta{Name: "pclass7"}, Value: int32(1000)}, metav1.CreateOptions{}) 1605 if err != nil && !apierrors.IsAlreadyExists(err) { 1606 framework.Failf("unexpected error while creating priority class: %v", err) 1607 } 1608 1609 hard := v1.ResourceList{} 1610 hard[v1.ResourcePods] = resource.MustParse("1") 1611 1612 ginkgo.By("Creating a ResourceQuota with priority class scope") 1613 resourceQuotaPriorityClass, err := createResourceQuota(ctx, f.ClientSet, f.Namespace.Name, newTestResourceQuotaWithScopeForPriorityClass("quota-priorityclass", hard, v1.ScopeSelectorOpNotIn, []string{"pclass7"})) 1614 framework.ExpectNoError(err) 1615 1616 ginkgo.By("Ensuring ResourceQuota status is calculated") 1617 usedResources := v1.ResourceList{} 1618 usedResources[v1.ResourcePods] = resource.MustParse("0") 1619 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, resourceQuotaPriorityClass.Name, usedResources) 1620 framework.ExpectNoError(err) 1621 1622 ginkgo.By("Creating a pod with priority class pclass7") 1623 podName := "testpod-pclass7" 1624 pod := newTestPodForQuotaWithPriority(f, podName, v1.ResourceList{}, v1.ResourceList{}, "pclass7") 1625 pod, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(ctx, pod, metav1.CreateOptions{}) 1626 framework.ExpectNoError(err) 1627 1628 ginkgo.By("Ensuring resource quota with priority class is not used") 1629 usedResources[v1.ResourcePods] = resource.MustParse("0") 1630 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, resourceQuotaPriorityClass.Name, usedResources) 1631 framework.ExpectNoError(err) 1632 1633 ginkgo.By("Deleting the pod") 1634 err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Delete(ctx, pod.Name, *metav1.NewDeleteOptions(0)) 1635 framework.ExpectNoError(err) 1636 }) 1637 1638 ginkgo.It("should verify ResourceQuota's priority class scope (quota set to pod count: 1) against a pod with different priority class (ScopeSelectorOpExists).", func(ctx context.Context) { 1639 1640 _, err := f.ClientSet.SchedulingV1().PriorityClasses().Create(ctx, &schedulingv1.PriorityClass{ObjectMeta: metav1.ObjectMeta{Name: "pclass8"}, Value: int32(1000)}, metav1.CreateOptions{}) 1641 if err != nil && !apierrors.IsAlreadyExists(err) { 1642 framework.Failf("unexpected error while creating priority class: %v", err) 1643 } 1644 1645 hard := v1.ResourceList{} 1646 hard[v1.ResourcePods] = resource.MustParse("1") 1647 1648 ginkgo.By("Creating a ResourceQuota with priority class scope") 1649 resourceQuotaPriorityClass, err := createResourceQuota(ctx, f.ClientSet, f.Namespace.Name, newTestResourceQuotaWithScopeForPriorityClass("quota-priorityclass", hard, v1.ScopeSelectorOpExists, []string{})) 1650 framework.ExpectNoError(err) 1651 1652 ginkgo.By("Ensuring ResourceQuota status is calculated") 1653 usedResources := v1.ResourceList{} 1654 usedResources[v1.ResourcePods] = resource.MustParse("0") 1655 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, resourceQuotaPriorityClass.Name, usedResources) 1656 framework.ExpectNoError(err) 1657 1658 ginkgo.By("Creating a pod with priority class pclass8") 1659 podName := "testpod-pclass8" 1660 pod := newTestPodForQuotaWithPriority(f, podName, v1.ResourceList{}, v1.ResourceList{}, "pclass8") 1661 pod, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(ctx, pod, metav1.CreateOptions{}) 1662 framework.ExpectNoError(err) 1663 1664 ginkgo.By("Ensuring resource quota with priority class is updated with the pod usage") 1665 usedResources[v1.ResourcePods] = resource.MustParse("1") 1666 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, resourceQuotaPriorityClass.Name, usedResources) 1667 framework.ExpectNoError(err) 1668 1669 ginkgo.By("Deleting the pod") 1670 err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Delete(ctx, pod.Name, *metav1.NewDeleteOptions(0)) 1671 framework.ExpectNoError(err) 1672 1673 ginkgo.By("Ensuring resource quota status released the pod usage") 1674 usedResources[v1.ResourcePods] = resource.MustParse("0") 1675 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, resourceQuotaPriorityClass.Name, usedResources) 1676 framework.ExpectNoError(err) 1677 }) 1678 1679 ginkgo.It("should verify ResourceQuota's priority class scope (cpu, memory quota set) against a pod with same priority class.", func(ctx context.Context) { 1680 1681 _, err := f.ClientSet.SchedulingV1().PriorityClasses().Create(ctx, &schedulingv1.PriorityClass{ObjectMeta: metav1.ObjectMeta{Name: "pclass9"}, Value: int32(1000)}, metav1.CreateOptions{}) 1682 if err != nil && !apierrors.IsAlreadyExists(err) { 1683 framework.Failf("unexpected error while creating priority class: %v", err) 1684 } 1685 1686 hard := v1.ResourceList{} 1687 hard[v1.ResourcePods] = resource.MustParse("1") 1688 hard[v1.ResourceRequestsCPU] = resource.MustParse("1") 1689 hard[v1.ResourceRequestsMemory] = resource.MustParse("1Gi") 1690 hard[v1.ResourceLimitsCPU] = resource.MustParse("3") 1691 hard[v1.ResourceLimitsMemory] = resource.MustParse("3Gi") 1692 1693 ginkgo.By("Creating a ResourceQuota with priority class scope") 1694 resourceQuotaPriorityClass, err := createResourceQuota(ctx, f.ClientSet, f.Namespace.Name, newTestResourceQuotaWithScopeForPriorityClass("quota-priorityclass", hard, v1.ScopeSelectorOpIn, []string{"pclass9"})) 1695 framework.ExpectNoError(err) 1696 1697 ginkgo.By("Ensuring ResourceQuota status is calculated") 1698 usedResources := v1.ResourceList{} 1699 usedResources[v1.ResourcePods] = resource.MustParse("0") 1700 usedResources[v1.ResourceRequestsCPU] = resource.MustParse("0") 1701 usedResources[v1.ResourceRequestsMemory] = resource.MustParse("0Gi") 1702 usedResources[v1.ResourceLimitsCPU] = resource.MustParse("0") 1703 usedResources[v1.ResourceLimitsMemory] = resource.MustParse("0Gi") 1704 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, resourceQuotaPriorityClass.Name, usedResources) 1705 framework.ExpectNoError(err) 1706 1707 ginkgo.By("Creating a pod with priority class") 1708 podName := "testpod-pclass9" 1709 request := v1.ResourceList{} 1710 request[v1.ResourceCPU] = resource.MustParse("1") 1711 request[v1.ResourceMemory] = resource.MustParse("1Gi") 1712 limit := v1.ResourceList{} 1713 limit[v1.ResourceCPU] = resource.MustParse("2") 1714 limit[v1.ResourceMemory] = resource.MustParse("2Gi") 1715 1716 pod := newTestPodForQuotaWithPriority(f, podName, request, limit, "pclass9") 1717 pod, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(ctx, pod, metav1.CreateOptions{}) 1718 framework.ExpectNoError(err) 1719 1720 ginkgo.By("Ensuring resource quota with priority class scope captures the pod usage") 1721 usedResources[v1.ResourcePods] = resource.MustParse("1") 1722 usedResources[v1.ResourceRequestsCPU] = resource.MustParse("1") 1723 usedResources[v1.ResourceRequestsMemory] = resource.MustParse("1Gi") 1724 usedResources[v1.ResourceLimitsCPU] = resource.MustParse("2") 1725 usedResources[v1.ResourceLimitsMemory] = resource.MustParse("2Gi") 1726 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, resourceQuotaPriorityClass.Name, usedResources) 1727 framework.ExpectNoError(err) 1728 1729 ginkgo.By("Deleting the pod") 1730 err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Delete(ctx, pod.Name, *metav1.NewDeleteOptions(0)) 1731 framework.ExpectNoError(err) 1732 1733 ginkgo.By("Ensuring resource quota status released the pod usage") 1734 usedResources[v1.ResourcePods] = resource.MustParse("0") 1735 usedResources[v1.ResourceRequestsCPU] = resource.MustParse("0") 1736 usedResources[v1.ResourceRequestsMemory] = resource.MustParse("0Gi") 1737 usedResources[v1.ResourceLimitsCPU] = resource.MustParse("0") 1738 usedResources[v1.ResourceLimitsMemory] = resource.MustParse("0Gi") 1739 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, resourceQuotaPriorityClass.Name, usedResources) 1740 framework.ExpectNoError(err) 1741 }) 1742 1743 }) 1744 1745 var _ = SIGDescribe("ResourceQuota", func() { 1746 f := framework.NewDefaultFramework("cross-namespace-pod-affinity") 1747 f.NamespacePodSecurityLevel = admissionapi.LevelBaseline 1748 ginkgo.It("should verify ResourceQuota with cross namespace pod affinity scope using scope-selectors.", func(ctx context.Context) { 1749 ginkgo.By("Creating a ResourceQuota with cross namespace pod affinity scope") 1750 quota, err := createResourceQuota( 1751 ctx, f.ClientSet, f.Namespace.Name, newTestResourceQuotaWithScopeSelector("quota-cross-namespace-pod-affinity", v1.ResourceQuotaScopeCrossNamespacePodAffinity)) 1752 framework.ExpectNoError(err) 1753 1754 ginkgo.By("Ensuring ResourceQuota status is calculated") 1755 wantUsedResources := v1.ResourceList{v1.ResourcePods: resource.MustParse("0")} 1756 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, quota.Name, wantUsedResources) 1757 framework.ExpectNoError(err) 1758 1759 ginkgo.By("Creating a pod that does not use cross namespace affinity") 1760 pod := newTestPodWithAffinityForQuota(f, "no-cross-namespace-affinity", &v1.Affinity{ 1761 PodAntiAffinity: &v1.PodAntiAffinity{ 1762 RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{{ 1763 TopologyKey: "region", 1764 }}}}) 1765 pod, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(ctx, pod, metav1.CreateOptions{}) 1766 framework.ExpectNoError(err) 1767 1768 ginkgo.By("Creating a pod that uses namespaces field") 1769 podWithNamespaces := newTestPodWithAffinityForQuota(f, "with-namespaces", &v1.Affinity{ 1770 PodAntiAffinity: &v1.PodAntiAffinity{ 1771 RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{{ 1772 TopologyKey: "region", 1773 Namespaces: []string{"ns1"}, 1774 }}}}) 1775 podWithNamespaces, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(ctx, podWithNamespaces, metav1.CreateOptions{}) 1776 framework.ExpectNoError(err) 1777 1778 ginkgo.By("Ensuring resource quota captures podWithNamespaces usage") 1779 wantUsedResources[v1.ResourcePods] = resource.MustParse("1") 1780 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, quota.Name, wantUsedResources) 1781 framework.ExpectNoError(err) 1782 1783 ginkgo.By("Creating a pod that uses namespaceSelector field") 1784 podWithNamespaceSelector := newTestPodWithAffinityForQuota(f, "with-namespace-selector", &v1.Affinity{ 1785 PodAntiAffinity: &v1.PodAntiAffinity{ 1786 RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{{ 1787 TopologyKey: "region", 1788 NamespaceSelector: &metav1.LabelSelector{ 1789 MatchExpressions: []metav1.LabelSelectorRequirement{ 1790 { 1791 Key: "team", 1792 Operator: metav1.LabelSelectorOpIn, 1793 Values: []string{"ads"}, 1794 }, 1795 }, 1796 }}}}}) 1797 podWithNamespaceSelector, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(ctx, podWithNamespaceSelector, metav1.CreateOptions{}) 1798 framework.ExpectNoError(err) 1799 1800 ginkgo.By("Ensuring resource quota captures podWithNamespaceSelector usage") 1801 wantUsedResources[v1.ResourcePods] = resource.MustParse("2") 1802 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, quota.Name, wantUsedResources) 1803 framework.ExpectNoError(err) 1804 1805 ginkgo.By("Deleting the pods") 1806 err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Delete(ctx, pod.Name, *metav1.NewDeleteOptions(0)) 1807 framework.ExpectNoError(err) 1808 err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Delete(ctx, podWithNamespaces.Name, *metav1.NewDeleteOptions(0)) 1809 framework.ExpectNoError(err) 1810 err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Delete(ctx, podWithNamespaceSelector.Name, *metav1.NewDeleteOptions(0)) 1811 framework.ExpectNoError(err) 1812 1813 ginkgo.By("Ensuring resource quota status released the pod usage") 1814 wantUsedResources[v1.ResourcePods] = resource.MustParse("0") 1815 err = waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, quota.Name, wantUsedResources) 1816 framework.ExpectNoError(err) 1817 }) 1818 }) 1819 1820 // newTestResourceQuotaWithScopeSelector returns a quota that enforces default constraints for testing with scopeSelectors 1821 func newTestResourceQuotaWithScopeSelector(name string, scope v1.ResourceQuotaScope) *v1.ResourceQuota { 1822 hard := v1.ResourceList{} 1823 hard[v1.ResourcePods] = resource.MustParse("5") 1824 switch scope { 1825 case v1.ResourceQuotaScopeTerminating, v1.ResourceQuotaScopeNotTerminating: 1826 hard[v1.ResourceRequestsCPU] = resource.MustParse("1") 1827 hard[v1.ResourceRequestsMemory] = resource.MustParse("500Mi") 1828 hard[v1.ResourceLimitsCPU] = resource.MustParse("2") 1829 hard[v1.ResourceLimitsMemory] = resource.MustParse("1Gi") 1830 } 1831 return &v1.ResourceQuota{ 1832 ObjectMeta: metav1.ObjectMeta{Name: name}, 1833 Spec: v1.ResourceQuotaSpec{Hard: hard, 1834 ScopeSelector: &v1.ScopeSelector{ 1835 MatchExpressions: []v1.ScopedResourceSelectorRequirement{ 1836 { 1837 ScopeName: scope, 1838 Operator: v1.ScopeSelectorOpExists}, 1839 }, 1840 }, 1841 }, 1842 } 1843 } 1844 1845 // newTestResourceQuotaWithScope returns a quota that enforces default constraints for testing with scopes 1846 func newTestResourceQuotaWithScope(name string, scope v1.ResourceQuotaScope) *v1.ResourceQuota { 1847 hard := v1.ResourceList{} 1848 hard[v1.ResourcePods] = resource.MustParse("5") 1849 switch scope { 1850 case v1.ResourceQuotaScopeTerminating, v1.ResourceQuotaScopeNotTerminating: 1851 hard[v1.ResourceRequestsCPU] = resource.MustParse("1") 1852 hard[v1.ResourceRequestsMemory] = resource.MustParse("500Mi") 1853 hard[v1.ResourceLimitsCPU] = resource.MustParse("2") 1854 hard[v1.ResourceLimitsMemory] = resource.MustParse("1Gi") 1855 } 1856 return &v1.ResourceQuota{ 1857 ObjectMeta: metav1.ObjectMeta{Name: name}, 1858 Spec: v1.ResourceQuotaSpec{Hard: hard, Scopes: []v1.ResourceQuotaScope{scope}}, 1859 } 1860 } 1861 1862 // newTestResourceQuotaWithScopeForPriorityClass returns a quota 1863 // that enforces default constraints for testing with ResourceQuotaScopePriorityClass scope 1864 func newTestResourceQuotaWithScopeForPriorityClass(name string, hard v1.ResourceList, op v1.ScopeSelectorOperator, values []string) *v1.ResourceQuota { 1865 return &v1.ResourceQuota{ 1866 ObjectMeta: metav1.ObjectMeta{Name: name}, 1867 Spec: v1.ResourceQuotaSpec{Hard: hard, 1868 ScopeSelector: &v1.ScopeSelector{ 1869 MatchExpressions: []v1.ScopedResourceSelectorRequirement{ 1870 { 1871 ScopeName: v1.ResourceQuotaScopePriorityClass, 1872 Operator: op, 1873 Values: values, 1874 }, 1875 }, 1876 }, 1877 }, 1878 } 1879 } 1880 1881 // newTestResourceQuota returns a quota that enforces default constraints for testing 1882 func newTestResourceQuota(name string) *v1.ResourceQuota { 1883 hard := v1.ResourceList{} 1884 hard[v1.ResourcePods] = resource.MustParse("5") 1885 hard[v1.ResourceServices] = resource.MustParse("10") 1886 hard[v1.ResourceServicesNodePorts] = resource.MustParse("1") 1887 hard[v1.ResourceServicesLoadBalancers] = resource.MustParse("1") 1888 hard[v1.ResourceReplicationControllers] = resource.MustParse("10") 1889 hard[v1.ResourceQuotas] = resource.MustParse("1") 1890 hard[v1.ResourceCPU] = resource.MustParse("1") 1891 hard[v1.ResourceMemory] = resource.MustParse("500Mi") 1892 hard[v1.ResourceConfigMaps] = resource.MustParse("10") 1893 hard[v1.ResourceSecrets] = resource.MustParse("10") 1894 hard[v1.ResourcePersistentVolumeClaims] = resource.MustParse("10") 1895 hard[v1.ResourceRequestsStorage] = resource.MustParse("10Gi") 1896 hard[v1.ResourceEphemeralStorage] = resource.MustParse("50Gi") 1897 hard[core.V1ResourceByStorageClass(classGold, v1.ResourcePersistentVolumeClaims)] = resource.MustParse("10") 1898 hard[core.V1ResourceByStorageClass(classGold, v1.ResourceRequestsStorage)] = resource.MustParse("10Gi") 1899 // test quota on discovered resource type 1900 hard[v1.ResourceName("count/replicasets.apps")] = resource.MustParse("5") 1901 // test quota on extended resource 1902 hard[v1.ResourceName(v1.DefaultResourceRequestsPrefix+extendedResourceName)] = resource.MustParse("3") 1903 return &v1.ResourceQuota{ 1904 ObjectMeta: metav1.ObjectMeta{Name: name}, 1905 Spec: v1.ResourceQuotaSpec{Hard: hard}, 1906 } 1907 } 1908 1909 // newTestPodForQuota returns a pod that has the specified requests and limits 1910 func newTestPodForQuota(f *framework.Framework, name string, requests v1.ResourceList, limits v1.ResourceList) *v1.Pod { 1911 return &v1.Pod{ 1912 ObjectMeta: metav1.ObjectMeta{ 1913 Name: name, 1914 }, 1915 Spec: v1.PodSpec{ 1916 // prevent disruption to other test workloads in parallel test runs by ensuring the quota 1917 // test pods don't get scheduled onto a node 1918 NodeSelector: map[string]string{ 1919 "x-test.k8s.io/unsatisfiable": "not-schedulable", 1920 }, 1921 Containers: []v1.Container{ 1922 { 1923 Name: "pause", 1924 Image: imageutils.GetPauseImageName(), 1925 Resources: v1.ResourceRequirements{ 1926 Requests: requests, 1927 Limits: limits, 1928 }, 1929 }, 1930 }, 1931 }, 1932 } 1933 } 1934 1935 // newTestPodForQuotaWithPriority returns a pod that has the specified requests, limits and priority class 1936 func newTestPodForQuotaWithPriority(f *framework.Framework, name string, requests v1.ResourceList, limits v1.ResourceList, pclass string) *v1.Pod { 1937 return &v1.Pod{ 1938 ObjectMeta: metav1.ObjectMeta{ 1939 Name: name, 1940 }, 1941 Spec: v1.PodSpec{ 1942 // prevent disruption to other test workloads in parallel test runs by ensuring the quota 1943 // test pods don't get scheduled onto a node 1944 NodeSelector: map[string]string{ 1945 "x-test.k8s.io/unsatisfiable": "not-schedulable", 1946 }, 1947 Containers: []v1.Container{ 1948 { 1949 Name: "pause", 1950 Image: imageutils.GetPauseImageName(), 1951 Resources: v1.ResourceRequirements{ 1952 Requests: requests, 1953 Limits: limits, 1954 }, 1955 }, 1956 }, 1957 PriorityClassName: pclass, 1958 }, 1959 } 1960 } 1961 1962 // newTestPodForQuota returns a pod that has the specified requests and limits 1963 func newTestPodWithAffinityForQuota(f *framework.Framework, name string, affinity *v1.Affinity) *v1.Pod { 1964 return &v1.Pod{ 1965 ObjectMeta: metav1.ObjectMeta{ 1966 Name: name, 1967 }, 1968 Spec: v1.PodSpec{ 1969 // prevent disruption to other test workloads in parallel test runs by ensuring the quota 1970 // test pods don't get scheduled onto a node 1971 NodeSelector: map[string]string{ 1972 "x-test.k8s.io/unsatisfiable": "not-schedulable", 1973 }, 1974 Affinity: affinity, 1975 Containers: []v1.Container{ 1976 { 1977 Name: "pause", 1978 Image: imageutils.GetPauseImageName(), 1979 Resources: v1.ResourceRequirements{}, 1980 }, 1981 }, 1982 }, 1983 } 1984 } 1985 1986 // newTestPersistentVolumeClaimForQuota returns a simple persistent volume claim 1987 func newTestPersistentVolumeClaimForQuota(name string) *v1.PersistentVolumeClaim { 1988 return &v1.PersistentVolumeClaim{ 1989 ObjectMeta: metav1.ObjectMeta{ 1990 Name: name, 1991 }, 1992 Spec: v1.PersistentVolumeClaimSpec{ 1993 AccessModes: []v1.PersistentVolumeAccessMode{ 1994 v1.ReadWriteOnce, 1995 v1.ReadOnlyMany, 1996 }, 1997 Resources: v1.VolumeResourceRequirements{ 1998 Requests: v1.ResourceList{ 1999 v1.ResourceName(v1.ResourceStorage): resource.MustParse("1Gi"), 2000 }, 2001 }, 2002 }, 2003 } 2004 } 2005 2006 // newTestReplicationControllerForQuota returns a simple replication controller 2007 func newTestReplicationControllerForQuota(name, image string, replicas int32) *v1.ReplicationController { 2008 return &v1.ReplicationController{ 2009 ObjectMeta: metav1.ObjectMeta{ 2010 Name: name, 2011 }, 2012 Spec: v1.ReplicationControllerSpec{ 2013 Replicas: pointer.Int32(replicas), 2014 Selector: map[string]string{ 2015 "name": name, 2016 }, 2017 Template: &v1.PodTemplateSpec{ 2018 ObjectMeta: metav1.ObjectMeta{ 2019 Labels: map[string]string{"name": name}, 2020 }, 2021 Spec: v1.PodSpec{ 2022 Containers: []v1.Container{ 2023 { 2024 Name: name, 2025 Image: image, 2026 }, 2027 }, 2028 }, 2029 }, 2030 }, 2031 } 2032 } 2033 2034 // newTestReplicaSetForQuota returns a simple replica set 2035 func newTestReplicaSetForQuota(name, image string, replicas int32) *appsv1.ReplicaSet { 2036 zero := int64(0) 2037 return &appsv1.ReplicaSet{ 2038 ObjectMeta: metav1.ObjectMeta{ 2039 Name: name, 2040 }, 2041 Spec: appsv1.ReplicaSetSpec{ 2042 Replicas: &replicas, 2043 Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"name": name}}, 2044 Template: v1.PodTemplateSpec{ 2045 ObjectMeta: metav1.ObjectMeta{ 2046 Labels: map[string]string{"name": name}, 2047 }, 2048 Spec: v1.PodSpec{ 2049 TerminationGracePeriodSeconds: &zero, 2050 Containers: []v1.Container{ 2051 { 2052 Name: name, 2053 Image: image, 2054 }, 2055 }, 2056 }, 2057 }, 2058 }, 2059 } 2060 } 2061 2062 // newTestServiceForQuota returns a simple service 2063 func newTestServiceForQuota(name string, serviceType v1.ServiceType, allocateLoadBalancerNodePorts bool) *v1.Service { 2064 var allocateNPs *bool 2065 // Only set allocateLoadBalancerNodePorts when service type is LB 2066 if serviceType == v1.ServiceTypeLoadBalancer { 2067 allocateNPs = &allocateLoadBalancerNodePorts 2068 } 2069 2070 return &v1.Service{ 2071 ObjectMeta: metav1.ObjectMeta{ 2072 Name: name, 2073 }, 2074 Spec: v1.ServiceSpec{ 2075 Type: serviceType, 2076 Ports: []v1.ServicePort{{ 2077 Port: 80, 2078 TargetPort: intstr.FromInt32(80), 2079 }}, 2080 AllocateLoadBalancerNodePorts: allocateNPs, 2081 }, 2082 } 2083 } 2084 2085 func newTestConfigMapForQuota(name string) *v1.ConfigMap { 2086 return &v1.ConfigMap{ 2087 ObjectMeta: metav1.ObjectMeta{ 2088 Name: name, 2089 }, 2090 Data: map[string]string{ 2091 "a": "b", 2092 }, 2093 } 2094 } 2095 2096 func newTestSecretForQuota(name string) *v1.Secret { 2097 return &v1.Secret{ 2098 ObjectMeta: metav1.ObjectMeta{ 2099 Name: name, 2100 }, 2101 Data: map[string][]byte{ 2102 "data-1": []byte("value-1\n"), 2103 "data-2": []byte("value-2\n"), 2104 "data-3": []byte("value-3\n"), 2105 }, 2106 } 2107 } 2108 2109 // createResourceQuota in the specified namespace 2110 func createResourceQuota(ctx context.Context, c clientset.Interface, namespace string, resourceQuota *v1.ResourceQuota) (*v1.ResourceQuota, error) { 2111 return c.CoreV1().ResourceQuotas(namespace).Create(ctx, resourceQuota, metav1.CreateOptions{}) 2112 } 2113 2114 // deleteResourceQuota with the specified name 2115 func deleteResourceQuota(ctx context.Context, c clientset.Interface, namespace, name string) error { 2116 return c.CoreV1().ResourceQuotas(namespace).Delete(ctx, name, metav1.DeleteOptions{}) 2117 } 2118 2119 // countResourceQuota counts the number of ResourceQuota in the specified namespace 2120 // On contended servers the service account controller can slow down, leading to the count changing during a run. 2121 // Wait up to 5s for the count to stabilize, assuming that updates come at a consistent rate, and are not held indefinitely. 2122 func countResourceQuota(ctx context.Context, c clientset.Interface, namespace string) (int, error) { 2123 found, unchanged := 0, 0 2124 return found, wait.PollWithContext(ctx, 1*time.Second, 30*time.Second, func(ctx context.Context) (bool, error) { 2125 resourceQuotas, err := c.CoreV1().ResourceQuotas(namespace).List(ctx, metav1.ListOptions{}) 2126 framework.ExpectNoError(err) 2127 if len(resourceQuotas.Items) == found { 2128 // loop until the number of resource quotas has stabilized for 5 seconds 2129 unchanged++ 2130 return unchanged > 4, nil 2131 } 2132 unchanged = 0 2133 found = len(resourceQuotas.Items) 2134 return false, nil 2135 }) 2136 } 2137 2138 // wait for resource quota status to show the expected used resources value 2139 func waitForResourceQuota(ctx context.Context, c clientset.Interface, ns, quotaName string, used v1.ResourceList) error { 2140 return wait.PollWithContext(ctx, framework.Poll, resourceQuotaTimeout, func(ctx context.Context) (bool, error) { 2141 resourceQuota, err := c.CoreV1().ResourceQuotas(ns).Get(ctx, quotaName, metav1.GetOptions{}) 2142 if err != nil { 2143 return false, err 2144 } 2145 // used may not yet be calculated 2146 if resourceQuota.Status.Used == nil { 2147 return false, nil 2148 } 2149 // verify that the quota shows the expected used resource values 2150 for k, v := range used { 2151 if actualValue, found := resourceQuota.Status.Used[k]; !found || (actualValue.Cmp(v) != 0) { 2152 framework.Logf("resource %s, expected %s, actual %s", k, v.String(), actualValue.String()) 2153 return false, nil 2154 } 2155 } 2156 return true, nil 2157 }) 2158 } 2159 2160 // updateResourceQuotaUntilUsageAppears updates the resource quota object until the usage is populated 2161 // for the specific resource name. 2162 func updateResourceQuotaUntilUsageAppears(ctx context.Context, c clientset.Interface, ns, quotaName string, resourceName v1.ResourceName) error { 2163 return wait.PollWithContext(ctx, framework.Poll, resourceQuotaTimeout, func(ctx context.Context) (bool, error) { 2164 resourceQuota, err := c.CoreV1().ResourceQuotas(ns).Get(ctx, quotaName, metav1.GetOptions{}) 2165 if err != nil { 2166 return false, err 2167 } 2168 // verify that the quota shows the expected used resource values 2169 _, ok := resourceQuota.Status.Used[resourceName] 2170 if ok { 2171 return true, nil 2172 } 2173 2174 current := resourceQuota.Spec.Hard[resourceName] 2175 current.Add(resource.MustParse("1")) 2176 resourceQuota.Spec.Hard[resourceName] = current 2177 _, err = c.CoreV1().ResourceQuotas(ns).Update(ctx, resourceQuota, metav1.UpdateOptions{}) 2178 // ignoring conflicts since someone else may already updated it. 2179 if apierrors.IsConflict(err) { 2180 return false, nil 2181 } 2182 return false, err 2183 }) 2184 } 2185 2186 func unstructuredToResourceQuota(obj *unstructured.Unstructured) (*v1.ResourceQuota, error) { 2187 json, err := runtime.Encode(unstructured.UnstructuredJSONScheme, obj) 2188 if err != nil { 2189 return nil, err 2190 } 2191 rq := &v1.ResourceQuota{} 2192 err = runtime.DecodeInto(clientscheme.Codecs.LegacyCodec(v1.SchemeGroupVersion), json, rq) 2193 2194 return rq, err 2195 }