k8s.io/kubernetes@v1.29.3/test/integration/evictions/evictions_test.go (about) 1 /* 2 Copyright 2015 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package evictions 18 19 import ( 20 "context" 21 "encoding/json" 22 "fmt" 23 "reflect" 24 "strings" 25 "sync" 26 "sync/atomic" 27 "testing" 28 "time" 29 30 v1 "k8s.io/api/core/v1" 31 policyv1 "k8s.io/api/policy/v1" 32 policyv1beta1 "k8s.io/api/policy/v1beta1" 33 apierrors "k8s.io/apimachinery/pkg/api/errors" 34 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 35 "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" 36 "k8s.io/apimachinery/pkg/runtime" 37 "k8s.io/apimachinery/pkg/runtime/schema" 38 "k8s.io/apimachinery/pkg/types" 39 utilerrors "k8s.io/apimachinery/pkg/util/errors" 40 "k8s.io/apimachinery/pkg/util/intstr" 41 "k8s.io/apimachinery/pkg/util/uuid" 42 "k8s.io/apimachinery/pkg/util/wait" 43 "k8s.io/apiserver/pkg/util/feature" 44 cacheddiscovery "k8s.io/client-go/discovery/cached/memory" 45 "k8s.io/client-go/dynamic" 46 "k8s.io/client-go/informers" 47 clientset "k8s.io/client-go/kubernetes" 48 policyv1client "k8s.io/client-go/kubernetes/typed/policy/v1" 49 restclient "k8s.io/client-go/rest" 50 "k8s.io/client-go/restmapper" 51 "k8s.io/client-go/scale" 52 "k8s.io/client-go/tools/cache" 53 featuregatetesting "k8s.io/component-base/featuregate/testing" 54 "k8s.io/klog/v2" 55 kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing" 56 podutil "k8s.io/kubernetes/pkg/api/v1/pod" 57 "k8s.io/kubernetes/pkg/controller/disruption" 58 "k8s.io/kubernetes/pkg/features" 59 "k8s.io/kubernetes/test/integration/framework" 60 "k8s.io/kubernetes/test/utils/ktesting" 61 ) 62 63 const ( 64 numOfEvictions = 10 65 ) 66 67 // TestConcurrentEvictionRequests is to make sure pod disruption budgets (PDB) controller is able to 68 // handle concurrent eviction requests. Original issue:#37605 69 func TestConcurrentEvictionRequests(t *testing.T) { 70 podNameFormat := "test-pod-%d" 71 72 _, ctx := ktesting.NewTestContext(t) 73 ctx, cancel := context.WithCancel(ctx) 74 75 closeFn, rm, informers, _, clientSet := rmSetup(ctx, t) 76 defer closeFn() 77 78 ns := framework.CreateNamespaceOrDie(clientSet, "concurrent-eviction-requests", t) 79 defer framework.DeleteNamespaceOrDie(clientSet, ns, t) 80 defer cancel() 81 82 informers.Start(ctx.Done()) 83 go rm.Run(ctx) 84 85 var gracePeriodSeconds int64 = 30 86 deleteOption := metav1.DeleteOptions{ 87 GracePeriodSeconds: &gracePeriodSeconds, 88 } 89 90 // Generate numOfEvictions pods to evict 91 for i := 0; i < numOfEvictions; i++ { 92 podName := fmt.Sprintf(podNameFormat, i) 93 pod := newPod(podName) 94 95 if _, err := clientSet.CoreV1().Pods(ns.Name).Create(context.TODO(), pod, metav1.CreateOptions{}); err != nil { 96 t.Errorf("Failed to create pod: %v", err) 97 } 98 pod.Status.Phase = v1.PodRunning 99 addPodConditionReady(pod) 100 if _, err := clientSet.CoreV1().Pods(ns.Name).UpdateStatus(context.TODO(), pod, metav1.UpdateOptions{}); err != nil { 101 t.Fatal(err) 102 } 103 } 104 105 waitToObservePods(t, informers.Core().V1().Pods().Informer(), numOfEvictions, v1.PodRunning) 106 107 pdb := newPDB() 108 if _, err := clientSet.PolicyV1().PodDisruptionBudgets(ns.Name).Create(context.TODO(), pdb, metav1.CreateOptions{}); err != nil { 109 t.Errorf("Failed to create PodDisruptionBudget: %v", err) 110 } 111 112 waitPDBStable(t, clientSet, ns.Name, pdb.Name, numOfEvictions) 113 114 var numberPodsEvicted uint32 115 errCh := make(chan error, 3*numOfEvictions) 116 var wg sync.WaitGroup 117 // spawn numOfEvictions goroutines to concurrently evict the pods 118 for i := 0; i < numOfEvictions; i++ { 119 wg.Add(1) 120 go func(id int, errCh chan error) { 121 defer wg.Done() 122 podName := fmt.Sprintf(podNameFormat, id) 123 eviction := newV1Eviction(ns.Name, podName, deleteOption) 124 125 err := wait.PollImmediate(5*time.Second, 60*time.Second, func() (bool, error) { 126 e := clientSet.PolicyV1().Evictions(ns.Name).Evict(context.TODO(), eviction) 127 switch { 128 case apierrors.IsTooManyRequests(e): 129 return false, nil 130 case apierrors.IsConflict(e): 131 return false, fmt.Errorf("Unexpected Conflict (409) error caused by failing to handle concurrent PDB updates: %v", e) 132 case e == nil: 133 return true, nil 134 default: 135 return false, e 136 } 137 }) 138 139 if err != nil { 140 errCh <- err 141 // should not return here otherwise we would leak the pod 142 } 143 144 _, err = clientSet.CoreV1().Pods(ns.Name).Get(context.TODO(), podName, metav1.GetOptions{}) 145 switch { 146 case apierrors.IsNotFound(err): 147 atomic.AddUint32(&numberPodsEvicted, 1) 148 // pod was evicted and deleted so return from goroutine immediately 149 return 150 case err == nil: 151 // this shouldn't happen if the pod was evicted successfully 152 errCh <- fmt.Errorf("Pod %q is expected to be evicted", podName) 153 default: 154 errCh <- err 155 } 156 157 // delete pod which still exists due to error 158 e := clientSet.CoreV1().Pods(ns.Name).Delete(context.TODO(), podName, deleteOption) 159 if e != nil { 160 errCh <- e 161 } 162 163 }(i, errCh) 164 } 165 166 wg.Wait() 167 168 close(errCh) 169 var errList []error 170 if err := clientSet.PolicyV1().PodDisruptionBudgets(ns.Name).Delete(context.TODO(), pdb.Name, deleteOption); err != nil { 171 errList = append(errList, fmt.Errorf("Failed to delete PodDisruptionBudget: %v", err)) 172 } 173 for err := range errCh { 174 errList = append(errList, err) 175 } 176 if len(errList) > 0 { 177 t.Fatal(utilerrors.NewAggregate(errList)) 178 } 179 180 if atomic.LoadUint32(&numberPodsEvicted) != numOfEvictions { 181 t.Fatalf("fewer number of successful evictions than expected : %d", numberPodsEvicted) 182 } 183 } 184 185 // TestTerminalPodEviction ensures that PDB is not checked for terminal pods. 186 func TestTerminalPodEviction(t *testing.T) { 187 _, ctx := ktesting.NewTestContext(t) 188 ctx, cancel := context.WithCancel(ctx) 189 190 closeFn, rm, informers, _, clientSet := rmSetup(ctx, t) 191 defer closeFn() 192 193 ns := framework.CreateNamespaceOrDie(clientSet, "terminalpod-eviction", t) 194 defer framework.DeleteNamespaceOrDie(clientSet, ns, t) 195 defer cancel() 196 197 informers.Start(ctx.Done()) 198 go rm.Run(ctx) 199 200 var gracePeriodSeconds int64 = 30 201 deleteOption := metav1.DeleteOptions{ 202 GracePeriodSeconds: &gracePeriodSeconds, 203 } 204 pod := newPod("test-terminal-pod1") 205 if _, err := clientSet.CoreV1().Pods(ns.Name).Create(context.TODO(), pod, metav1.CreateOptions{}); err != nil { 206 t.Errorf("Failed to create pod: %v", err) 207 } 208 209 pod.Status.Phase = v1.PodSucceeded 210 addPodConditionReady(pod) 211 if _, err := clientSet.CoreV1().Pods(ns.Name).UpdateStatus(context.TODO(), pod, metav1.UpdateOptions{}); err != nil { 212 t.Fatal(err) 213 } 214 215 waitToObservePods(t, informers.Core().V1().Pods().Informer(), 1, v1.PodSucceeded) 216 217 pdb := newPDB() 218 if _, err := clientSet.PolicyV1().PodDisruptionBudgets(ns.Name).Create(context.TODO(), pdb, metav1.CreateOptions{}); err != nil { 219 t.Errorf("Failed to create PodDisruptionBudget: %v", err) 220 } 221 222 waitPDBStable(t, clientSet, ns.Name, pdb.Name, 1) 223 224 pdbList, err := clientSet.PolicyV1().PodDisruptionBudgets(ns.Name).List(context.TODO(), metav1.ListOptions{}) 225 if err != nil { 226 t.Fatalf("Error while listing pod disruption budget") 227 } 228 oldPdb := pdbList.Items[0] 229 eviction := newV1Eviction(ns.Name, pod.Name, deleteOption) 230 err = wait.PollImmediate(5*time.Second, 60*time.Second, func() (bool, error) { 231 e := clientSet.PolicyV1().Evictions(ns.Name).Evict(context.TODO(), eviction) 232 switch { 233 case apierrors.IsTooManyRequests(e): 234 return false, nil 235 case apierrors.IsConflict(e): 236 return false, fmt.Errorf("Unexpected Conflict (409) error caused by failing to handle concurrent PDB updates: %v", e) 237 case e == nil: 238 return true, nil 239 default: 240 return false, e 241 } 242 }) 243 if err != nil { 244 t.Fatalf("Eviction of pod failed %v", err) 245 } 246 pdbList, err = clientSet.PolicyV1().PodDisruptionBudgets(ns.Name).List(context.TODO(), metav1.ListOptions{}) 247 if err != nil { 248 t.Fatalf("Error while listing pod disruption budget") 249 } 250 newPdb := pdbList.Items[0] 251 // We shouldn't see an update in pod disruption budget status' generation number as we are evicting terminal pods without checking for pod disruption. 252 if !reflect.DeepEqual(newPdb.Status.ObservedGeneration, oldPdb.Status.ObservedGeneration) { 253 t.Fatalf("Expected the pdb generation to be of same value %v but got %v", newPdb.Status.ObservedGeneration, oldPdb.Status.ObservedGeneration) 254 } 255 256 if err := clientSet.PolicyV1().PodDisruptionBudgets(ns.Name).Delete(context.TODO(), pdb.Name, deleteOption); err != nil { 257 t.Fatalf("Failed to delete pod disruption budget") 258 } 259 } 260 261 // TestEvictionVersions ensures the eviction endpoint accepts and returns the correct API versions 262 func TestEvictionVersions(t *testing.T) { 263 _, ctx := ktesting.NewTestContext(t) 264 ctx, cancel := context.WithCancel(ctx) 265 266 closeFn, rm, informers, config, clientSet := rmSetup(ctx, t) 267 defer closeFn() 268 defer cancel() 269 270 informers.Start(ctx.Done()) 271 go rm.Run(ctx) 272 273 ns := "default" 274 subresource := "eviction" 275 pod := newPod("test") 276 if _, err := clientSet.CoreV1().Pods(ns).Create(context.TODO(), pod, metav1.CreateOptions{}); err != nil { 277 t.Errorf("Failed to create pod: %v", err) 278 } 279 280 dynamicClient, err := dynamic.NewForConfig(config) 281 if err != nil { 282 t.Fatalf("Failed to create clientset: %v", err) 283 } 284 285 podClient := dynamicClient.Resource(schema.GroupVersionResource{Group: "", Version: "v1", Resource: "pods"}).Namespace(ns) 286 287 // get should not be supported 288 if _, err := podClient.Get(context.TODO(), pod.Name, metav1.GetOptions{}, subresource); !apierrors.IsMethodNotSupported(err) { 289 t.Fatalf("expected MethodNotSupported for GET, got %v", err) 290 } 291 292 // patch should not be supported 293 for _, patchType := range []types.PatchType{types.JSONPatchType, types.MergePatchType, types.StrategicMergePatchType, types.ApplyPatchType} { 294 if _, err := podClient.Patch(context.TODO(), pod.Name, patchType, []byte{}, metav1.PatchOptions{}, subresource); !apierrors.IsMethodNotSupported(err) { 295 t.Fatalf("expected MethodNotSupported for GET, got %v", err) 296 } 297 } 298 299 allowedEvictions := []runtime.Object{ 300 // v1beta1, no apiVersion/kind 301 &policyv1beta1.Eviction{ 302 TypeMeta: metav1.TypeMeta{}, 303 ObjectMeta: metav1.ObjectMeta{Name: pod.Name}, 304 DeleteOptions: &metav1.DeleteOptions{DryRun: []string{metav1.DryRunAll}}, 305 }, 306 // v1beta1, apiVersion/kind 307 &policyv1beta1.Eviction{ 308 TypeMeta: metav1.TypeMeta{APIVersion: "policy/v1beta1", Kind: "Eviction"}, 309 ObjectMeta: metav1.ObjectMeta{Name: pod.Name}, 310 DeleteOptions: &metav1.DeleteOptions{DryRun: []string{metav1.DryRunAll}}, 311 }, 312 // v1, no apiVersion/kind 313 &policyv1.Eviction{ 314 TypeMeta: metav1.TypeMeta{}, 315 ObjectMeta: metav1.ObjectMeta{Name: pod.Name}, 316 DeleteOptions: &metav1.DeleteOptions{DryRun: []string{metav1.DryRunAll}}, 317 }, 318 // v1, apiVersion/kind 319 &policyv1.Eviction{ 320 TypeMeta: metav1.TypeMeta{APIVersion: "policy/v1", Kind: "Eviction"}, 321 ObjectMeta: metav1.ObjectMeta{Name: pod.Name}, 322 DeleteOptions: &metav1.DeleteOptions{DryRun: []string{metav1.DryRunAll}}, 323 }, 324 } 325 v1Status := schema.GroupVersionKind{Version: "v1", Kind: "Status"} 326 for _, allowedEviction := range allowedEvictions { 327 data, _ := json.Marshal(allowedEviction) 328 u := &unstructured.Unstructured{} 329 json.Unmarshal(data, u) 330 result, err := podClient.Create(context.TODO(), u, metav1.CreateOptions{}, subresource) 331 if err != nil { 332 t.Fatalf("error posting %s: %v", string(data), err) 333 } 334 if result.GroupVersionKind() != v1Status { 335 t.Fatalf("expected v1 Status, got %#v", result) 336 } 337 } 338 339 // create unknown eviction version with apiVersion/kind should fail 340 u := &unstructured.Unstructured{Object: map[string]interface{}{ 341 "metadata": map[string]interface{}{"name": pod.Name}, 342 "apiVersion": "policy/v2", 343 "kind": "Eviction", 344 }} 345 if _, err := podClient.Create(context.TODO(), u, metav1.CreateOptions{}, subresource); err == nil { 346 t.Fatal("expected error posting unknown Eviction version, got none") 347 } else if !strings.Contains(err.Error(), "policy/v2") { 348 t.Fatalf("expected error about policy/v2, got %#v", err) 349 } 350 } 351 352 // TestEvictionWithFinalizers tests eviction with the use of finalizers 353 func TestEvictionWithFinalizers(t *testing.T) { 354 cases := map[string]struct { 355 enablePodDisruptionConditions bool 356 phase v1.PodPhase 357 dryRun bool 358 wantDisruptionTargetCond bool 359 }{ 360 "terminal pod with PodDisruptionConditions enabled": { 361 enablePodDisruptionConditions: true, 362 phase: v1.PodSucceeded, 363 wantDisruptionTargetCond: true, 364 }, 365 "terminal pod with PodDisruptionConditions disabled": { 366 enablePodDisruptionConditions: false, 367 phase: v1.PodSucceeded, 368 wantDisruptionTargetCond: false, 369 }, 370 "running pod with PodDisruptionConditions enabled": { 371 enablePodDisruptionConditions: true, 372 phase: v1.PodRunning, 373 wantDisruptionTargetCond: true, 374 }, 375 "running pod with PodDisruptionConditions disabled": { 376 enablePodDisruptionConditions: false, 377 phase: v1.PodRunning, 378 wantDisruptionTargetCond: false, 379 }, 380 "running pod with PodDisruptionConditions enabled should not update conditions in dry-run mode": { 381 enablePodDisruptionConditions: true, 382 phase: v1.PodRunning, 383 dryRun: true, 384 wantDisruptionTargetCond: false, 385 }, 386 } 387 for name, tc := range cases { 388 t.Run(name, func(t *testing.T) { 389 _, ctx := ktesting.NewTestContext(t) 390 ctx, cancel := context.WithCancel(ctx) 391 392 closeFn, rm, informers, _, clientSet := rmSetup(ctx, t) 393 defer closeFn() 394 395 ns := framework.CreateNamespaceOrDie(clientSet, "eviction-with-finalizers", t) 396 defer framework.DeleteNamespaceOrDie(clientSet, ns, t) 397 defer featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, features.PodDisruptionConditions, tc.enablePodDisruptionConditions)() 398 defer cancel() 399 400 informers.Start(ctx.Done()) 401 go rm.Run(ctx) 402 403 pod := newPod("pod") 404 pod.ObjectMeta.Finalizers = []string{"test.k8s.io/finalizer"} 405 if _, err := clientSet.CoreV1().Pods(ns.Name).Create(ctx, pod, metav1.CreateOptions{}); err != nil { 406 t.Errorf("Failed to create pod: %v", err) 407 } 408 409 pod.Status.Phase = tc.phase 410 addPodConditionReady(pod) 411 if _, err := clientSet.CoreV1().Pods(ns.Name).UpdateStatus(ctx, pod, metav1.UpdateOptions{}); err != nil { 412 t.Fatal(err) 413 } 414 415 waitToObservePods(t, informers.Core().V1().Pods().Informer(), 1, tc.phase) 416 deleteOption := metav1.DeleteOptions{} 417 if tc.dryRun { 418 deleteOption.DryRun = []string{metav1.DryRunAll} 419 } 420 421 eviction := newV1Eviction(ns.Name, pod.Name, deleteOption) 422 423 err := clientSet.PolicyV1().Evictions(ns.Name).Evict(ctx, eviction) 424 if err != nil { 425 t.Fatalf("Eviction of pod failed %v", err) 426 } 427 428 updatedPod, e := clientSet.CoreV1().Pods(ns.Name).Get(ctx, pod.Name, metav1.GetOptions{}) 429 if e != nil { 430 t.Fatalf("Failed to get the pod %q with error: %q", klog.KObj(pod), e) 431 } 432 _, cond := podutil.GetPodCondition(&updatedPod.Status, v1.PodConditionType(v1.DisruptionTarget)) 433 if tc.wantDisruptionTargetCond && cond == nil { 434 t.Errorf("Pod %q does not have the expected condition: %q", klog.KObj(updatedPod), v1.DisruptionTarget) 435 } else if !tc.wantDisruptionTargetCond && cond != nil { 436 t.Errorf("Pod %q has an unexpected condition: %q", klog.KObj(updatedPod), v1.DisruptionTarget) 437 } 438 }) 439 } 440 } 441 442 // TestEvictionWithUnhealthyPodEvictionPolicy tests eviction with a PDB that has a UnhealthyPodEvictionPolicy 443 func TestEvictionWithUnhealthyPodEvictionPolicy(t *testing.T) { 444 cases := map[string]struct { 445 enableUnhealthyPodEvictionPolicy bool 446 unhealthyPodEvictionPolicy *policyv1.UnhealthyPodEvictionPolicyType 447 isPodReady bool 448 }{ 449 "UnhealthyPodEvictionPolicy disabled and policy not set": { 450 enableUnhealthyPodEvictionPolicy: false, 451 unhealthyPodEvictionPolicy: nil, 452 isPodReady: true, 453 }, 454 "UnhealthyPodEvictionPolicy enabled but policy not set": { 455 enableUnhealthyPodEvictionPolicy: true, 456 unhealthyPodEvictionPolicy: nil, 457 isPodReady: true, 458 }, 459 "UnhealthyPodEvictionPolicy enabled but policy set to IfHealthyBudget with ready pod": { 460 enableUnhealthyPodEvictionPolicy: true, 461 unhealthyPodEvictionPolicy: unhealthyPolicyPtr(policyv1.IfHealthyBudget), 462 isPodReady: true, 463 }, 464 "UnhealthyPodEvictionPolicy enabled but policy set to AlwaysAllow with ready pod": { 465 enableUnhealthyPodEvictionPolicy: true, 466 unhealthyPodEvictionPolicy: unhealthyPolicyPtr(policyv1.AlwaysAllow), 467 isPodReady: true, 468 }, 469 "UnhealthyPodEvictionPolicy enabled but policy set to AlwaysAllow with unready pod": { 470 enableUnhealthyPodEvictionPolicy: true, 471 unhealthyPodEvictionPolicy: unhealthyPolicyPtr(policyv1.AlwaysAllow), 472 isPodReady: false, 473 }, 474 } 475 for name, tc := range cases { 476 t.Run(name, func(t *testing.T) { 477 _, ctx := ktesting.NewTestContext(t) 478 ctx, cancel := context.WithCancel(ctx) 479 480 defer featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, features.PDBUnhealthyPodEvictionPolicy, tc.enableUnhealthyPodEvictionPolicy)() 481 closeFn, rm, informers, _, clientSet := rmSetup(ctx, t) 482 defer closeFn() 483 484 ns := framework.CreateNamespaceOrDie(clientSet, "eviction-with-pdb-pod-healthy-policy", t) 485 defer framework.DeleteNamespaceOrDie(clientSet, ns, t) 486 defer cancel() 487 488 informers.Start(ctx.Done()) 489 go rm.Run(ctx) 490 491 pod := newPod("pod") 492 if _, err := clientSet.CoreV1().Pods(ns.Name).Create(context.TODO(), pod, metav1.CreateOptions{}); err != nil { 493 t.Errorf("Failed to create pod: %v", err) 494 } 495 496 pod.Status.Phase = v1.PodRunning 497 if tc.isPodReady { 498 addPodConditionReady(pod) 499 } 500 501 if _, err := clientSet.CoreV1().Pods(ns.Name).UpdateStatus(ctx, pod, metav1.UpdateOptions{}); err != nil { 502 t.Fatal(err) 503 } 504 505 waitToObservePods(t, informers.Core().V1().Pods().Informer(), 1, v1.PodRunning) 506 507 pdb := newPDB() 508 pdb.Spec.UnhealthyPodEvictionPolicy = tc.unhealthyPodEvictionPolicy 509 if _, err := clientSet.PolicyV1().PodDisruptionBudgets(ns.Name).Create(context.TODO(), pdb, metav1.CreateOptions{}); err != nil { 510 t.Errorf("Failed to create PodDisruptionBudget: %v", err) 511 } 512 513 if tc.isPodReady { 514 waitPDBStable(t, clientSet, ns.Name, pdb.Name, 1) 515 } else { 516 waitPDB(t, clientSet, ns.Name, pdb.Name, func(pdb *policyv1.PodDisruptionBudget) bool { 517 return pdb.Status.ExpectedPods == 1 518 }) 519 } 520 // Eviction API can potentially return http.StatusTooManyRequests (429) or http.StatusGatewayTimeout (504) with retryAfterSeconds == 10s 521 // Do not retry - we want to test that the first request succeeds and make sure it doesn't unnecessarily block the test for 10s 522 policyV1NoRetriesRESTClient := &noRetriesRESTClient{Interface: clientSet.PolicyV1().RESTClient()} 523 policyV1NoRetriesClient := policyv1client.New(policyV1NoRetriesRESTClient) 524 525 deleteOption := metav1.DeleteOptions{} 526 eviction := newV1Eviction(ns.Name, pod.Name, deleteOption) 527 err := policyV1NoRetriesClient.Evictions(ns.Name).Evict(ctx, eviction) 528 if err != nil { 529 t.Fatalf("Eviction of pod failed %v", err) 530 } 531 if policyV1NoRetriesRESTClient.postCalls != 1 { 532 t.Fatalf("expected a single POST call, got %d", policyV1NoRetriesRESTClient.postCalls) 533 } 534 535 waitToObservePods(t, informers.Core().V1().Pods().Informer(), 0, v1.PodRunning) 536 waitPDBStable(t, clientSet, ns.Name, pdb.Name, 0) 537 }) 538 } 539 } 540 541 // TestEvictionWithPrecondition tests eviction with delete preconditions 542 func TestEvictionWithPrecondition(t *testing.T) { 543 cases := map[string]struct { 544 enforceResourceVersion bool 545 injectWrongResourceVersion bool 546 enforceUID bool 547 injectWrongUID bool 548 shouldErr bool 549 }{ 550 "eviction enforcing resource version": { 551 enforceResourceVersion: true, 552 }, 553 "eviction enforcing UID": { 554 enforceUID: true, 555 }, 556 "eviction enforcing resource version and UID": { 557 enforceUID: true, 558 enforceResourceVersion: true, 559 }, 560 "eviction enforcing wrong resource version should fail": { 561 enforceResourceVersion: true, 562 injectWrongResourceVersion: true, 563 shouldErr: true, 564 }, 565 "eviction enforcing wrong UID should fail": { 566 enforceUID: true, 567 injectWrongUID: true, 568 shouldErr: true, 569 }, 570 } 571 for name, tc := range cases { 572 t.Run(name, func(t *testing.T) { 573 _, ctx := ktesting.NewTestContext(t) 574 ctx, cancel := context.WithCancel(ctx) 575 576 closeFn, rm, informers, _, clientSet := rmSetup(ctx, t) 577 defer closeFn() 578 579 ns := framework.CreateNamespaceOrDie(clientSet, "eviction-with-preconditions", t) 580 defer framework.DeleteNamespaceOrDie(clientSet, ns, t) 581 582 defer cancel() 583 informers.Start(ctx.Done()) 584 go rm.Run(ctx) 585 586 pod := newPod("pod") 587 pod, err := clientSet.CoreV1().Pods(ns.Name).Create(context.TODO(), pod, metav1.CreateOptions{}) 588 if err != nil { 589 t.Errorf("Failed to create pod: %q", err) 590 } 591 592 pod.Status.Phase = v1.PodRunning 593 addPodConditionReady(pod) 594 595 // generate a new resource version 596 updatedPod, err := clientSet.CoreV1().Pods(ns.Name).UpdateStatus(ctx, pod, metav1.UpdateOptions{}) 597 if err != nil { 598 t.Fatal(err) 599 } 600 601 waitToObservePods(t, informers.Core().V1().Pods().Informer(), 1, v1.PodRunning) 602 603 deleteOption := metav1.DeleteOptions{} 604 605 if tc.enforceResourceVersion || tc.enforceUID { 606 deleteOption.Preconditions = &metav1.Preconditions{} 607 } 608 609 if tc.enforceResourceVersion { 610 if tc.injectWrongResourceVersion { 611 deleteOption.Preconditions.ResourceVersion = &pod.ResourceVersion 612 } else { 613 deleteOption.Preconditions.ResourceVersion = &updatedPod.ResourceVersion 614 } 615 616 } 617 if tc.enforceUID { 618 if tc.injectWrongUID { 619 newUID := uuid.NewUUID() 620 deleteOption.Preconditions.UID = &newUID 621 } else { 622 deleteOption.Preconditions.UID = &updatedPod.UID 623 } 624 } 625 626 // Eviction API can potentially return http.StatusTooManyRequests (429) or http.StatusGatewayTimeout (504) with retryAfterSeconds == 10s 627 // Do not retry - we want to test that the first request succeeds and make sure it doesn't unnecessarily block the test for 10s 628 policyV1NoRetriesRESTClient := &noRetriesRESTClient{Interface: clientSet.PolicyV1().RESTClient()} 629 policyV1NoRetriesClient := policyv1client.New(policyV1NoRetriesRESTClient) 630 631 eviction := newV1Eviction(ns.Name, updatedPod.Name, deleteOption) 632 err = policyV1NoRetriesClient.Evictions(ns.Name).Evict(ctx, eviction) 633 if err != nil && !tc.shouldErr { 634 t.Fatalf("Eviction of pod failed %q", err) 635 } 636 if err == nil && tc.shouldErr { 637 t.Fatal("Eviction of pod should fail") 638 } 639 if policyV1NoRetriesRESTClient.postCalls != 1 { 640 t.Fatalf("expected a single POST call, got %d", policyV1NoRetriesRESTClient.postCalls) 641 } 642 }) 643 } 644 } 645 646 func newPod(podName string) *v1.Pod { 647 return &v1.Pod{ 648 ObjectMeta: metav1.ObjectMeta{ 649 Name: podName, 650 Labels: map[string]string{"app": "test-evictions"}, 651 }, 652 Spec: v1.PodSpec{ 653 Containers: []v1.Container{ 654 { 655 Name: "fake-name", 656 Image: "fakeimage", 657 }, 658 }, 659 }, 660 } 661 } 662 663 func addPodConditionReady(pod *v1.Pod) { 664 pod.Status.Conditions = append(pod.Status.Conditions, v1.PodCondition{ 665 Type: v1.PodReady, 666 Status: v1.ConditionTrue, 667 }) 668 } 669 670 func newPDB() *policyv1.PodDisruptionBudget { 671 return &policyv1.PodDisruptionBudget{ 672 ObjectMeta: metav1.ObjectMeta{ 673 Name: "test-pdb", 674 }, 675 Spec: policyv1.PodDisruptionBudgetSpec{ 676 MinAvailable: &intstr.IntOrString{ 677 Type: intstr.Int, 678 IntVal: 0, 679 }, 680 Selector: &metav1.LabelSelector{ 681 MatchLabels: map[string]string{"app": "test-evictions"}, 682 }, 683 }, 684 } 685 } 686 687 func newV1Eviction(ns, evictionName string, deleteOption metav1.DeleteOptions) *policyv1.Eviction { 688 return &policyv1.Eviction{ 689 TypeMeta: metav1.TypeMeta{ 690 APIVersion: "policy/v1", 691 Kind: "Eviction", 692 }, 693 ObjectMeta: metav1.ObjectMeta{ 694 Name: evictionName, 695 Namespace: ns, 696 }, 697 DeleteOptions: &deleteOption, 698 } 699 } 700 701 func rmSetup(ctx context.Context, t *testing.T) (kubeapiservertesting.TearDownFunc, *disruption.DisruptionController, informers.SharedInformerFactory, *restclient.Config, clientset.Interface) { 702 // Disable ServiceAccount admission plugin as we don't have serviceaccount controller running. 703 server := kubeapiservertesting.StartTestServerOrDie(t, nil, []string{"--disable-admission-plugins=ServiceAccount"}, framework.SharedEtcd()) 704 705 config := restclient.CopyConfig(server.ClientConfig) 706 clientSet, err := clientset.NewForConfig(config) 707 if err != nil { 708 t.Fatalf("Error in create clientset: %v", err) 709 } 710 resyncPeriod := 12 * time.Hour 711 informers := informers.NewSharedInformerFactory(clientset.NewForConfigOrDie(restclient.AddUserAgent(config, "pdb-informers")), resyncPeriod) 712 713 client := clientset.NewForConfigOrDie(restclient.AddUserAgent(config, "disruption-controller")) 714 715 discoveryClient := cacheddiscovery.NewMemCacheClient(clientSet.Discovery()) 716 mapper := restmapper.NewDeferredDiscoveryRESTMapper(discoveryClient) 717 718 scaleKindResolver := scale.NewDiscoveryScaleKindResolver(client.Discovery()) 719 scaleClient, err := scale.NewForConfig(config, mapper, dynamic.LegacyAPIPathResolverFunc, scaleKindResolver) 720 if err != nil { 721 t.Fatalf("Error in create scaleClient: %v", err) 722 } 723 724 rm := disruption.NewDisruptionController( 725 ctx, 726 informers.Core().V1().Pods(), 727 informers.Policy().V1().PodDisruptionBudgets(), 728 informers.Core().V1().ReplicationControllers(), 729 informers.Apps().V1().ReplicaSets(), 730 informers.Apps().V1().Deployments(), 731 informers.Apps().V1().StatefulSets(), 732 client, 733 mapper, 734 scaleClient, 735 client.Discovery(), 736 ) 737 return server.TearDownFn, rm, informers, config, clientSet 738 } 739 740 // wait for the podInformer to observe the pods. Call this function before 741 // running the RS controller to prevent the rc manager from creating new pods 742 // rather than adopting the existing ones. 743 func waitToObservePods(t *testing.T, podInformer cache.SharedIndexInformer, podNum int, phase v1.PodPhase) { 744 if err := wait.PollImmediate(2*time.Second, 60*time.Second, func() (bool, error) { 745 objects := podInformer.GetIndexer().List() 746 if len(objects) != podNum { 747 return false, nil 748 } 749 for _, obj := range objects { 750 pod := obj.(*v1.Pod) 751 if pod.Status.Phase != phase { 752 return false, nil 753 } 754 } 755 return true, nil 756 }); err != nil { 757 t.Fatal(err) 758 } 759 } 760 761 func waitPDBStable(t *testing.T, clientSet clientset.Interface, ns, pdbName string, podNum int32) { 762 waitPDB(t, clientSet, ns, pdbName, func(pdb *policyv1.PodDisruptionBudget) bool { 763 return pdb.Status.CurrentHealthy == podNum 764 }) 765 } 766 767 func waitPDB(t *testing.T, clientSet clientset.Interface, ns, pdbName string, condition func(budget *policyv1.PodDisruptionBudget) bool) { 768 if err := wait.PollImmediate(2*time.Second, 60*time.Second, func() (bool, error) { 769 pdb, err := clientSet.PolicyV1().PodDisruptionBudgets(ns).Get(context.TODO(), pdbName, metav1.GetOptions{}) 770 if err != nil { 771 return false, err 772 } 773 return condition(pdb), nil 774 }); err != nil { 775 t.Fatal(err) 776 } 777 } 778 779 func unhealthyPolicyPtr(unhealthyPodEvictionPolicy policyv1.UnhealthyPodEvictionPolicyType) *policyv1.UnhealthyPodEvictionPolicyType { 780 return &unhealthyPodEvictionPolicy 781 } 782 783 type noRetriesRESTClient struct { 784 mu sync.Mutex 785 postCalls int 786 restclient.Interface 787 } 788 789 func (n *noRetriesRESTClient) Post() *restclient.Request { 790 n.mu.Lock() 791 defer n.mu.Unlock() 792 n.postCalls++ 793 return n.Interface.Post().MaxRetries(0) 794 }