k8s.io/kubernetes@v1.29.3/test/integration/replicationcontroller/replicationcontroller_test.go (about) 1 /* 2 Copyright 2015 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package replicationcontroller 18 19 import ( 20 "context" 21 "fmt" 22 "reflect" 23 "testing" 24 "time" 25 26 v1 "k8s.io/api/core/v1" 27 apiequality "k8s.io/apimachinery/pkg/api/equality" 28 apierrors "k8s.io/apimachinery/pkg/api/errors" 29 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 30 "k8s.io/apimachinery/pkg/labels" 31 "k8s.io/apimachinery/pkg/util/uuid" 32 "k8s.io/apimachinery/pkg/util/wait" 33 utilfeature "k8s.io/apiserver/pkg/util/feature" 34 "k8s.io/client-go/informers" 35 clientset "k8s.io/client-go/kubernetes" 36 typedv1 "k8s.io/client-go/kubernetes/typed/core/v1" 37 restclient "k8s.io/client-go/rest" 38 "k8s.io/client-go/tools/cache" 39 "k8s.io/client-go/util/retry" 40 featuregatetesting "k8s.io/component-base/featuregate/testing" 41 "k8s.io/klog/v2/ktesting" 42 kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing" 43 podutil "k8s.io/kubernetes/pkg/api/v1/pod" 44 "k8s.io/kubernetes/pkg/controller/replication" 45 "k8s.io/kubernetes/pkg/features" 46 "k8s.io/kubernetes/test/integration/framework" 47 ) 48 49 const ( 50 interval = 100 * time.Millisecond 51 timeout = 60 * time.Second 52 ) 53 54 func labelMap() map[string]string { 55 return map[string]string{"foo": "bar"} 56 } 57 58 func newRC(name, namespace string, replicas int) *v1.ReplicationController { 59 replicasCopy := int32(replicas) 60 return &v1.ReplicationController{ 61 TypeMeta: metav1.TypeMeta{ 62 Kind: "ReplicationController", 63 APIVersion: "v1", 64 }, 65 ObjectMeta: metav1.ObjectMeta{ 66 Namespace: namespace, 67 Name: name, 68 }, 69 Spec: v1.ReplicationControllerSpec{ 70 Selector: labelMap(), 71 Replicas: &replicasCopy, 72 Template: &v1.PodTemplateSpec{ 73 ObjectMeta: metav1.ObjectMeta{ 74 Labels: labelMap(), 75 }, 76 Spec: v1.PodSpec{ 77 Containers: []v1.Container{ 78 { 79 Name: "fake-name", 80 Image: "fakeimage", 81 }, 82 }, 83 }, 84 }, 85 }, 86 } 87 } 88 89 func newMatchingPod(podName, namespace string) *v1.Pod { 90 return &v1.Pod{ 91 TypeMeta: metav1.TypeMeta{ 92 Kind: "Pod", 93 APIVersion: "v1", 94 }, 95 ObjectMeta: metav1.ObjectMeta{ 96 Name: podName, 97 Namespace: namespace, 98 Labels: labelMap(), 99 }, 100 Spec: v1.PodSpec{ 101 Containers: []v1.Container{ 102 { 103 Name: "fake-name", 104 Image: "fakeimage", 105 }, 106 }, 107 }, 108 Status: v1.PodStatus{ 109 Phase: v1.PodRunning, 110 }, 111 } 112 } 113 114 func rmSetup(t *testing.T) (kubeapiservertesting.TearDownFunc, *replication.ReplicationManager, informers.SharedInformerFactory, clientset.Interface) { 115 // Disable ServiceAccount admission plugin as we don't have serviceaccount controller running. 116 server := kubeapiservertesting.StartTestServerOrDie(t, nil, []string{"--disable-admission-plugins=ServiceAccount"}, framework.SharedEtcd()) 117 118 config := restclient.CopyConfig(server.ClientConfig) 119 clientSet, err := clientset.NewForConfig(config) 120 if err != nil { 121 t.Fatalf("Error in create clientset: %v", err) 122 } 123 resyncPeriod := 12 * time.Hour 124 informers := informers.NewSharedInformerFactory(clientset.NewForConfigOrDie(restclient.AddUserAgent(config, "rc-informers")), resyncPeriod) 125 126 logger, _ := ktesting.NewTestContext(t) 127 rm := replication.NewReplicationManager( 128 logger, 129 informers.Core().V1().Pods(), 130 informers.Core().V1().ReplicationControllers(), 131 clientset.NewForConfigOrDie(restclient.AddUserAgent(config, "replication-controller")), 132 replication.BurstReplicas, 133 ) 134 135 return server.TearDownFn, rm, informers, clientSet 136 } 137 138 // Run RC controller and informers 139 func runControllerAndInformers(t *testing.T, rm *replication.ReplicationManager, informers informers.SharedInformerFactory, podNum int) func() { 140 ctx, cancelFn := context.WithCancel(context.Background()) 141 informers.Start(ctx.Done()) 142 waitToObservePods(t, informers.Core().V1().Pods().Informer(), podNum) 143 go rm.Run(ctx, 5) 144 return cancelFn 145 } 146 147 // wait for the podInformer to observe the pods. Call this function before 148 // running the RC controller to prevent the rc manager from creating new pods 149 // rather than adopting the existing ones. 150 func waitToObservePods(t *testing.T, podInformer cache.SharedIndexInformer, podNum int) { 151 if err := wait.PollImmediate(interval, timeout, func() (bool, error) { 152 objects := podInformer.GetIndexer().List() 153 return len(objects) == podNum, nil 154 }); err != nil { 155 t.Fatalf("Error encountered when waiting for podInformer to observe the pods: %v", err) 156 } 157 } 158 159 func createRCsPods(t *testing.T, clientSet clientset.Interface, rcs []*v1.ReplicationController, pods []*v1.Pod) ([]*v1.ReplicationController, []*v1.Pod) { 160 var createdRCs []*v1.ReplicationController 161 var createdPods []*v1.Pod 162 for _, rc := range rcs { 163 createdRC, err := clientSet.CoreV1().ReplicationControllers(rc.Namespace).Create(context.TODO(), rc, metav1.CreateOptions{}) 164 if err != nil { 165 t.Fatalf("Failed to create replication controller %s: %v", rc.Name, err) 166 } 167 createdRCs = append(createdRCs, createdRC) 168 } 169 for _, pod := range pods { 170 createdPod, err := clientSet.CoreV1().Pods(pod.Namespace).Create(context.TODO(), pod, metav1.CreateOptions{}) 171 if err != nil { 172 t.Fatalf("Failed to create pod %s: %v", pod.Name, err) 173 } 174 createdPods = append(createdPods, createdPod) 175 } 176 177 return createdRCs, createdPods 178 } 179 180 // Verify .Status.Replicas is equal to .Spec.Replicas 181 func waitRCStable(t *testing.T, clientSet clientset.Interface, rc *v1.ReplicationController) { 182 rcClient := clientSet.CoreV1().ReplicationControllers(rc.Namespace) 183 if err := wait.PollImmediate(interval, timeout, func() (bool, error) { 184 newRC, err := rcClient.Get(context.TODO(), rc.Name, metav1.GetOptions{}) 185 if err != nil { 186 return false, err 187 } 188 return newRC.Status.Replicas == *rc.Spec.Replicas, nil 189 }); err != nil { 190 t.Fatalf("Failed to verify .Status.Replicas is equal to .Spec.Replicas for rc %s: %v", rc.Name, err) 191 } 192 } 193 194 // Update .Spec.Replicas to replicas and verify .Status.Replicas is changed accordingly 195 func scaleRC(t *testing.T, c clientset.Interface, rc *v1.ReplicationController, replicas int32) { 196 rcClient := c.CoreV1().ReplicationControllers(rc.Namespace) 197 rc = updateRC(t, rcClient, rc.Name, func(rc *v1.ReplicationController) { 198 *rc.Spec.Replicas = replicas 199 }) 200 waitRCStable(t, c, rc) 201 } 202 203 func updatePod(t *testing.T, podClient typedv1.PodInterface, podName string, updateFunc func(*v1.Pod)) *v1.Pod { 204 var pod *v1.Pod 205 if err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { 206 newPod, err := podClient.Get(context.TODO(), podName, metav1.GetOptions{}) 207 if err != nil { 208 return err 209 } 210 updateFunc(newPod) 211 pod, err = podClient.Update(context.TODO(), newPod, metav1.UpdateOptions{}) 212 return err 213 }); err != nil { 214 t.Fatalf("Failed to update pod %s: %v", podName, err) 215 } 216 return pod 217 } 218 219 func updatePodStatus(t *testing.T, podClient typedv1.PodInterface, pod *v1.Pod, updateStatusFunc func(*v1.Pod)) { 220 if err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { 221 newPod, err := podClient.Get(context.TODO(), pod.Name, metav1.GetOptions{}) 222 if err != nil { 223 return err 224 } 225 updateStatusFunc(newPod) 226 _, err = podClient.UpdateStatus(context.TODO(), newPod, metav1.UpdateOptions{}) 227 return err 228 }); err != nil { 229 t.Fatalf("Failed to update status of pod %s: %v", pod.Name, err) 230 } 231 } 232 233 func getPods(t *testing.T, podClient typedv1.PodInterface, labelMap map[string]string) *v1.PodList { 234 podSelector := labels.Set(labelMap).AsSelector() 235 options := metav1.ListOptions{LabelSelector: podSelector.String()} 236 pods, err := podClient.List(context.TODO(), options) 237 if err != nil { 238 t.Fatalf("Failed obtaining a list of pods that match the pod labels %v: %v", labelMap, err) 239 } 240 return pods 241 } 242 243 func updateRC(t *testing.T, rcClient typedv1.ReplicationControllerInterface, rcName string, updateFunc func(*v1.ReplicationController)) *v1.ReplicationController { 244 var rc *v1.ReplicationController 245 if err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { 246 newRC, err := rcClient.Get(context.TODO(), rcName, metav1.GetOptions{}) 247 if err != nil { 248 return err 249 } 250 updateFunc(newRC) 251 rc, err = rcClient.Update(context.TODO(), newRC, metav1.UpdateOptions{}) 252 return err 253 }); err != nil { 254 t.Fatalf("Failed to update rc %s: %v", rcName, err) 255 } 256 return rc 257 } 258 259 // Verify ControllerRef of a RC pod that has incorrect attributes is automatically patched by the RC 260 func testPodControllerRefPatch(t *testing.T, c clientset.Interface, pod *v1.Pod, ownerReference *metav1.OwnerReference, rc *v1.ReplicationController, expectedOwnerReferenceNum int) { 261 ns := rc.Namespace 262 podClient := c.CoreV1().Pods(ns) 263 updatePod(t, podClient, pod.Name, func(pod *v1.Pod) { 264 pod.OwnerReferences = []metav1.OwnerReference{*ownerReference} 265 }) 266 267 if err := wait.PollImmediate(interval, timeout, func() (bool, error) { 268 newPod, err := podClient.Get(context.TODO(), pod.Name, metav1.GetOptions{}) 269 if err != nil { 270 return false, err 271 } 272 return metav1.GetControllerOf(newPod) != nil, nil 273 }); err != nil { 274 t.Fatalf("Failed to verify ControllerRef for the pod %s is not nil: %v", pod.Name, err) 275 } 276 277 newPod, err := podClient.Get(context.TODO(), pod.Name, metav1.GetOptions{}) 278 if err != nil { 279 t.Fatalf("Failed to obtain pod %s: %v", pod.Name, err) 280 } 281 controllerRef := metav1.GetControllerOf(newPod) 282 if controllerRef.UID != rc.UID { 283 t.Fatalf("RC owner of the pod %s has a different UID: Expected %v, got %v", newPod.Name, rc.UID, controllerRef.UID) 284 } 285 ownerReferenceNum := len(newPod.GetOwnerReferences()) 286 if ownerReferenceNum != expectedOwnerReferenceNum { 287 t.Fatalf("Unexpected number of owner references for pod %s: Expected %d, got %d", newPod.Name, expectedOwnerReferenceNum, ownerReferenceNum) 288 } 289 } 290 291 func setPodsReadyCondition(t *testing.T, clientSet clientset.Interface, pods *v1.PodList, conditionStatus v1.ConditionStatus, lastTransitionTime time.Time) { 292 replicas := int32(len(pods.Items)) 293 var readyPods int32 294 err := wait.PollImmediate(interval, timeout, func() (bool, error) { 295 readyPods = 0 296 for i := range pods.Items { 297 pod := &pods.Items[i] 298 if podutil.IsPodReady(pod) { 299 readyPods++ 300 continue 301 } 302 pod.Status.Phase = v1.PodRunning 303 _, condition := podutil.GetPodCondition(&pod.Status, v1.PodReady) 304 if condition != nil { 305 condition.Status = conditionStatus 306 condition.LastTransitionTime = metav1.Time{Time: lastTransitionTime} 307 } else { 308 condition = &v1.PodCondition{ 309 Type: v1.PodReady, 310 Status: conditionStatus, 311 LastTransitionTime: metav1.Time{Time: lastTransitionTime}, 312 } 313 pod.Status.Conditions = append(pod.Status.Conditions, *condition) 314 } 315 _, err := clientSet.CoreV1().Pods(pod.Namespace).UpdateStatus(context.TODO(), pod, metav1.UpdateOptions{}) 316 if err != nil { 317 // When status fails to be updated, we continue to next pod 318 continue 319 } 320 readyPods++ 321 } 322 return readyPods >= replicas, nil 323 }) 324 if err != nil { 325 t.Fatalf("failed to mark all ReplicationController pods to ready: %v", err) 326 } 327 } 328 329 func testScalingUsingScaleSubresource(t *testing.T, c clientset.Interface, rc *v1.ReplicationController, replicas int32) { 330 ns := rc.Namespace 331 rcClient := c.CoreV1().ReplicationControllers(ns) 332 newRC, err := rcClient.Get(context.TODO(), rc.Name, metav1.GetOptions{}) 333 if err != nil { 334 t.Fatalf("Failed to obtain rc %s: %v", rc.Name, err) 335 } 336 scale, err := c.CoreV1().ReplicationControllers(ns).GetScale(context.TODO(), rc.Name, metav1.GetOptions{}) 337 if err != nil { 338 t.Fatalf("Failed to obtain scale subresource for rc %s: %v", rc.Name, err) 339 } 340 if scale.Spec.Replicas != *newRC.Spec.Replicas { 341 t.Fatalf("Scale subresource for rc %s does not match .Spec.Replicas: expected %d, got %d", rc.Name, *newRC.Spec.Replicas, scale.Spec.Replicas) 342 } 343 344 if err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { 345 scale, err := c.CoreV1().ReplicationControllers(ns).GetScale(context.TODO(), rc.Name, metav1.GetOptions{}) 346 if err != nil { 347 return err 348 } 349 scale.Spec.Replicas = replicas 350 _, err = c.CoreV1().ReplicationControllers(ns).UpdateScale(context.TODO(), rc.Name, scale, metav1.UpdateOptions{}) 351 return err 352 }); err != nil { 353 t.Fatalf("Failed to set .Spec.Replicas of scale subresource for rc %s: %v", rc.Name, err) 354 } 355 356 newRC, err = rcClient.Get(context.TODO(), rc.Name, metav1.GetOptions{}) 357 if err != nil { 358 t.Fatalf("Failed to obtain rc %s: %v", rc.Name, err) 359 } 360 if *newRC.Spec.Replicas != replicas { 361 t.Fatalf(".Spec.Replicas of rc %s does not match its scale subresource: expected %d, got %d", rc.Name, replicas, *newRC.Spec.Replicas) 362 } 363 } 364 365 func TestAdoption(t *testing.T) { 366 boolPtr := func(b bool) *bool { return &b } 367 testCases := []struct { 368 name string 369 existingOwnerReferences func(rc *v1.ReplicationController) []metav1.OwnerReference 370 expectedOwnerReferences func(rc *v1.ReplicationController) []metav1.OwnerReference 371 }{ 372 { 373 "pod refers rc as an owner, not a controller", 374 func(rc *v1.ReplicationController) []metav1.OwnerReference { 375 return []metav1.OwnerReference{{UID: rc.UID, Name: rc.Name, APIVersion: "v1", Kind: "ReplicationController"}} 376 }, 377 func(rc *v1.ReplicationController) []metav1.OwnerReference { 378 return []metav1.OwnerReference{{UID: rc.UID, Name: rc.Name, APIVersion: "v1", Kind: "ReplicationController", Controller: boolPtr(true), BlockOwnerDeletion: boolPtr(true)}} 379 }, 380 }, 381 { 382 "pod doesn't have owner references", 383 func(rc *v1.ReplicationController) []metav1.OwnerReference { 384 return []metav1.OwnerReference{} 385 }, 386 func(rc *v1.ReplicationController) []metav1.OwnerReference { 387 return []metav1.OwnerReference{{UID: rc.UID, Name: rc.Name, APIVersion: "v1", Kind: "ReplicationController", Controller: boolPtr(true), BlockOwnerDeletion: boolPtr(true)}} 388 }, 389 }, 390 { 391 "pod refers rc as a controller", 392 func(rc *v1.ReplicationController) []metav1.OwnerReference { 393 return []metav1.OwnerReference{{UID: rc.UID, Name: rc.Name, APIVersion: "v1", Kind: "ReplicationController", Controller: boolPtr(true)}} 394 }, 395 func(rc *v1.ReplicationController) []metav1.OwnerReference { 396 return []metav1.OwnerReference{{UID: rc.UID, Name: rc.Name, APIVersion: "v1", Kind: "ReplicationController", Controller: boolPtr(true)}} 397 }, 398 }, 399 { 400 "pod refers other rc as the controller, refers the rc as an owner", 401 func(rc *v1.ReplicationController) []metav1.OwnerReference { 402 return []metav1.OwnerReference{ 403 {UID: "1", Name: "anotherRC", APIVersion: "v1", Kind: "ReplicationController", Controller: boolPtr(true)}, 404 {UID: rc.UID, Name: rc.Name, APIVersion: "v1", Kind: "ReplicationController"}, 405 } 406 }, 407 func(rc *v1.ReplicationController) []metav1.OwnerReference { 408 return []metav1.OwnerReference{ 409 {UID: "1", Name: "anotherRC", APIVersion: "v1", Kind: "ReplicationController", Controller: boolPtr(true)}, 410 {UID: rc.UID, Name: rc.Name, APIVersion: "v1", Kind: "ReplicationController"}, 411 } 412 }, 413 }, 414 } 415 for i, tc := range testCases { 416 t.Run(tc.name, func(t *testing.T) { 417 closeFn, rm, informers, clientSet := rmSetup(t) 418 defer closeFn() 419 ns := framework.CreateNamespaceOrDie(clientSet, fmt.Sprintf("rc-adoption-%d", i), t) 420 defer framework.DeleteNamespaceOrDie(clientSet, ns, t) 421 422 rcClient := clientSet.CoreV1().ReplicationControllers(ns.Name) 423 podClient := clientSet.CoreV1().Pods(ns.Name) 424 const rcName = "rc" 425 rc, err := rcClient.Create(context.TODO(), newRC(rcName, ns.Name, 1), metav1.CreateOptions{}) 426 if err != nil { 427 t.Fatalf("Failed to create replication controllers: %v", err) 428 } 429 podName := fmt.Sprintf("pod%d", i) 430 pod := newMatchingPod(podName, ns.Name) 431 pod.OwnerReferences = tc.existingOwnerReferences(rc) 432 _, err = podClient.Create(context.TODO(), pod, metav1.CreateOptions{}) 433 if err != nil { 434 t.Fatalf("Failed to create Pod: %v", err) 435 } 436 437 stopControllers := runControllerAndInformers(t, rm, informers, 1) 438 defer stopControllers() 439 if err := wait.PollImmediate(interval, timeout, func() (bool, error) { 440 updatedPod, err := podClient.Get(context.TODO(), pod.Name, metav1.GetOptions{}) 441 if err != nil { 442 return false, err 443 } 444 445 e, a := tc.expectedOwnerReferences(rc), updatedPod.OwnerReferences 446 if reflect.DeepEqual(e, a) { 447 return true, nil 448 } 449 450 t.Logf("ownerReferences don't match, expect %v, got %v", e, a) 451 return false, nil 452 }); err != nil { 453 t.Fatalf("test %q failed: %v", tc.name, err) 454 } 455 }) 456 } 457 } 458 459 func TestSpecReplicasChange(t *testing.T) { 460 closeFn, rm, informers, c := rmSetup(t) 461 defer closeFn() 462 ns := framework.CreateNamespaceOrDie(c, "test-spec-replicas-change", t) 463 defer framework.DeleteNamespaceOrDie(c, ns, t) 464 stopControllers := runControllerAndInformers(t, rm, informers, 0) 465 defer stopControllers() 466 467 rc := newRC("rc", ns.Name, 2) 468 rcs, _ := createRCsPods(t, c, []*v1.ReplicationController{rc}, []*v1.Pod{}) 469 rc = rcs[0] 470 waitRCStable(t, c, rc) 471 472 // Update .Spec.Replicas and verify .Status.Replicas is changed accordingly 473 scaleRC(t, c, rc, 3) 474 scaleRC(t, c, rc, 0) 475 scaleRC(t, c, rc, 2) 476 477 // Add a template annotation change to test RC's status does update 478 // without .Spec.Replicas change 479 rcClient := c.CoreV1().ReplicationControllers(ns.Name) 480 var oldGeneration int64 481 newRC := updateRC(t, rcClient, rc.Name, func(rc *v1.ReplicationController) { 482 oldGeneration = rc.Generation 483 rc.Spec.Template.Annotations = map[string]string{"test": "annotation"} 484 }) 485 savedGeneration := newRC.Generation 486 if savedGeneration == oldGeneration { 487 t.Fatalf("Failed to verify .Generation has incremented for rc %s", rc.Name) 488 } 489 490 if err := wait.PollImmediate(interval, timeout, func() (bool, error) { 491 newRC, err := rcClient.Get(context.TODO(), rc.Name, metav1.GetOptions{}) 492 if err != nil { 493 return false, err 494 } 495 return newRC.Status.ObservedGeneration >= savedGeneration, nil 496 }); err != nil { 497 t.Fatalf("Failed to verify .Status.ObservedGeneration has incremented for rc %s: %v", rc.Name, err) 498 } 499 } 500 501 func TestLogarithmicScaleDown(t *testing.T) { 502 defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.LogarithmicScaleDown, true)() 503 closeFn, rm, informers, c := rmSetup(t) 504 defer closeFn() 505 ns := framework.CreateNamespaceOrDie(c, "test-spec-replicas-change", t) 506 defer framework.DeleteNamespaceOrDie(c, ns, t) 507 stopControllers := runControllerAndInformers(t, rm, informers, 0) 508 defer stopControllers() 509 510 rc := newRC("rc", ns.Name, 2) 511 rcs, _ := createRCsPods(t, c, []*v1.ReplicationController{rc}, []*v1.Pod{}) 512 rc = rcs[0] 513 waitRCStable(t, c, rc) 514 515 // get list of pods in the cluster 516 pods, err := c.CoreV1().Pods(ns.Name).List(context.TODO(), metav1.ListOptions{}) 517 if err != nil { 518 t.Fatalf("failed to get pods in namespace %s: %+v", ns.Name, err) 519 } 520 521 // Wait 10 seconds and scale up, the new pod should be in a new logarithmic rank from the first 2 522 time.Sleep(10 * time.Second) 523 scaleRC(t, c, rc, 3) 524 525 // scale back down, and confirm that the pods left in the namespace are the original ones 526 // (meaning the 3rd one was deleted) 527 scaleRC(t, c, rc, 2) 528 529 newPods, err := c.CoreV1().Pods(ns.Name).List(context.TODO(), metav1.ListOptions{}) 530 if err != nil { 531 t.Fatalf("failed to get pods in namespace %s: %+v", ns.Name, err) 532 } 533 534 if !apiequality.Semantic.DeepEqual(pods.Items, newPods.Items) { 535 t.Fatalf("expected pods %+v, got %+v", pods.Items, newPods.Items) 536 } 537 } 538 539 func TestDeletingAndFailedPods(t *testing.T) { 540 closeFn, rm, informers, c := rmSetup(t) 541 defer closeFn() 542 ns := framework.CreateNamespaceOrDie(c, "test-deleting-and-failed-pods", t) 543 defer framework.DeleteNamespaceOrDie(c, ns, t) 544 stopControllers := runControllerAndInformers(t, rm, informers, 0) 545 defer stopControllers() 546 547 rc := newRC("rc", ns.Name, 2) 548 rcs, _ := createRCsPods(t, c, []*v1.ReplicationController{rc}, []*v1.Pod{}) 549 rc = rcs[0] 550 waitRCStable(t, c, rc) 551 552 // Verify RC creates 2 pods 553 podClient := c.CoreV1().Pods(ns.Name) 554 pods := getPods(t, podClient, labelMap()) 555 if len(pods.Items) != 2 { 556 t.Fatalf("len(pods) = %d, want 2", len(pods.Items)) 557 } 558 559 // Set first pod as deleting pod 560 // Set finalizers for the pod to simulate pending deletion status 561 deletingPod := &pods.Items[0] 562 updatePod(t, podClient, deletingPod.Name, func(pod *v1.Pod) { 563 pod.Finalizers = []string{"fake.example.com/blockDeletion"} 564 }) 565 if err := c.CoreV1().Pods(ns.Name).Delete(context.TODO(), deletingPod.Name, metav1.DeleteOptions{}); err != nil { 566 t.Fatalf("Error deleting pod %s: %v", deletingPod.Name, err) 567 } 568 569 // Set second pod as failed pod 570 failedPod := &pods.Items[1] 571 updatePodStatus(t, podClient, failedPod, func(pod *v1.Pod) { 572 pod.Status.Phase = v1.PodFailed 573 }) 574 575 // Pool until 2 new pods have been created to replace deleting and failed pods 576 if err := wait.PollImmediate(interval, timeout, func() (bool, error) { 577 pods = getPods(t, podClient, labelMap()) 578 return len(pods.Items) == 4, nil 579 }); err != nil { 580 t.Fatalf("Failed to verify 2 new pods have been created (expected 4 pods): %v", err) 581 } 582 583 // Verify deleting and failed pods are among the four pods 584 foundDeletingPod := false 585 foundFailedPod := false 586 for _, pod := range pods.Items { 587 if pod.UID == deletingPod.UID { 588 foundDeletingPod = true 589 } 590 if pod.UID == failedPod.UID { 591 foundFailedPod = true 592 } 593 } 594 // Verify deleting pod exists 595 if !foundDeletingPod { 596 t.Fatalf("expected deleting pod %s exists, but it is not found", deletingPod.Name) 597 } 598 // Verify failed pod exists 599 if !foundFailedPod { 600 t.Fatalf("expected failed pod %s exists, but it is not found", failedPod.Name) 601 } 602 } 603 604 func TestOverlappingRCs(t *testing.T) { 605 closeFn, rm, informers, c := rmSetup(t) 606 defer closeFn() 607 ns := framework.CreateNamespaceOrDie(c, "test-overlapping-rcs", t) 608 defer framework.DeleteNamespaceOrDie(c, ns, t) 609 stopControllers := runControllerAndInformers(t, rm, informers, 0) 610 defer stopControllers() 611 612 // Create 2 RCs with identical selectors 613 for i := 0; i < 2; i++ { 614 // One RC has 1 replica, and another has 2 replicas 615 rc := newRC(fmt.Sprintf("rc-%d", i+1), ns.Name, i+1) 616 rcs, _ := createRCsPods(t, c, []*v1.ReplicationController{rc}, []*v1.Pod{}) 617 waitRCStable(t, c, rcs[0]) 618 } 619 620 // Expect 3 total Pods to be created 621 podClient := c.CoreV1().Pods(ns.Name) 622 pods := getPods(t, podClient, labelMap()) 623 if len(pods.Items) != 3 { 624 t.Errorf("len(pods) = %d, want 3", len(pods.Items)) 625 } 626 627 // Expect both RCs have .status.replicas = .spec.replicas 628 for i := 0; i < 2; i++ { 629 newRC, err := c.CoreV1().ReplicationControllers(ns.Name).Get(context.TODO(), fmt.Sprintf("rc-%d", i+1), metav1.GetOptions{}) 630 if err != nil { 631 t.Fatalf("failed to obtain rc rc-%d: %v", i+1, err) 632 } 633 if newRC.Status.Replicas != *newRC.Spec.Replicas { 634 t.Fatalf(".Status.Replicas %d is not equal to .Spec.Replicas %d", newRC.Status.Replicas, *newRC.Spec.Replicas) 635 } 636 } 637 } 638 639 func TestPodOrphaningAndAdoptionWhenLabelsChange(t *testing.T) { 640 closeFn, rm, informers, c := rmSetup(t) 641 defer closeFn() 642 ns := framework.CreateNamespaceOrDie(c, "test-pod-orphaning-and-adoption-when-labels-change", t) 643 defer framework.DeleteNamespaceOrDie(c, ns, t) 644 stopControllers := runControllerAndInformers(t, rm, informers, 0) 645 defer stopControllers() 646 647 rc := newRC("rc", ns.Name, 1) 648 rcs, _ := createRCsPods(t, c, []*v1.ReplicationController{rc}, []*v1.Pod{}) 649 rc = rcs[0] 650 waitRCStable(t, c, rc) 651 652 // Orphaning: RC should remove OwnerReference from a pod when the pod's labels change to not match its labels 653 podClient := c.CoreV1().Pods(ns.Name) 654 pods := getPods(t, podClient, labelMap()) 655 if len(pods.Items) != 1 { 656 t.Fatalf("len(pods) = %d, want 1", len(pods.Items)) 657 } 658 pod := &pods.Items[0] 659 660 // Start by verifying ControllerRef for the pod is not nil 661 if metav1.GetControllerOf(pod) == nil { 662 t.Fatalf("ControllerRef of pod %s is nil", pod.Name) 663 } 664 newLabelMap := map[string]string{"new-foo": "new-bar"} 665 updatePod(t, podClient, pod.Name, func(pod *v1.Pod) { 666 pod.Labels = newLabelMap 667 }) 668 if err := wait.PollImmediate(interval, timeout, func() (bool, error) { 669 newPod, err := podClient.Get(context.TODO(), pod.Name, metav1.GetOptions{}) 670 if err != nil { 671 return false, err 672 } 673 pod = newPod 674 return metav1.GetControllerOf(newPod) == nil, nil 675 }); err != nil { 676 t.Fatalf("Failed to verify ControllerRef for the pod %s is nil: %v", pod.Name, err) 677 } 678 679 // Adoption: RC should add ControllerRef to a pod when the pod's labels change to match its labels 680 updatePod(t, podClient, pod.Name, func(pod *v1.Pod) { 681 pod.Labels = labelMap() 682 }) 683 if err := wait.PollImmediate(interval, timeout, func() (bool, error) { 684 newPod, err := podClient.Get(context.TODO(), pod.Name, metav1.GetOptions{}) 685 if err != nil { 686 // If the pod is not found, it means the RC picks the pod for deletion (it is extra) 687 // Verify there is only one pod in namespace and it has ControllerRef to the RC 688 if !apierrors.IsNotFound(err) { 689 return false, err 690 } 691 692 pods := getPods(t, podClient, labelMap()) 693 if len(pods.Items) != 1 { 694 return false, fmt.Errorf("Expected 1 pod in current namespace, got %d", len(pods.Items)) 695 } 696 // Set the pod accordingly 697 pod = &pods.Items[0] 698 return true, nil 699 } 700 // Always update the pod so that we can save a GET call to API server later 701 pod = newPod 702 // If the pod is found, verify the pod has a ControllerRef 703 return metav1.GetControllerOf(newPod) != nil, nil 704 }); err != nil { 705 t.Fatalf("Failed to verify ControllerRef for pod %s is not nil: %v", pod.Name, err) 706 } 707 // Verify the pod has a ControllerRef to the RC 708 // Do nothing if the pod is nil (i.e., has been picked for deletion) 709 if pod != nil { 710 controllerRef := metav1.GetControllerOf(pod) 711 if controllerRef.UID != rc.UID { 712 t.Fatalf("RC owner of the pod %s has a different UID: Expected %v, got %v", pod.Name, rc.UID, controllerRef.UID) 713 } 714 } 715 } 716 717 func TestGeneralPodAdoption(t *testing.T) { 718 closeFn, rm, informers, c := rmSetup(t) 719 defer closeFn() 720 ns := framework.CreateNamespaceOrDie(c, "test-general-pod-adoption", t) 721 defer framework.DeleteNamespaceOrDie(c, ns, t) 722 stopControllers := runControllerAndInformers(t, rm, informers, 0) 723 defer stopControllers() 724 725 rc := newRC("rc", ns.Name, 1) 726 rcs, _ := createRCsPods(t, c, []*v1.ReplicationController{rc}, []*v1.Pod{}) 727 rc = rcs[0] 728 waitRCStable(t, c, rc) 729 730 podClient := c.CoreV1().Pods(ns.Name) 731 pods := getPods(t, podClient, labelMap()) 732 if len(pods.Items) != 1 { 733 t.Fatalf("len(pods) = %d, want 1", len(pods.Items)) 734 } 735 pod := &pods.Items[0] 736 var falseVar = false 737 738 // When the only OwnerReference of the pod points to another type of API object such as statefulset 739 // with Controller=false, the RC should add a second OwnerReference (ControllerRef) pointing to itself 740 // with Controller=true 741 ownerReference := metav1.OwnerReference{UID: uuid.NewUUID(), APIVersion: "apps/v1beta1", Kind: "StatefulSet", Name: rc.Name, Controller: &falseVar} 742 testPodControllerRefPatch(t, c, pod, &ownerReference, rc, 2) 743 744 // When the only OwnerReference of the pod points to the RC, but Controller=false 745 ownerReference = metav1.OwnerReference{UID: rc.UID, APIVersion: "v1", Kind: "ReplicationController", Name: rc.Name, Controller: &falseVar} 746 testPodControllerRefPatch(t, c, pod, &ownerReference, rc, 1) 747 } 748 749 func TestReadyAndAvailableReplicas(t *testing.T) { 750 closeFn, rm, informers, c := rmSetup(t) 751 defer closeFn() 752 ns := framework.CreateNamespaceOrDie(c, "test-ready-and-available-replicas", t) 753 defer framework.DeleteNamespaceOrDie(c, ns, t) 754 stopControllers := runControllerAndInformers(t, rm, informers, 0) 755 defer stopControllers() 756 757 rc := newRC("rc", ns.Name, 3) 758 rc.Spec.MinReadySeconds = 3600 759 rcs, _ := createRCsPods(t, c, []*v1.ReplicationController{rc}, []*v1.Pod{}) 760 rc = rcs[0] 761 waitRCStable(t, c, rc) 762 763 // First verify no pod is available 764 if rc.Status.AvailableReplicas != 0 { 765 t.Fatalf("Unexpected .Status.AvailableReplicas: Expected 0, saw %d", rc.Status.AvailableReplicas) 766 } 767 768 podClient := c.CoreV1().Pods(ns.Name) 769 pods := getPods(t, podClient, labelMap()) 770 if len(pods.Items) != 3 { 771 t.Fatalf("len(pods) = %d, want 3", len(pods.Items)) 772 } 773 774 // Separate 3 pods into their own list 775 firstPodList := &v1.PodList{Items: pods.Items[:1]} 776 secondPodList := &v1.PodList{Items: pods.Items[1:2]} 777 thirdPodList := &v1.PodList{Items: pods.Items[2:]} 778 // First pod: Running, but not Ready 779 // by setting the Ready condition to false with LastTransitionTime to be now 780 setPodsReadyCondition(t, c, firstPodList, v1.ConditionFalse, time.Now()) 781 // Second pod: Running and Ready, but not Available 782 // by setting LastTransitionTime to now 783 setPodsReadyCondition(t, c, secondPodList, v1.ConditionTrue, time.Now()) 784 // Third pod: Running, Ready, and Available 785 // by setting LastTransitionTime to more than 3600 seconds ago 786 setPodsReadyCondition(t, c, thirdPodList, v1.ConditionTrue, time.Now().Add(-120*time.Minute)) 787 788 rcClient := c.CoreV1().ReplicationControllers(ns.Name) 789 if err := wait.PollImmediate(interval, timeout, func() (bool, error) { 790 newRC, err := rcClient.Get(context.TODO(), rc.Name, metav1.GetOptions{}) 791 if err != nil { 792 return false, err 793 } 794 // Verify 3 pods exist, 2 pods are Ready, and 1 pod is Available 795 return newRC.Status.Replicas == 3 && newRC.Status.ReadyReplicas == 2 && newRC.Status.AvailableReplicas == 1, nil 796 }); err != nil { 797 t.Fatalf("Failed to verify number of Replicas, ReadyReplicas and AvailableReplicas of rc %s to be as expected: %v", rc.Name, err) 798 } 799 } 800 801 func TestRCScaleSubresource(t *testing.T) { 802 closeFn, rm, informers, c := rmSetup(t) 803 defer closeFn() 804 ns := framework.CreateNamespaceOrDie(c, "test-rc-scale-subresource", t) 805 defer framework.DeleteNamespaceOrDie(c, ns, t) 806 stopControllers := runControllerAndInformers(t, rm, informers, 0) 807 defer stopControllers() 808 809 rc := newRC("rc", ns.Name, 1) 810 rcs, _ := createRCsPods(t, c, []*v1.ReplicationController{rc}, []*v1.Pod{}) 811 rc = rcs[0] 812 waitRCStable(t, c, rc) 813 814 // Use scale subresource to scale up .Spec.Replicas to 3 815 testScalingUsingScaleSubresource(t, c, rc, 3) 816 // Use the scale subresource to scale down .Spec.Replicas to 0 817 testScalingUsingScaleSubresource(t, c, rc, 0) 818 } 819 820 func TestExtraPodsAdoptionAndDeletion(t *testing.T) { 821 closeFn, rm, informers, c := rmSetup(t) 822 defer closeFn() 823 ns := framework.CreateNamespaceOrDie(c, "test-extra-pods-adoption-and-deletion", t) 824 defer framework.DeleteNamespaceOrDie(c, ns, t) 825 826 rc := newRC("rc", ns.Name, 2) 827 // Create 3 pods, RC should adopt only 2 of them 828 podList := []*v1.Pod{} 829 for i := 0; i < 3; i++ { 830 pod := newMatchingPod(fmt.Sprintf("pod-%d", i+1), ns.Name) 831 pod.Labels = labelMap() 832 podList = append(podList, pod) 833 } 834 rcs, _ := createRCsPods(t, c, []*v1.ReplicationController{rc}, podList) 835 rc = rcs[0] 836 stopControllers := runControllerAndInformers(t, rm, informers, 3) 837 defer stopControllers() 838 waitRCStable(t, c, rc) 839 840 // Verify the extra pod is deleted eventually by determining whether number of 841 // all pods within namespace matches .spec.replicas of the RC (2 in this case) 842 podClient := c.CoreV1().Pods(ns.Name) 843 if err := wait.PollImmediate(interval, timeout, func() (bool, error) { 844 // All pods have labelMap as their labels 845 pods := getPods(t, podClient, labelMap()) 846 return int32(len(pods.Items)) == *rc.Spec.Replicas, nil 847 }); err != nil { 848 t.Fatalf("Failed to verify number of all pods within current namespace matches .spec.replicas of rc %s: %v", rc.Name, err) 849 } 850 } 851 852 func TestFullyLabeledReplicas(t *testing.T) { 853 closeFn, rm, informers, c := rmSetup(t) 854 defer closeFn() 855 ns := framework.CreateNamespaceOrDie(c, "test-fully-labeled-replicas", t) 856 defer framework.DeleteNamespaceOrDie(c, ns, t) 857 stopControllers := runControllerAndInformers(t, rm, informers, 0) 858 defer stopControllers() 859 860 extraLabelMap := map[string]string{"foo": "bar", "extraKey": "extraValue"} 861 rc := newRC("rc", ns.Name, 2) 862 rcs, _ := createRCsPods(t, c, []*v1.ReplicationController{rc}, []*v1.Pod{}) 863 rc = rcs[0] 864 waitRCStable(t, c, rc) 865 866 // Change RC's template labels to have extra labels, but not its selector 867 rcClient := c.CoreV1().ReplicationControllers(ns.Name) 868 updateRC(t, rcClient, rc.Name, func(rc *v1.ReplicationController) { 869 rc.Spec.Template.Labels = extraLabelMap 870 }) 871 872 // Set one of the pods to have extra labels 873 podClient := c.CoreV1().Pods(ns.Name) 874 pods := getPods(t, podClient, labelMap()) 875 if len(pods.Items) != 2 { 876 t.Fatalf("len(pods) = %d, want 2", len(pods.Items)) 877 } 878 fullyLabeledPod := &pods.Items[0] 879 updatePod(t, podClient, fullyLabeledPod.Name, func(pod *v1.Pod) { 880 pod.Labels = extraLabelMap 881 }) 882 883 // Verify only one pod is fully labeled 884 if err := wait.PollImmediate(interval, timeout, func() (bool, error) { 885 newRC, err := rcClient.Get(context.TODO(), rc.Name, metav1.GetOptions{}) 886 if err != nil { 887 return false, err 888 } 889 return (newRC.Status.Replicas == 2 && newRC.Status.FullyLabeledReplicas == 1), nil 890 }); err != nil { 891 t.Fatalf("Failed to verify only one pod is fully labeled: %v", err) 892 } 893 }