k8s.io/kubernetes@v1.29.3/test/e2e_node/container_lifecycle_test.go (about) 1 /* 2 Copyright 2023 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package e2enode 18 19 import ( 20 "context" 21 "fmt" 22 "time" 23 24 "github.com/onsi/ginkgo/v2" 25 "github.com/onsi/gomega" 26 v1 "k8s.io/api/core/v1" 27 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 28 runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" 29 admissionapi "k8s.io/pod-security-admission/api" 30 31 "k8s.io/kubernetes/test/e2e/framework" 32 e2epod "k8s.io/kubernetes/test/e2e/framework/pod" 33 imageutils "k8s.io/kubernetes/test/utils/image" 34 ) 35 36 const ( 37 PostStartPrefix = "PostStart" 38 PreStopPrefix = "PreStop" 39 ) 40 41 var containerRestartPolicyAlways = v1.ContainerRestartPolicyAlways 42 43 func prefixedName(namePrefix string, name string) string { 44 return fmt.Sprintf("%s-%s", namePrefix, name) 45 } 46 47 var _ = SIGDescribe(framework.WithNodeConformance(), "Containers Lifecycle", func() { 48 f := framework.NewDefaultFramework("containers-lifecycle-test") 49 f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged 50 51 ginkgo.It("should launch init container serially before a regular container", func() { 52 53 init1 := "init-1" 54 init2 := "init-2" 55 init3 := "init-3" 56 regular1 := "regular-1" 57 58 podSpec := &v1.Pod{ 59 ObjectMeta: metav1.ObjectMeta{ 60 Name: "initcontainer-test-pod", 61 }, 62 Spec: v1.PodSpec{ 63 RestartPolicy: v1.RestartPolicyNever, 64 InitContainers: []v1.Container{ 65 { 66 Name: init1, 67 Image: busyboxImage, 68 Command: ExecCommand(init1, execCommand{ 69 Delay: 1, 70 ExitCode: 0, 71 }), 72 }, 73 { 74 Name: init2, 75 Image: busyboxImage, 76 Command: ExecCommand(init2, execCommand{ 77 Delay: 1, 78 ExitCode: 0, 79 }), 80 }, 81 { 82 Name: init3, 83 Image: busyboxImage, 84 Command: ExecCommand(init3, execCommand{ 85 Delay: 1, 86 ExitCode: 0, 87 }), 88 }, 89 }, 90 Containers: []v1.Container{ 91 { 92 Name: regular1, 93 Image: busyboxImage, 94 Command: ExecCommand(regular1, execCommand{ 95 StartDelay: 5, 96 Delay: 1, 97 ExitCode: 0, 98 }), 99 StartupProbe: &v1.Probe{ 100 ProbeHandler: v1.ProbeHandler{ 101 Exec: &v1.ExecAction{ 102 Command: []string{ 103 "test", 104 "-f", 105 "started", 106 }, 107 }, 108 }, 109 }, 110 }, 111 }, 112 }, 113 } 114 115 preparePod(podSpec) 116 117 /// generates an out file output like: 118 // 119 // 1682076093 4905.79 init-1 Starting 0 120 // 1682076093 4905.80 init-1 Started 121 // 1682076093 4905.80 init-1 Delaying 1 122 // 1682076094 4906.80 init-1 Exiting 123 // 1682076095 4907.70 init-2 Starting 0 124 // 1682076095 4907.71 init-2 Started 125 // 1682076095 4907.71 init-2 Delaying 1 126 // 1682076096 4908.71 init-2 Exiting 127 // 1682076097 4909.74 init-3 Starting 0 128 // 1682076097 4909.74 init-3 Started 129 // 1682076097 4909.74 init-3 Delaying 1 130 // 1682076098 4910.75 init-3 Exiting 131 // 1682076099 4911.70 regular-1 Starting 5 132 // 1682076104 4916.71 regular-1 Started 133 // 1682076104 4916.71 regular-1 Delaying 1 134 // 1682076105 4917.72 regular-1 Exiting 135 136 client := e2epod.NewPodClient(f) 137 podSpec = client.Create(context.TODO(), podSpec) 138 ginkgo.By("Waiting for the pod to finish") 139 err := e2epod.WaitTimeoutForPodNoLongerRunningInNamespace(context.TODO(), f.ClientSet, podSpec.Name, podSpec.Namespace, 1*time.Minute) 140 framework.ExpectNoError(err) 141 142 ginkgo.By("Parsing results") 143 podSpec, err = client.Get(context.Background(), podSpec.Name, metav1.GetOptions{}) 144 framework.ExpectNoError(err) 145 results := parseOutput(context.TODO(), f, podSpec) 146 147 // which we then use to make assertions regarding container ordering 148 ginkgo.By("Analyzing results") 149 framework.ExpectNoError(results.StartsBefore(init1, init2)) 150 framework.ExpectNoError(results.ExitsBefore(init1, init2)) 151 152 framework.ExpectNoError(results.StartsBefore(init2, init3)) 153 framework.ExpectNoError(results.ExitsBefore(init2, init3)) 154 155 framework.ExpectNoError(results.StartsBefore(init3, regular1)) 156 framework.ExpectNoError(results.ExitsBefore(init3, regular1)) 157 }) 158 159 ginkgo.It("should not launch regular containers if an init container fails", func() { 160 161 init1 := "init-1" 162 regular1 := "regular-1" 163 164 podSpec := &v1.Pod{ 165 ObjectMeta: metav1.ObjectMeta{ 166 Name: "initcontainer-test-pod-failure", 167 }, 168 Spec: v1.PodSpec{ 169 RestartPolicy: v1.RestartPolicyNever, 170 InitContainers: []v1.Container{ 171 { 172 Name: init1, 173 Image: busyboxImage, 174 Command: ExecCommand(init1, execCommand{ 175 Delay: 1, 176 ExitCode: 1, 177 }), 178 }, 179 }, 180 Containers: []v1.Container{ 181 { 182 Name: regular1, 183 Image: busyboxImage, 184 Command: ExecCommand(regular1, execCommand{ 185 Delay: 1, 186 ExitCode: 0, 187 }), 188 }, 189 }, 190 }, 191 } 192 193 preparePod(podSpec) 194 195 client := e2epod.NewPodClient(f) 196 podSpec = client.Create(context.TODO(), podSpec) 197 ginkgo.By("Waiting for the pod to fail") 198 err := e2epod.WaitForPodFailedReason(context.TODO(), f.ClientSet, podSpec, "", 1*time.Minute) 199 framework.ExpectNoError(err) 200 201 ginkgo.By("Parsing results") 202 podSpec, err = client.Get(context.Background(), podSpec.Name, metav1.GetOptions{}) 203 framework.ExpectNoError(err) 204 results := parseOutput(context.TODO(), f, podSpec) 205 206 ginkgo.By("Analyzing results") 207 // init container should start and exit with an error, and the regular container should never start 208 framework.ExpectNoError(results.Starts(init1)) 209 framework.ExpectNoError(results.Exits(init1)) 210 211 framework.ExpectNoError(results.DoesntStart(regular1)) 212 }) 213 214 ginkgo.It("should run Init container to completion before call to PostStart of regular container", func() { 215 init1 := "init-1" 216 regular1 := "regular-1" 217 218 podSpec := &v1.Pod{ 219 ObjectMeta: metav1.ObjectMeta{ 220 Name: "initcontainer-test-pod-with-post-start", 221 }, 222 Spec: v1.PodSpec{ 223 RestartPolicy: v1.RestartPolicyNever, 224 InitContainers: []v1.Container{ 225 { 226 Name: init1, 227 Image: busyboxImage, 228 Command: ExecCommand(init1, execCommand{ 229 Delay: 1, 230 ExitCode: 0, 231 }), 232 }, 233 }, 234 Containers: []v1.Container{ 235 { 236 Name: regular1, 237 Image: busyboxImage, 238 Command: ExecCommand(regular1, execCommand{ 239 // Allocate sufficient time for its postStart hook 240 // to complete. 241 // Note that we've observed approximately a 2s 242 // delay before the postStart hook is called. 243 // 10s > 1s + 2s(estimated maximum delay) + other possible delays 244 Delay: 10, 245 ExitCode: 0, 246 }), 247 Lifecycle: &v1.Lifecycle{ 248 PostStart: &v1.LifecycleHandler{ 249 Exec: &v1.ExecAction{ 250 Command: ExecCommand(prefixedName(PostStartPrefix, regular1), execCommand{ 251 Delay: 1, 252 ExitCode: 0, 253 ContainerName: regular1, 254 }), 255 }, 256 }, 257 }, 258 }, 259 }, 260 }, 261 } 262 263 preparePod(podSpec) 264 265 client := e2epod.NewPodClient(f) 266 podSpec = client.Create(context.TODO(), podSpec) 267 ginkgo.By("Waiting for the pod to finish") 268 err := e2epod.WaitTimeoutForPodNoLongerRunningInNamespace(context.TODO(), f.ClientSet, podSpec.Name, podSpec.Namespace, 1*time.Minute) 269 framework.ExpectNoError(err) 270 271 ginkgo.By("Parsing results") 272 podSpec, err = client.Get(context.Background(), podSpec.Name, metav1.GetOptions{}) 273 framework.ExpectNoError(err) 274 results := parseOutput(context.TODO(), f, podSpec) 275 276 ginkgo.By("Analyzing results") 277 // init container should start and exit with an error, and the regular container should never start 278 framework.ExpectNoError(results.StartsBefore(init1, prefixedName(PostStartPrefix, regular1))) 279 framework.ExpectNoError(results.ExitsBefore(init1, prefixedName(PostStartPrefix, regular1))) 280 281 framework.ExpectNoError(results.RunTogether(regular1, prefixedName(PostStartPrefix, regular1))) 282 }) 283 284 ginkgo.It("should restart failing container when pod restartPolicy is Always", func() { 285 286 regular1 := "regular-1" 287 288 podSpec := &v1.Pod{ 289 ObjectMeta: metav1.ObjectMeta{ 290 Name: "container-must-be-restarted", 291 }, 292 Spec: v1.PodSpec{ 293 RestartPolicy: v1.RestartPolicyAlways, 294 Containers: []v1.Container{ 295 { 296 Name: regular1, 297 Image: busyboxImage, 298 Command: ExecCommand(regular1, execCommand{ 299 Delay: 1, 300 ExitCode: 1, 301 }), 302 }, 303 }, 304 }, 305 } 306 307 preparePod(podSpec) 308 309 client := e2epod.NewPodClient(f) 310 podSpec = client.Create(context.TODO(), podSpec) 311 ginkgo.By("Waiting for the pod, it will not finish") 312 err := WaitForPodContainerRestartCount(context.TODO(), f.ClientSet, podSpec.Namespace, podSpec.Name, 0, 3, 2*time.Minute) 313 framework.ExpectNoError(err) 314 315 ginkgo.By("Parsing results") 316 podSpec, err = client.Get(context.Background(), podSpec.Name, metav1.GetOptions{}) 317 framework.ExpectNoError(err) 318 results := parseOutput(context.TODO(), f, podSpec) 319 320 ginkgo.By("Analyzing results") 321 // container must be restarted 322 framework.ExpectNoError(results.Starts(regular1)) 323 framework.ExpectNoError(results.StartsBefore(regular1, regular1)) 324 framework.ExpectNoError(results.ExitsBefore(regular1, regular1)) 325 }) 326 327 ginkgo.It("should not launch second container before PostStart of the first container completed", func() { 328 329 regular1 := "regular-1" 330 regular2 := "regular-2" 331 332 podSpec := &v1.Pod{ 333 ObjectMeta: metav1.ObjectMeta{ 334 Name: "post-start-blocks-second-container", 335 }, 336 Spec: v1.PodSpec{ 337 RestartPolicy: v1.RestartPolicyNever, 338 Containers: []v1.Container{ 339 { 340 Name: regular1, 341 Image: busyboxImage, 342 Command: ExecCommand(regular1, execCommand{ 343 // Allocate sufficient time for its postStart hook 344 // to complete. 345 // Note that we've observed approximately a 2s 346 // delay before the postStart hook is called. 347 // 10s > 1s + 2s(estimated maximum delay) + other possible delays 348 Delay: 10, 349 ExitCode: 0, 350 }), 351 Lifecycle: &v1.Lifecycle{ 352 PostStart: &v1.LifecycleHandler{ 353 Exec: &v1.ExecAction{ 354 Command: ExecCommand(prefixedName(PostStartPrefix, regular1), execCommand{ 355 Delay: 1, 356 ExitCode: 0, 357 ContainerName: regular1, 358 }), 359 }, 360 }, 361 }, 362 }, 363 { 364 Name: regular2, 365 Image: busyboxImage, 366 Command: ExecCommand(regular2, execCommand{ 367 Delay: 1, 368 ExitCode: 0, 369 }), 370 }, 371 }, 372 }, 373 } 374 375 preparePod(podSpec) 376 377 client := e2epod.NewPodClient(f) 378 podSpec = client.Create(context.TODO(), podSpec) 379 ginkgo.By("Waiting for the pod to finish") 380 err := e2epod.WaitTimeoutForPodNoLongerRunningInNamespace(context.TODO(), f.ClientSet, podSpec.Name, podSpec.Namespace, 1*time.Minute) 381 framework.ExpectNoError(err) 382 383 ginkgo.By("Parsing results") 384 podSpec, err = client.Get(context.Background(), podSpec.Name, metav1.GetOptions{}) 385 framework.ExpectNoError(err) 386 results := parseOutput(context.TODO(), f, podSpec) 387 388 ginkgo.By("Analyzing results") 389 // second container should not start before the PostStart of a first container completed 390 framework.ExpectNoError(results.StartsBefore(prefixedName(PostStartPrefix, regular1), regular2)) 391 framework.ExpectNoError(results.ExitsBefore(prefixedName(PostStartPrefix, regular1), regular2)) 392 }) 393 394 ginkgo.When("have init container in a Pod with restartPolicy=Never", func() { 395 396 ginkgo.When("an init container fails to start because of a bad image", ginkgo.Ordered, func() { 397 398 init1 := "init1-1" 399 regular1 := "regular-1" 400 401 podSpec := &v1.Pod{ 402 ObjectMeta: metav1.ObjectMeta{ 403 Name: "bad-image", 404 }, 405 Spec: v1.PodSpec{ 406 RestartPolicy: v1.RestartPolicyNever, 407 InitContainers: []v1.Container{ 408 { 409 Name: init1, 410 Image: imageutils.GetE2EImage(imageutils.InvalidRegistryImage), 411 Command: ExecCommand(init1, execCommand{ 412 Delay: 600, 413 ExitCode: 0, 414 }), 415 }, 416 }, 417 Containers: []v1.Container{ 418 { 419 Name: regular1, 420 Image: busyboxImage, 421 Command: ExecCommand(regular1, execCommand{ 422 Delay: 1, 423 ExitCode: 0, 424 }), 425 }, 426 }, 427 }, 428 } 429 430 preparePod(podSpec) 431 var results containerOutputList 432 433 ginkgo.It("should mark a Pod as failed and produce log", func() { 434 client := e2epod.NewPodClient(f) 435 podSpec = client.Create(context.TODO(), podSpec) 436 437 err := WaitForPodInitContainerToFail(context.TODO(), f.ClientSet, podSpec.Namespace, podSpec.Name, 0, "ImagePullBackOff", f.Timeouts.PodStart) 438 framework.ExpectNoError(err) 439 440 podSpec, err = client.Get(context.Background(), podSpec.Name, metav1.GetOptions{}) 441 framework.ExpectNoError(err) 442 results = parseOutput(context.TODO(), f, podSpec) 443 }) 444 ginkgo.It("should not start an init container", func() { 445 framework.ExpectNoError(results.DoesntStart(init1)) 446 }) 447 ginkgo.It("should not start a regular container", func() { 448 framework.ExpectNoError(results.DoesntStart(regular1)) 449 }) 450 }) 451 }) 452 453 ginkgo.It("shouldn't restart init containers upon regular container restart", func() { 454 init1 := "init-1" 455 init2 := "init-2" 456 init3 := "init-3" 457 regular1 := "regular-1" 458 459 podSpec := &v1.Pod{ 460 ObjectMeta: metav1.ObjectMeta{ 461 Name: "initcontainer-test-pod", 462 }, 463 Spec: v1.PodSpec{ 464 RestartPolicy: v1.RestartPolicyAlways, 465 InitContainers: []v1.Container{ 466 { 467 Name: init1, 468 Image: busyboxImage, 469 Command: ExecCommand(init1, execCommand{ 470 Delay: 1, 471 ExitCode: 0, 472 }), 473 }, 474 { 475 Name: init2, 476 Image: busyboxImage, 477 Command: ExecCommand(init2, execCommand{ 478 Delay: 1, 479 ExitCode: 0, 480 }), 481 }, 482 { 483 Name: init3, 484 Image: busyboxImage, 485 Command: ExecCommand(init3, execCommand{ 486 Delay: 1, 487 ExitCode: 0, 488 }), 489 }, 490 }, 491 Containers: []v1.Container{ 492 { 493 Name: regular1, 494 Image: busyboxImage, 495 Command: ExecCommand(regular1, execCommand{ 496 Delay: 10, 497 ExitCode: -1, 498 }), 499 }, 500 }, 501 }, 502 } 503 504 preparePod(podSpec) 505 506 client := e2epod.NewPodClient(f) 507 podSpec = client.Create(context.TODO(), podSpec) 508 ginkgo.By("Waiting for the pod to restart a few times") 509 err := WaitForPodContainerRestartCount(context.TODO(), f.ClientSet, podSpec.Namespace, podSpec.Name, 0, 3, 2*time.Minute) 510 framework.ExpectNoError(err) 511 512 ginkgo.By("Parsing results") 513 podSpec, err = client.Get(context.Background(), podSpec.Name, metav1.GetOptions{}) 514 framework.ExpectNoError(err) 515 results := parseOutput(context.TODO(), f, podSpec) 516 517 ginkgo.By("Analyzing results") 518 framework.ExpectNoError(results.StartsBefore(init1, init2)) 519 framework.ExpectNoError(results.ExitsBefore(init1, init2)) 520 521 framework.ExpectNoError(results.StartsBefore(init2, init3)) 522 framework.ExpectNoError(results.ExitsBefore(init2, init3)) 523 524 framework.ExpectNoError(results.StartsBefore(init3, regular1)) 525 framework.ExpectNoError(results.ExitsBefore(init3, regular1)) 526 527 // ensure that the init containers never restarted 528 framework.ExpectNoError(results.HasNotRestarted(init1)) 529 framework.ExpectNoError(results.HasNotRestarted(init2)) 530 framework.ExpectNoError(results.HasNotRestarted(init3)) 531 // while the regular container did 532 framework.ExpectNoError(results.HasRestarted(regular1)) 533 }) 534 535 ginkgo.When("a pod cannot terminate gracefully", func() { 536 testPod := func(name string, gracePeriod int64) *v1.Pod { 537 return &v1.Pod{ 538 ObjectMeta: metav1.ObjectMeta{ 539 Name: name, 540 }, 541 Spec: v1.PodSpec{ 542 Containers: []v1.Container{ 543 { 544 Name: "busybox", 545 Image: imageutils.GetE2EImage(imageutils.BusyBox), 546 Command: []string{ 547 "sleep", 548 "10000", 549 }, 550 }, 551 }, 552 TerminationGracePeriodSeconds: &gracePeriod, 553 }, 554 } 555 } 556 557 // To account for the time it takes to delete the pod, we add a buffer. Its sized 558 // so that we allow up to 2x the grace time to delete the pod. Its extra large to 559 // reduce test flakes. 560 bufferSeconds := int64(30) 561 562 f.It("should respect termination grace period seconds", f.WithNodeConformance(), func() { 563 client := e2epod.NewPodClient(f) 564 gracePeriod := int64(30) 565 566 ginkgo.By("creating a pod with a termination grace period seconds") 567 pod := testPod("pod-termination-grace-period", gracePeriod) 568 pod = client.Create(context.TODO(), pod) 569 570 ginkgo.By("ensuring the pod is running") 571 err := e2epod.WaitForPodRunningInNamespace(context.TODO(), f.ClientSet, pod) 572 framework.ExpectNoError(err) 573 574 ginkgo.By("deleting the pod gracefully") 575 err = client.Delete(context.TODO(), pod.Name, metav1.DeleteOptions{}) 576 framework.ExpectNoError(err) 577 578 ginkgo.By("ensuring the pod is terminated within the grace period seconds + buffer seconds") 579 err = e2epod.WaitForPodNotFoundInNamespace(context.TODO(), f.ClientSet, pod.Name, pod.Namespace, time.Duration(gracePeriod+bufferSeconds)*time.Second) 580 framework.ExpectNoError(err) 581 }) 582 583 f.It("should respect termination grace period seconds with long-running preStop hook", f.WithNodeConformance(), func() { 584 client := e2epod.NewPodClient(f) 585 gracePeriod := int64(30) 586 587 ginkgo.By("creating a pod with a termination grace period seconds and long-running preStop hook") 588 pod := testPod("pod-termination-grace-period", gracePeriod) 589 pod.Spec.Containers[0].Lifecycle = &v1.Lifecycle{ 590 PreStop: &v1.LifecycleHandler{ 591 Exec: &v1.ExecAction{ 592 Command: []string{ 593 "sleep", 594 "10000", 595 }, 596 }, 597 }, 598 } 599 pod = client.Create(context.TODO(), pod) 600 601 ginkgo.By("ensuring the pod is running") 602 err := e2epod.WaitForPodRunningInNamespace(context.TODO(), f.ClientSet, pod) 603 framework.ExpectNoError(err) 604 605 ginkgo.By("deleting the pod gracefully") 606 err = client.Delete(context.TODO(), pod.Name, metav1.DeleteOptions{}) 607 framework.ExpectNoError(err) 608 609 ginkgo.By("ensuring the pod is terminated within the grace period seconds + buffer seconds") 610 err = e2epod.WaitForPodNotFoundInNamespace(context.TODO(), f.ClientSet, pod.Name, pod.Namespace, time.Duration(gracePeriod+bufferSeconds)*time.Second) 611 framework.ExpectNoError(err) 612 }) 613 }) 614 615 ginkgo.It("should call the container's preStop hook and terminate it if its startup probe fails", func() { 616 regular1 := "regular-1" 617 618 podSpec := &v1.Pod{ 619 ObjectMeta: metav1.ObjectMeta{ 620 Name: "test-pod", 621 }, 622 Spec: v1.PodSpec{ 623 RestartPolicy: v1.RestartPolicyNever, 624 Containers: []v1.Container{ 625 { 626 Name: regular1, 627 Image: busyboxImage, 628 Command: ExecCommand(regular1, execCommand{ 629 Delay: 100, 630 TerminationSeconds: 15, 631 ExitCode: 0, 632 }), 633 StartupProbe: &v1.Probe{ 634 ProbeHandler: v1.ProbeHandler{ 635 Exec: &v1.ExecAction{ 636 Command: []string{ 637 "sh", 638 "-c", 639 "exit 1", 640 }, 641 }, 642 }, 643 InitialDelaySeconds: 10, 644 FailureThreshold: 1, 645 }, 646 Lifecycle: &v1.Lifecycle{ 647 PreStop: &v1.LifecycleHandler{ 648 Exec: &v1.ExecAction{ 649 Command: ExecCommand(prefixedName(PreStopPrefix, regular1), execCommand{ 650 Delay: 1, 651 ExitCode: 0, 652 ContainerName: regular1, 653 }), 654 }, 655 }, 656 }, 657 }, 658 }, 659 }, 660 } 661 662 preparePod(podSpec) 663 664 client := e2epod.NewPodClient(f) 665 podSpec = client.Create(context.TODO(), podSpec) 666 667 ginkgo.By("Waiting for the pod to complete") 668 err := e2epod.WaitForPodNoLongerRunningInNamespace(context.TODO(), f.ClientSet, podSpec.Name, podSpec.Namespace) 669 framework.ExpectNoError(err) 670 671 ginkgo.By("Parsing results") 672 podSpec, err = client.Get(context.TODO(), podSpec.Name, metav1.GetOptions{}) 673 framework.ExpectNoError(err) 674 results := parseOutput(context.TODO(), f, podSpec) 675 676 ginkgo.By("Analyzing results") 677 framework.ExpectNoError(results.RunTogether(regular1, prefixedName(PreStopPrefix, regular1))) 678 framework.ExpectNoError(results.Starts(prefixedName(PreStopPrefix, regular1))) 679 framework.ExpectNoError(results.Exits(regular1)) 680 }) 681 682 ginkgo.It("should call the container's preStop hook and terminate it if its liveness probe fails", func() { 683 regular1 := "regular-1" 684 685 podSpec := &v1.Pod{ 686 ObjectMeta: metav1.ObjectMeta{ 687 Name: "test-pod", 688 }, 689 Spec: v1.PodSpec{ 690 RestartPolicy: v1.RestartPolicyNever, 691 Containers: []v1.Container{ 692 { 693 Name: regular1, 694 Image: busyboxImage, 695 Command: ExecCommand(regular1, execCommand{ 696 Delay: 100, 697 TerminationSeconds: 15, 698 ExitCode: 0, 699 }), 700 LivenessProbe: &v1.Probe{ 701 ProbeHandler: v1.ProbeHandler{ 702 Exec: &v1.ExecAction{ 703 Command: []string{ 704 "sh", 705 "-c", 706 "exit 1", 707 }, 708 }, 709 }, 710 InitialDelaySeconds: 10, 711 FailureThreshold: 1, 712 }, 713 Lifecycle: &v1.Lifecycle{ 714 PreStop: &v1.LifecycleHandler{ 715 Exec: &v1.ExecAction{ 716 Command: ExecCommand(prefixedName(PreStopPrefix, regular1), execCommand{ 717 Delay: 1, 718 ExitCode: 0, 719 ContainerName: regular1, 720 }), 721 }, 722 }, 723 }, 724 }, 725 }, 726 }, 727 } 728 729 preparePod(podSpec) 730 731 client := e2epod.NewPodClient(f) 732 podSpec = client.Create(context.TODO(), podSpec) 733 734 ginkgo.By("Waiting for the pod to complete") 735 err := e2epod.WaitForPodNoLongerRunningInNamespace(context.TODO(), f.ClientSet, podSpec.Name, podSpec.Namespace) 736 framework.ExpectNoError(err) 737 738 ginkgo.By("Parsing results") 739 podSpec, err = client.Get(context.TODO(), podSpec.Name, metav1.GetOptions{}) 740 framework.ExpectNoError(err) 741 results := parseOutput(context.TODO(), f, podSpec) 742 743 ginkgo.By("Analyzing results") 744 framework.ExpectNoError(results.RunTogether(regular1, prefixedName(PreStopPrefix, regular1))) 745 framework.ExpectNoError(results.Starts(prefixedName(PreStopPrefix, regular1))) 746 framework.ExpectNoError(results.Exits(regular1)) 747 }) 748 }) 749 750 var _ = SIGDescribe(framework.WithSerial(), "Containers Lifecycle", func() { 751 f := framework.NewDefaultFramework("containers-lifecycle-test-serial") 752 f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged 753 754 ginkgo.It("should restart the containers in right order after the node reboot", func(ctx context.Context) { 755 init1 := "init-1" 756 init2 := "init-2" 757 init3 := "init-3" 758 regular1 := "regular-1" 759 760 podLabels := map[string]string{ 761 "test": "containers-lifecycle-test-serial", 762 "namespace": f.Namespace.Name, 763 } 764 pod := &v1.Pod{ 765 ObjectMeta: metav1.ObjectMeta{ 766 Name: "initialized-pod", 767 Labels: podLabels, 768 }, 769 Spec: v1.PodSpec{ 770 RestartPolicy: v1.RestartPolicyAlways, 771 InitContainers: []v1.Container{ 772 { 773 Name: init1, 774 Image: busyboxImage, 775 Command: ExecCommand(init1, execCommand{ 776 Delay: 5, 777 ExitCode: 0, 778 }), 779 }, 780 { 781 Name: init2, 782 Image: busyboxImage, 783 Command: ExecCommand(init2, execCommand{ 784 Delay: 5, 785 ExitCode: 0, 786 }), 787 }, 788 { 789 Name: init3, 790 Image: busyboxImage, 791 Command: ExecCommand(init3, execCommand{ 792 Delay: 5, 793 ExitCode: 0, 794 }), 795 }, 796 }, 797 Containers: []v1.Container{ 798 { 799 Name: regular1, 800 Image: busyboxImage, 801 Command: ExecCommand(regular1, execCommand{ 802 Delay: 30, 803 ExitCode: 0, 804 }), 805 }, 806 }, 807 }, 808 } 809 preparePod(pod) 810 811 client := e2epod.NewPodClient(f) 812 pod = client.Create(ctx, pod) 813 ginkgo.By("Waiting for the pod to be initialized and run") 814 err := e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod) 815 framework.ExpectNoError(err) 816 817 ginkgo.By("Getting the current pod sandbox ID") 818 rs, _, err := getCRIClient() 819 framework.ExpectNoError(err) 820 821 sandboxes, err := rs.ListPodSandbox(ctx, &runtimeapi.PodSandboxFilter{ 822 LabelSelector: podLabels, 823 }) 824 framework.ExpectNoError(err) 825 gomega.Expect(sandboxes).To(gomega.HaveLen(1)) 826 podSandboxID := sandboxes[0].Id 827 828 ginkgo.By("Stopping the kubelet") 829 restartKubelet := stopKubelet() 830 gomega.Eventually(ctx, func() bool { 831 return kubeletHealthCheck(kubeletHealthCheckURL) 832 }, f.Timeouts.PodStart, f.Timeouts.Poll).Should(gomega.BeFalse()) 833 834 ginkgo.By("Stopping the pod sandbox to simulate the node reboot") 835 err = rs.StopPodSandbox(ctx, podSandboxID) 836 framework.ExpectNoError(err) 837 838 ginkgo.By("Restarting the kubelet") 839 restartKubelet() 840 gomega.Eventually(ctx, func() bool { 841 return kubeletHealthCheck(kubeletHealthCheckURL) 842 }, f.Timeouts.PodStart, f.Timeouts.Poll).Should(gomega.BeTrue()) 843 844 ginkgo.By("Waiting for the pod to be re-initialized and run") 845 err = e2epod.WaitForPodCondition(ctx, f.ClientSet, pod.Namespace, pod.Name, "re-initialized", f.Timeouts.PodStart, func(pod *v1.Pod) (bool, error) { 846 if pod.Status.ContainerStatuses[0].RestartCount < 2 { 847 return false, nil 848 } 849 if pod.Status.Phase != v1.PodRunning { 850 return false, nil 851 } 852 return true, nil 853 }) 854 framework.ExpectNoError(err) 855 856 ginkgo.By("Parsing results") 857 pod, err = client.Get(ctx, pod.Name, metav1.GetOptions{}) 858 framework.ExpectNoError(err) 859 results := parseOutput(context.TODO(), f, pod) 860 861 ginkgo.By("Analyzing results") 862 init1Started, err := results.FindIndex(init1, "Started", 0) 863 framework.ExpectNoError(err) 864 init2Started, err := results.FindIndex(init2, "Started", 0) 865 framework.ExpectNoError(err) 866 init3Started, err := results.FindIndex(init3, "Started", 0) 867 framework.ExpectNoError(err) 868 regular1Started, err := results.FindIndex(regular1, "Started", 0) 869 framework.ExpectNoError(err) 870 871 init1Restarted, err := results.FindIndex(init1, "Started", init1Started+1) 872 framework.ExpectNoError(err) 873 init2Restarted, err := results.FindIndex(init2, "Started", init2Started+1) 874 framework.ExpectNoError(err) 875 init3Restarted, err := results.FindIndex(init3, "Started", init3Started+1) 876 framework.ExpectNoError(err) 877 regular1Restarted, err := results.FindIndex(regular1, "Started", regular1Started+1) 878 framework.ExpectNoError(err) 879 880 framework.ExpectNoError(init1Started.IsBefore(init2Started)) 881 framework.ExpectNoError(init2Started.IsBefore(init3Started)) 882 framework.ExpectNoError(init3Started.IsBefore(regular1Started)) 883 884 framework.ExpectNoError(init1Restarted.IsBefore(init2Restarted)) 885 framework.ExpectNoError(init2Restarted.IsBefore(init3Restarted)) 886 framework.ExpectNoError(init3Restarted.IsBefore(regular1Restarted)) 887 }) 888 }) 889 890 var _ = SIGDescribe("[NodeAlphaFeature:SidecarContainers] Containers Lifecycle", func() { 891 f := framework.NewDefaultFramework("containers-lifecycle-test") 892 f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged 893 894 ginkgo.When("using a Pod with restartPolicy=Never, three init container and two restartable init containers", ginkgo.Ordered, func() { 895 896 init1 := "init-1" 897 restartableInit1 := "restartable-init-1" 898 init2 := "init-2" 899 restartableInit2 := "restartable-init-2" 900 init3 := "init-3" 901 regular1 := "regular-1" 902 903 podSpec := &v1.Pod{ 904 ObjectMeta: metav1.ObjectMeta{ 905 Name: "restartable-init-containers-start-serially", 906 }, 907 Spec: v1.PodSpec{ 908 RestartPolicy: v1.RestartPolicyNever, 909 InitContainers: []v1.Container{ 910 { 911 Name: init1, 912 Image: busyboxImage, 913 Command: ExecCommand(init1, execCommand{ 914 Delay: 1, 915 ExitCode: 0, 916 }), 917 }, 918 { 919 Name: restartableInit1, 920 Image: busyboxImage, 921 Command: ExecCommand(restartableInit1, execCommand{ 922 Delay: 600, 923 ExitCode: 0, 924 }), 925 RestartPolicy: &containerRestartPolicyAlways, 926 }, 927 { 928 Name: init2, 929 Image: busyboxImage, 930 Command: ExecCommand(init2, execCommand{ 931 Delay: 1, 932 ExitCode: 0, 933 }), 934 }, 935 { 936 Name: restartableInit2, 937 Image: busyboxImage, 938 Command: ExecCommand(restartableInit2, execCommand{ 939 Delay: 600, 940 ExitCode: 0, 941 }), 942 RestartPolicy: &containerRestartPolicyAlways, 943 }, 944 { 945 Name: init3, 946 Image: busyboxImage, 947 Command: ExecCommand(init3, execCommand{ 948 Delay: 1, 949 ExitCode: 0, 950 }), 951 }, 952 }, 953 Containers: []v1.Container{ 954 { 955 Name: regular1, 956 Image: busyboxImage, 957 Command: ExecCommand(regular1, execCommand{ 958 Delay: 1, 959 ExitCode: 0, 960 }), 961 }, 962 }, 963 }, 964 } 965 966 preparePod(podSpec) 967 var results containerOutputList 968 969 ginkgo.It("should finish and produce log", func() { 970 client := e2epod.NewPodClient(f) 971 podSpec = client.Create(context.TODO(), podSpec) 972 973 err := e2epod.WaitTimeoutForPodNoLongerRunningInNamespace(context.TODO(), f.ClientSet, podSpec.Name, podSpec.Namespace, 5*time.Minute) 974 framework.ExpectNoError(err) 975 976 podSpec, err := client.Get(context.Background(), podSpec.Name, metav1.GetOptions{}) 977 framework.ExpectNoError(err) 978 979 // pod should exit successfully 980 gomega.Expect(podSpec.Status.Phase).To(gomega.Equal(v1.PodSucceeded)) 981 982 results = parseOutput(context.TODO(), f, podSpec) 983 }) 984 985 ginkgo.It("should run the first init container to completion before starting first restartable init container", func() { 986 framework.ExpectNoError(results.StartsBefore(init1, restartableInit1)) 987 framework.ExpectNoError(results.ExitsBefore(init1, restartableInit1)) 988 }) 989 990 ginkgo.It("should start first restartable init container before starting second init container", func() { 991 framework.ExpectNoError(results.StartsBefore(restartableInit1, init2)) 992 }) 993 994 ginkgo.It("should run first init container and first restartable init container together", func() { 995 framework.ExpectNoError(results.RunTogether(restartableInit1, init2)) 996 }) 997 998 ginkgo.It("should run second init container to completion before starting second restartable init container", func() { 999 framework.ExpectNoError(results.StartsBefore(init2, restartableInit2)) 1000 framework.ExpectNoError(results.ExitsBefore(init2, restartableInit2)) 1001 }) 1002 1003 ginkgo.It("should start second restartable init container before third init container", func() { 1004 framework.ExpectNoError(results.StartsBefore(restartableInit2, init3)) 1005 }) 1006 1007 ginkgo.It("should run both restartable init containers and third init container together", func() { 1008 framework.ExpectNoError(results.RunTogether(restartableInit2, restartableInit1)) 1009 framework.ExpectNoError(results.RunTogether(restartableInit1, init3)) 1010 framework.ExpectNoError(results.RunTogether(restartableInit2, init3)) 1011 }) 1012 1013 ginkgo.It("should run third init container to completion before starting regular container", func() { 1014 framework.ExpectNoError(results.StartsBefore(init3, regular1)) 1015 framework.ExpectNoError(results.ExitsBefore(init3, regular1)) 1016 }) 1017 1018 ginkgo.It("should run both restartable init containers and a regular container together", func() { 1019 framework.ExpectNoError(results.RunTogether(restartableInit1, regular1)) 1020 framework.ExpectNoError(results.RunTogether(restartableInit2, regular1)) 1021 }) 1022 }) 1023 1024 ginkgo.When("using a restartable init container in a Pod with restartPolicy=Never", func() { 1025 ginkgo.When("a restartable init container runs continuously", ginkgo.Ordered, func() { 1026 1027 restartableInit1 := "restartable-init-1" 1028 regular1 := "regular-1" 1029 1030 podSpec := &v1.Pod{ 1031 ObjectMeta: metav1.ObjectMeta{ 1032 Name: "restartable-init-container-run-continuously", 1033 }, 1034 Spec: v1.PodSpec{ 1035 RestartPolicy: v1.RestartPolicyNever, 1036 InitContainers: []v1.Container{ 1037 { 1038 Name: restartableInit1, 1039 Image: busyboxImage, 1040 Command: ExecCommand(restartableInit1, execCommand{ 1041 Delay: 600, 1042 ExitCode: 0, 1043 }), 1044 RestartPolicy: &containerRestartPolicyAlways, 1045 }, 1046 }, 1047 Containers: []v1.Container{ 1048 { 1049 Name: regular1, 1050 Image: busyboxImage, 1051 Command: ExecCommand(regular1, execCommand{ 1052 Delay: 1, 1053 ExitCode: 0, 1054 }), 1055 }, 1056 }, 1057 }, 1058 } 1059 1060 preparePod(podSpec) 1061 var results containerOutputList 1062 1063 ginkgo.It("should complete a Pod successfully and produce log", func() { 1064 client := e2epod.NewPodClient(f) 1065 podSpec = client.Create(context.TODO(), podSpec) 1066 1067 err := e2epod.WaitTimeoutForPodNoLongerRunningInNamespace(context.TODO(), f.ClientSet, podSpec.Name, podSpec.Namespace, 5*time.Minute) 1068 framework.ExpectNoError(err) 1069 1070 podSpec, err := client.Get(context.TODO(), podSpec.Name, metav1.GetOptions{}) 1071 framework.ExpectNoError(err) 1072 1073 // pod should exit successfully 1074 gomega.Expect(podSpec.Status.Phase).To(gomega.Equal(v1.PodSucceeded)) 1075 1076 results = parseOutput(context.TODO(), f, podSpec) 1077 }) 1078 ginkgo.It("should not restart a restartable init container", func() { 1079 framework.ExpectNoError(results.DoesntStartAfter(restartableInit1, regular1)) 1080 }) 1081 ginkgo.It("should run a regular container to completion", func() { 1082 framework.ExpectNoError(results.Exits(regular1)) 1083 }) 1084 }) 1085 1086 ginkgo.When("a restartable init container fails to start because of a bad image", ginkgo.Ordered, func() { 1087 1088 restartableInit1 := "restartable-init-1" 1089 regular1 := "regular-1" 1090 1091 podSpec := &v1.Pod{ 1092 ObjectMeta: metav1.ObjectMeta{ 1093 Name: "restartable-init-runs-with-pod", 1094 }, 1095 Spec: v1.PodSpec{ 1096 RestartPolicy: v1.RestartPolicyNever, 1097 InitContainers: []v1.Container{ 1098 { 1099 Name: restartableInit1, 1100 Image: imageutils.GetE2EImage(imageutils.InvalidRegistryImage), 1101 Command: ExecCommand(restartableInit1, execCommand{ 1102 Delay: 600, 1103 ExitCode: 0, 1104 }), 1105 RestartPolicy: &containerRestartPolicyAlways, 1106 }, 1107 }, 1108 Containers: []v1.Container{ 1109 { 1110 Name: regular1, 1111 Image: busyboxImage, 1112 Command: ExecCommand(regular1, execCommand{ 1113 Delay: 1, 1114 ExitCode: 0, 1115 }), 1116 }, 1117 }, 1118 }, 1119 } 1120 1121 preparePod(podSpec) 1122 var results containerOutputList 1123 1124 ginkgo.It("should mark a Pod as failed and produce log", func() { 1125 client := e2epod.NewPodClient(f) 1126 podSpec = client.Create(context.TODO(), podSpec) 1127 1128 // restartable init container should be in image pull backoff 1129 err := WaitForPodInitContainerToFail(context.TODO(), f.ClientSet, podSpec.Namespace, podSpec.Name, 0, "ImagePullBackOff", f.Timeouts.PodStart) 1130 framework.ExpectNoError(err) 1131 1132 podSpec, err = client.Get(context.Background(), podSpec.Name, metav1.GetOptions{}) 1133 framework.ExpectNoError(err) 1134 results = parseOutput(context.TODO(), f, podSpec) 1135 }) 1136 ginkgo.It("should not start a restartable init container", func() { 1137 framework.ExpectNoError(results.DoesntStart(restartableInit1)) 1138 }) 1139 ginkgo.It("should not start a regular container", func() { 1140 framework.ExpectNoError(results.DoesntStart(regular1)) 1141 }) 1142 }) 1143 1144 // TODO: add a test case similar to one above, but with startup probe never succeeding 1145 1146 ginkgo.When("a restartable init container starts and exits with exit code 0 continuously", ginkgo.Ordered, func() { 1147 1148 restartableInit1 := "restartable-init-1" 1149 init1 := "init-1" 1150 regular1 := "regular-1" 1151 1152 podSpec := &v1.Pod{ 1153 ObjectMeta: metav1.ObjectMeta{ 1154 Name: "restartable-init-container-exit-0-continuously", 1155 }, 1156 Spec: v1.PodSpec{ 1157 RestartPolicy: v1.RestartPolicyNever, 1158 InitContainers: []v1.Container{ 1159 { 1160 Name: restartableInit1, 1161 Image: busyboxImage, 1162 Command: ExecCommand(restartableInit1, execCommand{ 1163 Delay: 1, 1164 ExitCode: 0, 1165 }), 1166 RestartPolicy: &containerRestartPolicyAlways, 1167 }, 1168 { 1169 Name: init1, 1170 Image: busyboxImage, 1171 Command: ExecCommand(init1, execCommand{ 1172 Delay: 5, 1173 ExitCode: 0, 1174 }), 1175 }, 1176 }, 1177 Containers: []v1.Container{ 1178 { 1179 Name: regular1, 1180 Image: busyboxImage, 1181 Command: ExecCommand(regular1, execCommand{ 1182 Delay: 60, 1183 ExitCode: 0, 1184 }), 1185 }, 1186 }, 1187 }, 1188 } 1189 1190 preparePod(podSpec) 1191 var results containerOutputList 1192 1193 ginkgo.It("should complete a Pod successfully and produce log", func() { 1194 client := e2epod.NewPodClient(f) 1195 podSpec = client.Create(context.TODO(), podSpec) 1196 1197 err := e2epod.WaitTimeoutForPodNoLongerRunningInNamespace(context.TODO(), f.ClientSet, podSpec.Name, podSpec.Namespace, 5*time.Minute) 1198 framework.ExpectNoError(err) 1199 1200 podSpec, err := client.Get(context.TODO(), podSpec.Name, metav1.GetOptions{}) 1201 framework.ExpectNoError(err) 1202 1203 // pod should exit successfully 1204 gomega.Expect(podSpec.Status.Phase).To(gomega.Equal(v1.PodSucceeded)) 1205 results = parseOutput(context.TODO(), f, podSpec) 1206 }) 1207 ginkgo.It("should restart a restartable init container before the regular container started", func() { 1208 framework.ExpectNoError(results.StartsBefore(restartableInit1, regular1)) 1209 }) 1210 ginkgo.It("should restart a restartable init container after the regular container started", func() { 1211 framework.ExpectNoError(results.StartsBefore(regular1, restartableInit1)) 1212 }) 1213 ginkgo.It("should run a regular container to completion", func() { 1214 framework.ExpectNoError(results.Exits(regular1)) 1215 }) 1216 }) 1217 1218 ginkgo.When("a restartable init container starts and exits with exit code 1 continuously", ginkgo.Ordered, func() { 1219 restartableInit1 := "restartable-init-1" 1220 init1 := "init-1" 1221 regular1 := "regular-1" 1222 1223 podSpec := &v1.Pod{ 1224 ObjectMeta: metav1.ObjectMeta{ 1225 Name: "restartable-init-container-exit-1-continuously", 1226 }, 1227 Spec: v1.PodSpec{ 1228 RestartPolicy: v1.RestartPolicyNever, 1229 InitContainers: []v1.Container{ 1230 { 1231 Name: restartableInit1, 1232 Image: busyboxImage, 1233 Command: ExecCommand(restartableInit1, execCommand{ 1234 Delay: 1, 1235 ExitCode: 1, 1236 }), 1237 RestartPolicy: &containerRestartPolicyAlways, 1238 }, 1239 { 1240 Name: init1, 1241 Image: busyboxImage, 1242 Command: ExecCommand(init1, execCommand{ 1243 Delay: 5, 1244 ExitCode: 0, 1245 }), 1246 }, 1247 }, 1248 Containers: []v1.Container{ 1249 { 1250 Name: regular1, 1251 Image: busyboxImage, 1252 Command: ExecCommand(regular1, execCommand{ 1253 Delay: 60, 1254 ExitCode: 0, 1255 }), 1256 }, 1257 }, 1258 }, 1259 } 1260 1261 preparePod(podSpec) 1262 var results containerOutputList 1263 1264 ginkgo.It("should complete a Pod successfully and produce log", func() { 1265 client := e2epod.NewPodClient(f) 1266 podSpec = client.Create(context.TODO(), podSpec) 1267 1268 err := e2epod.WaitTimeoutForPodNoLongerRunningInNamespace(context.TODO(), f.ClientSet, podSpec.Name, podSpec.Namespace, 5*time.Minute) 1269 framework.ExpectNoError(err) 1270 1271 podSpec, err := client.Get(context.TODO(), podSpec.Name, metav1.GetOptions{}) 1272 framework.ExpectNoError(err) 1273 1274 // pod should exit successfully 1275 gomega.Expect(podSpec.Status.Phase).To(gomega.Equal(v1.PodSucceeded)) 1276 1277 results = parseOutput(context.TODO(), f, podSpec) 1278 }) 1279 ginkgo.It("should restart a restartable init container before the regular container started", func() { 1280 framework.ExpectNoError(results.StartsBefore(restartableInit1, regular1)) 1281 }) 1282 ginkgo.It("should restart a restartable init container after the regular container started", func() { 1283 framework.ExpectNoError(results.StartsBefore(regular1, restartableInit1)) 1284 }) 1285 ginkgo.It("should run a regular container to completion", func() { 1286 framework.ExpectNoError(results.Exits(regular1)) 1287 }) 1288 }) 1289 1290 ginkgo.When("an Init container before restartable init container fails", ginkgo.Ordered, func() { 1291 1292 init1 := "init-1" 1293 restartableInit1 := "restartable-init-1" 1294 regular1 := "regular-1" 1295 1296 podSpec := &v1.Pod{ 1297 ObjectMeta: metav1.ObjectMeta{ 1298 Name: "init-container-fails-before-restartable-init-starts", 1299 }, 1300 Spec: v1.PodSpec{ 1301 RestartPolicy: v1.RestartPolicyNever, 1302 InitContainers: []v1.Container{ 1303 { 1304 Name: init1, 1305 Image: busyboxImage, 1306 Command: ExecCommand(init1, execCommand{ 1307 Delay: 1, 1308 ExitCode: 1, 1309 }), 1310 }, 1311 { 1312 Name: restartableInit1, 1313 Image: busyboxImage, 1314 Command: ExecCommand(restartableInit1, execCommand{ 1315 Delay: 600, 1316 ExitCode: 0, 1317 }), 1318 RestartPolicy: &containerRestartPolicyAlways, 1319 }, 1320 }, 1321 Containers: []v1.Container{ 1322 { 1323 Name: regular1, 1324 Image: busyboxImage, 1325 Command: ExecCommand(regular1, execCommand{ 1326 Delay: 600, 1327 ExitCode: 0, 1328 }), 1329 }, 1330 }, 1331 }, 1332 } 1333 1334 preparePod(podSpec) 1335 var results containerOutputList 1336 1337 ginkgo.It("should mark a Pod as failed and produce log", func() { 1338 client := e2epod.NewPodClient(f) 1339 podSpec = client.Create(context.TODO(), podSpec) 1340 1341 err := e2epod.WaitForPodFailedReason(context.TODO(), f.ClientSet, podSpec, "", 1*time.Minute) 1342 framework.ExpectNoError(err) 1343 1344 podSpec, err := client.Get(context.TODO(), podSpec.Name, metav1.GetOptions{}) 1345 framework.ExpectNoError(err) 1346 results = parseOutput(context.TODO(), f, podSpec) 1347 }) 1348 ginkgo.It("should mark an Init container as failed", func() { 1349 framework.ExpectNoError(results.Exits(init1)) 1350 }) 1351 ginkgo.It("should not start restartable init container", func() { 1352 framework.ExpectNoError(results.DoesntStart(restartableInit1)) 1353 }) 1354 }) 1355 1356 ginkgo.When("an Init container after restartable init container fails", ginkgo.Ordered, func() { 1357 1358 init1 := "init-1" 1359 restartableInit1 := "restartable-init-1" 1360 regular1 := "regular-1" 1361 1362 podSpec := &v1.Pod{ 1363 ObjectMeta: metav1.ObjectMeta{ 1364 Name: "restartable-init-container-fails-before-init-container", 1365 }, 1366 Spec: v1.PodSpec{ 1367 RestartPolicy: v1.RestartPolicyNever, 1368 InitContainers: []v1.Container{ 1369 { 1370 Name: restartableInit1, 1371 Image: busyboxImage, 1372 Command: ExecCommand(restartableInit1, execCommand{ 1373 Delay: 1, 1374 ExitCode: 1, 1375 }), 1376 RestartPolicy: &containerRestartPolicyAlways, 1377 }, 1378 { 1379 Name: init1, 1380 Image: busyboxImage, 1381 Command: ExecCommand(init1, execCommand{ 1382 Delay: 1, 1383 ExitCode: 1, 1384 }), 1385 }, 1386 }, 1387 Containers: []v1.Container{ 1388 { 1389 Name: regular1, 1390 Image: busyboxImage, 1391 Command: ExecCommand(regular1, execCommand{ 1392 Delay: 600, 1393 ExitCode: 0, 1394 }), 1395 }, 1396 }, 1397 }, 1398 } 1399 1400 preparePod(podSpec) 1401 var results containerOutputList 1402 1403 ginkgo.It("should mark a Pod as failed and produce log", func() { 1404 client := e2epod.NewPodClient(f) 1405 podSpec = client.Create(context.TODO(), podSpec) 1406 1407 err := e2epod.WaitForPodFailedReason(context.TODO(), f.ClientSet, podSpec, "", 1*time.Minute) 1408 framework.ExpectNoError(err) 1409 1410 podSpec, err := client.Get(context.TODO(), podSpec.Name, metav1.GetOptions{}) 1411 framework.ExpectNoError(err) 1412 results = parseOutput(context.TODO(), f, podSpec) 1413 }) 1414 ginkgo.It("should mark an Init container as failed", func() { 1415 framework.ExpectNoError(results.Exits(init1)) 1416 }) 1417 // TODO: how will we be able to test it if restartable init container 1418 // will never fail and there will be no termination log? Or will be? 1419 ginkgo.It("should be running restartable init container and a failed Init container in parallel", func() { 1420 framework.ExpectNoError(results.RunTogether(restartableInit1, init1)) 1421 }) 1422 }) 1423 }) 1424 1425 ginkgo.When("using a restartable init container in a Pod with restartPolicy=OnFailure", ginkgo.Ordered, func() { 1426 // this test case the same as for restartPolicy=Never 1427 ginkgo.When("a restartable init container runs continuously", func() { 1428 1429 restartableInit1 := "restartable-init-1" 1430 regular1 := "regular-1" 1431 1432 podSpec := &v1.Pod{ 1433 ObjectMeta: metav1.ObjectMeta{ 1434 Name: "restartable-init-container-run-continuously", 1435 }, 1436 Spec: v1.PodSpec{ 1437 RestartPolicy: v1.RestartPolicyOnFailure, 1438 InitContainers: []v1.Container{ 1439 { 1440 Name: restartableInit1, 1441 Image: busyboxImage, 1442 Command: ExecCommand(restartableInit1, execCommand{ 1443 Delay: 600, 1444 ExitCode: 0, 1445 }), 1446 RestartPolicy: &containerRestartPolicyAlways, 1447 }, 1448 }, 1449 Containers: []v1.Container{ 1450 { 1451 Name: regular1, 1452 Image: busyboxImage, 1453 Command: ExecCommand(regular1, execCommand{ 1454 Delay: 1, 1455 ExitCode: 0, 1456 }), 1457 }, 1458 }, 1459 }, 1460 } 1461 1462 preparePod(podSpec) 1463 var results containerOutputList 1464 1465 ginkgo.It("should complete a Pod successfully and produce log", func() { 1466 client := e2epod.NewPodClient(f) 1467 podSpec = client.Create(context.TODO(), podSpec) 1468 1469 err := e2epod.WaitTimeoutForPodNoLongerRunningInNamespace(context.TODO(), f.ClientSet, podSpec.Name, podSpec.Namespace, 5*time.Minute) 1470 framework.ExpectNoError(err) 1471 1472 podSpec, err := client.Get(context.TODO(), podSpec.Name, metav1.GetOptions{}) 1473 framework.ExpectNoError(err) 1474 1475 // pod should exit successfully 1476 gomega.Expect(podSpec.Status.Phase).To(gomega.Equal(v1.PodSucceeded)) 1477 1478 results = parseOutput(context.TODO(), f, podSpec) 1479 }) 1480 ginkgo.It("should not restart a restartable init container", func() { 1481 framework.ExpectNoError(results.DoesntStartAfter(restartableInit1, regular1)) 1482 }) 1483 ginkgo.It("should run a regular container to completion", func() { 1484 framework.ExpectNoError(results.Exits(regular1)) 1485 }) 1486 }) 1487 1488 ginkgo.When("a restartable init container fails to start because of a bad image", ginkgo.Ordered, func() { 1489 1490 restartableInit1 := "restartable-init-1" 1491 regular1 := "regular-1" 1492 1493 podSpec := &v1.Pod{ 1494 ObjectMeta: metav1.ObjectMeta{ 1495 Name: "restartable-init-runs-with-pod", 1496 }, 1497 Spec: v1.PodSpec{ 1498 RestartPolicy: v1.RestartPolicyOnFailure, 1499 InitContainers: []v1.Container{ 1500 { 1501 Name: restartableInit1, 1502 Image: imageutils.GetE2EImage(imageutils.InvalidRegistryImage), 1503 Command: ExecCommand(restartableInit1, execCommand{ 1504 Delay: 600, 1505 ExitCode: 0, 1506 }), 1507 RestartPolicy: &containerRestartPolicyAlways, 1508 }, 1509 }, 1510 Containers: []v1.Container{ 1511 { 1512 Name: regular1, 1513 Image: busyboxImage, 1514 Command: ExecCommand(regular1, execCommand{ 1515 Delay: 1, 1516 ExitCode: 0, 1517 }), 1518 }, 1519 }, 1520 }, 1521 } 1522 1523 preparePod(podSpec) 1524 var results containerOutputList 1525 1526 ginkgo.It("should mark a Pod as failed and produce log", func() { 1527 client := e2epod.NewPodClient(f) 1528 podSpec = client.Create(context.TODO(), podSpec) 1529 1530 // restartable init container should be in image pull backoff 1531 err := WaitForPodInitContainerToFail(context.TODO(), f.ClientSet, podSpec.Namespace, podSpec.Name, 0, "ImagePullBackOff", f.Timeouts.PodStart) 1532 framework.ExpectNoError(err) 1533 1534 podSpec, err = client.Get(context.Background(), podSpec.Name, metav1.GetOptions{}) 1535 framework.ExpectNoError(err) 1536 results = parseOutput(context.TODO(), f, podSpec) 1537 }) 1538 ginkgo.It("should not start a restartable init container", func() { 1539 framework.ExpectNoError(results.DoesntStart(restartableInit1)) 1540 }) 1541 ginkgo.It("should not start a regular container", func() { 1542 framework.ExpectNoError(results.DoesntStart(regular1)) 1543 }) 1544 }) 1545 1546 // TODO: add a test case similar to one above, but with startup probe never succeeding 1547 1548 // this test case the same as for restartPolicy=Never 1549 ginkgo.When("a restartable init container starts and exits with exit code 0 continuously", ginkgo.Ordered, func() { 1550 1551 restartableInit1 := "restartable-init-1" 1552 init1 := "init-1" 1553 regular1 := "regular-1" 1554 1555 podSpec := &v1.Pod{ 1556 ObjectMeta: metav1.ObjectMeta{ 1557 Name: "restartable-init-container-exit-0-continuously", 1558 }, 1559 Spec: v1.PodSpec{ 1560 RestartPolicy: v1.RestartPolicyOnFailure, 1561 InitContainers: []v1.Container{ 1562 { 1563 Name: restartableInit1, 1564 Image: busyboxImage, 1565 Command: ExecCommand(restartableInit1, execCommand{ 1566 Delay: 1, 1567 ExitCode: 0, 1568 }), 1569 RestartPolicy: &containerRestartPolicyAlways, 1570 }, 1571 { 1572 Name: init1, 1573 Image: busyboxImage, 1574 Command: ExecCommand(init1, execCommand{ 1575 Delay: 5, 1576 ExitCode: 0, 1577 }), 1578 }, 1579 }, 1580 Containers: []v1.Container{ 1581 { 1582 Name: regular1, 1583 Image: busyboxImage, 1584 Command: ExecCommand(regular1, execCommand{ 1585 Delay: 60, 1586 ExitCode: 0, 1587 }), 1588 }, 1589 }, 1590 }, 1591 } 1592 1593 preparePod(podSpec) 1594 var results containerOutputList 1595 1596 ginkgo.It("should complete a Pod successfully and produce log", func() { 1597 client := e2epod.NewPodClient(f) 1598 podSpec = client.Create(context.TODO(), podSpec) 1599 1600 err := e2epod.WaitTimeoutForPodNoLongerRunningInNamespace(context.TODO(), f.ClientSet, podSpec.Name, podSpec.Namespace, 5*time.Minute) 1601 framework.ExpectNoError(err) 1602 1603 podSpec, err := client.Get(context.TODO(), podSpec.Name, metav1.GetOptions{}) 1604 framework.ExpectNoError(err) 1605 1606 // pod should exit successfully 1607 gomega.Expect(podSpec.Status.Phase).To(gomega.Equal(v1.PodSucceeded)) 1608 1609 results = parseOutput(context.TODO(), f, podSpec) 1610 }) 1611 ginkgo.It("should restart a restartable init container before the regular container started", func() { 1612 framework.ExpectNoError(results.StartsBefore(restartableInit1, regular1)) 1613 }) 1614 ginkgo.It("should restart a restartable init container after the regular container started", func() { 1615 framework.ExpectNoError(results.StartsBefore(regular1, restartableInit1)) 1616 }) 1617 ginkgo.It("should run a regular container to completion", func() { 1618 framework.ExpectNoError(results.Exits(regular1)) 1619 }) 1620 }) 1621 1622 // this test case the same as for restartPolicy=Never 1623 ginkgo.When("a restartable init container starts and exits with exit code 1 continuously", ginkgo.Ordered, func() { 1624 1625 restartableInit1 := "restartable-init-1" 1626 init1 := "init-1" 1627 regular1 := "regular-1" 1628 1629 podSpec := &v1.Pod{ 1630 ObjectMeta: metav1.ObjectMeta{ 1631 Name: "restartable-init-container-exit-1-continuously", 1632 }, 1633 Spec: v1.PodSpec{ 1634 RestartPolicy: v1.RestartPolicyOnFailure, 1635 InitContainers: []v1.Container{ 1636 { 1637 Name: restartableInit1, 1638 Image: busyboxImage, 1639 Command: ExecCommand(restartableInit1, execCommand{ 1640 Delay: 1, 1641 ExitCode: 1, 1642 }), 1643 RestartPolicy: &containerRestartPolicyAlways, 1644 }, 1645 { 1646 Name: init1, 1647 Image: busyboxImage, 1648 Command: ExecCommand(init1, execCommand{ 1649 Delay: 5, 1650 ExitCode: 0, 1651 }), 1652 }, 1653 }, 1654 Containers: []v1.Container{ 1655 { 1656 Name: regular1, 1657 Image: busyboxImage, 1658 Command: ExecCommand(regular1, execCommand{ 1659 Delay: 60, 1660 ExitCode: 0, 1661 }), 1662 }, 1663 }, 1664 }, 1665 } 1666 1667 preparePod(podSpec) 1668 var results containerOutputList 1669 1670 ginkgo.It("should complete a Pod successfully and produce log", func() { 1671 client := e2epod.NewPodClient(f) 1672 podSpec = client.Create(context.TODO(), podSpec) 1673 1674 err := e2epod.WaitTimeoutForPodNoLongerRunningInNamespace(context.TODO(), f.ClientSet, podSpec.Name, podSpec.Namespace, 5*time.Minute) 1675 framework.ExpectNoError(err) 1676 1677 podSpec, err := client.Get(context.TODO(), podSpec.Name, metav1.GetOptions{}) 1678 framework.ExpectNoError(err) 1679 1680 // pod should exit successfully 1681 gomega.Expect(podSpec.Status.Phase).To(gomega.Equal(v1.PodSucceeded)) 1682 1683 results = parseOutput(context.TODO(), f, podSpec) 1684 }) 1685 ginkgo.It("should restart a restartable init container before the regular container started", func() { 1686 framework.ExpectNoError(results.StartsBefore(restartableInit1, regular1)) 1687 }) 1688 ginkgo.It("should restart a restartable init container after the regular container started", func() { 1689 framework.ExpectNoError(results.StartsBefore(regular1, restartableInit1)) 1690 }) 1691 ginkgo.It("should run a regular container to completion", func() { 1692 framework.ExpectNoError(results.Exits(regular1)) 1693 }) 1694 }) 1695 1696 ginkgo.When("an Init container before restartable init container continuously fails", ginkgo.Ordered, func() { 1697 1698 init1 := "init-1" 1699 restartableInit1 := "restartable-init-1" 1700 regular1 := "regular-1" 1701 1702 podSpec := &v1.Pod{ 1703 ObjectMeta: metav1.ObjectMeta{ 1704 Name: "init-container-fails-before-restartable-init-starts", 1705 }, 1706 Spec: v1.PodSpec{ 1707 RestartPolicy: v1.RestartPolicyOnFailure, 1708 InitContainers: []v1.Container{ 1709 { 1710 Name: init1, 1711 Image: busyboxImage, 1712 Command: ExecCommand(init1, execCommand{ 1713 Delay: 1, 1714 ExitCode: 1, 1715 }), 1716 }, 1717 { 1718 Name: restartableInit1, 1719 Image: busyboxImage, 1720 Command: ExecCommand(restartableInit1, execCommand{ 1721 Delay: 600, 1722 ExitCode: 0, 1723 }), 1724 RestartPolicy: &containerRestartPolicyAlways, 1725 }, 1726 }, 1727 Containers: []v1.Container{ 1728 { 1729 Name: regular1, 1730 Image: busyboxImage, 1731 Command: ExecCommand(regular1, execCommand{ 1732 Delay: 600, 1733 ExitCode: 0, 1734 }), 1735 }, 1736 }, 1737 }, 1738 } 1739 1740 preparePod(podSpec) 1741 var results containerOutputList 1742 1743 ginkgo.It("should continuously run Pod keeping it Pending", func() { 1744 client := e2epod.NewPodClient(f) 1745 podSpec = client.Create(context.TODO(), podSpec) 1746 1747 err := e2epod.WaitForPodCondition(context.TODO(), f.ClientSet, podSpec.Namespace, podSpec.Name, "pending and restarting 3 times", 5*time.Minute, func(pod *v1.Pod) (bool, error) { 1748 if pod.Status.Phase != v1.PodPending { 1749 return false, fmt.Errorf("pod should be in pending phase") 1750 } 1751 if len(pod.Status.InitContainerStatuses) < 1 { 1752 return false, nil 1753 } 1754 containerStatus := pod.Status.InitContainerStatuses[0] 1755 return containerStatus.RestartCount >= 3, nil 1756 }) 1757 framework.ExpectNoError(err) 1758 1759 podSpec, err := client.Get(context.TODO(), podSpec.Name, metav1.GetOptions{}) 1760 framework.ExpectNoError(err) 1761 results = parseOutput(context.TODO(), f, podSpec) 1762 }) 1763 ginkgo.It("should have Init container restartCount greater than 0", func() { 1764 framework.ExpectNoError(results.HasRestarted(init1)) 1765 }) 1766 ginkgo.It("should not start restartable init container", func() { 1767 framework.ExpectNoError(results.DoesntStart(restartableInit1)) 1768 }) 1769 }) 1770 1771 ginkgo.When("an Init container after restartable init container fails", ginkgo.Ordered, func() { 1772 1773 init1 := "init-1" 1774 restartableInit1 := "restartable-init-1" 1775 regular1 := "regular-1" 1776 1777 podSpec := &v1.Pod{ 1778 ObjectMeta: metav1.ObjectMeta{ 1779 Name: "restartable-init-container-fails-before-init-container", 1780 }, 1781 Spec: v1.PodSpec{ 1782 RestartPolicy: v1.RestartPolicyOnFailure, 1783 InitContainers: []v1.Container{ 1784 { 1785 Name: restartableInit1, 1786 Image: busyboxImage, 1787 Command: ExecCommand(restartableInit1, execCommand{ 1788 Delay: 1, 1789 ExitCode: 1, 1790 }), 1791 RestartPolicy: &containerRestartPolicyAlways, 1792 }, 1793 { 1794 Name: init1, 1795 Image: busyboxImage, 1796 Command: ExecCommand(init1, execCommand{ 1797 Delay: 1, 1798 ExitCode: 1, 1799 }), 1800 }, 1801 }, 1802 Containers: []v1.Container{ 1803 { 1804 Name: regular1, 1805 Image: busyboxImage, 1806 Command: ExecCommand(regular1, execCommand{ 1807 Delay: 600, 1808 ExitCode: 0, 1809 }), 1810 }, 1811 }, 1812 }, 1813 } 1814 1815 preparePod(podSpec) 1816 var results containerOutputList 1817 1818 ginkgo.It("should continuously run Pod keeping it Pending", func() { 1819 client := e2epod.NewPodClient(f) 1820 podSpec = client.Create(context.TODO(), podSpec) 1821 1822 err := e2epod.WaitForPodCondition(context.TODO(), f.ClientSet, podSpec.Namespace, podSpec.Name, "pending and restarting 3 times", 5*time.Minute, func(pod *v1.Pod) (bool, error) { 1823 if pod.Status.Phase != v1.PodPending { 1824 return false, fmt.Errorf("pod should be in pending phase") 1825 } 1826 if len(pod.Status.InitContainerStatuses) < 1 { 1827 return false, nil 1828 } 1829 containerStatus := pod.Status.InitContainerStatuses[0] 1830 return containerStatus.RestartCount >= 3, nil 1831 }) 1832 framework.ExpectNoError(err) 1833 1834 podSpec, err := client.Get(context.TODO(), podSpec.Name, metav1.GetOptions{}) 1835 framework.ExpectNoError(err) 1836 results = parseOutput(context.TODO(), f, podSpec) 1837 }) 1838 ginkgo.It("should have Init container restartCount greater than 0", func() { 1839 framework.ExpectNoError(results.HasRestarted(init1)) 1840 }) 1841 // TODO: how will we be able to test it if restartable init container will never fail and there will be no termination log? Or will be? 1842 ginkgo.It("should be running restartable init container and a failed Init container in parallel", func() { 1843 framework.ExpectNoError(results.RunTogether(restartableInit1, init1)) 1844 }) 1845 }) 1846 }) 1847 1848 ginkgo.When("using a restartable init container in a Pod with restartPolicy=Always", ginkgo.Ordered, func() { 1849 ginkgo.When("a restartable init container runs continuously", func() { 1850 1851 restartableInit1 := "restartable-init-1" 1852 regular1 := "regular-1" 1853 1854 podSpec := &v1.Pod{ 1855 ObjectMeta: metav1.ObjectMeta{ 1856 Name: "restartable-init-container-run-continuously", 1857 }, 1858 Spec: v1.PodSpec{ 1859 RestartPolicy: v1.RestartPolicyAlways, 1860 InitContainers: []v1.Container{ 1861 { 1862 Name: restartableInit1, 1863 Image: busyboxImage, 1864 Command: ExecCommand(restartableInit1, execCommand{ 1865 Delay: 600, 1866 ExitCode: 0, 1867 }), 1868 RestartPolicy: &containerRestartPolicyAlways, 1869 }, 1870 }, 1871 Containers: []v1.Container{ 1872 { 1873 Name: regular1, 1874 Image: busyboxImage, 1875 Command: ExecCommand(regular1, execCommand{ 1876 Delay: 1, 1877 ExitCode: 0, 1878 }), 1879 }, 1880 }, 1881 }, 1882 } 1883 1884 preparePod(podSpec) 1885 var results containerOutputList 1886 1887 // regular container should exit at least once so we can get it's termination log 1888 // this test case is different from restartPolicy=Never 1889 ginkgo.It("should keep running a Pod continuously and produce log", func() { /* check the regular container restartCount > 0 */ 1890 client := e2epod.NewPodClient(f) 1891 podSpec = client.Create(context.TODO(), podSpec) 1892 1893 err := WaitForPodContainerRestartCount(context.TODO(), f.ClientSet, podSpec.Namespace, podSpec.Name, 0, 2, 2*time.Minute) 1894 framework.ExpectNoError(err) 1895 1896 podSpec, err := client.Get(context.TODO(), podSpec.Name, metav1.GetOptions{}) 1897 framework.ExpectNoError(err) 1898 results = parseOutput(context.TODO(), f, podSpec) 1899 }) 1900 1901 ginkgo.It("should not restart a restartable init container", func() { 1902 framework.ExpectNoError(results.DoesntStartAfter(restartableInit1, regular1)) 1903 }) 1904 // this test case is different from restartPolicy=Never 1905 ginkgo.It("should start a regular container", func() { 1906 framework.ExpectNoError(results.HasRestarted(regular1)) 1907 }) 1908 }) 1909 1910 ginkgo.When("a restartable init container fails to start because of a bad image", ginkgo.Ordered, func() { 1911 1912 restartableInit1 := "restartable-init-1" 1913 regular1 := "regular-1" 1914 1915 podSpec := &v1.Pod{ 1916 ObjectMeta: metav1.ObjectMeta{ 1917 Name: "restartable-init-runs-with-pod", 1918 }, 1919 Spec: v1.PodSpec{ 1920 RestartPolicy: v1.RestartPolicyNever, 1921 InitContainers: []v1.Container{ 1922 { 1923 Name: restartableInit1, 1924 Image: imageutils.GetE2EImage(imageutils.InvalidRegistryImage), 1925 Command: ExecCommand(restartableInit1, execCommand{ 1926 Delay: 600, 1927 ExitCode: 0, 1928 }), 1929 RestartPolicy: &containerRestartPolicyAlways, 1930 }, 1931 }, 1932 Containers: []v1.Container{ 1933 { 1934 Name: regular1, 1935 Image: busyboxImage, 1936 Command: ExecCommand(regular1, execCommand{ 1937 Delay: 1, 1938 ExitCode: 0, 1939 }), 1940 }, 1941 }, 1942 }, 1943 } 1944 1945 preparePod(podSpec) 1946 var results containerOutputList 1947 1948 ginkgo.It("should continuously run Pod keeping it Pending and produce log", func() { 1949 client := e2epod.NewPodClient(f) 1950 podSpec = client.Create(context.TODO(), podSpec) 1951 1952 // restartable init container should be in image pull backoff 1953 err := WaitForPodInitContainerToFail(context.TODO(), f.ClientSet, podSpec.Namespace, podSpec.Name, 0, "ImagePullBackOff", f.Timeouts.PodStart) 1954 framework.ExpectNoError(err) 1955 1956 podSpec, err = client.Get(context.Background(), podSpec.Name, metav1.GetOptions{}) 1957 framework.ExpectNoError(err) 1958 results = parseOutput(context.TODO(), f, podSpec) 1959 }) 1960 ginkgo.It("should not start a restartable init container", func() { 1961 framework.ExpectNoError(results.DoesntStart(restartableInit1)) 1962 }) 1963 ginkgo.It("should not start a regular container", func() { 1964 framework.ExpectNoError(results.DoesntStart(regular1)) 1965 }) 1966 }) 1967 1968 // TODO: add a test case similar to one above, but with startup probe never succeeding 1969 1970 ginkgo.When("a restartable init container starts and exits with exit code 0 continuously", ginkgo.Ordered, func() { 1971 1972 restartableInit1 := "restartable-init-1" 1973 init1 := "init-1" 1974 regular1 := "regular-1" 1975 1976 podSpec := &v1.Pod{ 1977 ObjectMeta: metav1.ObjectMeta{ 1978 Name: "restartable-init-container-exit-0-continuously", 1979 }, 1980 Spec: v1.PodSpec{ 1981 RestartPolicy: v1.RestartPolicyAlways, 1982 InitContainers: []v1.Container{ 1983 { 1984 Name: restartableInit1, 1985 Image: busyboxImage, 1986 Command: ExecCommand(restartableInit1, execCommand{ 1987 Delay: 1, 1988 ExitCode: 0, 1989 }), 1990 RestartPolicy: &containerRestartPolicyAlways, 1991 }, 1992 { 1993 Name: init1, 1994 Image: busyboxImage, 1995 Command: ExecCommand(init1, execCommand{ 1996 Delay: 5, 1997 ExitCode: 0, 1998 }), 1999 }, 2000 }, 2001 Containers: []v1.Container{ 2002 { 2003 Name: regular1, 2004 Image: busyboxImage, 2005 Command: ExecCommand(regular1, execCommand{ 2006 Delay: 60, 2007 ExitCode: 0, 2008 }), 2009 }, 2010 }, 2011 }, 2012 } 2013 2014 preparePod(podSpec) 2015 var results containerOutputList 2016 2017 ginkgo.It("should keep running a Pod continuously and produce log", func() { /* check the regular container restartCount > 0 */ 2018 client := e2epod.NewPodClient(f) 2019 podSpec = client.Create(context.TODO(), podSpec) 2020 2021 err := WaitForPodContainerRestartCount(context.TODO(), f.ClientSet, podSpec.Namespace, podSpec.Name, 0, 1, 2*time.Minute) 2022 framework.ExpectNoError(err) 2023 2024 podSpec, err := client.Get(context.TODO(), podSpec.Name, metav1.GetOptions{}) 2025 framework.ExpectNoError(err) 2026 results = parseOutput(context.TODO(), f, podSpec) 2027 }) 2028 ginkgo.It("should restart a restartable init container before the regular container started", func() { 2029 framework.ExpectNoError(results.StartsBefore(restartableInit1, regular1)) 2030 }) 2031 ginkgo.It("should restart a restartable init container after the regular container started", func() { 2032 framework.ExpectNoError(results.StartsBefore(regular1, restartableInit1)) 2033 }) 2034 ginkgo.It("should start a regular container", func() { 2035 framework.ExpectNoError(results.Starts(regular1)) 2036 }) 2037 }) 2038 2039 // this test case the same as for restartPolicy=Never 2040 ginkgo.When("a restartable init container starts and exits with exit code 1 continuously", ginkgo.Ordered, func() { 2041 2042 restartableInit1 := "restartable-init-1" 2043 init1 := "init-1" 2044 regular1 := "regular-1" 2045 2046 podSpec := &v1.Pod{ 2047 ObjectMeta: metav1.ObjectMeta{ 2048 Name: "restartable-init-container-exit-1-continuously", 2049 }, 2050 Spec: v1.PodSpec{ 2051 RestartPolicy: v1.RestartPolicyAlways, 2052 InitContainers: []v1.Container{ 2053 { 2054 Name: restartableInit1, 2055 Image: busyboxImage, 2056 Command: ExecCommand(restartableInit1, execCommand{ 2057 Delay: 1, 2058 ExitCode: 1, 2059 }), 2060 RestartPolicy: &containerRestartPolicyAlways, 2061 }, 2062 { 2063 Name: init1, 2064 Image: busyboxImage, 2065 Command: ExecCommand(init1, execCommand{ 2066 Delay: 5, 2067 ExitCode: 0, 2068 }), 2069 }, 2070 }, 2071 Containers: []v1.Container{ 2072 { 2073 Name: regular1, 2074 Image: busyboxImage, 2075 Command: ExecCommand(regular1, execCommand{ 2076 Delay: 60, 2077 ExitCode: 0, 2078 }), 2079 }, 2080 }, 2081 }, 2082 } 2083 2084 preparePod(podSpec) 2085 var results containerOutputList 2086 2087 ginkgo.It("should keep running a Pod continuously and produce log", func() { /* check the regular container restartCount > 0 */ 2088 client := e2epod.NewPodClient(f) 2089 podSpec = client.Create(context.TODO(), podSpec) 2090 2091 err := WaitForPodContainerRestartCount(context.TODO(), f.ClientSet, podSpec.Namespace, podSpec.Name, 0, 1, 2*time.Minute) 2092 framework.ExpectNoError(err) 2093 2094 podSpec, err := client.Get(context.TODO(), podSpec.Name, metav1.GetOptions{}) 2095 framework.ExpectNoError(err) 2096 results = parseOutput(context.TODO(), f, podSpec) 2097 }) 2098 ginkgo.It("should restart a restartable init container before the regular container started", func() { 2099 framework.ExpectNoError(results.StartsBefore(restartableInit1, regular1)) 2100 }) 2101 ginkgo.It("should restart a restartable init container after the regular container started", func() { 2102 framework.ExpectNoError(results.StartsBefore(regular1, restartableInit1)) 2103 }) 2104 ginkgo.It("should start a regular container", func() { 2105 framework.ExpectNoError(results.Starts(regular1)) 2106 }) 2107 }) 2108 2109 ginkgo.When("an Init container before restartable init container continuously fails", ginkgo.Ordered, func() { 2110 2111 init1 := "init-1" 2112 restartableInit1 := "restartable-init-1" 2113 regular1 := "regular-1" 2114 2115 podSpec := &v1.Pod{ 2116 ObjectMeta: metav1.ObjectMeta{ 2117 Name: "init-container-fails-before-restartable-init-starts", 2118 }, 2119 Spec: v1.PodSpec{ 2120 RestartPolicy: v1.RestartPolicyAlways, 2121 InitContainers: []v1.Container{ 2122 { 2123 Name: init1, 2124 Image: busyboxImage, 2125 Command: ExecCommand(init1, execCommand{ 2126 Delay: 1, 2127 ExitCode: 1, 2128 }), 2129 }, 2130 { 2131 Name: restartableInit1, 2132 Image: busyboxImage, 2133 Command: ExecCommand(restartableInit1, execCommand{ 2134 Delay: 600, 2135 ExitCode: 0, 2136 }), 2137 RestartPolicy: &containerRestartPolicyAlways, 2138 }, 2139 }, 2140 Containers: []v1.Container{ 2141 { 2142 Name: regular1, 2143 Image: busyboxImage, 2144 Command: ExecCommand(regular1, execCommand{ 2145 Delay: 600, 2146 ExitCode: 0, 2147 }), 2148 }, 2149 }, 2150 }, 2151 } 2152 2153 preparePod(podSpec) 2154 var results containerOutputList 2155 2156 ginkgo.It("should continuously run Pod keeping it Pending", func() { 2157 client := e2epod.NewPodClient(f) 2158 podSpec = client.Create(context.TODO(), podSpec) 2159 2160 err := e2epod.WaitForPodCondition(context.TODO(), f.ClientSet, podSpec.Namespace, podSpec.Name, "pending and restarting 3 times", 5*time.Minute, func(pod *v1.Pod) (bool, error) { 2161 if pod.Status.Phase != v1.PodPending { 2162 return false, fmt.Errorf("pod should be in pending phase") 2163 } 2164 if len(pod.Status.InitContainerStatuses) < 1 { 2165 return false, nil 2166 } 2167 containerStatus := pod.Status.InitContainerStatuses[0] 2168 return containerStatus.RestartCount >= 3, nil 2169 }) 2170 framework.ExpectNoError(err) 2171 2172 podSpec, err := client.Get(context.TODO(), podSpec.Name, metav1.GetOptions{}) 2173 framework.ExpectNoError(err) 2174 results = parseOutput(context.TODO(), f, podSpec) 2175 }) 2176 ginkgo.It("should have Init container restartCount greater than 0", func() { 2177 framework.ExpectNoError(results.HasRestarted(init1)) 2178 }) 2179 ginkgo.It("should not start restartable init container", func() { 2180 framework.ExpectNoError(results.DoesntStart(restartableInit1)) 2181 }) 2182 }) 2183 2184 ginkgo.When("an Init container after restartable init container fails", ginkgo.Ordered, func() { 2185 2186 init1 := "init-1" 2187 restartableInit1 := "restartable-init-1" 2188 regular1 := "regular-1" 2189 2190 podSpec := &v1.Pod{ 2191 ObjectMeta: metav1.ObjectMeta{ 2192 Name: "restartable-init-container-fails-before-init-container", 2193 }, 2194 Spec: v1.PodSpec{ 2195 RestartPolicy: v1.RestartPolicyAlways, 2196 InitContainers: []v1.Container{ 2197 { 2198 Name: restartableInit1, 2199 Image: busyboxImage, 2200 Command: ExecCommand(restartableInit1, execCommand{ 2201 Delay: 1, 2202 ExitCode: 1, 2203 }), 2204 RestartPolicy: &containerRestartPolicyAlways, 2205 }, 2206 { 2207 Name: init1, 2208 Image: busyboxImage, 2209 Command: ExecCommand(init1, execCommand{ 2210 Delay: 1, 2211 ExitCode: 1, 2212 }), 2213 }, 2214 }, 2215 Containers: []v1.Container{ 2216 { 2217 Name: regular1, 2218 Image: busyboxImage, 2219 Command: ExecCommand(regular1, execCommand{ 2220 Delay: 600, 2221 ExitCode: 0, 2222 }), 2223 }, 2224 }, 2225 }, 2226 } 2227 2228 preparePod(podSpec) 2229 var results containerOutputList 2230 2231 ginkgo.It("should continuously run Pod keeping it Pending", func() { 2232 client := e2epod.NewPodClient(f) 2233 podSpec = client.Create(context.TODO(), podSpec) 2234 2235 err := e2epod.WaitForPodCondition(context.TODO(), f.ClientSet, podSpec.Namespace, podSpec.Name, "pending and restarting 3 times", 5*time.Minute, func(pod *v1.Pod) (bool, error) { 2236 if pod.Status.Phase != v1.PodPending { 2237 return false, fmt.Errorf("pod should be in pending phase") 2238 } 2239 if len(pod.Status.InitContainerStatuses) < 1 { 2240 return false, nil 2241 } 2242 containerStatus := pod.Status.InitContainerStatuses[0] 2243 return containerStatus.RestartCount >= 3, nil 2244 }) 2245 framework.ExpectNoError(err) 2246 2247 podSpec, err := client.Get(context.TODO(), podSpec.Name, metav1.GetOptions{}) 2248 framework.ExpectNoError(err) 2249 results = parseOutput(context.TODO(), f, podSpec) 2250 }) 2251 ginkgo.It("should have Init container restartCount greater than 0", func() { 2252 framework.ExpectNoError(results.HasRestarted(init1)) 2253 }) 2254 // TODO: how will we be able to test it if restartable init container will never fail and there will be no termination log? Or will be? 2255 ginkgo.It("should be running restartable init container and a failed Init container in parallel", func() { 2256 framework.ExpectNoError(results.RunTogether(restartableInit1, init1)) 2257 }) 2258 }) 2259 }) 2260 2261 ginkgo.It("should launch restartable init containers serially considering the startup probe", func() { 2262 2263 restartableInit1 := "restartable-init-1" 2264 restartableInit2 := "restartable-init-2" 2265 regular1 := "regular-1" 2266 2267 pod := &v1.Pod{ 2268 ObjectMeta: metav1.ObjectMeta{ 2269 Name: "restartable-init-containers-start-serially", 2270 }, 2271 Spec: v1.PodSpec{ 2272 RestartPolicy: v1.RestartPolicyNever, 2273 InitContainers: []v1.Container{ 2274 { 2275 Name: restartableInit1, 2276 Image: busyboxImage, 2277 Command: ExecCommand(restartableInit1, execCommand{ 2278 StartDelay: 10, 2279 Delay: 600, 2280 ExitCode: 0, 2281 }), 2282 StartupProbe: &v1.Probe{ 2283 ProbeHandler: v1.ProbeHandler{ 2284 Exec: &v1.ExecAction{ 2285 Command: []string{"test", "-f", "started"}, 2286 }, 2287 }, 2288 }, 2289 RestartPolicy: &containerRestartPolicyAlways, 2290 }, 2291 { 2292 Name: restartableInit2, 2293 Image: busyboxImage, 2294 Command: ExecCommand(restartableInit2, execCommand{ 2295 StartDelay: 10, 2296 Delay: 600, 2297 ExitCode: 0, 2298 }), 2299 StartupProbe: &v1.Probe{ 2300 ProbeHandler: v1.ProbeHandler{ 2301 Exec: &v1.ExecAction{ 2302 Command: []string{"test", "-f", "started"}, 2303 }, 2304 }, 2305 }, 2306 RestartPolicy: &containerRestartPolicyAlways, 2307 }, 2308 }, 2309 Containers: []v1.Container{ 2310 { 2311 Name: regular1, 2312 Image: busyboxImage, 2313 Command: ExecCommand(regular1, execCommand{ 2314 Delay: 1, 2315 ExitCode: 0, 2316 }), 2317 }, 2318 }, 2319 }, 2320 } 2321 2322 preparePod(pod) 2323 2324 client := e2epod.NewPodClient(f) 2325 pod = client.Create(context.TODO(), pod) 2326 2327 ginkgo.By("Waiting for the pod to finish") 2328 err := e2epod.WaitTimeoutForPodNoLongerRunningInNamespace(context.TODO(), f.ClientSet, pod.Name, pod.Namespace, 5*time.Minute) 2329 framework.ExpectNoError(err) 2330 2331 pod, err = client.Get(context.TODO(), pod.Name, metav1.GetOptions{}) 2332 framework.ExpectNoError(err) 2333 results := parseOutput(context.TODO(), f, pod) 2334 2335 ginkgo.By("Analyzing results") 2336 framework.ExpectNoError(results.StartsBefore(restartableInit1, restartableInit2)) 2337 framework.ExpectNoError(results.StartsBefore(restartableInit2, regular1)) 2338 }) 2339 2340 ginkgo.It("should call the container's preStop hook and not launch next container if the restartable init container's startup probe fails", func() { 2341 2342 restartableInit1 := "restartable-init-1" 2343 regular1 := "regular-1" 2344 2345 pod := &v1.Pod{ 2346 ObjectMeta: metav1.ObjectMeta{ 2347 Name: "restartable-init-container-failed-startup", 2348 }, 2349 Spec: v1.PodSpec{ 2350 RestartPolicy: v1.RestartPolicyAlways, 2351 InitContainers: []v1.Container{ 2352 { 2353 Name: restartableInit1, 2354 Image: busyboxImage, 2355 Command: ExecCommand(restartableInit1, execCommand{ 2356 Delay: 600, 2357 TerminationSeconds: 15, 2358 ExitCode: 0, 2359 }), 2360 StartupProbe: &v1.Probe{ 2361 InitialDelaySeconds: 5, 2362 FailureThreshold: 1, 2363 ProbeHandler: v1.ProbeHandler{ 2364 Exec: &v1.ExecAction{ 2365 Command: []string{ 2366 "sh", 2367 "-c", 2368 "exit 1", 2369 }, 2370 }, 2371 }, 2372 }, 2373 Lifecycle: &v1.Lifecycle{ 2374 PreStop: &v1.LifecycleHandler{ 2375 Exec: &v1.ExecAction{ 2376 Command: ExecCommand(prefixedName(PreStopPrefix, restartableInit1), execCommand{ 2377 Delay: 1, 2378 ExitCode: 0, 2379 ContainerName: restartableInit1, 2380 }), 2381 }, 2382 }, 2383 }, 2384 RestartPolicy: &containerRestartPolicyAlways, 2385 }, 2386 }, 2387 Containers: []v1.Container{ 2388 { 2389 Name: regular1, 2390 Image: busyboxImage, 2391 Command: ExecCommand(regular1, execCommand{ 2392 Delay: 1, 2393 ExitCode: 0, 2394 }), 2395 }, 2396 }, 2397 }, 2398 } 2399 2400 preparePod(pod) 2401 2402 client := e2epod.NewPodClient(f) 2403 pod = client.Create(context.TODO(), pod) 2404 2405 ginkgo.By("Waiting for the restartable init container to restart") 2406 err := WaitForPodInitContainerRestartCount(context.TODO(), f.ClientSet, pod.Namespace, pod.Name, 0, 2, 2*time.Minute) 2407 framework.ExpectNoError(err) 2408 2409 pod, err = client.Get(context.TODO(), pod.Name, metav1.GetOptions{}) 2410 framework.ExpectNoError(err) 2411 2412 if pod.Status.Phase != v1.PodPending { 2413 framework.Failf("pod %q is not pending, it's %q", pod.Name, pod.Status.Phase) 2414 } 2415 2416 results := parseOutput(context.TODO(), f, pod) 2417 2418 ginkgo.By("Analyzing results") 2419 framework.ExpectNoError(results.RunTogether(restartableInit1, prefixedName(PreStopPrefix, restartableInit1))) 2420 framework.ExpectNoError(results.Starts(prefixedName(PreStopPrefix, restartableInit1))) 2421 framework.ExpectNoError(results.Exits(restartableInit1)) 2422 framework.ExpectNoError(results.DoesntStart(regular1)) 2423 }) 2424 2425 ginkgo.It("should call the container's preStop hook and start the next container if the restartable init container's liveness probe fails", func() { 2426 2427 restartableInit1 := "restartable-init-1" 2428 regular1 := "regular-1" 2429 2430 pod := &v1.Pod{ 2431 ObjectMeta: metav1.ObjectMeta{ 2432 Name: "restartable-init-container-failed-startup", 2433 }, 2434 Spec: v1.PodSpec{ 2435 RestartPolicy: v1.RestartPolicyAlways, 2436 InitContainers: []v1.Container{ 2437 { 2438 Name: restartableInit1, 2439 Image: busyboxImage, 2440 Command: ExecCommand(restartableInit1, execCommand{ 2441 Delay: 600, 2442 TerminationSeconds: 15, 2443 ExitCode: 0, 2444 }), 2445 LivenessProbe: &v1.Probe{ 2446 InitialDelaySeconds: 5, 2447 FailureThreshold: 1, 2448 ProbeHandler: v1.ProbeHandler{ 2449 Exec: &v1.ExecAction{ 2450 Command: []string{ 2451 "sh", 2452 "-c", 2453 "exit 1", 2454 }, 2455 }, 2456 }, 2457 }, 2458 Lifecycle: &v1.Lifecycle{ 2459 PreStop: &v1.LifecycleHandler{ 2460 Exec: &v1.ExecAction{ 2461 Command: ExecCommand(prefixedName(PreStopPrefix, restartableInit1), execCommand{ 2462 Delay: 1, 2463 ExitCode: 0, 2464 ContainerName: restartableInit1, 2465 }), 2466 }, 2467 }, 2468 }, 2469 RestartPolicy: &containerRestartPolicyAlways, 2470 }, 2471 }, 2472 Containers: []v1.Container{ 2473 { 2474 Name: regular1, 2475 Image: busyboxImage, 2476 Command: ExecCommand(regular1, execCommand{ 2477 Delay: 1, 2478 ExitCode: 0, 2479 }), 2480 }, 2481 }, 2482 }, 2483 } 2484 2485 preparePod(pod) 2486 2487 client := e2epod.NewPodClient(f) 2488 pod = client.Create(context.TODO(), pod) 2489 2490 ginkgo.By("Waiting for the restartable init container to restart") 2491 err := WaitForPodInitContainerRestartCount(context.TODO(), f.ClientSet, pod.Namespace, pod.Name, 0, 2, 2*time.Minute) 2492 framework.ExpectNoError(err) 2493 2494 err = WaitForPodContainerRestartCount(context.TODO(), f.ClientSet, pod.Namespace, pod.Name, 0, 1, 2*time.Minute) 2495 framework.ExpectNoError(err) 2496 2497 pod, err = client.Get(context.TODO(), pod.Name, metav1.GetOptions{}) 2498 framework.ExpectNoError(err) 2499 2500 results := parseOutput(context.TODO(), f, pod) 2501 2502 ginkgo.By("Analyzing results") 2503 framework.ExpectNoError(results.RunTogether(restartableInit1, prefixedName(PreStopPrefix, restartableInit1))) 2504 framework.ExpectNoError(results.Starts(prefixedName(PreStopPrefix, restartableInit1))) 2505 framework.ExpectNoError(results.Exits(restartableInit1)) 2506 framework.ExpectNoError(results.Starts(regular1)) 2507 }) 2508 2509 ginkgo.It("should terminate sidecars in reverse order after all main containers have exited", func() { 2510 restartableInit1 := "restartable-init-1" 2511 restartableInit2 := "restartable-init-2" 2512 restartableInit3 := "restartable-init-3" 2513 regular1 := "regular-1" 2514 2515 pod := &v1.Pod{ 2516 ObjectMeta: metav1.ObjectMeta{ 2517 Name: "serialize-termination", 2518 }, 2519 Spec: v1.PodSpec{ 2520 RestartPolicy: v1.RestartPolicyNever, 2521 InitContainers: []v1.Container{ 2522 { 2523 Name: restartableInit1, 2524 Image: busyboxImage, 2525 RestartPolicy: &containerRestartPolicyAlways, 2526 Command: ExecCommand(restartableInit1, execCommand{ 2527 Delay: 60, 2528 TerminationSeconds: 5, 2529 ExitCode: 0, 2530 }), 2531 }, 2532 { 2533 Name: restartableInit2, 2534 Image: busyboxImage, 2535 RestartPolicy: &containerRestartPolicyAlways, 2536 Command: ExecCommand(restartableInit2, execCommand{ 2537 Delay: 60, 2538 TerminationSeconds: 5, 2539 ExitCode: 0, 2540 }), 2541 }, 2542 { 2543 Name: restartableInit3, 2544 Image: busyboxImage, 2545 RestartPolicy: &containerRestartPolicyAlways, 2546 Command: ExecCommand(restartableInit3, execCommand{ 2547 Delay: 60, 2548 TerminationSeconds: 5, 2549 ExitCode: 0, 2550 }), 2551 }, 2552 }, 2553 Containers: []v1.Container{ 2554 { 2555 Name: regular1, 2556 Image: busyboxImage, 2557 Command: ExecCommand(regular1, execCommand{ 2558 Delay: 5, 2559 ExitCode: 0, 2560 }), 2561 }, 2562 }, 2563 }, 2564 } 2565 2566 preparePod(pod) 2567 2568 client := e2epod.NewPodClient(f) 2569 pod = client.Create(context.TODO(), pod) 2570 2571 err := e2epod.WaitTimeoutForPodNoLongerRunningInNamespace(context.TODO(), f.ClientSet, pod.Name, pod.Namespace, 5*time.Minute) 2572 framework.ExpectNoError(err) 2573 2574 pod, err = client.Get(context.TODO(), pod.Name, metav1.GetOptions{}) 2575 framework.ExpectNoError(err) 2576 2577 results := parseOutput(context.TODO(), f, pod) 2578 2579 ginkgo.By("Analyzing results") 2580 framework.ExpectNoError(results.StartsBefore(restartableInit1, restartableInit2)) 2581 framework.ExpectNoError(results.StartsBefore(restartableInit1, restartableInit3)) 2582 framework.ExpectNoError(results.StartsBefore(restartableInit2, restartableInit3)) 2583 framework.ExpectNoError(results.StartsBefore(restartableInit1, regular1)) 2584 framework.ExpectNoError(results.StartsBefore(restartableInit2, regular1)) 2585 framework.ExpectNoError(results.StartsBefore(restartableInit3, regular1)) 2586 2587 // main containers exit first 2588 framework.ExpectNoError(results.ExitsBefore(regular1, restartableInit1)) 2589 framework.ExpectNoError(results.ExitsBefore(regular1, restartableInit2)) 2590 framework.ExpectNoError(results.ExitsBefore(regular1, restartableInit3)) 2591 // followed by sidecars in reverse order 2592 framework.ExpectNoError(results.ExitsBefore(restartableInit3, restartableInit2)) 2593 framework.ExpectNoError(results.ExitsBefore(restartableInit2, restartableInit1)) 2594 }) 2595 2596 ginkgo.It("should terminate sidecars simultaneously if prestop doesn't exit", func() { 2597 restartableInit1 := "restartable-init-1" 2598 restartableInit2 := "restartable-init-2" 2599 restartableInit3 := "restartable-init-3" 2600 regular1 := "regular-1" 2601 2602 makePrestop := func(containerName string) *v1.Lifecycle { 2603 return &v1.Lifecycle{ 2604 PreStop: &v1.LifecycleHandler{ 2605 Exec: &v1.ExecAction{ 2606 Command: ExecCommand(prefixedName(PreStopPrefix, containerName), execCommand{ 2607 ExitCode: 0, 2608 ContainerName: containerName, 2609 LoopForever: true, 2610 }), 2611 }, 2612 }, 2613 } 2614 } 2615 2616 pod := &v1.Pod{ 2617 ObjectMeta: metav1.ObjectMeta{ 2618 Name: "serialize-termination", 2619 }, 2620 Spec: v1.PodSpec{ 2621 RestartPolicy: v1.RestartPolicyNever, 2622 InitContainers: []v1.Container{ 2623 { 2624 Name: restartableInit1, 2625 Image: busyboxImage, 2626 RestartPolicy: &containerRestartPolicyAlways, 2627 Command: ExecCommand(restartableInit1, execCommand{ 2628 Delay: 60, 2629 TerminationSeconds: 5, 2630 ExitCode: 0, 2631 }), 2632 Lifecycle: makePrestop(restartableInit1), 2633 }, 2634 { 2635 Name: restartableInit2, 2636 Image: busyboxImage, 2637 RestartPolicy: &containerRestartPolicyAlways, 2638 Command: ExecCommand(restartableInit2, execCommand{ 2639 Delay: 60, 2640 TerminationSeconds: 5, 2641 ExitCode: 0, 2642 }), 2643 Lifecycle: makePrestop(restartableInit2), 2644 }, 2645 { 2646 Name: restartableInit3, 2647 Image: busyboxImage, 2648 RestartPolicy: &containerRestartPolicyAlways, 2649 Command: ExecCommand(restartableInit3, execCommand{ 2650 Delay: 60, 2651 TerminationSeconds: 5, 2652 ExitCode: 0, 2653 }), 2654 Lifecycle: makePrestop(restartableInit3), 2655 }, 2656 }, 2657 Containers: []v1.Container{ 2658 { 2659 Name: regular1, 2660 Image: busyboxImage, 2661 Command: ExecCommand(regular1, execCommand{ 2662 Delay: 5, 2663 ExitCode: 0, 2664 }), 2665 }, 2666 }, 2667 }, 2668 } 2669 2670 preparePod(pod) 2671 2672 client := e2epod.NewPodClient(f) 2673 pod = client.Create(context.TODO(), pod) 2674 2675 err := e2epod.WaitTimeoutForPodNoLongerRunningInNamespace(context.TODO(), f.ClientSet, pod.Name, pod.Namespace, 5*time.Minute) 2676 framework.ExpectNoError(err) 2677 2678 pod, err = client.Get(context.TODO(), pod.Name, metav1.GetOptions{}) 2679 framework.ExpectNoError(err) 2680 2681 results := parseOutput(context.TODO(), f, pod) 2682 2683 ginkgo.By("Analyzing results") 2684 framework.ExpectNoError(results.StartsBefore(restartableInit1, restartableInit2)) 2685 framework.ExpectNoError(results.StartsBefore(restartableInit1, restartableInit3)) 2686 framework.ExpectNoError(results.StartsBefore(restartableInit2, restartableInit3)) 2687 framework.ExpectNoError(results.StartsBefore(restartableInit1, regular1)) 2688 framework.ExpectNoError(results.StartsBefore(restartableInit2, regular1)) 2689 framework.ExpectNoError(results.StartsBefore(restartableInit3, regular1)) 2690 2691 ps1, err := results.TimeOfStart(prefixedName(PreStopPrefix, restartableInit1)) 2692 framework.ExpectNoError(err) 2693 ps2, err := results.TimeOfStart(prefixedName(PreStopPrefix, restartableInit2)) 2694 framework.ExpectNoError(err) 2695 ps3, err := results.TimeOfStart(prefixedName(PreStopPrefix, restartableInit3)) 2696 framework.ExpectNoError(err) 2697 2698 ps1Last, err := results.TimeOfLastLoop(prefixedName(PreStopPrefix, restartableInit1)) 2699 framework.ExpectNoError(err) 2700 ps2Last, err := results.TimeOfLastLoop(prefixedName(PreStopPrefix, restartableInit2)) 2701 framework.ExpectNoError(err) 2702 ps3Last, err := results.TimeOfLastLoop(prefixedName(PreStopPrefix, restartableInit3)) 2703 framework.ExpectNoError(err) 2704 2705 const simulToleration = 0.5 2706 // should all end together since they loop infinitely and exceed their grace period 2707 gomega.Expect(ps1Last-ps2Last).To(gomega.BeNumerically("~", 0, simulToleration), 2708 fmt.Sprintf("expected PostStart 1 & PostStart 2 to be killed at the same time, got %s", results)) 2709 gomega.Expect(ps1Last-ps3Last).To(gomega.BeNumerically("~", 0, simulToleration), 2710 fmt.Sprintf("expected PostStart 1 & PostStart 3 to be killed at the same time, got %s", results)) 2711 gomega.Expect(ps2Last-ps3Last).To(gomega.BeNumerically("~", 0, simulToleration), 2712 fmt.Sprintf("expected PostStart 2 & PostStart 3 to be killed at the same time, got %s", results)) 2713 2714 // 30 seconds + 2 second minimum grace for the SIGKILL 2715 const lifetimeToleration = 1 2716 gomega.Expect(ps1Last-ps1).To(gomega.BeNumerically("~", 32, lifetimeToleration), 2717 fmt.Sprintf("expected PostStart 1 to live for ~32 seconds, got %s", results)) 2718 gomega.Expect(ps2Last-ps2).To(gomega.BeNumerically("~", 32, lifetimeToleration), 2719 fmt.Sprintf("expected PostStart 2 to live for ~32 seconds, got %s", results)) 2720 gomega.Expect(ps3Last-ps3).To(gomega.BeNumerically("~", 32, lifetimeToleration), 2721 fmt.Sprintf("expected PostStart 3 to live for ~32 seconds, got %s", results)) 2722 2723 }) 2724 2725 ginkgo.It("should call sidecar container PreStop hook simultaneously", func() { 2726 restartableInit1 := "restartable-init-1" 2727 restartableInit2 := "restartable-init-2" 2728 restartableInit3 := "restartable-init-3" 2729 regular1 := "regular-1" 2730 2731 makePrestop := func(containerName string) *v1.Lifecycle { 2732 return &v1.Lifecycle{ 2733 PreStop: &v1.LifecycleHandler{ 2734 Exec: &v1.ExecAction{ 2735 Command: ExecCommand(prefixedName(PreStopPrefix, containerName), execCommand{ 2736 Delay: 1, 2737 ExitCode: 0, 2738 ContainerName: containerName, 2739 }), 2740 }, 2741 }, 2742 } 2743 } 2744 2745 pod := &v1.Pod{ 2746 ObjectMeta: metav1.ObjectMeta{ 2747 Name: "serialize-termination-simul-prestop", 2748 }, 2749 Spec: v1.PodSpec{ 2750 RestartPolicy: v1.RestartPolicyNever, 2751 InitContainers: []v1.Container{ 2752 { 2753 Name: restartableInit1, 2754 Image: busyboxImage, 2755 RestartPolicy: &containerRestartPolicyAlways, 2756 Command: ExecCommand(restartableInit1, execCommand{ 2757 Delay: 60, 2758 TerminationSeconds: 5, 2759 ExitCode: 0, 2760 }), 2761 Lifecycle: makePrestop(restartableInit1), 2762 }, 2763 { 2764 Name: restartableInit2, 2765 Image: busyboxImage, 2766 RestartPolicy: &containerRestartPolicyAlways, 2767 Command: ExecCommand(restartableInit2, execCommand{ 2768 Delay: 60, 2769 TerminationSeconds: 5, 2770 ExitCode: 0, 2771 }), 2772 Lifecycle: makePrestop(restartableInit2), 2773 }, 2774 { 2775 Name: restartableInit3, 2776 Image: busyboxImage, 2777 RestartPolicy: &containerRestartPolicyAlways, 2778 Command: ExecCommand(restartableInit3, execCommand{ 2779 Delay: 60, 2780 TerminationSeconds: 5, 2781 ExitCode: 0, 2782 }), 2783 Lifecycle: makePrestop(restartableInit3), 2784 }, 2785 }, 2786 Containers: []v1.Container{ 2787 { 2788 Name: regular1, 2789 Image: busyboxImage, 2790 Command: ExecCommand(regular1, execCommand{ 2791 Delay: 5, 2792 ExitCode: 0, 2793 }), 2794 }, 2795 }, 2796 }, 2797 } 2798 2799 preparePod(pod) 2800 2801 client := e2epod.NewPodClient(f) 2802 pod = client.Create(context.TODO(), pod) 2803 2804 err := e2epod.WaitTimeoutForPodNoLongerRunningInNamespace(context.TODO(), f.ClientSet, pod.Name, pod.Namespace, 5*time.Minute) 2805 framework.ExpectNoError(err) 2806 2807 pod, err = client.Get(context.TODO(), pod.Name, metav1.GetOptions{}) 2808 framework.ExpectNoError(err) 2809 2810 results := parseOutput(context.TODO(), f, pod) 2811 2812 ginkgo.By("Analyzing results") 2813 framework.ExpectNoError(results.StartsBefore(restartableInit1, restartableInit2)) 2814 framework.ExpectNoError(results.StartsBefore(restartableInit1, restartableInit3)) 2815 framework.ExpectNoError(results.StartsBefore(restartableInit2, restartableInit3)) 2816 framework.ExpectNoError(results.StartsBefore(restartableInit1, regular1)) 2817 framework.ExpectNoError(results.StartsBefore(restartableInit2, regular1)) 2818 framework.ExpectNoError(results.StartsBefore(restartableInit3, regular1)) 2819 2820 // main containers exit first 2821 framework.ExpectNoError(results.ExitsBefore(regular1, restartableInit1)) 2822 framework.ExpectNoError(results.ExitsBefore(regular1, restartableInit2)) 2823 framework.ExpectNoError(results.ExitsBefore(regular1, restartableInit3)) 2824 2825 // followed by sidecars in reverse order 2826 framework.ExpectNoError(results.ExitsBefore(restartableInit3, restartableInit2)) 2827 framework.ExpectNoError(results.ExitsBefore(restartableInit2, restartableInit1)) 2828 2829 // and the pre-stop hooks should have been called simultaneously 2830 ps1, err := results.TimeOfStart(prefixedName(PreStopPrefix, restartableInit1)) 2831 framework.ExpectNoError(err) 2832 ps2, err := results.TimeOfStart(prefixedName(PreStopPrefix, restartableInit2)) 2833 framework.ExpectNoError(err) 2834 ps3, err := results.TimeOfStart(prefixedName(PreStopPrefix, restartableInit3)) 2835 framework.ExpectNoError(err) 2836 2837 const toleration = 0.5 2838 gomega.Expect(ps1-ps2).To(gomega.BeNumerically("~", 0, toleration), 2839 fmt.Sprintf("expected PostStart 1 & PostStart 2 to start at the same time, got %s", results)) 2840 gomega.Expect(ps1-ps3).To(gomega.BeNumerically("~", 0, toleration), 2841 fmt.Sprintf("expected PostStart 1 & PostStart 3 to start at the same time, got %s", results)) 2842 gomega.Expect(ps2-ps3).To(gomega.BeNumerically("~", 0, toleration), 2843 fmt.Sprintf("expected PostStart 2 & PostStart 3 to start at the same time, got %s", results)) 2844 }) 2845 2846 ginkgo.It("should not hang in termination if terminated during initialization", func() { 2847 startInit := "start-init" 2848 restartableInit1 := "restartable-init-1" 2849 restartableInit2 := "restartable-init-2" 2850 restartableInit3 := "restartable-init-3" 2851 regular1 := "regular-1" 2852 2853 makePrestop := func(containerName string) *v1.Lifecycle { 2854 return &v1.Lifecycle{ 2855 PreStop: &v1.LifecycleHandler{ 2856 Exec: &v1.ExecAction{ 2857 Command: ExecCommand(prefixedName(PreStopPrefix, containerName), execCommand{ 2858 Delay: 1, 2859 ExitCode: 0, 2860 ContainerName: containerName, 2861 }), 2862 }, 2863 }, 2864 } 2865 } 2866 2867 pod := &v1.Pod{ 2868 ObjectMeta: metav1.ObjectMeta{ 2869 Name: "dont-hang-if-terminated-in-init", 2870 }, 2871 Spec: v1.PodSpec{ 2872 RestartPolicy: v1.RestartPolicyNever, 2873 InitContainers: []v1.Container{ 2874 { 2875 Name: startInit, 2876 Image: busyboxImage, 2877 Command: ExecCommand(startInit, execCommand{ 2878 Delay: 300, 2879 TerminationSeconds: 0, 2880 ExitCode: 0, 2881 }), 2882 }, 2883 { 2884 Name: restartableInit1, 2885 Image: busyboxImage, 2886 RestartPolicy: &containerRestartPolicyAlways, 2887 Command: ExecCommand(restartableInit1, execCommand{ 2888 Delay: 60, 2889 TerminationSeconds: 5, 2890 ExitCode: 0, 2891 }), 2892 Lifecycle: makePrestop(restartableInit1), 2893 }, 2894 { 2895 Name: restartableInit2, 2896 Image: busyboxImage, 2897 RestartPolicy: &containerRestartPolicyAlways, 2898 Command: ExecCommand(restartableInit2, execCommand{ 2899 Delay: 60, 2900 TerminationSeconds: 5, 2901 ExitCode: 0, 2902 }), 2903 Lifecycle: makePrestop(restartableInit2), 2904 }, 2905 { 2906 Name: restartableInit3, 2907 Image: busyboxImage, 2908 RestartPolicy: &containerRestartPolicyAlways, 2909 Command: ExecCommand(restartableInit3, execCommand{ 2910 Delay: 60, 2911 TerminationSeconds: 5, 2912 ExitCode: 0, 2913 }), 2914 Lifecycle: makePrestop(restartableInit3), 2915 }, 2916 }, 2917 Containers: []v1.Container{ 2918 { 2919 Name: regular1, 2920 Image: busyboxImage, 2921 Command: ExecCommand(regular1, execCommand{ 2922 Delay: 5, 2923 ExitCode: 0, 2924 }), 2925 }, 2926 }, 2927 }, 2928 } 2929 2930 preparePod(pod) 2931 2932 client := e2epod.NewPodClient(f) 2933 pod = client.Create(context.TODO(), pod) 2934 2935 err := e2epod.WaitForPodCondition(context.TODO(), f.ClientSet, pod.Namespace, pod.Name, "pod pending and init running", 2*time.Minute, func(pod *v1.Pod) (bool, error) { 2936 if pod.Status.Phase != v1.PodPending { 2937 return false, fmt.Errorf("pod should be in pending phase") 2938 } 2939 if len(pod.Status.InitContainerStatuses) < 1 { 2940 return false, nil 2941 } 2942 containerStatus := pod.Status.InitContainerStatuses[0] 2943 return *containerStatus.Started && containerStatus.State.Running != nil, nil 2944 }) 2945 framework.ExpectNoError(err) 2946 2947 // the init container is running, so we stop the pod before the sidecars even start 2948 start := time.Now() 2949 grace := int64(3) 2950 ginkgo.By("deleting the pod") 2951 err = client.Delete(context.TODO(), pod.Name, metav1.DeleteOptions{GracePeriodSeconds: &grace}) 2952 framework.ExpectNoError(err) 2953 ginkgo.By("waiting for the pod to disappear") 2954 err = e2epod.WaitForPodNotFoundInNamespace(context.TODO(), f.ClientSet, pod.Name, pod.Namespace, 120*time.Second) 2955 framework.ExpectNoError(err) 2956 2957 buffer := int64(2) 2958 deleteTime := time.Since(start).Seconds() 2959 // should delete quickly and not try to start/wait on any sidecars since they never started 2960 gomega.Expect(deleteTime).To(gomega.BeNumerically("<", grace+buffer), fmt.Sprintf("should delete in < %d seconds, took %f", grace+buffer, deleteTime)) 2961 }) 2962 }) 2963 2964 var _ = SIGDescribe("[NodeAlphaFeature:SidecarContainers]", framework.WithSerial(), "Containers Lifecycle", func() { 2965 f := framework.NewDefaultFramework("containers-lifecycle-test-serial") 2966 f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged 2967 2968 ginkgo.It("should restart the containers in right order after the node reboot", func(ctx context.Context) { 2969 init1 := "init-1" 2970 restartableInit2 := "restartable-init-2" 2971 init3 := "init-3" 2972 regular1 := "regular-1" 2973 2974 podLabels := map[string]string{ 2975 "test": "containers-lifecycle-test-serial", 2976 "namespace": f.Namespace.Name, 2977 } 2978 pod := &v1.Pod{ 2979 ObjectMeta: metav1.ObjectMeta{ 2980 Name: "initialized-pod", 2981 Labels: podLabels, 2982 }, 2983 Spec: v1.PodSpec{ 2984 RestartPolicy: v1.RestartPolicyAlways, 2985 InitContainers: []v1.Container{ 2986 { 2987 Name: init1, 2988 Image: busyboxImage, 2989 Command: ExecCommand(init1, execCommand{ 2990 Delay: 5, 2991 ExitCode: 0, 2992 }), 2993 }, 2994 { 2995 Name: restartableInit2, 2996 Image: busyboxImage, 2997 Command: ExecCommand(restartableInit2, execCommand{ 2998 Delay: 300, 2999 ExitCode: 0, 3000 }), 3001 RestartPolicy: &containerRestartPolicyAlways, 3002 }, 3003 { 3004 Name: init3, 3005 Image: busyboxImage, 3006 Command: ExecCommand(init3, execCommand{ 3007 Delay: 5, 3008 ExitCode: 0, 3009 }), 3010 }, 3011 }, 3012 Containers: []v1.Container{ 3013 { 3014 Name: regular1, 3015 Image: busyboxImage, 3016 Command: ExecCommand(regular1, execCommand{ 3017 Delay: 300, 3018 ExitCode: 0, 3019 }), 3020 }, 3021 }, 3022 }, 3023 } 3024 preparePod(pod) 3025 3026 client := e2epod.NewPodClient(f) 3027 pod = client.Create(ctx, pod) 3028 ginkgo.By("Waiting for the pod to be initialized and run") 3029 err := e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod) 3030 framework.ExpectNoError(err) 3031 3032 ginkgo.By("Getting the current pod sandbox ID") 3033 rs, _, err := getCRIClient() 3034 framework.ExpectNoError(err) 3035 3036 sandboxes, err := rs.ListPodSandbox(ctx, &runtimeapi.PodSandboxFilter{ 3037 LabelSelector: podLabels, 3038 }) 3039 framework.ExpectNoError(err) 3040 gomega.Expect(sandboxes).To(gomega.HaveLen(1)) 3041 podSandboxID := sandboxes[0].Id 3042 3043 ginkgo.By("Stopping the kubelet") 3044 restartKubelet := stopKubelet() 3045 gomega.Eventually(ctx, func() bool { 3046 return kubeletHealthCheck(kubeletHealthCheckURL) 3047 }, f.Timeouts.PodStart, f.Timeouts.Poll).Should(gomega.BeFalse()) 3048 3049 ginkgo.By("Stopping the pod sandbox to simulate the node reboot") 3050 err = rs.StopPodSandbox(ctx, podSandboxID) 3051 framework.ExpectNoError(err) 3052 3053 ginkgo.By("Restarting the kubelet") 3054 restartKubelet() 3055 gomega.Eventually(ctx, func() bool { 3056 return kubeletHealthCheck(kubeletHealthCheckURL) 3057 }, f.Timeouts.PodStart, f.Timeouts.Poll).Should(gomega.BeTrue()) 3058 3059 ginkgo.By("Waiting for the pod to be re-initialized and run") 3060 err = e2epod.WaitForPodCondition(ctx, f.ClientSet, pod.Namespace, pod.Name, "re-initialized", f.Timeouts.PodStart, func(pod *v1.Pod) (bool, error) { 3061 if pod.Status.ContainerStatuses[0].RestartCount < 1 { 3062 return false, nil 3063 } 3064 if pod.Status.Phase != v1.PodRunning { 3065 return false, nil 3066 } 3067 return true, nil 3068 }) 3069 framework.ExpectNoError(err) 3070 3071 ginkgo.By("Parsing results") 3072 pod, err = client.Get(ctx, pod.Name, metav1.GetOptions{}) 3073 framework.ExpectNoError(err) 3074 results := parseOutput(context.TODO(), f, pod) 3075 3076 ginkgo.By("Analyzing results") 3077 init1Started, err := results.FindIndex(init1, "Started", 0) 3078 framework.ExpectNoError(err) 3079 restartableInit2Started, err := results.FindIndex(restartableInit2, "Started", 0) 3080 framework.ExpectNoError(err) 3081 init3Started, err := results.FindIndex(init3, "Started", 0) 3082 framework.ExpectNoError(err) 3083 regular1Started, err := results.FindIndex(regular1, "Started", 0) 3084 framework.ExpectNoError(err) 3085 3086 init1Restarted, err := results.FindIndex(init1, "Started", init1Started+1) 3087 framework.ExpectNoError(err) 3088 restartableInit2Restarted, err := results.FindIndex(restartableInit2, "Started", restartableInit2Started+1) 3089 framework.ExpectNoError(err) 3090 init3Restarted, err := results.FindIndex(init3, "Started", init3Started+1) 3091 framework.ExpectNoError(err) 3092 regular1Restarted, err := results.FindIndex(regular1, "Started", regular1Started+1) 3093 framework.ExpectNoError(err) 3094 3095 framework.ExpectNoError(init1Started.IsBefore(restartableInit2Started)) 3096 framework.ExpectNoError(restartableInit2Started.IsBefore(init3Started)) 3097 framework.ExpectNoError(init3Started.IsBefore(regular1Started)) 3098 3099 framework.ExpectNoError(init1Restarted.IsBefore(restartableInit2Restarted)) 3100 framework.ExpectNoError(restartableInit2Restarted.IsBefore(init3Restarted)) 3101 framework.ExpectNoError(init3Restarted.IsBefore(regular1Restarted)) 3102 }) 3103 })