k8s.io/kubernetes@v1.29.3/test/e2e/common/node/init_container.go (about) 1 /* 2 Copyright 2016 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package node 18 19 import ( 20 "context" 21 "encoding/json" 22 "fmt" 23 "strconv" 24 "strings" 25 "time" 26 27 "github.com/onsi/ginkgo/v2" 28 "github.com/onsi/gomega" 29 30 v1 "k8s.io/api/core/v1" 31 "k8s.io/apimachinery/pkg/api/resource" 32 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 33 "k8s.io/apimachinery/pkg/fields" 34 "k8s.io/apimachinery/pkg/runtime" 35 "k8s.io/apimachinery/pkg/util/sets" 36 "k8s.io/apimachinery/pkg/util/uuid" 37 "k8s.io/apimachinery/pkg/watch" 38 "k8s.io/client-go/tools/cache" 39 watchtools "k8s.io/client-go/tools/watch" 40 podutil "k8s.io/kubernetes/pkg/api/v1/pod" 41 "k8s.io/kubernetes/pkg/client/conditions" 42 "k8s.io/kubernetes/test/e2e/framework" 43 e2epod "k8s.io/kubernetes/test/e2e/framework/pod" 44 imageutils "k8s.io/kubernetes/test/utils/image" 45 admissionapi "k8s.io/pod-security-admission/api" 46 ) 47 48 func recordEvents(events []watch.Event, f func(watch.Event) (bool, error)) func(watch.Event) (bool, error) { 49 return func(e watch.Event) (bool, error) { 50 events = append(events, e) 51 return f(e) 52 } 53 } 54 55 // invariantFunc is a func that checks for invariant. 56 type invariantFunc func(older, newer runtime.Object) error 57 58 // checkInvariants checks for invariant of the each events. 59 func checkInvariants(events []watch.Event, fns ...invariantFunc) error { 60 errs := sets.NewString() 61 for i := range events { 62 j := i + 1 63 if j >= len(events) { 64 continue 65 } 66 for _, fn := range fns { 67 if err := fn(events[i].Object, events[j].Object); err != nil { 68 errs.Insert(err.Error()) 69 } 70 } 71 } 72 if errs.Len() > 0 { 73 return fmt.Errorf("invariants violated:\n* %s", strings.Join(errs.List(), "\n* ")) 74 } 75 return nil 76 } 77 78 // containerInitInvariant checks for an init containers are initialized and invariant on both older and newer. 79 func containerInitInvariant(older, newer runtime.Object) error { 80 oldPod := older.(*v1.Pod) 81 newPod := newer.(*v1.Pod) 82 if len(oldPod.Spec.InitContainers) == 0 { 83 return nil 84 } 85 if len(oldPod.Spec.InitContainers) != len(newPod.Spec.InitContainers) { 86 return fmt.Errorf("init container list changed") 87 } 88 if oldPod.UID != newPod.UID { 89 return fmt.Errorf("two different pods exist in the condition: %s vs %s", oldPod.UID, newPod.UID) 90 } 91 if err := initContainersInvariants(oldPod); err != nil { 92 return err 93 } 94 if err := initContainersInvariants(newPod); err != nil { 95 return err 96 } 97 oldInit, _, _ := initialized(oldPod) 98 newInit, _, _ := initialized(newPod) 99 if oldInit && !newInit { 100 // TODO: we may in the future enable resetting initialized = false if the kubelet needs to restart it 101 // from scratch 102 return fmt.Errorf("pod cannot be initialized and then regress to not being initialized") 103 } 104 return nil 105 } 106 107 // initialized checks the state of all init containers in the pod. 108 func initialized(pod *v1.Pod) (ok bool, failed bool, err error) { 109 allInit := true 110 initFailed := false 111 for _, s := range pod.Status.InitContainerStatuses { 112 switch { 113 case initFailed && s.State.Waiting == nil: 114 return allInit, initFailed, fmt.Errorf("container %s is after a failed container but isn't waiting", s.Name) 115 case allInit && s.State.Waiting == nil: 116 return allInit, initFailed, fmt.Errorf("container %s is after an initializing container but isn't waiting", s.Name) 117 case s.State.Terminated == nil: 118 allInit = false 119 case s.State.Terminated.ExitCode != 0: 120 allInit = false 121 initFailed = true 122 case !s.Ready: 123 return allInit, initFailed, fmt.Errorf("container %s initialized but isn't marked as ready", s.Name) 124 } 125 } 126 return allInit, initFailed, nil 127 } 128 129 func initContainersInvariants(pod *v1.Pod) error { 130 allInit, initFailed, err := initialized(pod) 131 if err != nil { 132 return err 133 } 134 if !allInit || initFailed { 135 for _, s := range pod.Status.ContainerStatuses { 136 if s.State.Waiting == nil || s.RestartCount != 0 { 137 return fmt.Errorf("container %s is not waiting but initialization not complete", s.Name) 138 } 139 if s.State.Waiting.Reason != "PodInitializing" { 140 return fmt.Errorf("container %s should have reason PodInitializing: %s", s.Name, s.State.Waiting.Reason) 141 } 142 } 143 } 144 _, c := podutil.GetPodCondition(&pod.Status, v1.PodInitialized) 145 if c == nil { 146 return fmt.Errorf("pod does not have initialized condition") 147 } 148 if c.LastTransitionTime.IsZero() { 149 return fmt.Errorf("PodInitialized condition should always have a transition time") 150 } 151 switch { 152 case c.Status == v1.ConditionUnknown: 153 return fmt.Errorf("PodInitialized condition should never be Unknown") 154 case c.Status == v1.ConditionTrue && (initFailed || !allInit): 155 return fmt.Errorf("PodInitialized condition was True but all not all containers initialized") 156 case c.Status == v1.ConditionFalse && (!initFailed && allInit): 157 return fmt.Errorf("PodInitialized condition was False but all containers initialized") 158 } 159 return nil 160 } 161 162 var _ = SIGDescribe("InitContainer", framework.WithNodeConformance(), func() { 163 f := framework.NewDefaultFramework("init-container") 164 f.NamespacePodSecurityLevel = admissionapi.LevelBaseline 165 var podClient *e2epod.PodClient 166 ginkgo.BeforeEach(func() { 167 podClient = e2epod.NewPodClient(f) 168 }) 169 170 /* 171 Release: v1.12 172 Testname: init-container-starts-app-restartnever-pod 173 Description: Ensure that all InitContainers are started 174 and all containers in pod are voluntarily terminated with exit status 0, 175 and the system is not going to restart any of these containers 176 when Pod has restart policy as RestartNever. 177 */ 178 framework.ConformanceIt("should invoke init containers on a RestartNever pod", func(ctx context.Context) { 179 ginkgo.By("creating the pod") 180 name := "pod-init-" + string(uuid.NewUUID()) 181 value := strconv.Itoa(time.Now().Nanosecond()) 182 pod := &v1.Pod{ 183 ObjectMeta: metav1.ObjectMeta{ 184 Name: name, 185 Labels: map[string]string{ 186 "name": "foo", 187 "time": value, 188 }, 189 }, 190 Spec: v1.PodSpec{ 191 RestartPolicy: v1.RestartPolicyNever, 192 InitContainers: []v1.Container{ 193 { 194 Name: "init1", 195 Image: imageutils.GetE2EImage(imageutils.BusyBox), 196 Command: []string{"/bin/true"}, 197 }, 198 { 199 Name: "init2", 200 Image: imageutils.GetE2EImage(imageutils.BusyBox), 201 Command: []string{"/bin/true"}, 202 }, 203 }, 204 Containers: []v1.Container{ 205 { 206 Name: "run1", 207 Image: imageutils.GetE2EImage(imageutils.BusyBox), 208 Command: []string{"/bin/true"}, 209 }, 210 }, 211 }, 212 } 213 framework.Logf("PodSpec: initContainers in spec.initContainers") 214 startedPod := podClient.Create(ctx, pod) 215 216 fieldSelector := fields.OneTermEqualSelector("metadata.name", startedPod.Name).String() 217 w := &cache.ListWatch{ 218 WatchFunc: func(options metav1.ListOptions) (i watch.Interface, e error) { 219 options.FieldSelector = fieldSelector 220 return podClient.Watch(ctx, options) 221 }, 222 } 223 var events []watch.Event 224 ctx, cancel := watchtools.ContextWithOptionalTimeout(ctx, framework.PodStartTimeout) 225 defer cancel() 226 event, err := watchtools.Until(ctx, startedPod.ResourceVersion, w, 227 recordEvents(events, conditions.PodCompleted), 228 ) 229 framework.ExpectNoError(err) 230 231 checkInvariants(events, containerInitInvariant) 232 endPod := event.Object.(*v1.Pod) 233 gomega.Expect(endPod.Status.Phase).To(gomega.Equal(v1.PodSucceeded)) 234 _, init := podutil.GetPodCondition(&endPod.Status, v1.PodInitialized) 235 gomega.Expect(init).NotTo(gomega.BeNil()) 236 gomega.Expect(init.Status).To(gomega.Equal(v1.ConditionTrue)) 237 238 gomega.Expect(endPod.Status.InitContainerStatuses).To(gomega.HaveLen(2)) 239 for _, status := range endPod.Status.InitContainerStatuses { 240 if !status.Ready { 241 framework.Failf("init container %s should be in Ready status", status.Name) 242 } 243 gomega.Expect(status.State.Terminated).NotTo(gomega.BeNil()) 244 gomega.Expect(status.State.Terminated.ExitCode).To(gomega.BeZero()) 245 } 246 }) 247 248 /* 249 Release: v1.12 250 Testname: init-container-starts-app-restartalways-pod 251 Description: Ensure that all InitContainers are started 252 and all containers in pod started 253 and at least one container is still running or is in the process of being restarted 254 when Pod has restart policy as RestartAlways. 255 */ 256 framework.ConformanceIt("should invoke init containers on a RestartAlways pod", func(ctx context.Context) { 257 ginkgo.By("creating the pod") 258 name := "pod-init-" + string(uuid.NewUUID()) 259 value := strconv.Itoa(time.Now().Nanosecond()) 260 pod := &v1.Pod{ 261 ObjectMeta: metav1.ObjectMeta{ 262 Name: name, 263 Labels: map[string]string{ 264 "name": "foo", 265 "time": value, 266 }, 267 }, 268 Spec: v1.PodSpec{ 269 InitContainers: []v1.Container{ 270 { 271 Name: "init1", 272 Image: imageutils.GetE2EImage(imageutils.BusyBox), 273 Command: []string{"/bin/true"}, 274 }, 275 { 276 Name: "init2", 277 Image: imageutils.GetE2EImage(imageutils.BusyBox), 278 Command: []string{"/bin/true"}, 279 }, 280 }, 281 Containers: []v1.Container{ 282 { 283 Name: "run1", 284 Image: imageutils.GetPauseImageName(), 285 Resources: v1.ResourceRequirements{ 286 Limits: v1.ResourceList{ 287 v1.ResourceCPU: *resource.NewMilliQuantity(100, resource.DecimalSI), 288 }, 289 }, 290 }, 291 }, 292 }, 293 } 294 framework.Logf("PodSpec: initContainers in spec.initContainers") 295 startedPod := podClient.Create(ctx, pod) 296 297 fieldSelector := fields.OneTermEqualSelector("metadata.name", startedPod.Name).String() 298 w := &cache.ListWatch{ 299 WatchFunc: func(options metav1.ListOptions) (i watch.Interface, e error) { 300 options.FieldSelector = fieldSelector 301 return podClient.Watch(ctx, options) 302 }, 303 } 304 var events []watch.Event 305 ctx, cancel := watchtools.ContextWithOptionalTimeout(ctx, framework.PodStartTimeout) 306 defer cancel() 307 event, err := watchtools.Until(ctx, startedPod.ResourceVersion, w, recordEvents(events, conditions.PodRunning)) 308 framework.ExpectNoError(err) 309 310 checkInvariants(events, containerInitInvariant) 311 endPod := event.Object.(*v1.Pod) 312 gomega.Expect(endPod.Status.Phase).To(gomega.Equal(v1.PodRunning)) 313 _, init := podutil.GetPodCondition(&endPod.Status, v1.PodInitialized) 314 gomega.Expect(init).NotTo(gomega.BeNil()) 315 gomega.Expect(init.Status).To(gomega.Equal(v1.ConditionTrue)) 316 317 gomega.Expect(endPod.Status.InitContainerStatuses).To(gomega.HaveLen(2)) 318 for _, status := range endPod.Status.InitContainerStatuses { 319 if !status.Ready { 320 framework.Failf("init container %s should be in Ready status", status.Name) 321 } 322 gomega.Expect(status.State.Terminated).NotTo(gomega.BeNil()) 323 gomega.Expect(status.State.Terminated.ExitCode).To(gomega.BeZero()) 324 } 325 }) 326 327 /* 328 Release: v1.12 329 Testname: init-container-fails-stops-app-restartalways-pod 330 Description: Ensure that app container is not started 331 when all InitContainers failed to start 332 and Pod has restarted for few occurrences 333 and pod has restart policy as RestartAlways. 334 */ 335 framework.ConformanceIt("should not start app containers if init containers fail on a RestartAlways pod", func(ctx context.Context) { 336 ginkgo.By("creating the pod") 337 name := "pod-init-" + string(uuid.NewUUID()) 338 value := strconv.Itoa(time.Now().Nanosecond()) 339 340 pod := &v1.Pod{ 341 ObjectMeta: metav1.ObjectMeta{ 342 Name: name, 343 Labels: map[string]string{ 344 "name": "foo", 345 "time": value, 346 }, 347 }, 348 Spec: v1.PodSpec{ 349 InitContainers: []v1.Container{ 350 { 351 Name: "init1", 352 Image: imageutils.GetE2EImage(imageutils.BusyBox), 353 Command: []string{"/bin/false"}, 354 }, 355 { 356 Name: "init2", 357 Image: imageutils.GetE2EImage(imageutils.BusyBox), 358 Command: []string{"/bin/true"}, 359 }, 360 }, 361 Containers: []v1.Container{ 362 { 363 Name: "run1", 364 Image: imageutils.GetPauseImageName(), 365 Resources: v1.ResourceRequirements{ 366 Limits: v1.ResourceList{ 367 v1.ResourceCPU: *resource.NewMilliQuantity(100, resource.DecimalSI), 368 }, 369 }, 370 }, 371 }, 372 }, 373 } 374 framework.Logf("PodSpec: initContainers in spec.initContainers") 375 startedPod := podClient.Create(ctx, pod) 376 377 fieldSelector := fields.OneTermEqualSelector("metadata.name", startedPod.Name).String() 378 w := &cache.ListWatch{ 379 WatchFunc: func(options metav1.ListOptions) (i watch.Interface, e error) { 380 options.FieldSelector = fieldSelector 381 return podClient.Watch(ctx, options) 382 }, 383 } 384 385 var events []watch.Event 386 ctx, cancel := watchtools.ContextWithOptionalTimeout(ctx, framework.PodStartTimeout) 387 defer cancel() 388 event, err := watchtools.Until( 389 ctx, 390 startedPod.ResourceVersion, 391 w, 392 // check for the first container to fail at least once 393 func(evt watch.Event) (bool, error) { 394 switch t := evt.Object.(type) { 395 case *v1.Pod: 396 for _, status := range t.Status.ContainerStatuses { 397 if status.State.Waiting == nil { 398 return false, fmt.Errorf("container %q should not be out of waiting: %s", status.Name, toDebugJSON(status)) 399 } 400 if status.State.Waiting.Reason != "PodInitializing" { 401 return false, fmt.Errorf("container %q should have reason PodInitializing: %s", status.Name, toDebugJSON(status)) 402 } 403 } 404 if len(t.Status.InitContainerStatuses) != 2 { 405 return false, nil 406 } 407 status := t.Status.InitContainerStatuses[1] 408 if status.State.Waiting == nil { 409 return false, fmt.Errorf("second init container should not be out of waiting: %s", toDebugJSON(status)) 410 } 411 if status.State.Waiting.Reason != "PodInitializing" { 412 return false, fmt.Errorf("second init container should have reason PodInitializing: %s", toDebugJSON(status)) 413 } 414 status = t.Status.InitContainerStatuses[0] 415 if status.State.Terminated != nil && status.State.Terminated.ExitCode == 0 { 416 return false, fmt.Errorf("first init container should have exitCode != 0: %s", toDebugJSON(status)) 417 } 418 // continue until we see an attempt to restart the pod 419 return status.LastTerminationState.Terminated != nil, nil 420 default: 421 return false, fmt.Errorf("unexpected object: %#v", t) 422 } 423 }, 424 // verify we get two restarts 425 func(evt watch.Event) (bool, error) { 426 switch t := evt.Object.(type) { 427 case *v1.Pod: 428 status := t.Status.InitContainerStatuses[0] 429 if status.RestartCount < 3 { 430 return false, nil 431 } 432 framework.Logf("init container has failed twice: %#v", t) 433 // TODO: more conditions 434 return true, nil 435 default: 436 return false, fmt.Errorf("unexpected object: %#v", t) 437 } 438 }, 439 ) 440 framework.ExpectNoError(err) 441 442 checkInvariants(events, containerInitInvariant) 443 endPod := event.Object.(*v1.Pod) 444 gomega.Expect(endPod.Status.Phase).To(gomega.Equal(v1.PodPending)) 445 _, init := podutil.GetPodCondition(&endPod.Status, v1.PodInitialized) 446 gomega.Expect(init).NotTo(gomega.BeNil()) 447 gomega.Expect(init.Status).To(gomega.Equal(v1.ConditionFalse)) 448 gomega.Expect(init.Reason).To(gomega.Equal("ContainersNotInitialized")) 449 gomega.Expect(init.Message).To(gomega.Equal("containers with incomplete status: [init1 init2]")) 450 gomega.Expect(endPod.Status.InitContainerStatuses).To(gomega.HaveLen(2)) 451 }) 452 453 /* 454 Release: v1.12 455 Testname: init-container-fails-stops-app-restartnever-pod 456 Description: Ensure that app container is not started 457 when at least one InitContainer fails to start and Pod has restart policy as RestartNever. 458 */ 459 framework.ConformanceIt("should not start app containers and fail the pod if init containers fail on a RestartNever pod", func(ctx context.Context) { 460 ginkgo.By("creating the pod") 461 name := "pod-init-" + string(uuid.NewUUID()) 462 value := strconv.Itoa(time.Now().Nanosecond()) 463 pod := &v1.Pod{ 464 ObjectMeta: metav1.ObjectMeta{ 465 Name: name, 466 Labels: map[string]string{ 467 "name": "foo", 468 "time": value, 469 }, 470 }, 471 Spec: v1.PodSpec{ 472 RestartPolicy: v1.RestartPolicyNever, 473 InitContainers: []v1.Container{ 474 { 475 Name: "init1", 476 Image: imageutils.GetE2EImage(imageutils.BusyBox), 477 Command: []string{"/bin/true"}, 478 }, 479 { 480 Name: "init2", 481 Image: imageutils.GetE2EImage(imageutils.BusyBox), 482 Command: []string{"/bin/false"}, 483 }, 484 }, 485 Containers: []v1.Container{ 486 { 487 Name: "run1", 488 Image: imageutils.GetE2EImage(imageutils.BusyBox), 489 Command: []string{"/bin/true"}, 490 Resources: v1.ResourceRequirements{ 491 Limits: v1.ResourceList{ 492 v1.ResourceCPU: *resource.NewMilliQuantity(100, resource.DecimalSI), 493 }, 494 }, 495 }, 496 }, 497 }, 498 } 499 framework.Logf("PodSpec: initContainers in spec.initContainers") 500 startedPod := podClient.Create(ctx, pod) 501 502 fieldSelector := fields.OneTermEqualSelector("metadata.name", startedPod.Name).String() 503 w := &cache.ListWatch{ 504 WatchFunc: func(options metav1.ListOptions) (i watch.Interface, e error) { 505 options.FieldSelector = fieldSelector 506 return podClient.Watch(ctx, options) 507 }, 508 } 509 510 var events []watch.Event 511 ctx, cancel := watchtools.ContextWithOptionalTimeout(ctx, framework.PodStartTimeout) 512 defer cancel() 513 event, err := watchtools.Until( 514 ctx, startedPod.ResourceVersion, w, 515 recordEvents(events, 516 // check for the second container to fail at least once 517 func(evt watch.Event) (bool, error) { 518 switch t := evt.Object.(type) { 519 case *v1.Pod: 520 for _, status := range t.Status.ContainerStatuses { 521 if status.State.Waiting == nil { 522 return false, fmt.Errorf("container %q should not be out of waiting: %s", status.Name, toDebugJSON(status)) 523 } 524 if status.State.Waiting.Reason != "PodInitializing" { 525 return false, fmt.Errorf("container %q should have reason PodInitializing: %s", status.Name, toDebugJSON(status)) 526 } 527 } 528 if len(t.Status.InitContainerStatuses) != 2 { 529 return false, nil 530 } 531 status := t.Status.InitContainerStatuses[0] 532 if status.State.Terminated == nil { 533 if status.State.Waiting != nil && status.State.Waiting.Reason != "PodInitializing" { 534 return false, fmt.Errorf("second init container should have reason PodInitializing: %s", toDebugJSON(status)) 535 } 536 return false, nil 537 } 538 if status.State.Terminated != nil && status.State.Terminated.ExitCode != 0 { 539 return false, fmt.Errorf("first init container should have exitCode != 0: %s", toDebugJSON(status)) 540 } 541 status = t.Status.InitContainerStatuses[1] 542 if status.State.Terminated == nil { 543 return false, nil 544 } 545 if status.State.Terminated.ExitCode == 0 { 546 return false, fmt.Errorf("second init container should have failed: %s", toDebugJSON(status)) 547 } 548 return true, nil 549 default: 550 return false, fmt.Errorf("unexpected object: %#v", t) 551 } 552 }), 553 recordEvents(events, conditions.PodCompleted), 554 ) 555 framework.ExpectNoError(err) 556 557 checkInvariants(events, containerInitInvariant) 558 endPod := event.Object.(*v1.Pod) 559 560 gomega.Expect(endPod.Status.Phase).To(gomega.Equal(v1.PodFailed)) 561 _, init := podutil.GetPodCondition(&endPod.Status, v1.PodInitialized) 562 gomega.Expect(init).NotTo(gomega.BeNil()) 563 gomega.Expect(init.Status).To(gomega.Equal(v1.ConditionFalse)) 564 gomega.Expect(init.Reason).To(gomega.Equal("ContainersNotInitialized")) 565 gomega.Expect(init.Message).To(gomega.Equal("containers with incomplete status: [init2]")) 566 gomega.Expect(endPod.Status.InitContainerStatuses).To(gomega.HaveLen(2)) 567 gomega.Expect(endPod.Status.ContainerStatuses[0].State.Waiting).ToNot(gomega.BeNil()) 568 }) 569 }) 570 571 // toDebugJSON converts an object to its JSON representation for debug logging 572 // purposes instead of using a struct. 573 func toDebugJSON(obj interface{}) string { 574 m, err := json.Marshal(obj) 575 if err != nil { 576 return fmt.Sprintf("<error: %v>", err) 577 } 578 return string(m) 579 }