k8s.io/kubernetes@v1.29.3/test/e2e_node/runtime_conformance_test.go (about) 1 /* 2 Copyright 2016 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package e2enode 18 19 import ( 20 "context" 21 "fmt" 22 "os" 23 "path/filepath" 24 "time" 25 26 v1 "k8s.io/api/core/v1" 27 "k8s.io/kubernetes/pkg/kubelet/images" 28 "k8s.io/kubernetes/test/e2e/common/node" 29 "k8s.io/kubernetes/test/e2e/framework" 30 e2epod "k8s.io/kubernetes/test/e2e/framework/pod" 31 "k8s.io/kubernetes/test/e2e_node/services" 32 admissionapi "k8s.io/pod-security-admission/api" 33 34 "github.com/onsi/ginkgo/v2" 35 ) 36 37 var _ = SIGDescribe("Container Runtime Conformance Test", func() { 38 f := framework.NewDefaultFramework("runtime-conformance") 39 f.NamespacePodSecurityLevel = admissionapi.LevelBaseline 40 41 ginkgo.Describe("container runtime conformance blackbox test", func() { 42 43 ginkgo.Context("when running a container with a new image", func() { 44 // The service account only has pull permission 45 auth := ` 46 { 47 "auths": { 48 "https://gcr.io": { 49 "auth": "X2pzb25fa2V5OnsKICAidHlwZSI6ICJzZXJ2aWNlX2FjY291bnQiLAogICJwcm9qZWN0X2lkIjogImF1dGhlbnRpY2F0ZWQtaW1hZ2UtcHVsbGluZyIsCiAgInByaXZhdGVfa2V5X2lkIjogImI5ZjJhNjY0YWE5YjIwNDg0Y2MxNTg2MDYzZmVmZGExOTIyNGFjM2IiLAogICJwcml2YXRlX2tleSI6ICItLS0tLUJFR0lOIFBSSVZBVEUgS0VZLS0tLS1cbk1JSUV2UUlCQURBTkJna3Foa2lHOXcwQkFRRUZBQVNDQktjd2dnU2pBZ0VBQW9JQkFRQzdTSG5LVEVFaVlMamZcbkpmQVBHbUozd3JCY2VJNTBKS0xxS21GWE5RL3REWGJRK2g5YVl4aldJTDhEeDBKZTc0bVovS01uV2dYRjVLWlNcbm9BNktuSU85Yi9SY1NlV2VpSXRSekkzL1lYVitPNkNjcmpKSXl4anFWam5mVzJpM3NhMzd0OUE5VEZkbGZycm5cbjR6UkpiOWl4eU1YNGJMdHFGR3ZCMDNOSWl0QTNzVlo1ODhrb1FBZmgzSmhhQmVnTWorWjRSYko0aGVpQlFUMDNcbnZVbzViRWFQZVQ5RE16bHdzZWFQV2dydDZOME9VRGNBRTl4bGNJek11MjUzUG4vSzgySFpydEx4akd2UkhNVXhcbng0ZjhwSnhmQ3h4QlN3Z1NORit3OWpkbXR2b0wwRmE3ZGducFJlODZWRDY2ejNZenJqNHlLRXRqc2hLZHl5VWRcbkl5cVhoN1JSQWdNQkFBRUNnZ0VBT3pzZHdaeENVVlFUeEFka2wvSTVTRFVidi9NazRwaWZxYjJEa2FnbmhFcG9cbjFJajJsNGlWMTByOS9uenJnY2p5VlBBd3pZWk1JeDFBZVF0RDdoUzRHWmFweXZKWUc3NkZpWFpQUm9DVlB6b3VcbmZyOGRDaWFwbDV0enJDOWx2QXNHd29DTTdJWVRjZmNWdDdjRTEyRDNRS3NGNlo3QjJ6ZmdLS251WVBmK0NFNlRcbmNNMHkwaCtYRS9kMERvSERoVy96YU1yWEhqOFRvd2V1eXRrYmJzNGYvOUZqOVBuU2dET1lQd2xhbFZUcitGUWFcbkpSd1ZqVmxYcEZBUW14M0Jyd25rWnQzQ2lXV2lGM2QrSGk5RXRVYnRWclcxYjZnK1JRT0licWFtcis4YlJuZFhcbjZWZ3FCQWtKWjhSVnlkeFVQMGQxMUdqdU9QRHhCbkhCbmM0UW9rSXJFUUtCZ1FEMUNlaWN1ZGhXdGc0K2dTeGJcbnplanh0VjFONDFtZHVjQnpvMmp5b1dHbzNQVDh3ckJPL3lRRTM0cU9WSi9pZCs4SThoWjRvSWh1K0pBMDBzNmdcblRuSXErdi9kL1RFalk4MW5rWmlDa21SUFdiWHhhWXR4UjIxS1BYckxOTlFKS2ttOHRkeVh5UHFsOE1veUdmQ1dcbjJ2aVBKS05iNkhabnY5Q3lqZEo5ZzJMRG5RS0JnUUREcVN2eURtaGViOTIzSW96NGxlZ01SK205Z2xYVWdTS2dcbkVzZlllbVJmbU5XQitDN3ZhSXlVUm1ZNU55TXhmQlZXc3dXRldLYXhjK0krYnFzZmx6elZZdFpwMThNR2pzTURcbmZlZWZBWDZCWk1zVXQ3Qmw3WjlWSjg1bnRFZHFBQ0xwWitaLzN0SVJWdWdDV1pRMWhrbmxHa0dUMDI0SkVFKytcbk55SDFnM2QzUlFLQmdRQ1J2MXdKWkkwbVBsRklva0tGTkh1YTBUcDNLb1JTU1hzTURTVk9NK2xIckcxWHJtRjZcbkMwNGNTKzQ0N0dMUkxHOFVUaEpKbTRxckh0Ti9aK2dZOTYvMm1xYjRIakpORDM3TVhKQnZFYTN5ZUxTOHEvK1JcbjJGOU1LamRRaU5LWnhQcG84VzhOSlREWTVOa1BaZGh4a2pzSHdVNGRTNjZwMVRESUU0MGd0TFpaRFFLQmdGaldcbktyblFpTnEzOS9iNm5QOFJNVGJDUUFKbmR3anhTUU5kQTVmcW1rQTlhRk9HbCtqamsxQ1BWa0tNSWxLSmdEYkpcbk9heDl2OUc2Ui9NSTFIR1hmV3QxWU56VnRocjRIdHNyQTB0U3BsbWhwZ05XRTZWejZuQURqdGZQSnMyZUdqdlhcbmpQUnArdjhjY21MK3dTZzhQTGprM3ZsN2VlNXJsWWxNQndNdUdjUHhBb0dBZWRueGJXMVJMbVZubEFpSEx1L0xcbmxtZkF3RFdtRWlJMFVnK1BMbm9Pdk81dFE1ZDRXMS94RU44bFA0cWtzcGtmZk1Rbk5oNFNZR0VlQlQzMlpxQ1RcbkpSZ2YwWGpveXZ2dXA5eFhqTWtYcnBZL3ljMXpmcVRaQzBNTzkvMVVjMWJSR2RaMmR5M2xSNU5XYXA3T1h5Zk9cblBQcE5Gb1BUWGd2M3FDcW5sTEhyR3pNPVxuLS0tLS1FTkQgUFJJVkFURSBLRVktLS0tLVxuIiwKICAiY2xpZW50X2VtYWlsIjogImltYWdlLXB1bGxpbmdAYXV0aGVudGljYXRlZC1pbWFnZS1wdWxsaW5nLmlhbS5nc2VydmljZWFjY291bnQuY29tIiwKICAiY2xpZW50X2lkIjogIjExMzc5NzkxNDUzMDA3MzI3ODcxMiIsCiAgImF1dGhfdXJpIjogImh0dHBzOi8vYWNjb3VudHMuZ29vZ2xlLmNvbS9vL29hdXRoMi9hdXRoIiwKICAidG9rZW5fdXJpIjogImh0dHBzOi8vYWNjb3VudHMuZ29vZ2xlLmNvbS9vL29hdXRoMi90b2tlbiIsCiAgImF1dGhfcHJvdmlkZXJfeDUwOV9jZXJ0X3VybCI6ICJodHRwczovL3d3dy5nb29nbGVhcGlzLmNvbS9vYXV0aDIvdjEvY2VydHMiLAogICJjbGllbnRfeDUwOV9jZXJ0X3VybCI6ICJodHRwczovL3d3dy5nb29nbGVhcGlzLmNvbS9yb2JvdC92MS9tZXRhZGF0YS94NTA5L2ltYWdlLXB1bGxpbmclNDBhdXRoZW50aWNhdGVkLWltYWdlLXB1bGxpbmcuaWFtLmdzZXJ2aWNlYWNjb3VudC5jb20iCn0=", 50 "email": "image-pulling@authenticated-image-pulling.iam.gserviceaccount.com" 51 } 52 } 53 }` 54 // The following images are not added into NodePrePullImageList, because this test is 55 // testing image pulling, these images don't need to be prepulled. The ImagePullPolicy 56 // is v1.PullAlways, so it won't be blocked by framework image pre-pull list check. 57 for _, testCase := range []struct { 58 description string 59 image string 60 phase v1.PodPhase 61 waiting bool 62 }{ 63 { 64 description: "should be able to pull from private registry with credential provider", 65 image: "gcr.io/authenticated-image-pulling/alpine:3.7", 66 phase: v1.PodRunning, 67 waiting: false, 68 }, 69 } { 70 testCase := testCase 71 f.It(testCase.description+"", f.WithNodeConformance(), func(ctx context.Context) { 72 name := "image-pull-test" 73 command := []string{"/bin/sh", "-c", "while true; do sleep 1; done"} 74 container := node.ConformanceContainer{ 75 PodClient: e2epod.NewPodClient(f), 76 Container: v1.Container{ 77 Name: name, 78 Image: testCase.image, 79 Command: command, 80 // PullAlways makes sure that the image will always be pulled even if it is present before the test. 81 ImagePullPolicy: v1.PullAlways, 82 }, 83 RestartPolicy: v1.RestartPolicyNever, 84 } 85 86 configFile := filepath.Join(services.KubeletRootDirectory, "config.json") 87 err := os.WriteFile(configFile, []byte(auth), 0644) 88 framework.ExpectNoError(err) 89 defer os.Remove(configFile) 90 91 // checkContainerStatus checks whether the container status matches expectation. 92 checkContainerStatus := func(ctx context.Context) error { 93 status, err := container.GetStatus(ctx) 94 if err != nil { 95 return fmt.Errorf("failed to get container status: %w", err) 96 } 97 // We need to check container state first. The default pod status is pending, If we check 98 // pod phase first, and the expected pod phase is Pending, the container status may not 99 // even show up when we check it. 100 // Check container state 101 if !testCase.waiting { 102 if status.State.Running == nil { 103 return fmt.Errorf("expected container state: Running, got: %q", 104 node.GetContainerState(status.State)) 105 } 106 } 107 if testCase.waiting { 108 if status.State.Waiting == nil { 109 return fmt.Errorf("expected container state: Waiting, got: %q", 110 node.GetContainerState(status.State)) 111 } 112 reason := status.State.Waiting.Reason 113 if reason != images.ErrImagePull.Error() && 114 reason != images.ErrImagePullBackOff.Error() { 115 return fmt.Errorf("unexpected waiting reason: %q", reason) 116 } 117 } 118 // Check pod phase 119 phase, err := container.GetPhase(ctx) 120 if err != nil { 121 return fmt.Errorf("failed to get pod phase: %w", err) 122 } 123 if phase != testCase.phase { 124 return fmt.Errorf("expected pod phase: %q, got: %q", testCase.phase, phase) 125 } 126 return nil 127 } 128 // The image registry is not stable, which sometimes causes the test to fail. Add retry mechanism to make this 129 // less flaky. 130 const flakeRetry = 3 131 for i := 1; i <= flakeRetry; i++ { 132 var err error 133 ginkgo.By("create the container") 134 container.Create(ctx) 135 ginkgo.By("check the container status") 136 for start := time.Now(); time.Since(start) < node.ContainerStatusRetryTimeout; time.Sleep(node.ContainerStatusPollInterval) { 137 if err = checkContainerStatus(ctx); err == nil { 138 break 139 } 140 } 141 ginkgo.By("delete the container") 142 _ = container.Delete(ctx) 143 if err == nil { 144 break 145 } 146 if i < flakeRetry { 147 framework.Logf("No.%d attempt failed: %v, retrying...", i, err) 148 } else { 149 framework.Failf("All %d attempts failed: %v", flakeRetry, err) 150 } 151 } 152 }) 153 } 154 }) 155 }) 156 })