k8s.io/kubernetes@v1.29.3/test/e2e_node/image_list.go (about) 1 /* 2 Copyright 2016 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package e2enode 18 19 import ( 20 "context" 21 "fmt" 22 "os" 23 "os/user" 24 "runtime" 25 "sync" 26 "time" 27 28 "k8s.io/klog/v2" 29 30 utilerrors "k8s.io/apimachinery/pkg/util/errors" 31 "k8s.io/apimachinery/pkg/util/sets" 32 internalapi "k8s.io/cri-api/pkg/apis" 33 runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" 34 commontest "k8s.io/kubernetes/test/e2e/common" 35 e2egpu "k8s.io/kubernetes/test/e2e/framework/gpu" 36 e2emanifest "k8s.io/kubernetes/test/e2e/framework/manifest" 37 e2epod "k8s.io/kubernetes/test/e2e/framework/pod" 38 e2etestfiles "k8s.io/kubernetes/test/e2e/framework/testfiles" 39 imageutils "k8s.io/kubernetes/test/utils/image" 40 ) 41 42 const ( 43 // Number of attempts to pull an image. 44 maxImagePullRetries = 5 45 // Sleep duration between image pull retry attempts. 46 imagePullRetryDelay = time.Second 47 // Number of parallel count to pull images. 48 maxParallelImagePullCount = 5 49 ) 50 51 // NodePrePullImageList is a list of images used in node e2e test. These images will be prepulled 52 // before test running so that the image pulling won't fail in actual test. 53 var NodePrePullImageList = sets.NewString( 54 imageutils.GetE2EImage(imageutils.Agnhost), 55 "gcr.io/cadvisor/cadvisor:v0.47.2", 56 busyboxImage, 57 "registry.k8s.io/e2e-test-images/busybox@sha256:a9155b13325b2abef48e71de77bb8ac015412a566829f621d06bfae5c699b1b9", 58 imageutils.GetE2EImage(imageutils.Nginx), 59 imageutils.GetE2EImage(imageutils.Perl), 60 imageutils.GetE2EImage(imageutils.Nonewprivs), 61 imageutils.GetPauseImageName(), 62 imageutils.GetE2EImage(imageutils.NodePerfNpbEp), 63 imageutils.GetE2EImage(imageutils.NodePerfNpbIs), 64 imageutils.GetE2EImage(imageutils.Etcd), 65 ) 66 67 // updateImageAllowList updates the e2epod.ImagePrePullList with 68 // 1. the hard coded lists 69 // 2. the ones passed in from framework.TestContext.ExtraEnvs 70 // So this function needs to be called after the extra envs are applied. 71 func updateImageAllowList(ctx context.Context) { 72 // Architecture-specific image 73 if !isRunningOnArm64() { 74 // NodePerfTfWideDeep is only supported on x86_64, pulling in arm64 will fail 75 NodePrePullImageList = NodePrePullImageList.Insert(imageutils.GetE2EImage(imageutils.NodePerfTfWideDeep)) 76 } 77 // Union NodePrePullImageList and PrePulledImages into the framework image pre-pull list. 78 e2epod.ImagePrePullList = NodePrePullImageList.Union(commontest.PrePulledImages) 79 // Images from extra envs 80 e2epod.ImagePrePullList.Insert(getNodeProblemDetectorImage()) 81 if sriovDevicePluginImage, err := getSRIOVDevicePluginImage(); err != nil { 82 klog.Errorln(err) 83 } else { 84 e2epod.ImagePrePullList.Insert(sriovDevicePluginImage) 85 } 86 if gpuDevicePluginImage, err := getGPUDevicePluginImage(ctx); err != nil { 87 klog.Errorln(err) 88 } else { 89 e2epod.ImagePrePullList.Insert(gpuDevicePluginImage) 90 } 91 if samplePluginImage, err := getContainerImageFromE2ETestDaemonset(SampleDevicePluginDSYAML); err != nil { 92 klog.Errorln(err) 93 } else { 94 e2epod.ImagePrePullList.Insert(samplePluginImage) 95 } 96 if samplePluginImageCtrlReg, err := getContainerImageFromE2ETestDaemonset(SampleDevicePluginControlRegistrationDSYAML); err != nil { 97 klog.Errorln(err) 98 } else { 99 e2epod.ImagePrePullList.Insert(samplePluginImageCtrlReg) 100 } 101 } 102 103 func isRunningOnArm64() bool { 104 return runtime.GOARCH == "arm64" 105 } 106 107 func getNodeProblemDetectorImage() string { 108 const defaultImage string = "registry.k8s.io/node-problem-detector/node-problem-detector:v0.8.13" 109 image := os.Getenv("NODE_PROBLEM_DETECTOR_IMAGE") 110 if image == "" { 111 image = defaultImage 112 } 113 return image 114 } 115 116 // puller represents a generic image puller 117 type puller interface { 118 // Pull pulls an image by name 119 Pull(image string) ([]byte, error) 120 // Name returns the name of the specific puller implementation 121 Name() string 122 } 123 124 type remotePuller struct { 125 imageService internalapi.ImageManagerService 126 } 127 128 func (rp *remotePuller) Name() string { 129 return "CRI" 130 } 131 132 func (rp *remotePuller) Pull(image string) ([]byte, error) { 133 resp, err := rp.imageService.ImageStatus(context.Background(), &runtimeapi.ImageSpec{Image: image}, false) 134 if err == nil && resp.GetImage() != nil { 135 return nil, nil 136 } 137 _, err = rp.imageService.PullImage(context.Background(), &runtimeapi.ImageSpec{Image: image}, nil, nil) 138 return nil, err 139 } 140 141 func getPuller() (puller, error) { 142 _, is, err := getCRIClient() 143 if err != nil { 144 return nil, err 145 } 146 return &remotePuller{ 147 imageService: is, 148 }, nil 149 } 150 151 // PrePullAllImages pre-fetches all images tests depend on so that we don't fail in an actual test. 152 func PrePullAllImages() error { 153 puller, err := getPuller() 154 if err != nil { 155 return err 156 } 157 usr, err := user.Current() 158 if err != nil { 159 return err 160 } 161 images := e2epod.ImagePrePullList.List() 162 klog.V(4).Infof("Pre-pulling images with %s %+v", puller.Name(), images) 163 164 imageCh := make(chan int, len(images)) 165 for i := range images { 166 imageCh <- i 167 } 168 close(imageCh) 169 170 pullErrs := make([]error, len(images)) 171 ctx, cancel := context.WithCancel(context.Background()) 172 defer cancel() 173 174 parallelImagePullCount := maxParallelImagePullCount 175 if len(images) < parallelImagePullCount { 176 parallelImagePullCount = len(images) 177 } 178 179 var wg sync.WaitGroup 180 wg.Add(parallelImagePullCount) 181 for i := 0; i < parallelImagePullCount; i++ { 182 go func() { 183 defer wg.Done() 184 185 for i := range imageCh { 186 var ( 187 pullErr error 188 output []byte 189 ) 190 for retryCount := 0; retryCount < maxImagePullRetries; retryCount++ { 191 select { 192 case <-ctx.Done(): 193 return 194 default: 195 } 196 197 if retryCount > 0 { 198 time.Sleep(imagePullRetryDelay) 199 } 200 if output, pullErr = puller.Pull(images[i]); pullErr == nil { 201 break 202 } 203 klog.Warningf("Failed to pull %s as user %q, retrying in %s (%d of %d): %v", 204 images[i], usr.Username, imagePullRetryDelay.String(), retryCount+1, maxImagePullRetries, pullErr) 205 } 206 if pullErr != nil { 207 klog.Warningf("Could not pre-pull image %s %v output: %s", images[i], pullErr, output) 208 pullErrs[i] = pullErr 209 cancel() 210 return 211 } 212 } 213 }() 214 } 215 216 wg.Wait() 217 return utilerrors.NewAggregate(pullErrs) 218 } 219 220 // getGPUDevicePluginImage returns the image of GPU device plugin. 221 func getGPUDevicePluginImage(ctx context.Context) (string, error) { 222 ds, err := e2emanifest.DaemonSetFromURL(ctx, e2egpu.GPUDevicePluginDSYAML) 223 if err != nil { 224 return "", fmt.Errorf("failed to parse the device plugin image: %w", err) 225 } 226 if ds == nil { 227 return "", fmt.Errorf("failed to parse the device plugin image: the extracted DaemonSet is nil") 228 } 229 if len(ds.Spec.Template.Spec.Containers) < 1 { 230 return "", fmt.Errorf("failed to parse the device plugin image: cannot extract the container from YAML") 231 } 232 return ds.Spec.Template.Spec.Containers[0].Image, nil 233 } 234 235 func getContainerImageFromE2ETestDaemonset(dsYamlPath string) (string, error) { 236 data, err := e2etestfiles.Read(dsYamlPath) 237 if err != nil { 238 return "", fmt.Errorf("failed to read the daemonset yaml: %w", err) 239 } 240 241 ds, err := e2emanifest.DaemonSetFromData(data) 242 if err != nil { 243 return "", fmt.Errorf("failed to parse daemonset yaml: %w", err) 244 } 245 246 if len(ds.Spec.Template.Spec.Containers) < 1 { 247 return "", fmt.Errorf("failed to parse the container image: cannot extract the container from YAML") 248 } 249 return ds.Spec.Template.Spec.Containers[0].Image, nil 250 } 251 252 // getSRIOVDevicePluginImage returns the image of SRIOV device plugin. 253 func getSRIOVDevicePluginImage() (string, error) { 254 data, err := e2etestfiles.Read(SRIOVDevicePluginDSYAML) 255 if err != nil { 256 return "", fmt.Errorf("failed to read the device plugin manifest: %w", err) 257 } 258 ds, err := e2emanifest.DaemonSetFromData(data) 259 if err != nil { 260 return "", fmt.Errorf("failed to parse the device plugin image: %w", err) 261 } 262 if ds == nil { 263 return "", fmt.Errorf("failed to parse the device plugin image: the extracted DaemonSet is nil") 264 } 265 if len(ds.Spec.Template.Spec.Containers) < 1 { 266 return "", fmt.Errorf("failed to parse the device plugin image: cannot extract the container from YAML") 267 } 268 return ds.Spec.Template.Spec.Containers[0].Image, nil 269 }