k8s.io/kubernetes@v1.29.3/test/e2e_node/remote/utils.go (about) 1 /* 2 Copyright 2016 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package remote 18 19 import ( 20 "fmt" 21 "path/filepath" 22 23 "k8s.io/klog/v2" 24 25 "k8s.io/kubernetes/test/e2e_node/builder" 26 ) 27 28 // utils.go contains functions used across test suites. 29 30 const ( 31 cniVersion = "v1.3.0" 32 cniDirectory = "cni/bin" // The CNI tarball places binaries under directory under "cni/bin". 33 cniConfDirectory = "cni/net.d" 34 35 ecrCredentialProviderVersion = "v1.27.1" 36 ) 37 38 const cniConfig = `{ 39 "name": "mynet", 40 "type": "bridge", 41 "bridge": "mynet0", 42 "isDefaultGateway": true, 43 "forceAddress": false, 44 "ipMasq": true, 45 "hairpinMode": true, 46 "ipam": { 47 "type": "host-local", 48 "subnet": "10.10.0.0/16" 49 } 50 } 51 ` 52 53 const credentialGCPProviderConfig = `kind: CredentialProviderConfig 54 apiVersion: kubelet.config.k8s.io/v1 55 providers: 56 - name: gcp-credential-provider 57 apiVersion: credentialprovider.kubelet.k8s.io/v1 58 matchImages: 59 - "gcr.io" 60 - "*.gcr.io" 61 - "container.cloud.google.com" 62 - "*.pkg.dev" 63 defaultCacheDuration: 1m` 64 65 const credentialAWSProviderConfig = `kind: CredentialProviderConfig 66 apiVersion: kubelet.config.k8s.io/v1 67 providers: 68 - name: ecr-credential-provider 69 apiVersion: credentialprovider.kubelet.k8s.io/v1 70 matchImages: 71 - "*.dkr.ecr.*.amazonaws.com" 72 - "*.dkr.ecr.*.amazonaws.com.cn" 73 - "*.dkr.ecr-fips.*.amazonaws.com" 74 - "*.dkr.ecr.us-iso-east-1.c2s.ic.gov" 75 - "*.dkr.ecr.us-isob-east-1.sc2s.sgov.gov" 76 defaultCacheDuration: 12h` 77 78 func getCNIURL() string { 79 cniArch := "amd64" 80 if builder.IsTargetArchArm64() { 81 cniArch = "arm64" 82 } 83 cniURL := fmt.Sprintf("https://storage.googleapis.com/k8s-artifacts-cni/release/%s/cni-plugins-linux-%s-%s.tgz", cniVersion, cniArch, cniVersion) 84 return cniURL 85 86 } 87 88 // Install the cni plugin and add basic bridge configuration to the 89 // configuration directory. 90 func setupCNI(host, workspace string) error { 91 klog.V(2).Infof("Install CNI on %q", host) 92 cniPath := filepath.Join(workspace, cniDirectory) 93 klog.V(2).Infof("Install CNI on path %q", cniPath) 94 cmd := getSSHCommand(" ; ", 95 fmt.Sprintf("mkdir -p %s", cniPath), 96 fmt.Sprintf("curl -s -L %s | tar -xz -C %s", getCNIURL(), cniPath), 97 ) 98 if output, err := SSH(host, "sh", "-c", cmd); err != nil { 99 return fmt.Errorf("failed to install cni plugin on %q: %v output: %q", host, err, output) 100 } 101 102 // The added CNI network config is not needed for kubenet. It is only 103 // used when testing the CNI network plugin, but is added in both cases 104 // for consistency and simplicity. 105 klog.V(2).Infof("Adding CNI configuration on %q", host) 106 cniConfigPath := filepath.Join(workspace, cniConfDirectory) 107 cmd = getSSHCommand(" ; ", 108 fmt.Sprintf("mkdir -p %s", cniConfigPath), 109 fmt.Sprintf("echo %s > %s", quote(cniConfig), filepath.Join(cniConfigPath, "mynet.conf")), 110 ) 111 if output, err := SSH(host, "sh", "-c", cmd); err != nil { 112 return fmt.Errorf("failed to write cni configuration on %q: %v output: %q", host, err, output) 113 } 114 return nil 115 } 116 117 func getECRCredentialProviderURL() string { 118 arch := "amd64" 119 if builder.IsTargetArchArm64() { 120 arch = "arm64" 121 } 122 return fmt.Sprintf("https://artifacts.k8s.io/binaries/cloud-provider-aws/%s/linux/%s/ecr-credential-provider-linux-%s", ecrCredentialProviderVersion, arch, arch) 123 } 124 125 func setupECRCredentialProvider(host, workspace string) error { 126 klog.V(2).Infof("Install ecr-credential-provider on %q at %q", host, workspace) 127 cmd := getSSHCommand(" ; ", 128 fmt.Sprintf("curl -s -Lo %s/ecr-credential-provider %s", workspace, getECRCredentialProviderURL()), 129 fmt.Sprintf("chmod a+x %s/ecr-credential-provider", workspace), 130 ) 131 if output, err := SSH(host, "sh", "-c", cmd); err != nil { 132 return fmt.Errorf("failed to install ecr-credential-provider plugin on %q: %v output: %q", host, err, output) 133 } 134 return nil 135 } 136 137 func configureCredentialProvider(host, workspace string) error { 138 klog.V(2).Infof("Configuring kubelet credential provider on %q", host) 139 140 credentialProviderConfig := credentialGCPProviderConfig 141 if GetSSHUser() == "ec2-user" { 142 credentialProviderConfig = credentialAWSProviderConfig 143 err := setupECRCredentialProvider(host, workspace) 144 if err != nil { 145 return err 146 } 147 } 148 149 cmd := getSSHCommand(" ; ", 150 fmt.Sprintf("echo %s > %s", quote(credentialProviderConfig), filepath.Join(workspace, "credential-provider.yaml")), 151 ) 152 if output, err := SSH(host, "sh", "-c", cmd); err != nil { 153 return fmt.Errorf("failed to write credential provider configuration on %q: %v output: %q", host, err, output) 154 } 155 156 return nil 157 } 158 159 // configureFirewall configures iptable firewall rules. 160 func configureFirewall(host string) error { 161 klog.V(2).Infof("Configure iptables firewall rules on %q", host) 162 163 // Since the goal is to enable connectivity without taking into account current rule, 164 // we can just prepend the accept rules directly without any check 165 cmd := getSSHCommand("&&", 166 "iptables -I INPUT 1 -w -p tcp -j ACCEPT", 167 "iptables -I INPUT 1 -w -p udp -j ACCEPT", 168 "iptables -I INPUT 1 -w -p icmp -j ACCEPT", 169 "iptables -I FORWARD 1 -w -p tcp -j ACCEPT", 170 "iptables -I FORWARD 1 -w -p udp -j ACCEPT", 171 "iptables -I FORWARD 1 -w -p icmp -j ACCEPT", 172 ) 173 output, err := SSH(host, "sh", "-c", cmd) 174 if err != nil { 175 return fmt.Errorf("failed to configured firewall on %q: %v output: %v", host, err, output) 176 } 177 return nil 178 } 179 180 // cleanupNodeProcesses kills all running node processes may conflict with the test. 181 func cleanupNodeProcesses(host string) { 182 klog.V(2).Infof("Killing any existing node processes on %q", host) 183 cmd := getSSHCommand(" ; ", 184 "pkill kubelet", 185 "pkill kube-apiserver", 186 "pkill etcd", 187 "pkill e2e_node.test", 188 ) 189 // No need to log an error if pkill fails since pkill will fail if the commands are not running. 190 // If we are unable to stop existing running k8s processes, we should see messages in the kubelet/apiserver/etcd 191 // logs about failing to bind the required ports. 192 SSH(host, "sh", "-c", cmd) 193 } 194 195 // Quotes a shell literal so it can be nested within another shell scope. 196 func quote(s string) string { 197 return fmt.Sprintf("'\"'\"'%s'\"'\"'", s) 198 }