github.com/cilium/cilium@v1.16.2/test/k8s/config.go (about) 1 // SPDX-License-Identifier: Apache-2.0 2 // Copyright Authors of Cilium 3 4 package k8sTest 5 6 import ( 7 "context" 8 "fmt" 9 "os" 10 11 . "github.com/onsi/gomega" 12 13 . "github.com/cilium/cilium/test/ginkgo-ext" 14 "github.com/cilium/cilium/test/helpers" 15 ) 16 17 // Some basic checks of the CiliumNodeConfig plumbing. 18 // 19 // This bit is an end-to-end test to ensure that the Helm 20 // chart is working as intended. There is more complicated 21 // tests of the logic in unit tests. 22 // 23 // It creates a CiliumNodeConfig, ensure it doesn't apply, 24 // then labels one node and ensure that, yes, it did apply 25 var _ = SkipDescribeIf(func() bool { 26 // right now, the 5.4 job is the fastest, so use only that 27 // but only if we're on jenkins 28 return helpers.DoesNotRunOn54Kernel() && helpers.RunsOnJenkins() 29 }, "K8sAgentPerNodeConfigTest", func() { 30 var ( 31 kubectl *helpers.Kubectl 32 33 ciliumFilename string 34 ) 35 BeforeEach(func() { 36 kubectl = helpers.CreateKubectl(helpers.K8s1VMName(), logger) 37 38 res := kubectl.ExecShort("kubectl label --overwrite --all node io.cilium.testing-") 39 Expect(res.GetErr("unlabel node")).To(BeNil()) 40 41 ciliumFilename = helpers.TimestampFilename("cilium.yaml") 42 RedeployCilium(kubectl, ciliumFilename, map[string]string{}) 43 }) 44 45 AfterEach(func() { 46 res := kubectl.ExecShort("kubectl label --overwrite --all node io.cilium.testing-") 47 Expect(res.GetErr("unlabel node")).To(BeNil()) 48 UninstallCiliumFromManifest(kubectl, ciliumFilename) 49 kubectl.CloseSSHClient() 50 }) 51 52 It("Correctly computes config overrides with CNC v2alpha1", func() { 53 pods, err := kubectl.GetPodsNodes(helpers.CiliumNamespace, helpers.CiliumSelector) 54 Expect(err).To(BeNil(), "error finding cilium pods") 55 56 // pick a node 57 var nodeName string 58 for _, v := range pods { 59 nodeName = v 60 break 61 } 62 if nodeName == "" { 63 Fail("No cilium pods were found") 64 } 65 66 recreatePods := func() { 67 By("Deleting all cilium pods") 68 kubectl.DeleteResource("-n kube-system pod", "-l "+helpers.CiliumAgentLabel) 69 // Wait for pods to be up and running 70 kubectl.WaitForCiliumReadiness(0, "not all cilium pods returned") 71 } 72 73 // returns a map of nodename -> config key 74 getNodeConfigKeys := func(key string) map[string]string { 75 podtonode, err := kubectl.GetPodsNodes(helpers.CiliumNamespace, helpers.CiliumSelector) 76 Expect(err).To(BeNil(), "error finding cilium pods") 77 78 ress, err := kubectl.ExecInPods(context.TODO(), helpers.CiliumNamespace, helpers.CiliumSelector, 79 fmt.Sprintf("/bin/sh -c 'cat /tmp/cilium/config-map/%s || true'", key)) 80 Expect(err).NotTo(HaveOccurred()) 81 out := map[string]string{} 82 for p, res := range ress { 83 Expect(res.GetErr(p)).NotTo(HaveOccurred()) 84 out[podtonode[p]] = res.Stdout() 85 } 86 return out 87 } 88 89 By("Creating a CiliumNodeConfig v2alpha1") 90 // Create a CiliumNodeConfig that does not apply to any nodes 91 cnc := ` 92 apiVersion: cilium.io/v2alpha1 93 kind: CiliumNodeConfig 94 metadata: 95 namespace: kube-system 96 name: testing 97 spec: 98 nodeSelector: 99 matchLabels: 100 io.cilium.testing: foo 101 defaults: 102 test-key: foo 103 ` 104 f, err := os.CreateTemp("", "pernodeconfig-") 105 Expect(err).Should(BeNil()) 106 defer os.Remove(f.Name()) 107 108 _, err = f.WriteString(cnc) 109 Expect(err).Should(BeNil()) 110 111 res := kubectl.Apply(helpers.ApplyOptions{ 112 FilePath: f.Name(), 113 Namespace: "kube-system", 114 }) 115 Expect(res.GetErr("apply")).To(BeNil()) 116 117 // ensure no pods have this key 118 recreatePods() 119 nodeConfigKeys := getNodeConfigKeys("test-key") 120 Expect(nodeConfigKeys).To(HaveKey(nodeName)) 121 Expect(nodeConfigKeys).To(HaveEach("")) //ensure that all elements are "" 122 123 // Now, label 1 node and check the only it gets this config key 124 By("Labeling node %s with io.cilium.testing=foo", nodeName) 125 res = kubectl.ExecShort(fmt.Sprintf("kubectl label node %s io.cilium.testing=foo", nodeName)) 126 Expect(res.GetErr("label node")).To(BeNil()) 127 recreatePods() 128 By("Ensuring only node %s has the config override", nodeName) 129 nodeConfigKeys = getNodeConfigKeys("test-key") 130 // ensure it is zero 131 Expect(nodeConfigKeys).To(HaveKeyWithValue(nodeName, "foo")) 132 // If there are other nodes, make sure it doesn't have the config key 133 if len(nodeConfigKeys) > 1 { 134 delete(nodeConfigKeys, nodeName) 135 Expect(nodeConfigKeys).To(HaveEach("")) 136 } 137 }) 138 139 It("Correctly computes config overrides with CNC v2", func() { 140 pods, err := kubectl.GetPodsNodes(helpers.CiliumNamespace, helpers.CiliumSelector) 141 Expect(err).To(BeNil(), "error finding cilium pods") 142 143 // pick a node 144 var nodeName string 145 for _, v := range pods { 146 nodeName = v 147 break 148 } 149 if nodeName == "" { 150 Fail("No cilium pods were found") 151 } 152 153 recreatePods := func() { 154 By("Deleting all cilium pods") 155 kubectl.DeleteResource("-n kube-system pod", "-l "+helpers.CiliumAgentLabel) 156 // Wait for pods to be up and running 157 kubectl.WaitForCiliumReadiness(0, "not all cilium pods returned") 158 } 159 160 // returns a map of nodename -> config key 161 getNodeConfigKeys := func(key string) map[string]string { 162 podtonode, err := kubectl.GetPodsNodes(helpers.CiliumNamespace, helpers.CiliumSelector) 163 Expect(err).To(BeNil(), "error finding cilium pods") 164 165 ress, err := kubectl.ExecInPods(context.TODO(), helpers.CiliumNamespace, helpers.CiliumSelector, 166 fmt.Sprintf("/bin/sh -c 'cat /tmp/cilium/config-map/%s || true'", key)) 167 Expect(err).NotTo(HaveOccurred()) 168 out := map[string]string{} 169 for p, res := range ress { 170 Expect(res.GetErr(p)).NotTo((HaveOccurred())) 171 out[podtonode[p]] = res.Stdout() 172 } 173 return out 174 } 175 176 By("Creating a CiliumNodeConfig v2") 177 // Create a CiliumNodeConfig that does not apply to any nodes 178 cnc := ` 179 apiVersion: cilium.io/v2 180 kind: CiliumNodeConfig 181 metadata: 182 namespace: kube-system 183 name: testing-v2 184 spec: 185 nodeSelector: 186 matchLabels: 187 io.cilium.testing: bar 188 defaults: 189 test-key: bar 190 ` 191 f, err := os.CreateTemp("", "pernodeconfig-") 192 Expect(err).Should(BeNil()) 193 defer os.Remove(f.Name()) 194 195 _, err = f.WriteString(cnc) 196 Expect(err).Should(BeNil()) 197 198 res := kubectl.Apply(helpers.ApplyOptions{ 199 FilePath: f.Name(), 200 Namespace: "kube-system", 201 }) 202 Expect(res.GetErr("apply")).To(BeNil()) 203 204 // ensure no pods have this key 205 recreatePods() 206 nodeConfigKeys := getNodeConfigKeys("test-key") 207 Expect(nodeConfigKeys).To(HaveKey(nodeName)) 208 Expect(nodeConfigKeys).To(HaveEach("")) //ensure that all elements are "" 209 210 // Now, label 1 node and check the only it gets this config key 211 By("Labeling node %s with io.cilium.testing=bar", nodeName) 212 res = kubectl.ExecShort(fmt.Sprintf("kubectl label node %s io.cilium.testing=bar", nodeName)) 213 Expect(res.GetErr("label node")).To(BeNil()) 214 recreatePods() 215 By("Ensuring only node %s has the config override", nodeName) 216 nodeConfigKeys = getNodeConfigKeys("test-key") 217 // ensure it is zero 218 Expect(nodeConfigKeys).To(HaveKeyWithValue(nodeName, "bar")) 219 // If there are other nodes, make sure it doesn't have the config key 220 if len(nodeConfigKeys) > 1 { 221 delete(nodeConfigKeys, nodeName) 222 Expect(nodeConfigKeys).To(HaveEach("")) 223 } 224 }) 225 })