github.com/zhyoulun/cilium@v1.6.12/test/k8sT/Chaos.go (about) 1 // Copyright 2017-2019 Authors of Cilium 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package k8sTest 16 17 import ( 18 "context" 19 "fmt" 20 21 . "github.com/cilium/cilium/test/ginkgo-ext" 22 "github.com/cilium/cilium/test/helpers" 23 24 . "github.com/onsi/gomega" 25 ) 26 27 var _ = Describe("K8sChaosTest", func() { 28 29 var ( 30 kubectl *helpers.Kubectl 31 demoDSPath = helpers.ManifestGet("demo_ds.yaml") 32 testDSService = "testds-service" 33 ) 34 35 BeforeAll(func() { 36 kubectl = helpers.CreateKubectl(helpers.K8s1VMName(), logger) 37 DeployCiliumAndDNS(kubectl) 38 }) 39 40 AfterFailed(func() { 41 kubectl.CiliumReport(helpers.KubeSystemNamespace, 42 "cilium service list", 43 "cilium endpoint list") 44 }) 45 46 JustAfterEach(func() { 47 kubectl.ValidateNoErrorsInLogs(CurrentGinkgoTestDescription().Duration) 48 }) 49 50 AfterAll(func() { 51 ExpectAllPodsTerminated(kubectl) 52 kubectl.CloseSSHClient() 53 }) 54 55 Context("Connectivity demo application", func() { 56 BeforeEach(func() { 57 kubectl.ApplyDefault(demoDSPath).ExpectSuccess("DS deployment cannot be applied") 58 59 err := kubectl.WaitforPods( 60 helpers.DefaultNamespace, fmt.Sprintf("-l zgroup=testDS"), helpers.HelperTimeout) 61 Expect(err).Should(BeNil(), "Pods are not ready after timeout") 62 }) 63 64 AfterEach(func() { 65 kubectl.Delete(demoDSPath).ExpectSuccess( 66 "%s deployment cannot be deleted", demoDSPath) 67 ExpectAllPodsTerminated(kubectl) 68 69 }) 70 71 // connectivityTest performs a few test inside: 72 // - tests connectivity of all client pods to the backend pods directly via ping 73 // - tests connectivity of all client pods to the ClusterIP of the test-ds service via curl 74 // - tests connectivity of all client pods to the DNS name for the test-ds service via curl 75 connectivityTest := func() { 76 pods, err := kubectl.GetPodNames(helpers.DefaultNamespace, "zgroup=testDSClient") 77 Expect(err).To(BeNil(), "Cannot get pods names") 78 Expect(len(pods)).To(BeNumerically(">", 0), "No pods available to test connectivity") 79 80 dsPods, err := kubectl.GetPodsIPs(helpers.DefaultNamespace, "zgroup=testDS") 81 Expect(err).To(BeNil(), "Cannot get daemonset pods IPS") 82 Expect(len(dsPods)).To(BeNumerically(">", 0), "No pods available to test connectivity") 83 84 By("Waiting for kube-dns entry for service testds-service") 85 err = kubectl.WaitForKubeDNSEntry(testDSService, helpers.DefaultNamespace) 86 ExpectWithOffset(1, err).To(BeNil(), "DNS entry is not ready after timeout") 87 88 By("Getting ClusterIP For testds-service") 89 host, _, err := kubectl.GetServiceHostPort(helpers.DefaultNamespace, "testds-service") 90 ExpectWithOffset(1, err).To(BeNil(), "unable to get ClusterIP and port for service testds-service") 91 92 for _, pod := range pods { 93 for _, ip := range dsPods { 94 By("Pinging testds pod with IP %q from client pod %q", ip, pod) 95 res := kubectl.ExecPodCmd( 96 helpers.DefaultNamespace, pod, helpers.Ping(ip)) 97 log.Debugf("Pod %s ping %v", pod, ip) 98 ExpectWithOffset(1, res).To(helpers.CMDSuccess(), 99 "Cannot ping from %q to %q", pod, ip) 100 } 101 102 By("Curling testds-service via ClusterIP %q", host) 103 res := kubectl.ExecPodCmd( 104 helpers.DefaultNamespace, pod, helpers.CurlFail("http://%s:80/", host)) 105 ExpectWithOffset(1, res).To(helpers.CMDSuccess(), 106 "Cannot curl from %q to testds-service via ClusterIP", pod) 107 108 By("Curling testds-service via DNS hostname") 109 res = kubectl.ExecPodCmd( 110 helpers.DefaultNamespace, pod, helpers.CurlFail("http://%s:80/", testDSService)) 111 ExpectWithOffset(1, res).To(helpers.CMDSuccess(), 112 "Cannot curl from %q to testds-service via DNS hostname", pod) 113 } 114 } 115 116 It("Endpoint can still connect while Cilium is not running", func() { 117 By("Waiting for deployed pods to be ready") 118 err := kubectl.WaitforPods( 119 helpers.DefaultNamespace, 120 fmt.Sprintf("-l zgroup=testDSClient"), helpers.HelperTimeout) 121 Expect(err).Should(BeNil(), "Pods are not ready after timeout") 122 123 err = kubectl.CiliumEndpointWaitReady() 124 Expect(err).To(BeNil(), "Endpoints are not ready after timeout") 125 126 By("Checking connectivity before restarting Cilium") 127 connectivityTest() 128 129 By("Deleting cilium pods") 130 res := kubectl.Exec(fmt.Sprintf("%s -n %s delete pods -l k8s-app=cilium", 131 helpers.KubectlCmd, helpers.KubeSystemNamespace)) 132 res.ExpectSuccess() 133 134 ExpectAllPodsTerminated(kubectl) 135 136 ExpectCiliumReady(kubectl) 137 err = kubectl.CiliumEndpointWaitReady() 138 Expect(err).To(BeNil(), "Endpoints are not ready after Cilium restarts") 139 140 By("Checking connectivity after restarting Cilium") 141 connectivityTest() 142 143 By("Uninstall cilium pods") 144 145 res = kubectl.DeleteResource( 146 "ds", fmt.Sprintf("-n %s cilium", helpers.KubeSystemNamespace)) 147 res.ExpectSuccess("Cilium DS cannot be deleted") 148 149 ExpectAllPodsTerminated(kubectl) 150 151 By("Checking connectivity after uninstalling Cilium") 152 connectivityTest() 153 154 By("Install cilium pods") 155 156 err = kubectl.CiliumInstall([]string{}) 157 Expect(err).To(BeNil(), "Cilium cannot be installed") 158 159 ExpectCiliumReady(kubectl) 160 161 err = kubectl.CiliumEndpointWaitReady() 162 Expect(err).To(BeNil(), "Endpoints are not ready after timeout") 163 164 By("Checking connectivity after reinstalling Cilium") 165 connectivityTest() 166 }) 167 }) 168 169 Context("Restart with long lived connections", func() { 170 171 var ( 172 netperfManifest = helpers.ManifestGet("netperf-deployment.yaml") 173 netperfPolicy = helpers.ManifestGet("netperf-policy.yaml") 174 netperfServiceName = "netperf-service" 175 podsIps map[string]string 176 netperfClient = "netperf-client" 177 netperfServer = "netperf-server" 178 ) 179 180 BeforeAll(func() { 181 kubectl.ApplyDefault(netperfManifest).ExpectSuccess("Netperf cannot be deployed") 182 183 err := kubectl.WaitforPods( 184 helpers.DefaultNamespace, 185 fmt.Sprintf("-l zgroup=testapp"), helpers.HelperTimeout) 186 Expect(err).Should(BeNil(), "Pods are not ready after timeout") 187 188 podsIps, err = kubectl.GetPodsIPs(helpers.DefaultNamespace, "zgroup=testapp") 189 Expect(err).To(BeNil(), "Cannot get pods ips") 190 191 _, _, err = kubectl.GetServiceHostPort(helpers.DefaultNamespace, netperfServiceName) 192 Expect(err).To(BeNil(), "cannot get service netperf ip") 193 }) 194 195 AfterAll(func() { 196 _ = kubectl.Delete(netperfManifest) 197 }) 198 199 AfterEach(func() { 200 _ = kubectl.Delete(netperfPolicy) 201 }) 202 203 restartCilium := func() { 204 ciliumFilter := "k8s-app=cilium" 205 206 By("Deleting all cilium pods") 207 res := kubectl.Exec(fmt.Sprintf( 208 "%s -n %s delete pods -l %s", 209 helpers.KubectlCmd, helpers.KubeSystemNamespace, ciliumFilter)) 210 res.ExpectSuccess("Failed to delete cilium pods") 211 212 By("Waiting cilium pods to terminate") 213 ExpectAllPodsTerminated(kubectl) 214 215 By("Waiting for cilium pods to be ready") 216 err := kubectl.WaitforPods( 217 helpers.KubeSystemNamespace, fmt.Sprintf("-l %s", ciliumFilter), helpers.HelperTimeout) 218 Expect(err).Should(BeNil(), "Pods are not ready after timeout") 219 220 err = kubectl.CiliumEndpointWaitReady() 221 Expect(err).To(BeNil(), "Endpoints are not ready after timeout") 222 } 223 224 It("TCP connection is not dropped when cilium restarts", func() { 225 ctx, cancel := context.WithCancel(context.Background()) 226 defer cancel() 227 res := kubectl.ExecPodCmdBackground( 228 ctx, 229 helpers.DefaultNamespace, 230 netperfClient, 231 fmt.Sprintf("netperf -l 300 -t TCP_STREAM -H %s", podsIps[netperfServer])) 232 233 restartCilium() 234 235 By("Stopping netperf client test") 236 cancel() 237 res.WaitUntilFinish() 238 res.ExpectSuccess("Failed while cilium was restarting") 239 }) 240 241 It("L3/L4 policies still work while Cilium is restarted", func() { 242 243 ctx, cancel := context.WithCancel(context.Background()) 244 defer cancel() 245 res := kubectl.ExecPodCmdBackground( 246 ctx, 247 helpers.DefaultNamespace, 248 netperfClient, 249 fmt.Sprintf("netperf -l 300 -t TCP_STREAM -H %s", podsIps[netperfServer])) 250 251 By("Installing the L3-L4 Policy") 252 _, err := kubectl.CiliumPolicyAction( 253 helpers.DefaultNamespace, netperfPolicy, helpers.KubectlApply, helpers.HelperTimeout) 254 Expect(err).Should(BeNil(), "Cannot install %q policy", netperfPolicy) 255 256 restartCilium() 257 258 By("Stopping netperf client test") 259 cancel() 260 res.WaitUntilFinish() 261 res.ExpectSuccess("Failed while cilium was restarting") 262 }) 263 }) 264 })