github.com/cilium/cilium@v1.16.2/test/k8s/datapath_configuration.go (about) 1 // SPDX-License-Identifier: Apache-2.0 2 // Copyright Authors of Cilium 3 4 package k8sTest 5 6 import ( 7 "context" 8 "fmt" 9 "net" 10 "regexp" 11 "strconv" 12 "strings" 13 "sync" 14 "time" 15 16 . "github.com/onsi/gomega" 17 18 "github.com/cilium/cilium/pkg/defaults" 19 "github.com/cilium/cilium/test/config" 20 . "github.com/cilium/cilium/test/ginkgo-ext" 21 "github.com/cilium/cilium/test/helpers" 22 ) 23 24 var _ = Describe("K8sDatapathConfig", func() { 25 const ( 26 bpffsDir string = defaults.BPFFSRoot + "/" + defaults.TCGlobalsPath + "/" 27 ) 28 29 var ( 30 kubectl *helpers.Kubectl 31 monitorLog = "monitor-aggregation.log" 32 ) 33 34 BeforeAll(func() { 35 kubectl = helpers.CreateKubectl(helpers.K8s1VMName(), logger) 36 deploymentManager.SetKubectl(kubectl) 37 }) 38 39 AfterEach(func() { 40 deploymentManager.DeleteAll() 41 ExpectAllPodsTerminated(kubectl) 42 }) 43 44 AfterFailed(func() { 45 kubectl.CiliumReport("cilium-dbg status", "cilium-dbg endpoint list") 46 }) 47 48 AfterAll(func() { 49 kubectl.ScaleDownDNS() 50 ExpectAllPodsTerminated(kubectl) 51 deploymentManager.DeleteCilium() 52 kubectl.ScaleUpDNS() 53 kubectl.CloseSSHClient() 54 }) 55 56 JustAfterEach(func() { 57 kubectl.ValidateNoErrorsInLogs(CurrentGinkgoTestDescription().Duration) 58 }) 59 60 Context("MonitorAggregation", func() { 61 It("Checks that monitor aggregation restricts notifications", func() { 62 deploymentManager.DeployCilium(map[string]string{ 63 "bpf.monitorAggregation": "medium", 64 "bpf.monitorInterval": "60s", 65 "bpf.monitorFlags": "syn", 66 }, DeployCiliumOptionsAndDNS) 67 68 monitorRes, monitorCancel, targetIP := monitorConnectivityAcrossNodes(kubectl) 69 defer monitorCancel() 70 71 var monitorOutput []byte 72 searchMonitorLog := func(expr *regexp.Regexp) bool { 73 monitorOutput = monitorRes.CombineOutput().Bytes() 74 egressMatches := expr.FindAllIndex(monitorOutput, -1) 75 return len(egressMatches) > 0 76 } 77 78 By("Checking that ICMP notifications in egress direction were observed") 79 expEgress := fmt.Sprintf("ICMPv4.*DstIP=%s", targetIP) 80 expEgressRegex := regexp.MustCompile(expEgress) 81 Eventually(func() bool { 82 return searchMonitorLog(expEgressRegex) 83 }, helpers.HelperTimeout, time.Second).Should(BeTrue(), "Egress ICMPv4 flow (%q) not found in monitor log\n%s", expEgress, monitorOutput) 84 85 By("Checking that ICMP notifications in ingress direction were observed") 86 expIngress := fmt.Sprintf("ICMPv4.*SrcIP=%s", targetIP) 87 expIngressRegex := regexp.MustCompile(expIngress) 88 Eventually(func() bool { 89 return searchMonitorLog(expIngressRegex) 90 }, helpers.HelperTimeout, time.Second).Should(BeTrue(), "Ingress ICMPv4 flow (%q) not found in monitor log\n%s", expIngress, monitorOutput) 91 92 By("Checking the set of TCP notifications received matches expectations") 93 // | TCP Flags | Direction | Report? | Why? 94 // +===========+===========+=========+===== 95 // | SYN | -> | Y | monitorFlags=SYN 96 // | SYN / ACK | <- | Y | monitorFlags=SYN 97 // | ACK | -> | N | monitorFlags=(!ACK) 98 // | ACK | ... | N | monitorFlags=(!ACK) 99 // | ACK | <- | N | monitorFlags=(!ACK) 100 // | FIN | -> | Y | monitorAggregation=medium 101 // | FIN / ACK | <- | Y | monitorAggregation=medium 102 // | ACK | -> | Y | monitorAggregation=medium 103 egressPktCount := 3 104 ingressPktCount := 2 105 Eventually(func() error { 106 monitorOutput = monitorRes.CombineOutput().Bytes() 107 return checkMonitorOutput(monitorOutput, egressPktCount, ingressPktCount) 108 }, helpers.HelperTimeout, time.Second).Should(BeNil(), "Monitor log did not contain %d ingress and %d egress TCP notifications\n%s", 109 ingressPktCount, egressPktCount, monitorOutput) 110 111 helpers.WriteToReportFile(monitorOutput, monitorLog) 112 }) 113 114 It("Checks that monitor aggregation flags send notifications", func() { 115 deploymentManager.DeployCilium(map[string]string{ 116 "bpf.monitorAggregation": "medium", 117 "bpf.monitorInterval": "60s", 118 "bpf.monitorFlags": "psh", 119 }, DeployCiliumOptionsAndDNS) 120 monitorRes, monitorCancel, _ := monitorConnectivityAcrossNodes(kubectl) 121 defer monitorCancel() 122 123 var monitorOutput []byte 124 By("Checking the set of TCP notifications received matches expectations") 125 // | TCP Flags | Direction | Report? | Why? 126 // +===========+===========+=========+===== 127 // | SYN | -> | Y | monitorAggregation=medium 128 // | SYN / ACK | <- | Y | monitorAggregation=medium 129 // | ACK | -> | N | monitorFlags=(!ACK) 130 // | ACK | ... | N | monitorFlags=(!ACK) 131 // | PSH | -> | Y | monitorFlags=(PSH) 132 // | PSH | <- | Y | monitorFlags=(PSH) 133 // | FIN | -> | Y | monitorAggregation=medium 134 // | FIN / ACK | <- | Y | monitorAggregation=medium 135 // | ACK | -> | Y | monitorAggregation=medium 136 egressPktCount := 4 137 ingressPktCount := 3 138 Eventually(func() error { 139 monitorOutput = monitorRes.CombineOutput().Bytes() 140 return checkMonitorOutput(monitorOutput, egressPktCount, ingressPktCount) 141 }, helpers.HelperTimeout, time.Second).Should(BeNil(), "monitor aggregation did not result in correct number of TCP notifications\n%s", monitorOutput) 142 helpers.WriteToReportFile(monitorOutput, monitorLog) 143 }) 144 }) 145 146 Context("Encapsulation", func() { 147 var ( 148 tmpEchoPodPath string 149 outside, outside6 string 150 ) 151 152 BeforeAll(func() { 153 if helpers.ExistNodeWithoutCilium() && !helpers.SupportIPv6ToOutside() { 154 // Deploy echoserver on the node which does not run Cilium to test 155 // IPv6 connectivity to the outside world. The pod will run in 156 // the host netns, so no CNI is required for the pod on that host. 157 echoPodPath := helpers.ManifestGet(kubectl.BasePath(), "echoserver-hostnetns.yaml") 158 res := kubectl.ExecMiddle("mktemp") 159 res.ExpectSuccess() 160 tmpEchoPodPath = strings.Trim(res.Stdout(), "\n") 161 kubectl.ExecMiddle(fmt.Sprintf("sed 's/NODE_WITHOUT_CILIUM/%s/' %s > %s", 162 helpers.GetFirstNodeWithoutCilium(), echoPodPath, tmpEchoPodPath)).ExpectSuccess() 163 kubectl.ApplyDefault(tmpEchoPodPath).ExpectSuccess("Cannot install echoserver application") 164 Expect(kubectl.WaitforPods(helpers.DefaultNamespace, "-l name=echoserver-hostnetns", 165 helpers.HelperTimeout)).Should(BeNil()) 166 var err error 167 outside, err = kubectl.GetNodeIPByLabel(kubectl.GetFirstNodeWithoutCiliumLabel(), false) 168 Expect(err).Should(BeNil()) 169 outside6, err = kubectl.GetNodeIPv6ByLabel(kubectl.GetFirstNodeWithoutCiliumLabel(), false) 170 Expect(err).Should(BeNil()) 171 outside6 = net.JoinHostPort(outside6, "80") 172 } else { 173 outside = "http://google.com" 174 outside6 = "http://google.com" 175 } 176 }) 177 178 AfterAll(func() { 179 if tmpEchoPodPath != "" { 180 kubectl.Delete(tmpEchoPodPath) 181 } 182 }) 183 184 enableVXLANTunneling := func(options map[string]string) { 185 options["tunnelProtocol"] = "vxlan" 186 if helpers.RunsOnGKE() { 187 // We need to disable gke.enabled as it disables tunneling. 188 options["gke.enabled"] = "false" 189 options["endpointRoutes.enabled"] = "true" 190 } 191 } 192 193 It("Check iptables masquerading with random-fully", func() { 194 options := map[string]string{ 195 "bpf.masquerade": "false", 196 "enableIPv6Masquerade": "true", 197 "iptablesRandomFully": "true", 198 } 199 enableVXLANTunneling(options) 200 deploymentManager.DeployCilium(options, DeployCiliumOptionsAndDNS) 201 Expect(testPodConnectivityAcrossNodes(kubectl)).Should(BeTrue(), "Connectivity test between nodes failed") 202 203 By("Test iptables masquerading") 204 Expect(testPodHTTPToOutside(kubectl, outside, false, false, false)). 205 Should(BeTrue(), "IPv4 connectivity test to %s", outside) 206 if helpers.ExistNodeWithoutCilium() || helpers.SupportIPv6ToOutside() { 207 Expect(testPodHTTPToOutside(kubectl, outside6, false, false, true)). 208 Should(BeTrue(), "IPv6 connectivity test to %s failed", outside6) 209 } 210 }) 211 }) 212 213 SkipContextIf(func() bool { 214 return helpers.RunsWithKubeProxyReplacement() || helpers.GetCurrentIntegration() != "" 215 }, "IPv6 masquerading", func() { 216 var ( 217 k8s1EndpointIPs map[string]string 218 testDSK8s1IPv6 string = "fd03::310" 219 220 demoYAML string 221 ) 222 223 BeforeAll(func() { 224 deploymentManager.DeployCilium(map[string]string{ 225 "routingMode": "native", 226 "autoDirectNodeRoutes": "true", 227 "ipv6NativeRoutingCIDR": helpers.IPv6NativeRoutingCIDR, 228 }, DeployCiliumOptionsAndDNS) 229 230 demoYAML = helpers.ManifestGet(kubectl.BasePath(), "demo_ds.yaml") 231 res := kubectl.ApplyDefault(demoYAML) 232 Expect(res).Should(helpers.CMDSuccess(), "Unable to apply %s", demoYAML) 233 waitPodsDs(kubectl, []string{testDS, testDSClient, testDSK8s2}) 234 235 pod, err := kubectl.GetCiliumPodOnNode(helpers.K8s1) 236 Expect(err).Should(BeNil(), "Cannot get cilium pod on node %s", helpers.K8s1) 237 k8s1EndpointIPs = kubectl.CiliumEndpointIPv6(pod, "-l k8s:zgroup=testDS,k8s:io.kubernetes.pod.namespace=default") 238 239 k8s1Backends := []string{} 240 for _, epIP := range k8s1EndpointIPs { 241 k8s1Backends = append(k8s1Backends, net.JoinHostPort(epIP, "80")) 242 } 243 244 ciliumAddService(kubectl, 31080, net.JoinHostPort(testDSK8s1IPv6, "80"), k8s1Backends, "ClusterIP", "Cluster") 245 }) 246 247 It("across K8s nodes, skipped due to native routing CIDR", func() { 248 // Because a native routing CIDR is set, the 249 // IPv6 for packets routed to another node are 250 // _not_ masqueraded. Retrieve the address for 251 // the client, and make sure the echo server 252 // receives it unchanged. 253 pod, err := kubectl.GetCiliumPodOnNode(helpers.K8s2) 254 Expect(err).Should(BeNil(), "Cannot get cilium pod on node %s", helpers.K8s2) 255 k8s2EndpointIPs := kubectl.CiliumEndpointIPv6(pod, fmt.Sprintf("-l k8s:%s,k8s:io.kubernetes.pod.namespace=default", testDSK8s2)) 256 k8s2ClientIPv6 := "" 257 for _, epIP := range k8s2EndpointIPs { 258 k8s2ClientIPv6 = epIP 259 break 260 } 261 Expect(k8s2ClientIPv6).ShouldNot(BeEmpty(), "Cannot get client IPv6") 262 263 url := fmt.Sprintf(`"http://[%s]:80/"`, testDSK8s1IPv6) 264 testCurlFromPodWithSourceIPCheck(kubectl, testDSK8s2, url, 5, k8s2ClientIPv6) 265 266 for _, epIP := range k8s1EndpointIPs { 267 url = fmt.Sprintf(`"http://[%s]:80/"`, epIP) 268 testCurlFromPodWithSourceIPCheck(kubectl, testDSK8s2, url, 5, k8s2ClientIPv6) 269 } 270 }) 271 272 // Note: At the time we add the test below, it does not 273 // run on the CI because the only job which is running 274 // with a 3rd, non-K8s node (net-next) also has KPR, 275 // and skips the context we're in. Run locally. 276 // TODO: uncomment once the above has changed 277 //SkipItIf(helpers.DoesNotExistNodeWithoutCilium, "for external traffic", func() { 278 // // A native routing CIDR is set, but it does 279 // // not prevent masquerading for packets going 280 // // outside of the K8s cluster. Check that the 281 // // echo server sees the IPv6 address of the 282 // // client's node. 283 // url := fmt.Sprintf(`"http://[%s]:80/"`, ni.outsideIPv6) 284 // testCurlFromPodWithSourceIPCheck(kubectl, testDSK8s2, url, 5, ni.primaryK8s2IPv6) 285 286 // for _, epIP := range k8s1EndpointIPs { 287 // url = fmt.Sprintf(`"http://[%s]:80/"`, epIP) 288 // testCurlFromPodWithSourceIPCheck(kubectl, testDSK8s2, url, 5, ni.primaryK8s2IPv6) 289 // } 290 //}) 291 292 AfterAll(func() { 293 ciliumDelService(kubectl, 31080) 294 _ = kubectl.Delete(demoYAML) 295 }) 296 }) 297 298 SkipContextIf(func() bool { 299 return helpers.DoesNotExistNodeWithoutCilium() 300 }, "Check BPF masquerading with ip-masq-agent", func() { 301 var ( 302 tmpEchoPodPath string 303 nodeIP string 304 ) 305 306 BeforeAll(func() { 307 // Deploy echoserver on the node which does not run Cilium to test 308 // BPF masquerading. The pod will run in the host netns, so no CNI 309 // is required for the pod on that host. 310 echoPodPath := helpers.ManifestGet(kubectl.BasePath(), "echoserver-hostnetns.yaml") 311 res := kubectl.ExecMiddle("mktemp") 312 res.ExpectSuccess() 313 tmpEchoPodPath = strings.Trim(res.Stdout(), "\n") 314 kubectl.ExecMiddle(fmt.Sprintf("sed 's/NODE_WITHOUT_CILIUM/%s/' %s > %s", 315 helpers.GetFirstNodeWithoutCilium(), echoPodPath, tmpEchoPodPath)).ExpectSuccess() 316 kubectl.ApplyDefault(tmpEchoPodPath).ExpectSuccess("Cannot install echoserver application") 317 Expect(kubectl.WaitforPods(helpers.DefaultNamespace, "-l name=echoserver-hostnetns", 318 helpers.HelperTimeout)).Should(BeNil()) 319 var err error 320 nodeIP, err = kubectl.GetNodeIPByLabel(kubectl.GetFirstNodeWithoutCiliumLabel(), false) 321 Expect(err).Should(BeNil()) 322 }) 323 324 AfterAll(func() { 325 if tmpEchoPodPath != "" { 326 kubectl.Delete(tmpEchoPodPath) 327 } 328 }) 329 330 testIPMasqAgent := func() { 331 Expect(testPodHTTPToOutside(kubectl, 332 fmt.Sprintf("http://%s:80", nodeIP), false, true, false)).Should(BeTrue(), 333 "IPv4 Connectivity test to http://%s failed", nodeIP) 334 335 // remove nonMasqueradeCIDRs from the ConfigMap 336 kubectl.Patch(helpers.CiliumNamespace, "configMap", "ip-masq-agent", `{"data":{"config":"{\"nonMasqueradeCIDRs\":[]}"}}`) 337 // Wait until the ip-masq-agent config update is handled by the agent 338 time.Sleep(90 * time.Second) 339 340 // Check that requests to the echoserver from client pods are masqueraded. 341 Expect(testPodHTTPToOutside(kubectl, 342 fmt.Sprintf("http://%s:80", nodeIP), true, false, false)).Should(BeTrue(), 343 "IPv4 Connectivity test to http://%s failed", nodeIP) 344 345 Expect(testPodHTTPToOutside(kubectl, 346 fmt.Sprintf("http://%s:80", nodeIP), true, false, true)).Should(BeTrue(), 347 "IPv6 Connectivity test to http://%s failed", nodeIP) 348 } 349 350 It("DirectRouting", func() { 351 deploymentManager.DeployCilium(map[string]string{ 352 "ipMasqAgent.enabled": "true", 353 "routingMode": "native", 354 "autoDirectNodeRoutes": "true", 355 "ipMasqAgent.config.nonMasqueradeCIDRs": fmt.Sprintf("{%s/32}", nodeIP), 356 }, DeployCiliumOptionsAndDNS) 357 358 testIPMasqAgent() 359 }) 360 361 It("DirectRouting, IPv4 only", func() { 362 deploymentManager.DeployCilium(map[string]string{ 363 "ipMasqAgent.enabled": "true", 364 "routingMode": "native", 365 "autoDirectNodeRoutes": "true", 366 "ipMasqAgent.config.nonMasqueradeCIDRs": fmt.Sprintf("{%s/32}", nodeIP), 367 "ipv6.enabled": "false", 368 }, DeployCiliumOptionsAndDNS) 369 370 testIPMasqAgent() 371 }) 372 373 It("VXLAN", func() { 374 deploymentManager.DeployCilium(map[string]string{ 375 "ipMasqAgent.enabled": "true", 376 "tunnelProtocol": "vxlan", 377 "ipMasqAgent.config.nonMasqueradeCIDRs": fmt.Sprintf("{%s/32}", nodeIP), 378 }, DeployCiliumOptionsAndDNS) 379 380 testIPMasqAgent() 381 }) 382 }) 383 384 SkipContextIf(helpers.DoesNotRunOnNetNextKernel, "WireGuard encryption strict mode", func() { 385 BeforeAll(func() { 386 kubectl.DeleteResource("ciliumnode", "--all --wait") 387 }) 388 AfterEach(func() { 389 kubectl.DeleteResource("ciliumnode", "--all --wait") 390 }) 391 392 testStrictWireguard := func(interNodeDev string) { 393 // Disable the Cilium operator 394 // In combination with the used CES option, this will stop any 395 // endpoint propagation to other nodes. 396 kubectl.SetCiliumOperatorReplicas(0) 397 randomNamespace := deploymentManager.DeployRandomNamespaceShared(DemoDaemonSet) 398 deploymentManager.WaitUntilReady() 399 400 k8s1NodeName, k8s1IP := kubectl.GetNodeInfo(helpers.K8s1) 401 k8s2NodeName, k8s2IP := kubectl.GetNodeInfo(helpers.K8s2) 402 403 // Fetch srcPod (testDSClient@k8s1) 404 srcPod, srcPodJSON := fetchPodsWithOffset(kubectl, randomNamespace, "client", "zgroup=testDSClient", k8s2IP, true, 0) 405 srcPodIP, err := srcPodJSON.Filter("{.status.podIP}") 406 ExpectWithOffset(1, err).Should(BeNil(), "Failure to retrieve pod IP %s", srcPod) 407 srcHost, err := srcPodJSON.Filter("{.status.hostIP}") 408 ExpectWithOffset(1, err).Should(BeNil(), "Failure to retrieve host of pod %s", srcPod) 409 // Sanity check 410 ExpectWithOffset(1, srcHost.String()).Should(Equal(k8s1IP)) 411 // Fetch srcPod IPv6 412 ciliumPodK8s1, err := kubectl.GetCiliumPodOnNode(helpers.K8s1) 413 ExpectWithOffset(1, err).Should(BeNil(), "Unable to fetch cilium pod on k8s1") 414 endpointIPs := kubectl.CiliumEndpointIPv6(ciliumPodK8s1, "-l k8s:zgroup=testDSClient") 415 // Sanity check 416 ExpectWithOffset(1, len(endpointIPs)).Should(Equal(1), "BUG: more than one DS client on %s", ciliumPodK8s1) 417 418 // Fetch dstPod (testDS@k8s2) 419 dstPod, dstPodJSON := fetchPodsWithOffset(kubectl, randomNamespace, "server", "zgroup=testDS", k8s1IP, true, 0) 420 dstPodIP, err := dstPodJSON.Filter("{.status.podIP}") 421 ExpectWithOffset(1, err).Should(BeNil(), "Failure to retrieve IP of pod %s", dstPod) 422 dstHost, err := dstPodJSON.Filter("{.status.hostIP}") 423 ExpectWithOffset(1, err).Should(BeNil(), "Failure to retrieve host of pod %s", dstPod) 424 // Sanity check 425 ExpectWithOffset(1, dstHost.String()).Should(Equal(k8s2IP)) 426 // Fetch dstPod IPv6 427 ciliumPodK8s2, err := kubectl.GetCiliumPodOnNode(helpers.K8s2) 428 ExpectWithOffset(1, err).Should(BeNil(), "Unable to fetch cilium pod on k8s2") 429 endpointIPs = kubectl.CiliumEndpointIPv6(ciliumPodK8s2, "-l k8s:zgroup=testDS") 430 // Sanity check 431 ExpectWithOffset(1, len(endpointIPs)).Should(Equal(1), "BUG: more than one DS server on %s", ciliumPodK8s2) 432 433 // Fetch svc IP 434 dsSvcJSON := kubectl.Get(randomNamespace, "svc testds-service -o json") 435 dsSvcIP, err := dsSvcJSON.Filter("{.spec.clusterIP}") 436 ExpectWithOffset(1, err).Should(BeNil(), "Failure to retrieve IP of service testds-service") 437 438 checkNoLeakP2P := func(srcPod, srcIP, dstIP string, shouldBeSuccessful bool) { 439 cmd := fmt.Sprintf("tcpdump -vv -i %s --immediate-mode -n 'host %s and host %s' -c 1", interNodeDev, srcIP, dstIP) 440 res1, cancel1, err := kubectl.ExecInHostNetNSInBackground(context.TODO(), k8s1NodeName, cmd) 441 ExpectWithOffset(2, err).Should(BeNil(), "Cannot exec tcpdump in bg") 442 res2, cancel2, err := kubectl.ExecInHostNetNSInBackground(context.TODO(), k8s2NodeName, cmd) 443 ExpectWithOffset(2, err).Should(BeNil(), "Cannot exec tcpdump in bg") 444 445 // HTTP connectivity test (pod2pod) 446 447 cmdRes := kubectl.ExecPodCmd(randomNamespace, srcPod, helpers.CurlFail("http://%s/", net.JoinHostPort(dstIP, "80"))) 448 if shouldBeSuccessful { 449 cmdRes.ExpectSuccess("Failed to curl dst pod") 450 } else { 451 cmdRes.ExpectFail("Should not be able to curl dst pod") 452 } 453 454 // Check that no unencrypted pod2pod traffic was captured on the direct routing device 455 cancel1() 456 cancel2() 457 ExpectWithOffset(2, res1.CombineOutput().String()).Should(Not(ContainSubstring("1 packet captured"))) 458 ExpectWithOffset(2, res2.CombineOutput().String()).Should(Not(ContainSubstring("1 packet captured"))) 459 } 460 461 checkNoLeakP2RemoteService := func(srcPod, dstPod, srcIP, dstService string, shouldBeSuccessful bool) { 462 cmd := fmt.Sprintf("tcpdump -vv -i %s --immediate-mode -n 'src %s or dst %s' -c 1", interNodeDev, srcIP, srcIP) 463 res1, cancel1, err := kubectl.ExecInHostNetNSInBackground(context.TODO(), k8s1NodeName, cmd) 464 ExpectWithOffset(2, err).Should(BeNil(), "Cannot exec tcpdump in bg") 465 res2, cancel2, err := kubectl.ExecInHostNetNSInBackground(context.TODO(), k8s2NodeName, cmd) 466 ExpectWithOffset(2, err).Should(BeNil(), "Cannot exec tcpdump in bg") 467 468 // HTTP connectivity test (pod2pod) 469 470 cmdRes, err := kubectl.ExecUntilMatch(randomNamespace, srcPod, helpers.CurlTimeout("http://%s/", time.Second, net.JoinHostPort(dstService, "80")), dstPod) 471 if shouldBeSuccessful { 472 cmdRes.ExpectSuccess("Failed to curl svc on dst pod") 473 ExpectWithOffset(2, err).Should(BeNil(), "Failed to curl svc on dst pod") 474 } else { 475 ExpectWithOffset(2, err).ShouldNot(BeNil(), "Should not be able to curl svc on dst pod") 476 } 477 478 // Check that no unencrypted pod2pod traffic was captured on the direct routing device 479 cancel1() 480 cancel2() 481 ExpectWithOffset(2, res1.CombineOutput().String()).Should(Not(ContainSubstring("1 packet captured"))) 482 ExpectWithOffset(2, res2.CombineOutput().String()).Should(Not(ContainSubstring("1 packet captured"))) 483 } 484 485 checkNoLeakP2P(srcPod, srcPodIP.String(), dstPodIP.String(), false) 486 checkNoLeakP2RemoteService(srcPod, dstPod, srcPodIP.String(), dsSvcIP.String(), false) 487 488 // Check that the src pod can reach the remote host 489 kubectl.ExecPodCmd(randomNamespace, srcPod, helpers.Ping(k8s2IP)). 490 ExpectSuccess("Failed to ping k8s2 host from src pod") 491 492 // Check that the remote host can reach the dst pod 493 kubectl.ExecInHostNetNS(context.TODO(), k8s1NodeName, 494 helpers.CurlFail("http://%s:80/", dstPodIP)).ExpectSuccess("Failed to curl dst pod from k8s1") 495 496 // Re-enable operator so that all endpoints are eventually updated 497 kubectl.SetCiliumOperatorReplicas(2) 498 499 // Due to IPCache update delays, it can take up to a few seconds 500 // before both nodes have added the new pod IPs to their allowedIPs 501 // list, which can cause flakes in CI. Therefore wait for the 502 // IPs to be present on both nodes before performing the test 503 waitForAllowedIP := func(ciliumPod, ip string) { 504 jsonpath := fmt.Sprintf(`{.encryption.wireguard.interfaces[*].peers[*].allowed-ips[?(@=='%s')]}`, ip) 505 ciliumCmd := fmt.Sprintf(`cilium-dbg debuginfo --output jsonpath="%s"`, jsonpath) 506 expected := fmt.Sprintf("jsonpath=%s", ip) 507 err := kubectl.CiliumExecUntilMatch(ciliumPod, ciliumCmd, expected) 508 Expect(err).To(BeNil(), "ip %q not in allowedIPs of pod %q", ip, ciliumPod) 509 } 510 511 waitForAllowedIP(ciliumPodK8s1, fmt.Sprintf("%s/32", dstPodIP)) 512 waitForAllowedIP(ciliumPodK8s2, fmt.Sprintf("%s/32", srcPodIP)) 513 514 checkNoLeakP2P(srcPod, srcPodIP.String(), dstPodIP.String(), true) 515 516 checkNoLeakP2RemoteService(srcPod, dstPod, srcPodIP.String(), dsSvcIP.String(), true) 517 } 518 519 It("Pod-to-pod traffic is encrypted in native routing mode with per-endpoint routes", func() { 520 deploymentManager.DeployCilium(map[string]string{ 521 "routingMode": "native", 522 "autoDirectNodeRoutes": "true", 523 "ipv4NativeRoutingCIDR": "10.244.0.0/16", 524 "ipv6.enabled": "false", 525 "endpointRoutes.enabled": "true", 526 "encryption.enabled": "true", 527 "encryption.type": "wireguard", 528 "l7Proxy": "false", 529 "ipam.operator.clusterPoolIPv4PodCIDRList": "10.244.0.0/16", 530 "encryption.strictMode.enabled": "true", 531 "encryption.strictMode.cidr": "10.244.0.0/16", 532 }, DeployCiliumOptionsAndDNS) 533 534 privateIface, err := kubectl.GetPrivateIface(helpers.K8s1) 535 Expect(err).Should(BeNil(), "Cannot determine private iface") 536 testStrictWireguard(privateIface) 537 }) 538 It("Pod-to-pod traffic is encrypted in native routing mode with per-endpoint routes and overlapping node and pod CIDRs", func() { 539 deploymentManager.DeployCilium(map[string]string{ 540 "routingMode": "native", 541 "autoDirectNodeRoutes": "true", 542 "ipv4NativeRoutingCIDR": "192.168.56.0/24", 543 "ipv6.enabled": "false", 544 "endpointRoutes.enabled": "true", 545 "encryption.enabled": "true", 546 "encryption.type": "wireguard", 547 "l7Proxy": "false", 548 "ipam.operator.clusterPoolIPv4PodCIDRList": "192.168.56.128/25", 549 "ipam.operator.clusterPoolIPv4MaskSize": "26", 550 "encryption.strictMode.enabled": "true", 551 "encryption.strictMode.cidr": "192.168.56.0/24", 552 "encryption.strictMode.allowRemoteNodeIdentities": "true", 553 }, DeployCiliumOptionsAndDNS) 554 555 privateIface, err := kubectl.GetPrivateIface(helpers.K8s1) 556 Expect(err).Should(BeNil(), "Cannot determine private iface") 557 testStrictWireguard(privateIface) 558 }) 559 }) 560 561 Context("IPv4Only", func() { 562 It("Check connectivity with IPv6 disabled", func() { 563 deploymentManager.DeployCilium(map[string]string{ 564 "ipv4.enabled": "true", 565 "ipv6.enabled": "false", 566 }, DeployCiliumOptionsAndDNS) 567 Expect(testPodConnectivityAcrossNodes(kubectl)).Should(BeTrue(), "Connectivity test between nodes failed") 568 }) 569 }) 570 571 Context("Host firewall", func() { 572 BeforeAll(func() { 573 kubectl.Exec("kubectl label nodes --all status=lockdown") 574 575 // Need to install Cilium w/ host fw prior to the host policy preparation 576 // step. 577 deploymentManager.DeployCilium(map[string]string{ 578 "hostFirewall.enabled": "true", 579 }, DeployCiliumOptionsAndDNS) 580 581 hostPolicy := helpers.ManifestGet(kubectl.BasePath(), "host-policies.yaml") 582 prepareHostPolicyEnforcement(kubectl, hostPolicy) 583 }) 584 585 AfterAll(func() { 586 kubectl.Exec("kubectl label nodes --all status-") 587 }) 588 589 SkipItIf(func() bool { 590 return !helpers.IsIntegration(helpers.CIIntegrationGKE) 591 }, "Check connectivity with IPv6 disabled", func() { 592 deploymentManager.DeployCilium(map[string]string{ 593 "ipv4.enabled": "true", 594 "ipv6.enabled": "false", 595 "hostFirewall.enabled": "true", 596 }, DeployCiliumOptionsAndDNS) 597 Expect(testPodConnectivityAcrossNodes(kubectl)).Should(BeTrue(), "Connectivity test between nodes failed") 598 }) 599 600 SkipItIf(helpers.RunsOnAKS, "With VXLAN", func() { 601 options := map[string]string{ 602 "hostFirewall.enabled": "true", 603 } 604 if helpers.RunsOnGKE() { 605 options["gke.enabled"] = "false" 606 options["tunnelProtocol"] = "vxlan" 607 } 608 deploymentManager.DeployCilium(options, DeployCiliumOptionsAndDNS) 609 testHostFirewall(kubectl) 610 }) 611 612 SkipItIf(func() bool { 613 return helpers.RunsOnAKS() 614 }, "With VXLAN and endpoint routes", func() { 615 options := map[string]string{ 616 "hostFirewall.enabled": "true", 617 "endpointRoutes.enabled": "true", 618 } 619 if helpers.RunsOnGKE() { 620 options["gke.enabled"] = "false" 621 options["tunnelProtocol"] = "vxlan" 622 } 623 deploymentManager.DeployCilium(options, DeployCiliumOptionsAndDNS) 624 testHostFirewall(kubectl) 625 }) 626 627 It("With native routing", func() { 628 options := map[string]string{ 629 "hostFirewall.enabled": "true", 630 "routingMode": "native", 631 } 632 // We don't want to run with per-endpoint routes (enabled by 633 // gke.enabled) for this test. 634 if helpers.RunsOnGKE() { 635 options["gke.enabled"] = "false" 636 } else { 637 options["autoDirectNodeRoutes"] = "true" 638 } 639 deploymentManager.DeployCilium(options, DeployCiliumOptionsAndDNS) 640 testHostFirewall(kubectl) 641 }) 642 643 It("With native routing and endpoint routes", func() { 644 options := map[string]string{ 645 "hostFirewall.enabled": "true", 646 "routingMode": "native", 647 "endpointRoutes.enabled": "true", 648 } 649 if !helpers.RunsOnGKE() { 650 options["autoDirectNodeRoutes"] = "true" 651 } 652 deploymentManager.DeployCilium(options, DeployCiliumOptionsAndDNS) 653 testHostFirewall(kubectl) 654 }) 655 }) 656 657 SkipContextIf(helpers.DoesNotRunOnNetNextKernel, "High-scale IPcache", func() { 658 const hsIPcacheFile = "high-scale-ipcache.yaml" 659 660 AfterEach(func() { 661 hsIPcacheYAML := helpers.ManifestGet(kubectl.BasePath(), hsIPcacheFile) 662 _ = kubectl.Delete(hsIPcacheYAML) 663 }) 664 665 testHighScaleIPcache := func(tunnelProto string, epRoutesConfig string) { 666 options := map[string]string{ 667 "highScaleIPcache.enabled": "true", 668 "routingMode": "native", 669 "bpf.monitorAggregation": "none", 670 "ipv6.enabled": "false", 671 "wellKnownIdentities.enabled": "true", 672 "tunnelProtocol": tunnelProto, 673 "endpointRoutes.enabled": epRoutesConfig, 674 } 675 if !helpers.RunsOnGKE() { 676 options["autoDirectNodeRoutes"] = "true" 677 } 678 if helpers.RunsWithKubeProxy() { 679 options["kubeProxyReplacement"] = "false" 680 } else if helpers.RunsWithKubeProxyReplacement() { 681 options["loadBalancer.mode"] = "dsr" 682 options["loadBalancer.dsrDispatch"] = "geneve" 683 } 684 deploymentManager.DeployCilium(options, DeployCiliumOptionsAndDNS) 685 686 cmd := fmt.Sprintf("bpftool map update pinned %scilium_world_cidrs4 key 0 0 0 0 0 0 0 0 value 1", bpffsDir) 687 kubectl.CiliumExecMustSucceedOnAll(context.TODO(), cmd) 688 689 hsIPcacheYAML := helpers.ManifestGet(kubectl.BasePath(), hsIPcacheFile) 690 kubectl.Create(hsIPcacheYAML).ExpectSuccess("Unable to create resource %q", hsIPcacheYAML) 691 692 // We need a longer timeout here because of the larger number of 693 // pods that need to be deployed. 694 err := kubectl.WaitforPods(helpers.DefaultNamespace, "-l type=client", 2*helpers.HelperTimeout) 695 Expect(err).ToNot(HaveOccurred(), "Client pods not ready after timeout") 696 } 697 698 It("Test ingress policy enforcement with GENEVE and endpoint routes", func() { 699 testHighScaleIPcache("geneve", "true") 700 }) 701 }) 702 703 Context("Iptables", func() { 704 SkipItIf(func() bool { 705 return helpers.IsIntegration(helpers.CIIntegrationGKE) || helpers.DoesNotRunWithKubeProxyReplacement() 706 }, "Skip conntrack for pod traffic", func() { 707 deploymentManager.DeployCilium(map[string]string{ 708 "routingMode": "native", 709 "autoDirectNodeRoutes": "true", 710 "installNoConntrackIptablesRules": "true", 711 }, DeployCiliumOptionsAndDNS) 712 713 ciliumPod, err := kubectl.GetCiliumPodOnNode(helpers.K8s1) 714 ExpectWithOffset(1, err).Should(BeNil(), "Unable to determine cilium pod on node %s", helpers.K8s1) 715 716 _, err = kubectl.ExecInHostNetNSByLabel(context.TODO(), helpers.K8s1, "conntrack -F") 717 if err != nil { 718 ExpectWithOffset(1, err).Should(BeNil(), "Cannot flush conntrack table") 719 } 720 721 Expect(testPodConnectivityAcrossNodes(kubectl)).Should(BeTrue(), "Connectivity test between nodes failed") 722 723 cmd := fmt.Sprintf("iptables -w 60 -t raw -C CILIUM_PRE_raw -s %s -m comment --comment 'cilium: NOTRACK for pod traffic' -j CT --notrack", helpers.IPv4NativeRoutingCIDR) 724 res := kubectl.ExecPodCmd(helpers.CiliumNamespace, ciliumPod, cmd) 725 res.ExpectSuccess("Missing '-j CT --notrack' iptables rule") 726 727 cmd = fmt.Sprintf("iptables -w 60 -t raw -C CILIUM_PRE_raw -d %s -m comment --comment 'cilium: NOTRACK for pod traffic' -j CT --notrack", helpers.IPv4NativeRoutingCIDR) 728 res = kubectl.ExecPodCmd(helpers.CiliumNamespace, ciliumPod, cmd) 729 res.ExpectSuccess("Missing '-j CT --notrack' iptables rule") 730 731 cmd = fmt.Sprintf("iptables -w 60 -t raw -C CILIUM_OUTPUT_raw -s %s -m comment --comment 'cilium: NOTRACK for pod traffic' -j CT --notrack", helpers.IPv4NativeRoutingCIDR) 732 res = kubectl.ExecPodCmd(helpers.CiliumNamespace, ciliumPod, cmd) 733 res.ExpectSuccess("Missing '-j CT --notrack' iptables rule") 734 735 cmd = fmt.Sprintf("iptables -w 60 -t raw -C CILIUM_OUTPUT_raw -d %s -m comment --comment 'cilium: NOTRACK for pod traffic' -j CT --notrack", helpers.IPv4NativeRoutingCIDR) 736 res = kubectl.ExecPodCmd(helpers.CiliumNamespace, ciliumPod, cmd) 737 res.ExpectSuccess("Missing '-j CT --notrack' iptables rule") 738 739 cmd = fmt.Sprintf("conntrack -L -s %s -d %s | wc -l", helpers.IPv4NativeRoutingCIDR, helpers.IPv4NativeRoutingCIDR) 740 resStr, err := kubectl.ExecInHostNetNSByLabel(context.TODO(), helpers.K8s1, cmd) 741 if err != nil { 742 ExpectWithOffset(1, err).Should(BeNil(), "Cannot list conntrack entries") 743 } 744 Expect(strings.TrimSpace(resStr)).To(Equal("0"), "Unexpected conntrack entries") 745 }) 746 }) 747 }) 748 749 // To avoid flakes, we need to perform some prep work before we enable host 750 // policy enforcement. 751 // 752 // When we first enable the host firewall, Cilium will for the first time track 753 // all hostns connections on all nodes. Because those connections are already 754 // established, the first packet we see from them may be a reply packet. For 755 // that reason, it's possible for Cilium to create conntrack entries in the 756 // wrong direction. As a consequence, once host policies are enforced, we will 757 // allow the forward path through and enforce policies on replies. 758 // For example, consider a connection to the kube-apiserver, a:52483 -> b:6443. 759 // If the first packet we track is the SYN+ACK, we will create a conntrack 760 // entry TCP OUT b:6443 -> a:52483. All traffic a:52483 -> b:6443 will be 761 // considered reply traffic and we will enforce policies on b:6443 -> a:52483. 762 // If there are any L4 policy rules, this 52483 destination port is unlikely to 763 // be allowed through. 764 // 765 // That situation unfortunately doesn't resolve on its own because Linux will 766 // consider the connections to be in LAST-ACK state on the server side and will 767 // keep them around indefinitely. 768 // 769 // To fix that, we need to force the termination of those connections. One way 770 // to do that is to enforce policies just long enough that all such connections 771 // will end up in a closing state (LAST-ACK or TIME-WAIT). Once that is the 772 // case, we remove the policies to allow everything through and enable proper 773 // termination of those connections. 774 // This function implements that process. 775 func prepareHostPolicyEnforcement(kubectl *helpers.Kubectl, policy string) { 776 By(fmt.Sprintf("Applying policies %s for 1min", policy)) 777 _, err := kubectl.CiliumClusterwidePolicyAction(policy, helpers.KubectlApply, helpers.HelperTimeout) 778 ExpectWithOffset(1, err).Should(BeNil(), fmt.Sprintf("Error creating resource %s: %s", policy, err)) 779 780 time.Sleep(1 * time.Minute) 781 782 _, err = kubectl.CiliumClusterwidePolicyAction(policy, helpers.KubectlDelete, helpers.HelperTimeout) 783 ExpectWithOffset(1, err).Should(BeNil(), fmt.Sprintf("Error deleting resource %s: %s", policy, err)) 784 785 By("Deleted the policies, waiting for connection terminations") 786 time.Sleep(30 * time.Second) 787 } 788 789 func testHostFirewall(kubectl *helpers.Kubectl) { 790 randomNs := deploymentManager.DeployRandomNamespaceShared(DemoHostFirewall) 791 deploymentManager.WaitUntilReady() 792 793 demoHostPolicies := helpers.ManifestGet(kubectl.BasePath(), "host-policies.yaml") 794 By(fmt.Sprintf("Applying policies %s", demoHostPolicies)) 795 _, err := kubectl.CiliumClusterwidePolicyAction(demoHostPolicies, helpers.KubectlApply, helpers.HelperTimeout) 796 ExpectWithOffset(1, err).Should(BeNil(), fmt.Sprintf("Error creating resource %s: %s", demoHostPolicies, err)) 797 defer func() { 798 _, err := kubectl.CiliumClusterwidePolicyAction(demoHostPolicies, helpers.KubectlDelete, helpers.HelperTimeout) 799 ExpectWithOffset(1, err).Should(BeNil(), fmt.Sprintf("Error deleting resource %s: %s", demoHostPolicies, err)) 800 }() 801 802 var wg sync.WaitGroup 803 wg.Add(1) 804 go func() { 805 defer GinkgoRecover() 806 defer wg.Done() 807 By("Checking host policies on ingress from local pod") 808 testHostFirewallWithPath(kubectl, randomNs, "zgroup=testClient", "zgroup=testServerHost", false) 809 }() 810 wg.Add(1) 811 go func() { 812 defer GinkgoRecover() 813 defer wg.Done() 814 By("Checking host policies on ingress from remote pod") 815 testHostFirewallWithPath(kubectl, randomNs, "zgroup=testClient", "zgroup=testServerHost", true) 816 }() 817 wg.Add(1) 818 go func() { 819 defer GinkgoRecover() 820 defer wg.Done() 821 By("Checking host policies on egress to local pod") 822 testHostFirewallWithPath(kubectl, randomNs, "zgroup=testClientHost", "zgroup=testServer", false) 823 }() 824 wg.Add(1) 825 go func() { 826 defer GinkgoRecover() 827 defer wg.Done() 828 By("Checking host policies on egress to remote pod") 829 testHostFirewallWithPath(kubectl, randomNs, "zgroup=testClientHost", "zgroup=testServer", true) 830 }() 831 wg.Add(1) 832 go func() { 833 defer GinkgoRecover() 834 defer wg.Done() 835 By("Checking host policies on ingress from remote node") 836 testHostFirewallWithPath(kubectl, randomNs, "zgroup=testServerHost", "zgroup=testClientHost", true) 837 }() 838 wg.Add(1) 839 go func() { 840 defer GinkgoRecover() 841 defer wg.Done() 842 By("Checking host policies on egress to remote node") 843 testHostFirewallWithPath(kubectl, randomNs, "zgroup=testClientHost", "zgroup=testServerHost", true) 844 }() 845 wg.Wait() 846 } 847 848 func testHostFirewallWithPath(kubectl *helpers.Kubectl, randomNs, client, server string, crossNodes bool) { 849 srcPod, srcPodJSON := fetchPodsWithOffset(kubectl, randomNs, "client", client, "", crossNodes, 3) 850 srcHost, err := srcPodJSON.Filter("{.status.hostIP}") 851 ExpectWithOffset(2, err).Should(BeNil(), "Failure to retrieve host of pod %s", srcPod) 852 853 dstPod, dstPodJSON := fetchPodsWithOffset(kubectl, randomNs, "server", server, srcHost.String(), crossNodes, 3) 854 podIP, err := dstPodJSON.Filter("{.status.podIP}") 855 ExpectWithOffset(2, err).Should(BeNil(), "Failure to retrieve IP of pod %s", dstPod) 856 targetIP := podIP.String() 857 858 res := kubectl.ExecPodCmd(randomNs, srcPod, helpers.CurlFail("http://%s:80/", targetIP)) 859 ExpectWithOffset(2, res).Should(helpers.CMDSuccess(), 860 "Failed to reach %s:80 from %s", targetIP, srcPod) 861 862 res = kubectl.ExecPodCmd(randomNs, srcPod, helpers.CurlFail("tftp://%s:69/hello", targetIP)) 863 ExpectWithOffset(2, res).ShouldNot(helpers.CMDSuccess(), 864 "Managed to reach %s:69 from %s", targetIP, srcPod) 865 } 866 867 func testPodConnectivityAcrossNodes(kubectl *helpers.Kubectl) bool { 868 result, _ := testPodConnectivityAndReturnIP(kubectl, true, 1) 869 return result 870 } 871 872 func fetchPodsWithOffset(kubectl *helpers.Kubectl, namespace, name, filter, hostIPAntiAffinity string, requireMultiNode bool, callOffset int) (targetPod string, targetPodJSON *helpers.CmdRes) { 873 callOffset++ 874 875 // Fetch pod (names) with the specified filter 876 err := kubectl.WaitforPods(namespace, fmt.Sprintf("-l %s", filter), helpers.HelperTimeout) 877 ExpectWithOffset(callOffset, err).Should(BeNil(), "Failure while waiting for connectivity test pods to start") 878 pods, err := kubectl.GetPodNames(namespace, filter) 879 ExpectWithOffset(callOffset, err).Should(BeNil(), "Failure while retrieving pod name for %s", filter) 880 if requireMultiNode { 881 if config.CiliumTestConfig.Multinode { 882 ExpectWithOffset(callOffset, len(pods)).Should(BeNumerically(">", 1), 883 fmt.Sprintf("This test requires at least two %s instances, but only one was found", name)) 884 } else { 885 By("Ignoring the requirement for clients on multiple nodes") 886 requireMultiNode = false 887 } 888 } 889 890 // Fetch the json description of one of the pods 891 targetPod = pods[0] 892 targetPodJSON = kubectl.Get( 893 namespace, 894 fmt.Sprintf("pod %s -o json", targetPod)) 895 896 // If multinode / antiaffinity is required, ensure that the target is 897 // not on the same node as "hostIPAntiAffinity". 898 if requireMultiNode && hostIPAntiAffinity != "" { 899 targetHost, err := targetPodJSON.Filter("{.status.hostIP}") 900 ExpectWithOffset(callOffset, err).Should(BeNil(), "Failure to retrieve host of pod %s", targetPod) 901 902 if targetHost.String() == hostIPAntiAffinity { 903 targetPod = pods[1] 904 targetPodJSON = kubectl.Get( 905 namespace, 906 fmt.Sprintf("pod %s -o json", targetPod)) 907 } 908 } else if !requireMultiNode && hostIPAntiAffinity != "" { 909 targetHost, err := targetPodJSON.Filter("{.status.hostIP}") 910 ExpectWithOffset(callOffset, err).Should(BeNil(), "Failure to retrieve host of pod %s", targetPod) 911 912 if targetHost.String() != hostIPAntiAffinity { 913 targetPod = pods[1] 914 targetPodJSON = kubectl.Get( 915 namespace, 916 fmt.Sprintf("pod %s -o json", targetPod)) 917 } 918 } 919 return targetPod, targetPodJSON 920 } 921 922 func applyL3Policy(kubectl *helpers.Kubectl, ns string) (withdrawPolicy func()) { 923 demoPolicyL3 := helpers.ManifestGet(kubectl.BasePath(), "l3-policy-demo.yaml") 924 By(fmt.Sprintf("Applying policy %s", demoPolicyL3)) 925 _, err := kubectl.CiliumPolicyAction(ns, demoPolicyL3, helpers.KubectlApply, helpers.HelperTimeout) 926 ExpectWithOffset(1, err).Should(BeNil(), fmt.Sprintf("Error creating resource %s: %s", demoPolicyL3, err)) 927 928 return func() { 929 _, err := kubectl.CiliumPolicyAction(ns, demoPolicyL3, helpers.KubectlDelete, helpers.HelperTimeout) 930 ExpectWithOffset(1, err).Should(BeNil(), fmt.Sprintf("Error deleting resource %s: %s", demoPolicyL3, err)) 931 } 932 } 933 934 func testPodConnectivityAndReturnIP(kubectl *helpers.Kubectl, requireMultiNode bool, callOffset int) (bool, string) { 935 callOffset++ 936 937 randomNamespace := deploymentManager.DeployRandomNamespaceShared(DemoDaemonSet) 938 withdrawPolicy := applyL3Policy(kubectl, randomNamespace) 939 defer withdrawPolicy() 940 deploymentManager.WaitUntilReady() 941 942 By("Checking pod connectivity between nodes") 943 srcPod, srcPodJSON := fetchPodsWithOffset(kubectl, randomNamespace, "client", "zgroup=testDSClient", "", requireMultiNode, callOffset) 944 srcHost, err := srcPodJSON.Filter("{.status.hostIP}") 945 ExpectWithOffset(callOffset, err).Should(BeNil(), "Failure to retrieve host of pod %s", srcPod) 946 947 dstPod, dstPodJSON := fetchPodsWithOffset(kubectl, randomNamespace, "server", "zgroup=testDS", srcHost.String(), requireMultiNode, callOffset) 948 podIP, err := dstPodJSON.Filter("{.status.podIP}") 949 ExpectWithOffset(callOffset, err).Should(BeNil(), "Failure to retrieve IP of pod %s", dstPod) 950 targetIP := podIP.String() 951 952 // ICMP connectivity test 953 res := kubectl.ExecPodCmd(randomNamespace, srcPod, helpers.Ping(targetIP)) 954 if !res.WasSuccessful() { 955 return false, targetIP 956 } 957 958 // HTTP connectivity test 959 res = kubectl.ExecPodCmd(randomNamespace, srcPod, 960 helpers.CurlFail("http://%s:80/", targetIP)) 961 return res.WasSuccessful(), targetIP 962 } 963 964 func testPodHTTPToOutside(kubectl *helpers.Kubectl, outsideURL string, expectNodeIP, expectPodIP, ipv6 bool) bool { 965 var hostIPs map[string]string 966 var podIPs map[string]string 967 968 // IPv6 is not supported when the source IP should be checked. It could be 969 // supported with more work, but it doesn't make sense as in those cases, 970 // we can simply pass the IPv6 target address as outsideURL. 971 if ipv6 && expectPodIP { 972 panic("IPv6 not supported with source IP checking.") 973 } 974 975 namespace := deploymentManager.DeployRandomNamespaceShared(DemoDaemonSet) 976 withdrawPolicy := applyL3Policy(kubectl, namespace) 977 defer withdrawPolicy() 978 deploymentManager.WaitUntilReady() 979 980 label := "zgroup=testDSClient" 981 pods, err := kubectl.GetPodNames(namespace, label) 982 ExpectWithOffset(1, err).Should(BeNil(), "Cannot retrieve pod names by label %s", label) 983 984 cmd := outsideURL 985 if ipv6 { 986 cmd = fmt.Sprintf("-6 %s", cmd) 987 } 988 cmd = helpers.CurlWithRetries(cmd, 10, true) 989 990 if expectNodeIP || expectPodIP { 991 cmd += " | grep client_address=" 992 hostIPs, err = kubectl.GetPodsHostIPs(namespace, label) 993 ExpectWithOffset(1, err).Should(BeNil(), "Cannot retrieve pod host IPs") 994 if expectPodIP { 995 podIPs, err = kubectl.GetPodsIPs(namespace, label) 996 ExpectWithOffset(1, err).Should(BeNil(), "Cannot retrieve pod IPs") 997 } 998 } 999 1000 for _, pod := range pods { 1001 By("Making ten curl requests from %q to %q", pod, outsideURL) 1002 1003 hostIP := hostIPs[pod] 1004 podIP := podIPs[pod] 1005 1006 if expectPodIP { 1007 // Make pods reachable from the host which doesn't run Cilium 1008 kubectl.AddIPRoute(helpers.GetFirstNodeWithoutCilium(), podIP, hostIP, false). 1009 ExpectSuccess("Failed to add ip route") 1010 defer func() { 1011 kubectl.DelIPRoute(helpers.GetFirstNodeWithoutCilium(), podIP, hostIP). 1012 ExpectSuccess("Failed to del ip route") 1013 }() 1014 } 1015 1016 for i := 1; i <= 10; i++ { 1017 res := kubectl.ExecPodCmd(namespace, pod, cmd) 1018 ExpectWithOffset(1, res).Should(helpers.CMDSuccess(), 1019 "Pod %q can not connect to %q", pod, outsideURL) 1020 1021 if expectNodeIP || expectPodIP { 1022 // Parse the IPs to avoid issues with 4-in-6 formats 1023 sourceIP := net.ParseIP(strings.TrimSpace( 1024 strings.Split(res.Stdout(), "=")[1])).String() 1025 if expectNodeIP { 1026 Expect(sourceIP).To(Equal(hostIP), "Expected node IP") 1027 } 1028 if expectPodIP { 1029 Expect(sourceIP).To(Equal(podIP), "Expected pod IP") 1030 } 1031 } 1032 } 1033 } 1034 1035 return true 1036 } 1037 1038 func monitorConnectivityAcrossNodes(kubectl *helpers.Kubectl) (monitorRes *helpers.CmdRes, monitorCancel func(), targetIP string) { 1039 requireMultinode := config.CiliumTestConfig.Multinode 1040 if !config.CiliumTestConfig.Multinode { 1041 By("Performing multinode connectivity check within a single node") 1042 } 1043 1044 ciliumPodK8s1, err := kubectl.GetCiliumPodOnNode(helpers.K8s1) 1045 ExpectWithOffset(1, err).Should(BeNil(), "Cannot get cilium pod on k8s1") 1046 1047 By(fmt.Sprintf("Launching cilium-dbg monitor on %q", ciliumPodK8s1)) 1048 monitorRes, monitorCancel = kubectl.MonitorStart(ciliumPodK8s1) 1049 result, targetIP := testPodConnectivityAndReturnIP(kubectl, requireMultinode, 2) 1050 ExpectWithOffset(1, result).Should(BeTrue(), "Connectivity test between nodes failed") 1051 1052 return monitorRes, monitorCancel, targetIP 1053 } 1054 1055 func checkMonitorOutput(monitorOutput []byte, egressPktCount, ingressPktCount int) error { 1056 // Multiple connection attempts may be made, we need to 1057 // narrow down to the last connection close, then match 1058 // the ephemeral port + flags to ensure that the 1059 // notifications match the table above. 1060 egressTCPExpr := `TCP.*DstPort=80.*FIN=true` 1061 egressTCPRegex := regexp.MustCompile(egressTCPExpr) 1062 egressTCPMatches := egressTCPRegex.FindAll(monitorOutput, -1) 1063 if len(egressTCPMatches) == 0 { 1064 return fmt.Errorf("could not locate TCP FIN in monitor log") 1065 } 1066 finalMatch := egressTCPMatches[len(egressTCPMatches)-1] 1067 portRegex := regexp.MustCompile(`SrcPort=([0-9]*)`) 1068 // FindSubmatch should return ["SrcPort=12345" "12345"] 1069 portBytes := portRegex.FindSubmatch(finalMatch)[1] 1070 1071 By("Looking for TCP notifications using the ephemeral port %q", portBytes) 1072 port, err := strconv.Atoi(string(portBytes)) 1073 if err != nil { 1074 return fmt.Errorf("ephemeral port %q could not be converted to integer: %w", 1075 string(portBytes), err) 1076 } 1077 1078 expEgress := fmt.Sprintf("SrcPort=%d", port) 1079 expEgressRegex := regexp.MustCompile(expEgress) 1080 egressMatches := expEgressRegex.FindAllIndex(monitorOutput, -1) 1081 if len(egressMatches) != egressPktCount { 1082 return fmt.Errorf("monitor logs contained unexpected number (%d) of egress notifications matching %q", 1083 len(egressMatches), expEgress) 1084 } 1085 1086 expIngress := fmt.Sprintf("DstPort=%d", port) 1087 expIngressRegex := regexp.MustCompile(expIngress) 1088 ingressMatches := expIngressRegex.FindAllIndex(monitorOutput, -1) 1089 if len(ingressMatches) != ingressPktCount { 1090 return fmt.Errorf("monitor log contained unexpected number (%d) of ingress notifications matching %q", 1091 len(ingressMatches), expIngress) 1092 } 1093 1094 return nil 1095 }