github.com/cilium/cilium@v1.16.2/test/k8s/datapath_configuration.go (about)

     1  // SPDX-License-Identifier: Apache-2.0
     2  // Copyright Authors of Cilium
     3  
     4  package k8sTest
     5  
     6  import (
     7  	"context"
     8  	"fmt"
     9  	"net"
    10  	"regexp"
    11  	"strconv"
    12  	"strings"
    13  	"sync"
    14  	"time"
    15  
    16  	. "github.com/onsi/gomega"
    17  
    18  	"github.com/cilium/cilium/pkg/defaults"
    19  	"github.com/cilium/cilium/test/config"
    20  	. "github.com/cilium/cilium/test/ginkgo-ext"
    21  	"github.com/cilium/cilium/test/helpers"
    22  )
    23  
    24  var _ = Describe("K8sDatapathConfig", func() {
    25  	const (
    26  		bpffsDir string = defaults.BPFFSRoot + "/" + defaults.TCGlobalsPath + "/"
    27  	)
    28  
    29  	var (
    30  		kubectl    *helpers.Kubectl
    31  		monitorLog = "monitor-aggregation.log"
    32  	)
    33  
    34  	BeforeAll(func() {
    35  		kubectl = helpers.CreateKubectl(helpers.K8s1VMName(), logger)
    36  		deploymentManager.SetKubectl(kubectl)
    37  	})
    38  
    39  	AfterEach(func() {
    40  		deploymentManager.DeleteAll()
    41  		ExpectAllPodsTerminated(kubectl)
    42  	})
    43  
    44  	AfterFailed(func() {
    45  		kubectl.CiliumReport("cilium-dbg status", "cilium-dbg endpoint list")
    46  	})
    47  
    48  	AfterAll(func() {
    49  		kubectl.ScaleDownDNS()
    50  		ExpectAllPodsTerminated(kubectl)
    51  		deploymentManager.DeleteCilium()
    52  		kubectl.ScaleUpDNS()
    53  		kubectl.CloseSSHClient()
    54  	})
    55  
    56  	JustAfterEach(func() {
    57  		kubectl.ValidateNoErrorsInLogs(CurrentGinkgoTestDescription().Duration)
    58  	})
    59  
    60  	Context("MonitorAggregation", func() {
    61  		It("Checks that monitor aggregation restricts notifications", func() {
    62  			deploymentManager.DeployCilium(map[string]string{
    63  				"bpf.monitorAggregation": "medium",
    64  				"bpf.monitorInterval":    "60s",
    65  				"bpf.monitorFlags":       "syn",
    66  			}, DeployCiliumOptionsAndDNS)
    67  
    68  			monitorRes, monitorCancel, targetIP := monitorConnectivityAcrossNodes(kubectl)
    69  			defer monitorCancel()
    70  
    71  			var monitorOutput []byte
    72  			searchMonitorLog := func(expr *regexp.Regexp) bool {
    73  				monitorOutput = monitorRes.CombineOutput().Bytes()
    74  				egressMatches := expr.FindAllIndex(monitorOutput, -1)
    75  				return len(egressMatches) > 0
    76  			}
    77  
    78  			By("Checking that ICMP notifications in egress direction were observed")
    79  			expEgress := fmt.Sprintf("ICMPv4.*DstIP=%s", targetIP)
    80  			expEgressRegex := regexp.MustCompile(expEgress)
    81  			Eventually(func() bool {
    82  				return searchMonitorLog(expEgressRegex)
    83  			}, helpers.HelperTimeout, time.Second).Should(BeTrue(), "Egress ICMPv4 flow (%q) not found in monitor log\n%s", expEgress, monitorOutput)
    84  
    85  			By("Checking that ICMP notifications in ingress direction were observed")
    86  			expIngress := fmt.Sprintf("ICMPv4.*SrcIP=%s", targetIP)
    87  			expIngressRegex := regexp.MustCompile(expIngress)
    88  			Eventually(func() bool {
    89  				return searchMonitorLog(expIngressRegex)
    90  			}, helpers.HelperTimeout, time.Second).Should(BeTrue(), "Ingress ICMPv4 flow (%q) not found in monitor log\n%s", expIngress, monitorOutput)
    91  
    92  			By("Checking the set of TCP notifications received matches expectations")
    93  			// | TCP Flags | Direction | Report? | Why?
    94  			// +===========+===========+=========+=====
    95  			// | SYN       |    ->     |    Y    | monitorFlags=SYN
    96  			// | SYN / ACK |    <-     |    Y    | monitorFlags=SYN
    97  			// | ACK       |    ->     |    N    | monitorFlags=(!ACK)
    98  			// | ACK       |    ...    |    N    | monitorFlags=(!ACK)
    99  			// | ACK       |    <-     |    N    | monitorFlags=(!ACK)
   100  			// | FIN       |    ->     |    Y    | monitorAggregation=medium
   101  			// | FIN / ACK |    <-     |    Y    | monitorAggregation=medium
   102  			// | ACK       |    ->     |    Y    | monitorAggregation=medium
   103  			egressPktCount := 3
   104  			ingressPktCount := 2
   105  			Eventually(func() error {
   106  				monitorOutput = monitorRes.CombineOutput().Bytes()
   107  				return checkMonitorOutput(monitorOutput, egressPktCount, ingressPktCount)
   108  			}, helpers.HelperTimeout, time.Second).Should(BeNil(), "Monitor log did not contain %d ingress and %d egress TCP notifications\n%s",
   109  				ingressPktCount, egressPktCount, monitorOutput)
   110  
   111  			helpers.WriteToReportFile(monitorOutput, monitorLog)
   112  		})
   113  
   114  		It("Checks that monitor aggregation flags send notifications", func() {
   115  			deploymentManager.DeployCilium(map[string]string{
   116  				"bpf.monitorAggregation": "medium",
   117  				"bpf.monitorInterval":    "60s",
   118  				"bpf.monitorFlags":       "psh",
   119  			}, DeployCiliumOptionsAndDNS)
   120  			monitorRes, monitorCancel, _ := monitorConnectivityAcrossNodes(kubectl)
   121  			defer monitorCancel()
   122  
   123  			var monitorOutput []byte
   124  			By("Checking the set of TCP notifications received matches expectations")
   125  			// | TCP Flags | Direction | Report? | Why?
   126  			// +===========+===========+=========+=====
   127  			// | SYN       |    ->     |    Y    | monitorAggregation=medium
   128  			// | SYN / ACK |    <-     |    Y    | monitorAggregation=medium
   129  			// | ACK       |    ->     |    N    | monitorFlags=(!ACK)
   130  			// | ACK       |    ...    |    N    | monitorFlags=(!ACK)
   131  			// | PSH       |    ->     |    Y    | monitorFlags=(PSH)
   132  			// | PSH       |    <-     |    Y    | monitorFlags=(PSH)
   133  			// | FIN       |    ->     |    Y    | monitorAggregation=medium
   134  			// | FIN / ACK |    <-     |    Y    | monitorAggregation=medium
   135  			// | ACK       |    ->     |    Y    | monitorAggregation=medium
   136  			egressPktCount := 4
   137  			ingressPktCount := 3
   138  			Eventually(func() error {
   139  				monitorOutput = monitorRes.CombineOutput().Bytes()
   140  				return checkMonitorOutput(monitorOutput, egressPktCount, ingressPktCount)
   141  			}, helpers.HelperTimeout, time.Second).Should(BeNil(), "monitor aggregation did not result in correct number of TCP notifications\n%s", monitorOutput)
   142  			helpers.WriteToReportFile(monitorOutput, monitorLog)
   143  		})
   144  	})
   145  
   146  	Context("Encapsulation", func() {
   147  		var (
   148  			tmpEchoPodPath    string
   149  			outside, outside6 string
   150  		)
   151  
   152  		BeforeAll(func() {
   153  			if helpers.ExistNodeWithoutCilium() && !helpers.SupportIPv6ToOutside() {
   154  				// Deploy echoserver on the node which does not run Cilium to test
   155  				// IPv6 connectivity to the outside world. The pod will run in
   156  				// the host netns, so no CNI is required for the pod on that host.
   157  				echoPodPath := helpers.ManifestGet(kubectl.BasePath(), "echoserver-hostnetns.yaml")
   158  				res := kubectl.ExecMiddle("mktemp")
   159  				res.ExpectSuccess()
   160  				tmpEchoPodPath = strings.Trim(res.Stdout(), "\n")
   161  				kubectl.ExecMiddle(fmt.Sprintf("sed 's/NODE_WITHOUT_CILIUM/%s/' %s > %s",
   162  					helpers.GetFirstNodeWithoutCilium(), echoPodPath, tmpEchoPodPath)).ExpectSuccess()
   163  				kubectl.ApplyDefault(tmpEchoPodPath).ExpectSuccess("Cannot install echoserver application")
   164  				Expect(kubectl.WaitforPods(helpers.DefaultNamespace, "-l name=echoserver-hostnetns",
   165  					helpers.HelperTimeout)).Should(BeNil())
   166  				var err error
   167  				outside, err = kubectl.GetNodeIPByLabel(kubectl.GetFirstNodeWithoutCiliumLabel(), false)
   168  				Expect(err).Should(BeNil())
   169  				outside6, err = kubectl.GetNodeIPv6ByLabel(kubectl.GetFirstNodeWithoutCiliumLabel(), false)
   170  				Expect(err).Should(BeNil())
   171  				outside6 = net.JoinHostPort(outside6, "80")
   172  			} else {
   173  				outside = "http://google.com"
   174  				outside6 = "http://google.com"
   175  			}
   176  		})
   177  
   178  		AfterAll(func() {
   179  			if tmpEchoPodPath != "" {
   180  				kubectl.Delete(tmpEchoPodPath)
   181  			}
   182  		})
   183  
   184  		enableVXLANTunneling := func(options map[string]string) {
   185  			options["tunnelProtocol"] = "vxlan"
   186  			if helpers.RunsOnGKE() {
   187  				// We need to disable gke.enabled as it disables tunneling.
   188  				options["gke.enabled"] = "false"
   189  				options["endpointRoutes.enabled"] = "true"
   190  			}
   191  		}
   192  
   193  		It("Check iptables masquerading with random-fully", func() {
   194  			options := map[string]string{
   195  				"bpf.masquerade":       "false",
   196  				"enableIPv6Masquerade": "true",
   197  				"iptablesRandomFully":  "true",
   198  			}
   199  			enableVXLANTunneling(options)
   200  			deploymentManager.DeployCilium(options, DeployCiliumOptionsAndDNS)
   201  			Expect(testPodConnectivityAcrossNodes(kubectl)).Should(BeTrue(), "Connectivity test between nodes failed")
   202  
   203  			By("Test iptables masquerading")
   204  			Expect(testPodHTTPToOutside(kubectl, outside, false, false, false)).
   205  				Should(BeTrue(), "IPv4 connectivity test to %s", outside)
   206  			if helpers.ExistNodeWithoutCilium() || helpers.SupportIPv6ToOutside() {
   207  				Expect(testPodHTTPToOutside(kubectl, outside6, false, false, true)).
   208  					Should(BeTrue(), "IPv6 connectivity test to %s failed", outside6)
   209  			}
   210  		})
   211  	})
   212  
   213  	SkipContextIf(func() bool {
   214  		return helpers.RunsWithKubeProxyReplacement() || helpers.GetCurrentIntegration() != ""
   215  	}, "IPv6 masquerading", func() {
   216  		var (
   217  			k8s1EndpointIPs map[string]string
   218  			testDSK8s1IPv6  string = "fd03::310"
   219  
   220  			demoYAML string
   221  		)
   222  
   223  		BeforeAll(func() {
   224  			deploymentManager.DeployCilium(map[string]string{
   225  				"routingMode":           "native",
   226  				"autoDirectNodeRoutes":  "true",
   227  				"ipv6NativeRoutingCIDR": helpers.IPv6NativeRoutingCIDR,
   228  			}, DeployCiliumOptionsAndDNS)
   229  
   230  			demoYAML = helpers.ManifestGet(kubectl.BasePath(), "demo_ds.yaml")
   231  			res := kubectl.ApplyDefault(demoYAML)
   232  			Expect(res).Should(helpers.CMDSuccess(), "Unable to apply %s", demoYAML)
   233  			waitPodsDs(kubectl, []string{testDS, testDSClient, testDSK8s2})
   234  
   235  			pod, err := kubectl.GetCiliumPodOnNode(helpers.K8s1)
   236  			Expect(err).Should(BeNil(), "Cannot get cilium pod on node %s", helpers.K8s1)
   237  			k8s1EndpointIPs = kubectl.CiliumEndpointIPv6(pod, "-l k8s:zgroup=testDS,k8s:io.kubernetes.pod.namespace=default")
   238  
   239  			k8s1Backends := []string{}
   240  			for _, epIP := range k8s1EndpointIPs {
   241  				k8s1Backends = append(k8s1Backends, net.JoinHostPort(epIP, "80"))
   242  			}
   243  
   244  			ciliumAddService(kubectl, 31080, net.JoinHostPort(testDSK8s1IPv6, "80"), k8s1Backends, "ClusterIP", "Cluster")
   245  		})
   246  
   247  		It("across K8s nodes, skipped due to native routing CIDR", func() {
   248  			// Because a native routing CIDR is set, the
   249  			// IPv6 for packets routed to another node are
   250  			// _not_ masqueraded. Retrieve the address for
   251  			// the client, and make sure the echo server
   252  			// receives it unchanged.
   253  			pod, err := kubectl.GetCiliumPodOnNode(helpers.K8s2)
   254  			Expect(err).Should(BeNil(), "Cannot get cilium pod on node %s", helpers.K8s2)
   255  			k8s2EndpointIPs := kubectl.CiliumEndpointIPv6(pod, fmt.Sprintf("-l k8s:%s,k8s:io.kubernetes.pod.namespace=default", testDSK8s2))
   256  			k8s2ClientIPv6 := ""
   257  			for _, epIP := range k8s2EndpointIPs {
   258  				k8s2ClientIPv6 = epIP
   259  				break
   260  			}
   261  			Expect(k8s2ClientIPv6).ShouldNot(BeEmpty(), "Cannot get client IPv6")
   262  
   263  			url := fmt.Sprintf(`"http://[%s]:80/"`, testDSK8s1IPv6)
   264  			testCurlFromPodWithSourceIPCheck(kubectl, testDSK8s2, url, 5, k8s2ClientIPv6)
   265  
   266  			for _, epIP := range k8s1EndpointIPs {
   267  				url = fmt.Sprintf(`"http://[%s]:80/"`, epIP)
   268  				testCurlFromPodWithSourceIPCheck(kubectl, testDSK8s2, url, 5, k8s2ClientIPv6)
   269  			}
   270  		})
   271  
   272  		// Note: At the time we add the test below, it does not
   273  		// run on the CI because the only job which is running
   274  		// with a 3rd, non-K8s node (net-next) also has KPR,
   275  		// and skips the context we're in. Run locally.
   276  		// TODO: uncomment once the above has changed
   277  		//SkipItIf(helpers.DoesNotExistNodeWithoutCilium, "for external traffic", func() {
   278  		//	// A native routing CIDR is set, but it does
   279  		//	// not prevent masquerading for packets going
   280  		//	// outside of the K8s cluster. Check that the
   281  		//	// echo server sees the IPv6 address of the
   282  		//	// client's node.
   283  		//	url := fmt.Sprintf(`"http://[%s]:80/"`, ni.outsideIPv6)
   284  		//	testCurlFromPodWithSourceIPCheck(kubectl, testDSK8s2, url, 5, ni.primaryK8s2IPv6)
   285  
   286  		//	for _, epIP := range k8s1EndpointIPs {
   287  		//		url = fmt.Sprintf(`"http://[%s]:80/"`, epIP)
   288  		//		testCurlFromPodWithSourceIPCheck(kubectl, testDSK8s2, url, 5, ni.primaryK8s2IPv6)
   289  		//	}
   290  		//})
   291  
   292  		AfterAll(func() {
   293  			ciliumDelService(kubectl, 31080)
   294  			_ = kubectl.Delete(demoYAML)
   295  		})
   296  	})
   297  
   298  	SkipContextIf(func() bool {
   299  		return helpers.DoesNotExistNodeWithoutCilium()
   300  	}, "Check BPF masquerading with ip-masq-agent", func() {
   301  		var (
   302  			tmpEchoPodPath string
   303  			nodeIP         string
   304  		)
   305  
   306  		BeforeAll(func() {
   307  			// Deploy echoserver on the node which does not run Cilium to test
   308  			// BPF masquerading. The pod will run in the host netns, so no CNI
   309  			// is required for the pod on that host.
   310  			echoPodPath := helpers.ManifestGet(kubectl.BasePath(), "echoserver-hostnetns.yaml")
   311  			res := kubectl.ExecMiddle("mktemp")
   312  			res.ExpectSuccess()
   313  			tmpEchoPodPath = strings.Trim(res.Stdout(), "\n")
   314  			kubectl.ExecMiddle(fmt.Sprintf("sed 's/NODE_WITHOUT_CILIUM/%s/' %s > %s",
   315  				helpers.GetFirstNodeWithoutCilium(), echoPodPath, tmpEchoPodPath)).ExpectSuccess()
   316  			kubectl.ApplyDefault(tmpEchoPodPath).ExpectSuccess("Cannot install echoserver application")
   317  			Expect(kubectl.WaitforPods(helpers.DefaultNamespace, "-l name=echoserver-hostnetns",
   318  				helpers.HelperTimeout)).Should(BeNil())
   319  			var err error
   320  			nodeIP, err = kubectl.GetNodeIPByLabel(kubectl.GetFirstNodeWithoutCiliumLabel(), false)
   321  			Expect(err).Should(BeNil())
   322  		})
   323  
   324  		AfterAll(func() {
   325  			if tmpEchoPodPath != "" {
   326  				kubectl.Delete(tmpEchoPodPath)
   327  			}
   328  		})
   329  
   330  		testIPMasqAgent := func() {
   331  			Expect(testPodHTTPToOutside(kubectl,
   332  				fmt.Sprintf("http://%s:80", nodeIP), false, true, false)).Should(BeTrue(),
   333  				"IPv4 Connectivity test to http://%s failed", nodeIP)
   334  
   335  			// remove nonMasqueradeCIDRs from the ConfigMap
   336  			kubectl.Patch(helpers.CiliumNamespace, "configMap", "ip-masq-agent", `{"data":{"config":"{\"nonMasqueradeCIDRs\":[]}"}}`)
   337  			// Wait until the ip-masq-agent config update is handled by the agent
   338  			time.Sleep(90 * time.Second)
   339  
   340  			// Check that requests to the echoserver from client pods are masqueraded.
   341  			Expect(testPodHTTPToOutside(kubectl,
   342  				fmt.Sprintf("http://%s:80", nodeIP), true, false, false)).Should(BeTrue(),
   343  				"IPv4 Connectivity test to http://%s failed", nodeIP)
   344  
   345  			Expect(testPodHTTPToOutside(kubectl,
   346  				fmt.Sprintf("http://%s:80", nodeIP), true, false, true)).Should(BeTrue(),
   347  				"IPv6 Connectivity test to http://%s failed", nodeIP)
   348  		}
   349  
   350  		It("DirectRouting", func() {
   351  			deploymentManager.DeployCilium(map[string]string{
   352  				"ipMasqAgent.enabled":                   "true",
   353  				"routingMode":                           "native",
   354  				"autoDirectNodeRoutes":                  "true",
   355  				"ipMasqAgent.config.nonMasqueradeCIDRs": fmt.Sprintf("{%s/32}", nodeIP),
   356  			}, DeployCiliumOptionsAndDNS)
   357  
   358  			testIPMasqAgent()
   359  		})
   360  
   361  		It("DirectRouting, IPv4 only", func() {
   362  			deploymentManager.DeployCilium(map[string]string{
   363  				"ipMasqAgent.enabled":                   "true",
   364  				"routingMode":                           "native",
   365  				"autoDirectNodeRoutes":                  "true",
   366  				"ipMasqAgent.config.nonMasqueradeCIDRs": fmt.Sprintf("{%s/32}", nodeIP),
   367  				"ipv6.enabled":                          "false",
   368  			}, DeployCiliumOptionsAndDNS)
   369  
   370  			testIPMasqAgent()
   371  		})
   372  
   373  		It("VXLAN", func() {
   374  			deploymentManager.DeployCilium(map[string]string{
   375  				"ipMasqAgent.enabled":                   "true",
   376  				"tunnelProtocol":                        "vxlan",
   377  				"ipMasqAgent.config.nonMasqueradeCIDRs": fmt.Sprintf("{%s/32}", nodeIP),
   378  			}, DeployCiliumOptionsAndDNS)
   379  
   380  			testIPMasqAgent()
   381  		})
   382  	})
   383  
   384  	SkipContextIf(helpers.DoesNotRunOnNetNextKernel, "WireGuard encryption strict mode", func() {
   385  		BeforeAll(func() {
   386  			kubectl.DeleteResource("ciliumnode", "--all --wait")
   387  		})
   388  		AfterEach(func() {
   389  			kubectl.DeleteResource("ciliumnode", "--all --wait")
   390  		})
   391  
   392  		testStrictWireguard := func(interNodeDev string) {
   393  			// Disable the Cilium operator
   394  			// In combination with the used CES option, this will stop any
   395  			// endpoint propagation to other nodes.
   396  			kubectl.SetCiliumOperatorReplicas(0)
   397  			randomNamespace := deploymentManager.DeployRandomNamespaceShared(DemoDaemonSet)
   398  			deploymentManager.WaitUntilReady()
   399  
   400  			k8s1NodeName, k8s1IP := kubectl.GetNodeInfo(helpers.K8s1)
   401  			k8s2NodeName, k8s2IP := kubectl.GetNodeInfo(helpers.K8s2)
   402  
   403  			// Fetch srcPod (testDSClient@k8s1)
   404  			srcPod, srcPodJSON := fetchPodsWithOffset(kubectl, randomNamespace, "client", "zgroup=testDSClient", k8s2IP, true, 0)
   405  			srcPodIP, err := srcPodJSON.Filter("{.status.podIP}")
   406  			ExpectWithOffset(1, err).Should(BeNil(), "Failure to retrieve pod IP %s", srcPod)
   407  			srcHost, err := srcPodJSON.Filter("{.status.hostIP}")
   408  			ExpectWithOffset(1, err).Should(BeNil(), "Failure to retrieve host of pod %s", srcPod)
   409  			// Sanity check
   410  			ExpectWithOffset(1, srcHost.String()).Should(Equal(k8s1IP))
   411  			// Fetch srcPod IPv6
   412  			ciliumPodK8s1, err := kubectl.GetCiliumPodOnNode(helpers.K8s1)
   413  			ExpectWithOffset(1, err).Should(BeNil(), "Unable to fetch cilium pod on k8s1")
   414  			endpointIPs := kubectl.CiliumEndpointIPv6(ciliumPodK8s1, "-l k8s:zgroup=testDSClient")
   415  			// Sanity check
   416  			ExpectWithOffset(1, len(endpointIPs)).Should(Equal(1), "BUG: more than one DS client on %s", ciliumPodK8s1)
   417  
   418  			// Fetch dstPod (testDS@k8s2)
   419  			dstPod, dstPodJSON := fetchPodsWithOffset(kubectl, randomNamespace, "server", "zgroup=testDS", k8s1IP, true, 0)
   420  			dstPodIP, err := dstPodJSON.Filter("{.status.podIP}")
   421  			ExpectWithOffset(1, err).Should(BeNil(), "Failure to retrieve IP of pod %s", dstPod)
   422  			dstHost, err := dstPodJSON.Filter("{.status.hostIP}")
   423  			ExpectWithOffset(1, err).Should(BeNil(), "Failure to retrieve host of pod %s", dstPod)
   424  			// Sanity check
   425  			ExpectWithOffset(1, dstHost.String()).Should(Equal(k8s2IP))
   426  			// Fetch dstPod IPv6
   427  			ciliumPodK8s2, err := kubectl.GetCiliumPodOnNode(helpers.K8s2)
   428  			ExpectWithOffset(1, err).Should(BeNil(), "Unable to fetch cilium pod on k8s2")
   429  			endpointIPs = kubectl.CiliumEndpointIPv6(ciliumPodK8s2, "-l k8s:zgroup=testDS")
   430  			// Sanity check
   431  			ExpectWithOffset(1, len(endpointIPs)).Should(Equal(1), "BUG: more than one DS server on %s", ciliumPodK8s2)
   432  
   433  			// Fetch svc IP
   434  			dsSvcJSON := kubectl.Get(randomNamespace, "svc testds-service -o json")
   435  			dsSvcIP, err := dsSvcJSON.Filter("{.spec.clusterIP}")
   436  			ExpectWithOffset(1, err).Should(BeNil(), "Failure to retrieve IP of service testds-service")
   437  
   438  			checkNoLeakP2P := func(srcPod, srcIP, dstIP string, shouldBeSuccessful bool) {
   439  				cmd := fmt.Sprintf("tcpdump -vv -i %s --immediate-mode -n 'host %s and host %s' -c 1", interNodeDev, srcIP, dstIP)
   440  				res1, cancel1, err := kubectl.ExecInHostNetNSInBackground(context.TODO(), k8s1NodeName, cmd)
   441  				ExpectWithOffset(2, err).Should(BeNil(), "Cannot exec tcpdump in bg")
   442  				res2, cancel2, err := kubectl.ExecInHostNetNSInBackground(context.TODO(), k8s2NodeName, cmd)
   443  				ExpectWithOffset(2, err).Should(BeNil(), "Cannot exec tcpdump in bg")
   444  
   445  				// HTTP connectivity test (pod2pod)
   446  
   447  				cmdRes := kubectl.ExecPodCmd(randomNamespace, srcPod, helpers.CurlFail("http://%s/", net.JoinHostPort(dstIP, "80")))
   448  				if shouldBeSuccessful {
   449  					cmdRes.ExpectSuccess("Failed to curl dst pod")
   450  				} else {
   451  					cmdRes.ExpectFail("Should not be able to curl dst pod")
   452  				}
   453  
   454  				// Check that no unencrypted pod2pod traffic was captured on the direct routing device
   455  				cancel1()
   456  				cancel2()
   457  				ExpectWithOffset(2, res1.CombineOutput().String()).Should(Not(ContainSubstring("1 packet captured")))
   458  				ExpectWithOffset(2, res2.CombineOutput().String()).Should(Not(ContainSubstring("1 packet captured")))
   459  			}
   460  
   461  			checkNoLeakP2RemoteService := func(srcPod, dstPod, srcIP, dstService string, shouldBeSuccessful bool) {
   462  				cmd := fmt.Sprintf("tcpdump -vv -i %s --immediate-mode -n 'src %s or dst %s' -c 1", interNodeDev, srcIP, srcIP)
   463  				res1, cancel1, err := kubectl.ExecInHostNetNSInBackground(context.TODO(), k8s1NodeName, cmd)
   464  				ExpectWithOffset(2, err).Should(BeNil(), "Cannot exec tcpdump in bg")
   465  				res2, cancel2, err := kubectl.ExecInHostNetNSInBackground(context.TODO(), k8s2NodeName, cmd)
   466  				ExpectWithOffset(2, err).Should(BeNil(), "Cannot exec tcpdump in bg")
   467  
   468  				// HTTP connectivity test (pod2pod)
   469  
   470  				cmdRes, err := kubectl.ExecUntilMatch(randomNamespace, srcPod, helpers.CurlTimeout("http://%s/", time.Second, net.JoinHostPort(dstService, "80")), dstPod)
   471  				if shouldBeSuccessful {
   472  					cmdRes.ExpectSuccess("Failed to curl svc on dst pod")
   473  					ExpectWithOffset(2, err).Should(BeNil(), "Failed to curl svc on dst pod")
   474  				} else {
   475  					ExpectWithOffset(2, err).ShouldNot(BeNil(), "Should not be able to curl svc on dst pod")
   476  				}
   477  
   478  				// Check that no unencrypted pod2pod traffic was captured on the direct routing device
   479  				cancel1()
   480  				cancel2()
   481  				ExpectWithOffset(2, res1.CombineOutput().String()).Should(Not(ContainSubstring("1 packet captured")))
   482  				ExpectWithOffset(2, res2.CombineOutput().String()).Should(Not(ContainSubstring("1 packet captured")))
   483  			}
   484  
   485  			checkNoLeakP2P(srcPod, srcPodIP.String(), dstPodIP.String(), false)
   486  			checkNoLeakP2RemoteService(srcPod, dstPod, srcPodIP.String(), dsSvcIP.String(), false)
   487  
   488  			// Check that the src pod can reach the remote host
   489  			kubectl.ExecPodCmd(randomNamespace, srcPod, helpers.Ping(k8s2IP)).
   490  				ExpectSuccess("Failed to ping k8s2 host from src pod")
   491  
   492  			// Check that the remote host can reach the dst pod
   493  			kubectl.ExecInHostNetNS(context.TODO(), k8s1NodeName,
   494  				helpers.CurlFail("http://%s:80/", dstPodIP)).ExpectSuccess("Failed to curl dst pod from k8s1")
   495  
   496  			// Re-enable operator so that all endpoints are eventually updated
   497  			kubectl.SetCiliumOperatorReplicas(2)
   498  
   499  			// Due to IPCache update delays, it can take up to a few seconds
   500  			// before both nodes have added the new pod IPs to their allowedIPs
   501  			// list, which can cause flakes in CI. Therefore wait for the
   502  			// IPs to be present on both nodes before performing the test
   503  			waitForAllowedIP := func(ciliumPod, ip string) {
   504  				jsonpath := fmt.Sprintf(`{.encryption.wireguard.interfaces[*].peers[*].allowed-ips[?(@=='%s')]}`, ip)
   505  				ciliumCmd := fmt.Sprintf(`cilium-dbg debuginfo --output jsonpath="%s"`, jsonpath)
   506  				expected := fmt.Sprintf("jsonpath=%s", ip)
   507  				err := kubectl.CiliumExecUntilMatch(ciliumPod, ciliumCmd, expected)
   508  				Expect(err).To(BeNil(), "ip %q not in allowedIPs of pod %q", ip, ciliumPod)
   509  			}
   510  
   511  			waitForAllowedIP(ciliumPodK8s1, fmt.Sprintf("%s/32", dstPodIP))
   512  			waitForAllowedIP(ciliumPodK8s2, fmt.Sprintf("%s/32", srcPodIP))
   513  
   514  			checkNoLeakP2P(srcPod, srcPodIP.String(), dstPodIP.String(), true)
   515  
   516  			checkNoLeakP2RemoteService(srcPod, dstPod, srcPodIP.String(), dsSvcIP.String(), true)
   517  		}
   518  
   519  		It("Pod-to-pod traffic is encrypted in native routing mode with per-endpoint routes", func() {
   520  			deploymentManager.DeployCilium(map[string]string{
   521  				"routingMode":                              "native",
   522  				"autoDirectNodeRoutes":                     "true",
   523  				"ipv4NativeRoutingCIDR":                    "10.244.0.0/16",
   524  				"ipv6.enabled":                             "false",
   525  				"endpointRoutes.enabled":                   "true",
   526  				"encryption.enabled":                       "true",
   527  				"encryption.type":                          "wireguard",
   528  				"l7Proxy":                                  "false",
   529  				"ipam.operator.clusterPoolIPv4PodCIDRList": "10.244.0.0/16",
   530  				"encryption.strictMode.enabled":            "true",
   531  				"encryption.strictMode.cidr":               "10.244.0.0/16",
   532  			}, DeployCiliumOptionsAndDNS)
   533  
   534  			privateIface, err := kubectl.GetPrivateIface(helpers.K8s1)
   535  			Expect(err).Should(BeNil(), "Cannot determine private iface")
   536  			testStrictWireguard(privateIface)
   537  		})
   538  		It("Pod-to-pod traffic is encrypted in native routing mode with per-endpoint routes and overlapping node and pod CIDRs", func() {
   539  			deploymentManager.DeployCilium(map[string]string{
   540  				"routingMode":                                     "native",
   541  				"autoDirectNodeRoutes":                            "true",
   542  				"ipv4NativeRoutingCIDR":                           "192.168.56.0/24",
   543  				"ipv6.enabled":                                    "false",
   544  				"endpointRoutes.enabled":                          "true",
   545  				"encryption.enabled":                              "true",
   546  				"encryption.type":                                 "wireguard",
   547  				"l7Proxy":                                         "false",
   548  				"ipam.operator.clusterPoolIPv4PodCIDRList":        "192.168.56.128/25",
   549  				"ipam.operator.clusterPoolIPv4MaskSize":           "26",
   550  				"encryption.strictMode.enabled":                   "true",
   551  				"encryption.strictMode.cidr":                      "192.168.56.0/24",
   552  				"encryption.strictMode.allowRemoteNodeIdentities": "true",
   553  			}, DeployCiliumOptionsAndDNS)
   554  
   555  			privateIface, err := kubectl.GetPrivateIface(helpers.K8s1)
   556  			Expect(err).Should(BeNil(), "Cannot determine private iface")
   557  			testStrictWireguard(privateIface)
   558  		})
   559  	})
   560  
   561  	Context("IPv4Only", func() {
   562  		It("Check connectivity with IPv6 disabled", func() {
   563  			deploymentManager.DeployCilium(map[string]string{
   564  				"ipv4.enabled": "true",
   565  				"ipv6.enabled": "false",
   566  			}, DeployCiliumOptionsAndDNS)
   567  			Expect(testPodConnectivityAcrossNodes(kubectl)).Should(BeTrue(), "Connectivity test between nodes failed")
   568  		})
   569  	})
   570  
   571  	Context("Host firewall", func() {
   572  		BeforeAll(func() {
   573  			kubectl.Exec("kubectl label nodes --all status=lockdown")
   574  
   575  			// Need to install Cilium w/ host fw prior to the host policy preparation
   576  			// step.
   577  			deploymentManager.DeployCilium(map[string]string{
   578  				"hostFirewall.enabled": "true",
   579  			}, DeployCiliumOptionsAndDNS)
   580  
   581  			hostPolicy := helpers.ManifestGet(kubectl.BasePath(), "host-policies.yaml")
   582  			prepareHostPolicyEnforcement(kubectl, hostPolicy)
   583  		})
   584  
   585  		AfterAll(func() {
   586  			kubectl.Exec("kubectl label nodes --all status-")
   587  		})
   588  
   589  		SkipItIf(func() bool {
   590  			return !helpers.IsIntegration(helpers.CIIntegrationGKE)
   591  		}, "Check connectivity with IPv6 disabled", func() {
   592  			deploymentManager.DeployCilium(map[string]string{
   593  				"ipv4.enabled":         "true",
   594  				"ipv6.enabled":         "false",
   595  				"hostFirewall.enabled": "true",
   596  			}, DeployCiliumOptionsAndDNS)
   597  			Expect(testPodConnectivityAcrossNodes(kubectl)).Should(BeTrue(), "Connectivity test between nodes failed")
   598  		})
   599  
   600  		SkipItIf(helpers.RunsOnAKS, "With VXLAN", func() {
   601  			options := map[string]string{
   602  				"hostFirewall.enabled": "true",
   603  			}
   604  			if helpers.RunsOnGKE() {
   605  				options["gke.enabled"] = "false"
   606  				options["tunnelProtocol"] = "vxlan"
   607  			}
   608  			deploymentManager.DeployCilium(options, DeployCiliumOptionsAndDNS)
   609  			testHostFirewall(kubectl)
   610  		})
   611  
   612  		SkipItIf(func() bool {
   613  			return helpers.RunsOnAKS()
   614  		}, "With VXLAN and endpoint routes", func() {
   615  			options := map[string]string{
   616  				"hostFirewall.enabled":   "true",
   617  				"endpointRoutes.enabled": "true",
   618  			}
   619  			if helpers.RunsOnGKE() {
   620  				options["gke.enabled"] = "false"
   621  				options["tunnelProtocol"] = "vxlan"
   622  			}
   623  			deploymentManager.DeployCilium(options, DeployCiliumOptionsAndDNS)
   624  			testHostFirewall(kubectl)
   625  		})
   626  
   627  		It("With native routing", func() {
   628  			options := map[string]string{
   629  				"hostFirewall.enabled": "true",
   630  				"routingMode":          "native",
   631  			}
   632  			// We don't want to run with per-endpoint routes (enabled by
   633  			// gke.enabled) for this test.
   634  			if helpers.RunsOnGKE() {
   635  				options["gke.enabled"] = "false"
   636  			} else {
   637  				options["autoDirectNodeRoutes"] = "true"
   638  			}
   639  			deploymentManager.DeployCilium(options, DeployCiliumOptionsAndDNS)
   640  			testHostFirewall(kubectl)
   641  		})
   642  
   643  		It("With native routing and endpoint routes", func() {
   644  			options := map[string]string{
   645  				"hostFirewall.enabled":   "true",
   646  				"routingMode":            "native",
   647  				"endpointRoutes.enabled": "true",
   648  			}
   649  			if !helpers.RunsOnGKE() {
   650  				options["autoDirectNodeRoutes"] = "true"
   651  			}
   652  			deploymentManager.DeployCilium(options, DeployCiliumOptionsAndDNS)
   653  			testHostFirewall(kubectl)
   654  		})
   655  	})
   656  
   657  	SkipContextIf(helpers.DoesNotRunOnNetNextKernel, "High-scale IPcache", func() {
   658  		const hsIPcacheFile = "high-scale-ipcache.yaml"
   659  
   660  		AfterEach(func() {
   661  			hsIPcacheYAML := helpers.ManifestGet(kubectl.BasePath(), hsIPcacheFile)
   662  			_ = kubectl.Delete(hsIPcacheYAML)
   663  		})
   664  
   665  		testHighScaleIPcache := func(tunnelProto string, epRoutesConfig string) {
   666  			options := map[string]string{
   667  				"highScaleIPcache.enabled":    "true",
   668  				"routingMode":                 "native",
   669  				"bpf.monitorAggregation":      "none",
   670  				"ipv6.enabled":                "false",
   671  				"wellKnownIdentities.enabled": "true",
   672  				"tunnelProtocol":              tunnelProto,
   673  				"endpointRoutes.enabled":      epRoutesConfig,
   674  			}
   675  			if !helpers.RunsOnGKE() {
   676  				options["autoDirectNodeRoutes"] = "true"
   677  			}
   678  			if helpers.RunsWithKubeProxy() {
   679  				options["kubeProxyReplacement"] = "false"
   680  			} else if helpers.RunsWithKubeProxyReplacement() {
   681  				options["loadBalancer.mode"] = "dsr"
   682  				options["loadBalancer.dsrDispatch"] = "geneve"
   683  			}
   684  			deploymentManager.DeployCilium(options, DeployCiliumOptionsAndDNS)
   685  
   686  			cmd := fmt.Sprintf("bpftool map update pinned %scilium_world_cidrs4 key 0 0 0 0 0 0 0 0 value 1", bpffsDir)
   687  			kubectl.CiliumExecMustSucceedOnAll(context.TODO(), cmd)
   688  
   689  			hsIPcacheYAML := helpers.ManifestGet(kubectl.BasePath(), hsIPcacheFile)
   690  			kubectl.Create(hsIPcacheYAML).ExpectSuccess("Unable to create resource %q", hsIPcacheYAML)
   691  
   692  			// We need a longer timeout here because of the larger number of
   693  			// pods that need to be deployed.
   694  			err := kubectl.WaitforPods(helpers.DefaultNamespace, "-l type=client", 2*helpers.HelperTimeout)
   695  			Expect(err).ToNot(HaveOccurred(), "Client pods not ready after timeout")
   696  		}
   697  
   698  		It("Test ingress policy enforcement with GENEVE and endpoint routes", func() {
   699  			testHighScaleIPcache("geneve", "true")
   700  		})
   701  	})
   702  
   703  	Context("Iptables", func() {
   704  		SkipItIf(func() bool {
   705  			return helpers.IsIntegration(helpers.CIIntegrationGKE) || helpers.DoesNotRunWithKubeProxyReplacement()
   706  		}, "Skip conntrack for pod traffic", func() {
   707  			deploymentManager.DeployCilium(map[string]string{
   708  				"routingMode":                     "native",
   709  				"autoDirectNodeRoutes":            "true",
   710  				"installNoConntrackIptablesRules": "true",
   711  			}, DeployCiliumOptionsAndDNS)
   712  
   713  			ciliumPod, err := kubectl.GetCiliumPodOnNode(helpers.K8s1)
   714  			ExpectWithOffset(1, err).Should(BeNil(), "Unable to determine cilium pod on node %s", helpers.K8s1)
   715  
   716  			_, err = kubectl.ExecInHostNetNSByLabel(context.TODO(), helpers.K8s1, "conntrack -F")
   717  			if err != nil {
   718  				ExpectWithOffset(1, err).Should(BeNil(), "Cannot flush conntrack table")
   719  			}
   720  
   721  			Expect(testPodConnectivityAcrossNodes(kubectl)).Should(BeTrue(), "Connectivity test between nodes failed")
   722  
   723  			cmd := fmt.Sprintf("iptables -w 60 -t raw -C CILIUM_PRE_raw -s %s -m comment --comment 'cilium: NOTRACK for pod traffic' -j CT --notrack", helpers.IPv4NativeRoutingCIDR)
   724  			res := kubectl.ExecPodCmd(helpers.CiliumNamespace, ciliumPod, cmd)
   725  			res.ExpectSuccess("Missing '-j CT --notrack' iptables rule")
   726  
   727  			cmd = fmt.Sprintf("iptables -w 60 -t raw -C CILIUM_PRE_raw -d %s -m comment --comment 'cilium: NOTRACK for pod traffic' -j CT --notrack", helpers.IPv4NativeRoutingCIDR)
   728  			res = kubectl.ExecPodCmd(helpers.CiliumNamespace, ciliumPod, cmd)
   729  			res.ExpectSuccess("Missing '-j CT --notrack' iptables rule")
   730  
   731  			cmd = fmt.Sprintf("iptables -w 60 -t raw -C CILIUM_OUTPUT_raw -s %s -m comment --comment 'cilium: NOTRACK for pod traffic' -j CT --notrack", helpers.IPv4NativeRoutingCIDR)
   732  			res = kubectl.ExecPodCmd(helpers.CiliumNamespace, ciliumPod, cmd)
   733  			res.ExpectSuccess("Missing '-j CT --notrack' iptables rule")
   734  
   735  			cmd = fmt.Sprintf("iptables -w 60 -t raw -C CILIUM_OUTPUT_raw -d %s -m comment --comment 'cilium: NOTRACK for pod traffic' -j CT --notrack", helpers.IPv4NativeRoutingCIDR)
   736  			res = kubectl.ExecPodCmd(helpers.CiliumNamespace, ciliumPod, cmd)
   737  			res.ExpectSuccess("Missing '-j CT --notrack' iptables rule")
   738  
   739  			cmd = fmt.Sprintf("conntrack -L -s %s -d %s | wc -l", helpers.IPv4NativeRoutingCIDR, helpers.IPv4NativeRoutingCIDR)
   740  			resStr, err := kubectl.ExecInHostNetNSByLabel(context.TODO(), helpers.K8s1, cmd)
   741  			if err != nil {
   742  				ExpectWithOffset(1, err).Should(BeNil(), "Cannot list conntrack entries")
   743  			}
   744  			Expect(strings.TrimSpace(resStr)).To(Equal("0"), "Unexpected conntrack entries")
   745  		})
   746  	})
   747  })
   748  
   749  // To avoid flakes, we need to perform some prep work before we enable host
   750  // policy enforcement.
   751  //
   752  // When we first enable the host firewall, Cilium will for the first time track
   753  // all hostns connections on all nodes. Because those connections are already
   754  // established, the first packet we see from them may be a reply packet. For
   755  // that reason, it's possible for Cilium to create conntrack entries in the
   756  // wrong direction. As a consequence, once host policies are enforced, we will
   757  // allow the forward path through and enforce policies on replies.
   758  // For example, consider a connection to the kube-apiserver, a:52483 -> b:6443.
   759  // If the first packet we track is the SYN+ACK, we will create a conntrack
   760  // entry TCP OUT b:6443 -> a:52483. All traffic a:52483 -> b:6443 will be
   761  // considered reply traffic and we will enforce policies on b:6443 -> a:52483.
   762  // If there are any L4 policy rules, this 52483 destination port is unlikely to
   763  // be allowed through.
   764  //
   765  // That situation unfortunately doesn't resolve on its own because Linux will
   766  // consider the connections to be in LAST-ACK state on the server side and will
   767  // keep them around indefinitely.
   768  //
   769  // To fix that, we need to force the termination of those connections. One way
   770  // to do that is to enforce policies just long enough that all such connections
   771  // will end up in a closing state (LAST-ACK or TIME-WAIT). Once that is the
   772  // case, we remove the policies to allow everything through and enable proper
   773  // termination of those connections.
   774  // This function implements that process.
   775  func prepareHostPolicyEnforcement(kubectl *helpers.Kubectl, policy string) {
   776  	By(fmt.Sprintf("Applying policies %s for 1min", policy))
   777  	_, err := kubectl.CiliumClusterwidePolicyAction(policy, helpers.KubectlApply, helpers.HelperTimeout)
   778  	ExpectWithOffset(1, err).Should(BeNil(), fmt.Sprintf("Error creating resource %s: %s", policy, err))
   779  
   780  	time.Sleep(1 * time.Minute)
   781  
   782  	_, err = kubectl.CiliumClusterwidePolicyAction(policy, helpers.KubectlDelete, helpers.HelperTimeout)
   783  	ExpectWithOffset(1, err).Should(BeNil(), fmt.Sprintf("Error deleting resource %s: %s", policy, err))
   784  
   785  	By("Deleted the policies, waiting for connection terminations")
   786  	time.Sleep(30 * time.Second)
   787  }
   788  
   789  func testHostFirewall(kubectl *helpers.Kubectl) {
   790  	randomNs := deploymentManager.DeployRandomNamespaceShared(DemoHostFirewall)
   791  	deploymentManager.WaitUntilReady()
   792  
   793  	demoHostPolicies := helpers.ManifestGet(kubectl.BasePath(), "host-policies.yaml")
   794  	By(fmt.Sprintf("Applying policies %s", demoHostPolicies))
   795  	_, err := kubectl.CiliumClusterwidePolicyAction(demoHostPolicies, helpers.KubectlApply, helpers.HelperTimeout)
   796  	ExpectWithOffset(1, err).Should(BeNil(), fmt.Sprintf("Error creating resource %s: %s", demoHostPolicies, err))
   797  	defer func() {
   798  		_, err := kubectl.CiliumClusterwidePolicyAction(demoHostPolicies, helpers.KubectlDelete, helpers.HelperTimeout)
   799  		ExpectWithOffset(1, err).Should(BeNil(), fmt.Sprintf("Error deleting resource %s: %s", demoHostPolicies, err))
   800  	}()
   801  
   802  	var wg sync.WaitGroup
   803  	wg.Add(1)
   804  	go func() {
   805  		defer GinkgoRecover()
   806  		defer wg.Done()
   807  		By("Checking host policies on ingress from local pod")
   808  		testHostFirewallWithPath(kubectl, randomNs, "zgroup=testClient", "zgroup=testServerHost", false)
   809  	}()
   810  	wg.Add(1)
   811  	go func() {
   812  		defer GinkgoRecover()
   813  		defer wg.Done()
   814  		By("Checking host policies on ingress from remote pod")
   815  		testHostFirewallWithPath(kubectl, randomNs, "zgroup=testClient", "zgroup=testServerHost", true)
   816  	}()
   817  	wg.Add(1)
   818  	go func() {
   819  		defer GinkgoRecover()
   820  		defer wg.Done()
   821  		By("Checking host policies on egress to local pod")
   822  		testHostFirewallWithPath(kubectl, randomNs, "zgroup=testClientHost", "zgroup=testServer", false)
   823  	}()
   824  	wg.Add(1)
   825  	go func() {
   826  		defer GinkgoRecover()
   827  		defer wg.Done()
   828  		By("Checking host policies on egress to remote pod")
   829  		testHostFirewallWithPath(kubectl, randomNs, "zgroup=testClientHost", "zgroup=testServer", true)
   830  	}()
   831  	wg.Add(1)
   832  	go func() {
   833  		defer GinkgoRecover()
   834  		defer wg.Done()
   835  		By("Checking host policies on ingress from remote node")
   836  		testHostFirewallWithPath(kubectl, randomNs, "zgroup=testServerHost", "zgroup=testClientHost", true)
   837  	}()
   838  	wg.Add(1)
   839  	go func() {
   840  		defer GinkgoRecover()
   841  		defer wg.Done()
   842  		By("Checking host policies on egress to remote node")
   843  		testHostFirewallWithPath(kubectl, randomNs, "zgroup=testClientHost", "zgroup=testServerHost", true)
   844  	}()
   845  	wg.Wait()
   846  }
   847  
   848  func testHostFirewallWithPath(kubectl *helpers.Kubectl, randomNs, client, server string, crossNodes bool) {
   849  	srcPod, srcPodJSON := fetchPodsWithOffset(kubectl, randomNs, "client", client, "", crossNodes, 3)
   850  	srcHost, err := srcPodJSON.Filter("{.status.hostIP}")
   851  	ExpectWithOffset(2, err).Should(BeNil(), "Failure to retrieve host of pod %s", srcPod)
   852  
   853  	dstPod, dstPodJSON := fetchPodsWithOffset(kubectl, randomNs, "server", server, srcHost.String(), crossNodes, 3)
   854  	podIP, err := dstPodJSON.Filter("{.status.podIP}")
   855  	ExpectWithOffset(2, err).Should(BeNil(), "Failure to retrieve IP of pod %s", dstPod)
   856  	targetIP := podIP.String()
   857  
   858  	res := kubectl.ExecPodCmd(randomNs, srcPod, helpers.CurlFail("http://%s:80/", targetIP))
   859  	ExpectWithOffset(2, res).Should(helpers.CMDSuccess(),
   860  		"Failed to reach %s:80 from %s", targetIP, srcPod)
   861  
   862  	res = kubectl.ExecPodCmd(randomNs, srcPod, helpers.CurlFail("tftp://%s:69/hello", targetIP))
   863  	ExpectWithOffset(2, res).ShouldNot(helpers.CMDSuccess(),
   864  		"Managed to reach %s:69 from %s", targetIP, srcPod)
   865  }
   866  
   867  func testPodConnectivityAcrossNodes(kubectl *helpers.Kubectl) bool {
   868  	result, _ := testPodConnectivityAndReturnIP(kubectl, true, 1)
   869  	return result
   870  }
   871  
   872  func fetchPodsWithOffset(kubectl *helpers.Kubectl, namespace, name, filter, hostIPAntiAffinity string, requireMultiNode bool, callOffset int) (targetPod string, targetPodJSON *helpers.CmdRes) {
   873  	callOffset++
   874  
   875  	// Fetch pod (names) with the specified filter
   876  	err := kubectl.WaitforPods(namespace, fmt.Sprintf("-l %s", filter), helpers.HelperTimeout)
   877  	ExpectWithOffset(callOffset, err).Should(BeNil(), "Failure while waiting for connectivity test pods to start")
   878  	pods, err := kubectl.GetPodNames(namespace, filter)
   879  	ExpectWithOffset(callOffset, err).Should(BeNil(), "Failure while retrieving pod name for %s", filter)
   880  	if requireMultiNode {
   881  		if config.CiliumTestConfig.Multinode {
   882  			ExpectWithOffset(callOffset, len(pods)).Should(BeNumerically(">", 1),
   883  				fmt.Sprintf("This test requires at least two %s instances, but only one was found", name))
   884  		} else {
   885  			By("Ignoring the requirement for clients on multiple nodes")
   886  			requireMultiNode = false
   887  		}
   888  	}
   889  
   890  	// Fetch the json description of one of the pods
   891  	targetPod = pods[0]
   892  	targetPodJSON = kubectl.Get(
   893  		namespace,
   894  		fmt.Sprintf("pod %s -o json", targetPod))
   895  
   896  	// If multinode / antiaffinity is required, ensure that the target is
   897  	// not on the same node as "hostIPAntiAffinity".
   898  	if requireMultiNode && hostIPAntiAffinity != "" {
   899  		targetHost, err := targetPodJSON.Filter("{.status.hostIP}")
   900  		ExpectWithOffset(callOffset, err).Should(BeNil(), "Failure to retrieve host of pod %s", targetPod)
   901  
   902  		if targetHost.String() == hostIPAntiAffinity {
   903  			targetPod = pods[1]
   904  			targetPodJSON = kubectl.Get(
   905  				namespace,
   906  				fmt.Sprintf("pod %s -o json", targetPod))
   907  		}
   908  	} else if !requireMultiNode && hostIPAntiAffinity != "" {
   909  		targetHost, err := targetPodJSON.Filter("{.status.hostIP}")
   910  		ExpectWithOffset(callOffset, err).Should(BeNil(), "Failure to retrieve host of pod %s", targetPod)
   911  
   912  		if targetHost.String() != hostIPAntiAffinity {
   913  			targetPod = pods[1]
   914  			targetPodJSON = kubectl.Get(
   915  				namespace,
   916  				fmt.Sprintf("pod %s -o json", targetPod))
   917  		}
   918  	}
   919  	return targetPod, targetPodJSON
   920  }
   921  
   922  func applyL3Policy(kubectl *helpers.Kubectl, ns string) (withdrawPolicy func()) {
   923  	demoPolicyL3 := helpers.ManifestGet(kubectl.BasePath(), "l3-policy-demo.yaml")
   924  	By(fmt.Sprintf("Applying policy %s", demoPolicyL3))
   925  	_, err := kubectl.CiliumPolicyAction(ns, demoPolicyL3, helpers.KubectlApply, helpers.HelperTimeout)
   926  	ExpectWithOffset(1, err).Should(BeNil(), fmt.Sprintf("Error creating resource %s: %s", demoPolicyL3, err))
   927  
   928  	return func() {
   929  		_, err := kubectl.CiliumPolicyAction(ns, demoPolicyL3, helpers.KubectlDelete, helpers.HelperTimeout)
   930  		ExpectWithOffset(1, err).Should(BeNil(), fmt.Sprintf("Error deleting resource %s: %s", demoPolicyL3, err))
   931  	}
   932  }
   933  
   934  func testPodConnectivityAndReturnIP(kubectl *helpers.Kubectl, requireMultiNode bool, callOffset int) (bool, string) {
   935  	callOffset++
   936  
   937  	randomNamespace := deploymentManager.DeployRandomNamespaceShared(DemoDaemonSet)
   938  	withdrawPolicy := applyL3Policy(kubectl, randomNamespace)
   939  	defer withdrawPolicy()
   940  	deploymentManager.WaitUntilReady()
   941  
   942  	By("Checking pod connectivity between nodes")
   943  	srcPod, srcPodJSON := fetchPodsWithOffset(kubectl, randomNamespace, "client", "zgroup=testDSClient", "", requireMultiNode, callOffset)
   944  	srcHost, err := srcPodJSON.Filter("{.status.hostIP}")
   945  	ExpectWithOffset(callOffset, err).Should(BeNil(), "Failure to retrieve host of pod %s", srcPod)
   946  
   947  	dstPod, dstPodJSON := fetchPodsWithOffset(kubectl, randomNamespace, "server", "zgroup=testDS", srcHost.String(), requireMultiNode, callOffset)
   948  	podIP, err := dstPodJSON.Filter("{.status.podIP}")
   949  	ExpectWithOffset(callOffset, err).Should(BeNil(), "Failure to retrieve IP of pod %s", dstPod)
   950  	targetIP := podIP.String()
   951  
   952  	// ICMP connectivity test
   953  	res := kubectl.ExecPodCmd(randomNamespace, srcPod, helpers.Ping(targetIP))
   954  	if !res.WasSuccessful() {
   955  		return false, targetIP
   956  	}
   957  
   958  	// HTTP connectivity test
   959  	res = kubectl.ExecPodCmd(randomNamespace, srcPod,
   960  		helpers.CurlFail("http://%s:80/", targetIP))
   961  	return res.WasSuccessful(), targetIP
   962  }
   963  
   964  func testPodHTTPToOutside(kubectl *helpers.Kubectl, outsideURL string, expectNodeIP, expectPodIP, ipv6 bool) bool {
   965  	var hostIPs map[string]string
   966  	var podIPs map[string]string
   967  
   968  	// IPv6 is not supported when the source IP should be checked. It could be
   969  	// supported with more work, but it doesn't make sense as in those cases,
   970  	// we can simply pass the IPv6 target address as outsideURL.
   971  	if ipv6 && expectPodIP {
   972  		panic("IPv6 not supported with source IP checking.")
   973  	}
   974  
   975  	namespace := deploymentManager.DeployRandomNamespaceShared(DemoDaemonSet)
   976  	withdrawPolicy := applyL3Policy(kubectl, namespace)
   977  	defer withdrawPolicy()
   978  	deploymentManager.WaitUntilReady()
   979  
   980  	label := "zgroup=testDSClient"
   981  	pods, err := kubectl.GetPodNames(namespace, label)
   982  	ExpectWithOffset(1, err).Should(BeNil(), "Cannot retrieve pod names by label %s", label)
   983  
   984  	cmd := outsideURL
   985  	if ipv6 {
   986  		cmd = fmt.Sprintf("-6 %s", cmd)
   987  	}
   988  	cmd = helpers.CurlWithRetries(cmd, 10, true)
   989  
   990  	if expectNodeIP || expectPodIP {
   991  		cmd += " | grep client_address="
   992  		hostIPs, err = kubectl.GetPodsHostIPs(namespace, label)
   993  		ExpectWithOffset(1, err).Should(BeNil(), "Cannot retrieve pod host IPs")
   994  		if expectPodIP {
   995  			podIPs, err = kubectl.GetPodsIPs(namespace, label)
   996  			ExpectWithOffset(1, err).Should(BeNil(), "Cannot retrieve pod IPs")
   997  		}
   998  	}
   999  
  1000  	for _, pod := range pods {
  1001  		By("Making ten curl requests from %q to %q", pod, outsideURL)
  1002  
  1003  		hostIP := hostIPs[pod]
  1004  		podIP := podIPs[pod]
  1005  
  1006  		if expectPodIP {
  1007  			// Make pods reachable from the host which doesn't run Cilium
  1008  			kubectl.AddIPRoute(helpers.GetFirstNodeWithoutCilium(), podIP, hostIP, false).
  1009  				ExpectSuccess("Failed to add ip route")
  1010  			defer func() {
  1011  				kubectl.DelIPRoute(helpers.GetFirstNodeWithoutCilium(), podIP, hostIP).
  1012  					ExpectSuccess("Failed to del ip route")
  1013  			}()
  1014  		}
  1015  
  1016  		for i := 1; i <= 10; i++ {
  1017  			res := kubectl.ExecPodCmd(namespace, pod, cmd)
  1018  			ExpectWithOffset(1, res).Should(helpers.CMDSuccess(),
  1019  				"Pod %q can not connect to %q", pod, outsideURL)
  1020  
  1021  			if expectNodeIP || expectPodIP {
  1022  				// Parse the IPs to avoid issues with 4-in-6 formats
  1023  				sourceIP := net.ParseIP(strings.TrimSpace(
  1024  					strings.Split(res.Stdout(), "=")[1])).String()
  1025  				if expectNodeIP {
  1026  					Expect(sourceIP).To(Equal(hostIP), "Expected node IP")
  1027  				}
  1028  				if expectPodIP {
  1029  					Expect(sourceIP).To(Equal(podIP), "Expected pod IP")
  1030  				}
  1031  			}
  1032  		}
  1033  	}
  1034  
  1035  	return true
  1036  }
  1037  
  1038  func monitorConnectivityAcrossNodes(kubectl *helpers.Kubectl) (monitorRes *helpers.CmdRes, monitorCancel func(), targetIP string) {
  1039  	requireMultinode := config.CiliumTestConfig.Multinode
  1040  	if !config.CiliumTestConfig.Multinode {
  1041  		By("Performing multinode connectivity check within a single node")
  1042  	}
  1043  
  1044  	ciliumPodK8s1, err := kubectl.GetCiliumPodOnNode(helpers.K8s1)
  1045  	ExpectWithOffset(1, err).Should(BeNil(), "Cannot get cilium pod on k8s1")
  1046  
  1047  	By(fmt.Sprintf("Launching cilium-dbg monitor on %q", ciliumPodK8s1))
  1048  	monitorRes, monitorCancel = kubectl.MonitorStart(ciliumPodK8s1)
  1049  	result, targetIP := testPodConnectivityAndReturnIP(kubectl, requireMultinode, 2)
  1050  	ExpectWithOffset(1, result).Should(BeTrue(), "Connectivity test between nodes failed")
  1051  
  1052  	return monitorRes, monitorCancel, targetIP
  1053  }
  1054  
  1055  func checkMonitorOutput(monitorOutput []byte, egressPktCount, ingressPktCount int) error {
  1056  	// Multiple connection attempts may be made, we need to
  1057  	// narrow down to the last connection close, then match
  1058  	// the ephemeral port + flags to ensure that the
  1059  	// notifications match the table above.
  1060  	egressTCPExpr := `TCP.*DstPort=80.*FIN=true`
  1061  	egressTCPRegex := regexp.MustCompile(egressTCPExpr)
  1062  	egressTCPMatches := egressTCPRegex.FindAll(monitorOutput, -1)
  1063  	if len(egressTCPMatches) == 0 {
  1064  		return fmt.Errorf("could not locate TCP FIN in monitor log")
  1065  	}
  1066  	finalMatch := egressTCPMatches[len(egressTCPMatches)-1]
  1067  	portRegex := regexp.MustCompile(`SrcPort=([0-9]*)`)
  1068  	// FindSubmatch should return ["SrcPort=12345" "12345"]
  1069  	portBytes := portRegex.FindSubmatch(finalMatch)[1]
  1070  
  1071  	By("Looking for TCP notifications using the ephemeral port %q", portBytes)
  1072  	port, err := strconv.Atoi(string(portBytes))
  1073  	if err != nil {
  1074  		return fmt.Errorf("ephemeral port %q could not be converted to integer: %w",
  1075  			string(portBytes), err)
  1076  	}
  1077  
  1078  	expEgress := fmt.Sprintf("SrcPort=%d", port)
  1079  	expEgressRegex := regexp.MustCompile(expEgress)
  1080  	egressMatches := expEgressRegex.FindAllIndex(monitorOutput, -1)
  1081  	if len(egressMatches) != egressPktCount {
  1082  		return fmt.Errorf("monitor logs contained unexpected number (%d) of egress notifications matching %q",
  1083  			len(egressMatches), expEgress)
  1084  	}
  1085  
  1086  	expIngress := fmt.Sprintf("DstPort=%d", port)
  1087  	expIngressRegex := regexp.MustCompile(expIngress)
  1088  	ingressMatches := expIngressRegex.FindAllIndex(monitorOutput, -1)
  1089  	if len(ingressMatches) != ingressPktCount {
  1090  		return fmt.Errorf("monitor log contained unexpected number (%d) of ingress notifications matching %q",
  1091  			len(ingressMatches), expIngress)
  1092  	}
  1093  
  1094  	return nil
  1095  }