github.com/cilium/cilium@v1.16.2/test/k8s/bgp.go (about)

     1  // SPDX-License-Identifier: Apache-2.0
     2  // Copyright Authors of Cilium
     3  
     4  package k8sTest
     5  
     6  import (
     7  	"context"
     8  	"fmt"
     9  	"os"
    10  	"path/filepath"
    11  	"text/template"
    12  	"time"
    13  
    14  	. "github.com/onsi/gomega"
    15  
    16  	. "github.com/cilium/cilium/test/ginkgo-ext"
    17  	"github.com/cilium/cilium/test/helpers"
    18  )
    19  
    20  var _ = SkipDescribeIf(
    21  	func() bool {
    22  		// The 5.4 CI job is intended to catch BPF complexity regressions and
    23  		// as such doesn't need to execute this test suite.
    24  		return helpers.RunsOn54Kernel() ||
    25  			// Test requests to the LB are going to be sent from the node which
    26  			// doesn't run Cilium.
    27  			helpers.DoesNotExistNodeWithoutCilium()
    28  	}, "K8sDatapathBGPTests", func() {
    29  		var (
    30  			kubectl        *helpers.Kubectl
    31  			ciliumFilename string
    32  			ni             *helpers.NodesInfo
    33  			err            error
    34  		)
    35  
    36  		BeforeAll(func() {
    37  			kubectl = helpers.CreateKubectl(helpers.K8s1VMName(), logger)
    38  			ni, err = helpers.GetNodesInfo(kubectl)
    39  			Expect(err).Should(BeNil(), "Cannot get nodes info")
    40  			ciliumFilename = helpers.TimestampFilename("cilium.yaml")
    41  		})
    42  
    43  		JustAfterEach(func() {
    44  			kubectl.ValidateNoErrorsInLogs(CurrentGinkgoTestDescription().Duration)
    45  		})
    46  
    47  		AfterAll(func() {
    48  			UninstallCiliumFromManifest(kubectl, ciliumFilename)
    49  			kubectl.CloseSSHClient()
    50  		})
    51  
    52  		Context("Tests LoadBalancer", func() {
    53  			var (
    54  				frr      string // BGP router
    55  				routerIP string
    56  
    57  				bgpConfigMap string
    58  
    59  				demoDS    string
    60  				lbSvcName = "test-lb"
    61  
    62  				ciliumPodK8s1, ciliumPodK8s2 string
    63  				testStartTime                time.Time
    64  			)
    65  
    66  			BeforeAll(func() {
    67  				frr = applyFRRTemplate(kubectl, ni)
    68  				kubectl.ApplyDefault(frr).ExpectSuccess("Unable to apply rendered template %s", frr)
    69  
    70  				err := kubectl.WaitForSinglePod("kube-system", "frr", 30*time.Second)
    71  				ExpectWithOffset(1, err).ShouldNot(HaveOccurred(), "Failed to wait for FRR Pod")
    72  
    73  				frrPod, err := kubectl.GetPodsIPs(helpers.KubeSystemNamespace, "app=frr")
    74  				ExpectWithOffset(1, err).ShouldNot(HaveOccurred(), "Cannot determine FRR Pod IPs")
    75  
    76  				var ok bool
    77  				routerIP, ok = frrPod["frr"]
    78  				Expect(ok).To(BeTrue())
    79  
    80  				bgpConfigMap = applyBGPCMTemplate(kubectl, routerIP)
    81  				kubectl.ApplyDefault(bgpConfigMap).ExpectSuccess("Unable to apply BGP ConfigMap %s", bgpConfigMap)
    82  
    83  				DeployCiliumOptionsAndDNS(kubectl, ciliumFilename,
    84  					map[string]string{
    85  						"bgp.enabled":                 "true",
    86  						"bgp.announce.loadbalancerIP": "true",
    87  
    88  						"debug.verbose":        "datapath", // https://github.com/cilium/cilium/issues/16399
    89  						"routingMode":          "native",
    90  						"autoDirectNodeRoutes": "true",
    91  					})
    92  
    93  				demoDS = helpers.ManifestGet(kubectl.BasePath(), "demo_ds.yaml")
    94  				kubectl.ApplyDefault(demoDS).ExpectSuccess("Unable to apply %s", demoDS)
    95  
    96  				ciliumPodK8s1, err = kubectl.GetCiliumPodOnNode(helpers.K8s1)
    97  				ExpectWithOffset(1, err).ShouldNot(HaveOccurred(), "Cannot determine cilium pod name")
    98  				ciliumPodK8s2, err = kubectl.GetCiliumPodOnNode(helpers.K8s2)
    99  				ExpectWithOffset(1, err).ShouldNot(HaveOccurred(), "Cannot determine cilium pod name")
   100  				testStartTime = time.Now()
   101  			})
   102  
   103  			AfterAll(func() {
   104  				kubectl.Delete(frr)
   105  				kubectl.Delete(bgpConfigMap)
   106  				kubectl.Delete(demoDS)
   107  				// Delete temp files
   108  				os.Remove(frr)
   109  				os.Remove(bgpConfigMap)
   110  				ExpectAllPodsTerminated(kubectl)
   111  			})
   112  
   113  			AfterFailed(func() {
   114  				res := kubectl.CiliumExecContext(
   115  					context.TODO(),
   116  					ciliumPodK8s1,
   117  					fmt.Sprintf(
   118  						"hubble observe debug-events --since %v -o jsonpb",
   119  						testStartTime.Format(time.RFC3339),
   120  					),
   121  				)
   122  				helpers.WriteToReportFile(
   123  					res.CombineOutput().Bytes(),
   124  					"tests-loadbalancer-hubble-observe-debug-events-k8s1.log",
   125  				)
   126  				res = kubectl.CiliumExecContext(
   127  					context.TODO(),
   128  					ciliumPodK8s2,
   129  					fmt.Sprintf(
   130  						"hubble observe debug-events --since %v -o jsonpb",
   131  						testStartTime.Format(time.RFC3339),
   132  					),
   133  				)
   134  				helpers.WriteToReportFile(
   135  					res.CombineOutput().Bytes(),
   136  					"tests-loadbalancer-hubble-observe-debug-events-k8s2.log",
   137  				)
   138  			})
   139  
   140  			It("Connectivity to endpoint via LB", func() {
   141  				By("Waiting until the Operator has assigned the LB IP")
   142  				lbIP, err := kubectl.GetLoadBalancerIP(
   143  					helpers.DefaultNamespace, lbSvcName, 30*time.Second)
   144  				Expect(err).Should(BeNil(), "Cannot retrieve LB IP for test-lb")
   145  
   146  				By("Waiting until the Agents have announced the LB IP via BGP")
   147  				Eventually(func() string {
   148  					return kubectl.ExecInHostNetNS(
   149  						context.TODO(),
   150  						ni.OutsideNodeName,
   151  						"ip route",
   152  					).GetStdOut().String()
   153  				}, 30*time.Second, 1*time.Second).Should(ContainSubstring(lbIP),
   154  					"BGP router does not have route for LB IP")
   155  
   156  				// Check connectivity from outside
   157  				url := "http://" + lbIP
   158  				testCurlFromOutside(kubectl, ni, url, 10, false)
   159  
   160  				// Patch service to add a LB source range to disallow requests
   161  				// from the outsideNode
   162  				kubectl.Patch(helpers.DefaultNamespace, "service", lbSvcName,
   163  					`{"spec": {"loadBalancerSourceRanges": ["1.1.1.0/24"]}}`)
   164  				time.Sleep(5 * time.Second)
   165  				testCurlFailFromOutside(kubectl, ni, url, 1)
   166  				// Patch again, but this time add outsideNode IP addr
   167  				kubectl.Patch(helpers.DefaultNamespace, "service", lbSvcName,
   168  					fmt.Sprintf(
   169  						`{"spec": {"loadBalancerSourceRanges": ["1.1.1.0/24", "%s/32"]}}`,
   170  						ni.OutsideIP))
   171  				time.Sleep(5 * time.Second)
   172  				testCurlFromOutside(kubectl, ni, url, 10, false)
   173  			})
   174  		})
   175  	})
   176  
   177  func applyFRRTemplate(kubectl *helpers.Kubectl, ni *helpers.NodesInfo) string {
   178  	tmpl := helpers.ManifestGet(kubectl.BasePath(), "frr.yaml.tmpl")
   179  	content, err := os.ReadFile(tmpl)
   180  	ExpectWithOffset(1, err).ToNot(HaveOccurred())
   181  	ExpectWithOffset(1, content).ToNot(BeEmpty())
   182  
   183  	render, err := os.CreateTemp(os.TempDir(), "frr-")
   184  	ExpectWithOffset(1, err).ToNot(HaveOccurred())
   185  	defer render.Close()
   186  
   187  	t := template.Must(template.New("").Parse(string(content)))
   188  	err = t.Execute(render, struct {
   189  		OutsideNodeName string
   190  		Nodes           []string
   191  	}{
   192  		OutsideNodeName: ni.OutsideNodeName,
   193  		Nodes:           []string{ni.K8s1IP, ni.K8s2IP},
   194  	})
   195  	ExpectWithOffset(1, err).ToNot(HaveOccurred())
   196  
   197  	path, err := filepath.Abs(render.Name())
   198  	ExpectWithOffset(1, err).ToNot(HaveOccurred())
   199  	return path
   200  }
   201  
   202  func applyBGPCMTemplate(kubectl *helpers.Kubectl, ip string) string {
   203  	tmpl := helpers.ManifestGet(kubectl.BasePath(), "bgp-configmap.yaml.tmpl")
   204  	content, err := os.ReadFile(tmpl)
   205  	ExpectWithOffset(1, err).ToNot(HaveOccurred())
   206  	ExpectWithOffset(1, content).ToNot(BeEmpty())
   207  
   208  	render, err := os.CreateTemp(os.TempDir(), "bgp-cm-")
   209  	ExpectWithOffset(1, err).ToNot(HaveOccurred())
   210  	defer render.Close()
   211  
   212  	t := template.Must(template.New("").Parse(string(content)))
   213  	err = t.Execute(render, struct {
   214  		RouterIP string
   215  	}{
   216  		RouterIP: ip,
   217  	})
   218  	ExpectWithOffset(1, err).ToNot(HaveOccurred())
   219  
   220  	path, err := filepath.Abs(render.Name())
   221  	ExpectWithOffset(1, err).ToNot(HaveOccurred())
   222  	return path
   223  }