github.com/cilium/cilium@v1.16.2/test/k8s/bgp.go (about) 1 // SPDX-License-Identifier: Apache-2.0 2 // Copyright Authors of Cilium 3 4 package k8sTest 5 6 import ( 7 "context" 8 "fmt" 9 "os" 10 "path/filepath" 11 "text/template" 12 "time" 13 14 . "github.com/onsi/gomega" 15 16 . "github.com/cilium/cilium/test/ginkgo-ext" 17 "github.com/cilium/cilium/test/helpers" 18 ) 19 20 var _ = SkipDescribeIf( 21 func() bool { 22 // The 5.4 CI job is intended to catch BPF complexity regressions and 23 // as such doesn't need to execute this test suite. 24 return helpers.RunsOn54Kernel() || 25 // Test requests to the LB are going to be sent from the node which 26 // doesn't run Cilium. 27 helpers.DoesNotExistNodeWithoutCilium() 28 }, "K8sDatapathBGPTests", func() { 29 var ( 30 kubectl *helpers.Kubectl 31 ciliumFilename string 32 ni *helpers.NodesInfo 33 err error 34 ) 35 36 BeforeAll(func() { 37 kubectl = helpers.CreateKubectl(helpers.K8s1VMName(), logger) 38 ni, err = helpers.GetNodesInfo(kubectl) 39 Expect(err).Should(BeNil(), "Cannot get nodes info") 40 ciliumFilename = helpers.TimestampFilename("cilium.yaml") 41 }) 42 43 JustAfterEach(func() { 44 kubectl.ValidateNoErrorsInLogs(CurrentGinkgoTestDescription().Duration) 45 }) 46 47 AfterAll(func() { 48 UninstallCiliumFromManifest(kubectl, ciliumFilename) 49 kubectl.CloseSSHClient() 50 }) 51 52 Context("Tests LoadBalancer", func() { 53 var ( 54 frr string // BGP router 55 routerIP string 56 57 bgpConfigMap string 58 59 demoDS string 60 lbSvcName = "test-lb" 61 62 ciliumPodK8s1, ciliumPodK8s2 string 63 testStartTime time.Time 64 ) 65 66 BeforeAll(func() { 67 frr = applyFRRTemplate(kubectl, ni) 68 kubectl.ApplyDefault(frr).ExpectSuccess("Unable to apply rendered template %s", frr) 69 70 err := kubectl.WaitForSinglePod("kube-system", "frr", 30*time.Second) 71 ExpectWithOffset(1, err).ShouldNot(HaveOccurred(), "Failed to wait for FRR Pod") 72 73 frrPod, err := kubectl.GetPodsIPs(helpers.KubeSystemNamespace, "app=frr") 74 ExpectWithOffset(1, err).ShouldNot(HaveOccurred(), "Cannot determine FRR Pod IPs") 75 76 var ok bool 77 routerIP, ok = frrPod["frr"] 78 Expect(ok).To(BeTrue()) 79 80 bgpConfigMap = applyBGPCMTemplate(kubectl, routerIP) 81 kubectl.ApplyDefault(bgpConfigMap).ExpectSuccess("Unable to apply BGP ConfigMap %s", bgpConfigMap) 82 83 DeployCiliumOptionsAndDNS(kubectl, ciliumFilename, 84 map[string]string{ 85 "bgp.enabled": "true", 86 "bgp.announce.loadbalancerIP": "true", 87 88 "debug.verbose": "datapath", // https://github.com/cilium/cilium/issues/16399 89 "routingMode": "native", 90 "autoDirectNodeRoutes": "true", 91 }) 92 93 demoDS = helpers.ManifestGet(kubectl.BasePath(), "demo_ds.yaml") 94 kubectl.ApplyDefault(demoDS).ExpectSuccess("Unable to apply %s", demoDS) 95 96 ciliumPodK8s1, err = kubectl.GetCiliumPodOnNode(helpers.K8s1) 97 ExpectWithOffset(1, err).ShouldNot(HaveOccurred(), "Cannot determine cilium pod name") 98 ciliumPodK8s2, err = kubectl.GetCiliumPodOnNode(helpers.K8s2) 99 ExpectWithOffset(1, err).ShouldNot(HaveOccurred(), "Cannot determine cilium pod name") 100 testStartTime = time.Now() 101 }) 102 103 AfterAll(func() { 104 kubectl.Delete(frr) 105 kubectl.Delete(bgpConfigMap) 106 kubectl.Delete(demoDS) 107 // Delete temp files 108 os.Remove(frr) 109 os.Remove(bgpConfigMap) 110 ExpectAllPodsTerminated(kubectl) 111 }) 112 113 AfterFailed(func() { 114 res := kubectl.CiliumExecContext( 115 context.TODO(), 116 ciliumPodK8s1, 117 fmt.Sprintf( 118 "hubble observe debug-events --since %v -o jsonpb", 119 testStartTime.Format(time.RFC3339), 120 ), 121 ) 122 helpers.WriteToReportFile( 123 res.CombineOutput().Bytes(), 124 "tests-loadbalancer-hubble-observe-debug-events-k8s1.log", 125 ) 126 res = kubectl.CiliumExecContext( 127 context.TODO(), 128 ciliumPodK8s2, 129 fmt.Sprintf( 130 "hubble observe debug-events --since %v -o jsonpb", 131 testStartTime.Format(time.RFC3339), 132 ), 133 ) 134 helpers.WriteToReportFile( 135 res.CombineOutput().Bytes(), 136 "tests-loadbalancer-hubble-observe-debug-events-k8s2.log", 137 ) 138 }) 139 140 It("Connectivity to endpoint via LB", func() { 141 By("Waiting until the Operator has assigned the LB IP") 142 lbIP, err := kubectl.GetLoadBalancerIP( 143 helpers.DefaultNamespace, lbSvcName, 30*time.Second) 144 Expect(err).Should(BeNil(), "Cannot retrieve LB IP for test-lb") 145 146 By("Waiting until the Agents have announced the LB IP via BGP") 147 Eventually(func() string { 148 return kubectl.ExecInHostNetNS( 149 context.TODO(), 150 ni.OutsideNodeName, 151 "ip route", 152 ).GetStdOut().String() 153 }, 30*time.Second, 1*time.Second).Should(ContainSubstring(lbIP), 154 "BGP router does not have route for LB IP") 155 156 // Check connectivity from outside 157 url := "http://" + lbIP 158 testCurlFromOutside(kubectl, ni, url, 10, false) 159 160 // Patch service to add a LB source range to disallow requests 161 // from the outsideNode 162 kubectl.Patch(helpers.DefaultNamespace, "service", lbSvcName, 163 `{"spec": {"loadBalancerSourceRanges": ["1.1.1.0/24"]}}`) 164 time.Sleep(5 * time.Second) 165 testCurlFailFromOutside(kubectl, ni, url, 1) 166 // Patch again, but this time add outsideNode IP addr 167 kubectl.Patch(helpers.DefaultNamespace, "service", lbSvcName, 168 fmt.Sprintf( 169 `{"spec": {"loadBalancerSourceRanges": ["1.1.1.0/24", "%s/32"]}}`, 170 ni.OutsideIP)) 171 time.Sleep(5 * time.Second) 172 testCurlFromOutside(kubectl, ni, url, 10, false) 173 }) 174 }) 175 }) 176 177 func applyFRRTemplate(kubectl *helpers.Kubectl, ni *helpers.NodesInfo) string { 178 tmpl := helpers.ManifestGet(kubectl.BasePath(), "frr.yaml.tmpl") 179 content, err := os.ReadFile(tmpl) 180 ExpectWithOffset(1, err).ToNot(HaveOccurred()) 181 ExpectWithOffset(1, content).ToNot(BeEmpty()) 182 183 render, err := os.CreateTemp(os.TempDir(), "frr-") 184 ExpectWithOffset(1, err).ToNot(HaveOccurred()) 185 defer render.Close() 186 187 t := template.Must(template.New("").Parse(string(content))) 188 err = t.Execute(render, struct { 189 OutsideNodeName string 190 Nodes []string 191 }{ 192 OutsideNodeName: ni.OutsideNodeName, 193 Nodes: []string{ni.K8s1IP, ni.K8s2IP}, 194 }) 195 ExpectWithOffset(1, err).ToNot(HaveOccurred()) 196 197 path, err := filepath.Abs(render.Name()) 198 ExpectWithOffset(1, err).ToNot(HaveOccurred()) 199 return path 200 } 201 202 func applyBGPCMTemplate(kubectl *helpers.Kubectl, ip string) string { 203 tmpl := helpers.ManifestGet(kubectl.BasePath(), "bgp-configmap.yaml.tmpl") 204 content, err := os.ReadFile(tmpl) 205 ExpectWithOffset(1, err).ToNot(HaveOccurred()) 206 ExpectWithOffset(1, content).ToNot(BeEmpty()) 207 208 render, err := os.CreateTemp(os.TempDir(), "bgp-cm-") 209 ExpectWithOffset(1, err).ToNot(HaveOccurred()) 210 defer render.Close() 211 212 t := template.Must(template.New("").Parse(string(content))) 213 err = t.Execute(render, struct { 214 RouterIP string 215 }{ 216 RouterIP: ip, 217 }) 218 ExpectWithOffset(1, err).ToNot(HaveOccurred()) 219 220 path, err := filepath.Abs(render.Name()) 221 ExpectWithOffset(1, err).ToNot(HaveOccurred()) 222 return path 223 }