github.com/cilium/cilium@v1.16.2/test/k8s/hubble.go (about) 1 // SPDX-License-Identifier: Apache-2.0 2 // Copyright Authors of Cilium 3 4 package k8sTest 5 6 import ( 7 "context" 8 "fmt" 9 "net" 10 "strconv" 11 "strings" 12 13 . "github.com/onsi/gomega" 14 "google.golang.org/protobuf/encoding/protojson" 15 16 observerpb "github.com/cilium/cilium/api/v1/observer" 17 "github.com/cilium/cilium/pkg/annotation" 18 "github.com/cilium/cilium/pkg/hubble/defaults" 19 "github.com/cilium/cilium/pkg/identity" 20 . "github.com/cilium/cilium/test/ginkgo-ext" 21 "github.com/cilium/cilium/test/helpers" 22 ) 23 24 var _ = Describe("K8sAgentHubbleTest", func() { 25 // We want to run Hubble tests both with and without our kube-proxy 26 // replacement, as the trace events depend on it. We thus run the tests 27 // on GKE and our 4.19 pipeline. 28 SkipContextIf(func() bool { 29 return helpers.RunsOnNetNextKernel() || helpers.RunsOnAKS() 30 }, "Hubble Observe", func() { 31 var ( 32 kubectl *helpers.Kubectl 33 ciliumFilename string 34 k8s1NodeName string 35 ciliumPodK8s1 string 36 37 hubbleRelayNamespace = helpers.CiliumNamespace 38 hubbleRelayService = "hubble-relay" 39 hubbleRelayAddress string 40 41 demoPath string 42 43 app1Service = "app1-service" 44 app1Labels = "id=app1,zgroup=testapp" 45 apps = []string{helpers.App1, helpers.App2, helpers.App3} 46 prometheusPort = "9965" 47 48 namespaceForTest string 49 appPods map[string]string 50 app1ClusterIP string 51 app1Port int 52 ) 53 54 addVisibilityAnnotation := func(ns, podLabels, direction, port, l4proto, l7proto string) { 55 visibilityAnnotation := fmt.Sprintf("<%s/%s/%s/%s>", direction, port, l4proto, l7proto) 56 By("Adding visibility annotation %s on pod with labels %s", visibilityAnnotation, podLabels) 57 58 // Prints <node>=<ns>/<podname> for each pod the annotation was applied to 59 res := kubectl.Exec(fmt.Sprintf("%s annotate pod -n %s -l %s %s=%q"+ 60 " -o 'jsonpath={.spec.nodeName}={.metadata.namespace}/{.metadata.name}{\"\\n\"}'", 61 helpers.KubectlCmd, 62 ns, app1Labels, 63 annotation.ProxyVisibility, visibilityAnnotation)) 64 res.ExpectSuccess("adding proxy visibility annotation failed") 65 66 // For each pod, check that the Cilium proxy-statistics contain the new annotation 67 expectedProxyState := strings.ToLower(visibilityAnnotation) 68 for node, podName := range res.KVOutput() { 69 ciliumPod, err := kubectl.GetCiliumPodOnNodeByName(node) 70 Expect(err).To(BeNil()) 71 72 // Extract annotation from endpoint model of pod. It does not have the l4proto, so we insert it manually. 73 cmd := fmt.Sprintf("cilium-dbg endpoint get pod-name:%s"+ 74 " -o jsonpath='{range [*].status.policy.proxy-statistics[*]}<{.location}/{.port}/%s/{.protocol}>{\"\\n\"}{end}'", 75 podName, strings.ToLower(l4proto)) 76 err = kubectl.CiliumExecUntilMatch(ciliumPod, cmd, expectedProxyState) 77 Expect(err).To(BeNil(), "timed out waiting for endpoint to regenerate for visibility annotation") 78 } 79 } 80 81 removeVisibilityAnnotation := func(ns, podLabels string) { 82 By("Removing visibility annotation on pod with labels %s", app1Labels) 83 res := kubectl.Exec(fmt.Sprintf("%s annotate pod -n %s -l %s %s-", helpers.KubectlCmd, ns, podLabels, annotation.ProxyVisibility)) 84 res.ExpectSuccess("removing proxy visibility annotation failed") 85 } 86 87 getFlowsFromRelay := func(args string) []*observerpb.GetFlowsResponse { 88 args = fmt.Sprintf("--server %s %s", hubbleRelayAddress, args) 89 90 var result []*observerpb.GetFlowsResponse 91 hubbleObserve := func() error { 92 res := kubectl.HubbleObserve(ciliumPodK8s1, args) 93 res.ExpectSuccess("hubble observe invocation failed: %q", res.OutputPrettyPrint()) 94 95 lines := res.ByLines() 96 flows := make([]*observerpb.GetFlowsResponse, 0, len(lines)) 97 for _, line := range lines { 98 if len(line) == 0 { 99 continue 100 } 101 102 f := &observerpb.GetFlowsResponse{} 103 if err := protojson.Unmarshal([]byte(line), f); err != nil { 104 return fmt.Errorf("failed to decode in %q: %w", lines, err) 105 } 106 flows = append(flows, f) 107 } 108 109 if len(flows) == 0 { 110 return fmt.Errorf("no flows returned for query %q", args) 111 } 112 113 result = flows 114 return nil 115 } 116 117 Eventually(hubbleObserve, helpers.MidCommandTimeout).Should(BeNil()) 118 return result 119 } 120 121 BeforeAll(func() { 122 kubectl = helpers.CreateKubectl(helpers.K8s1VMName(), logger) 123 k8s1NodeName, _ = kubectl.GetNodeInfo(helpers.K8s1) 124 125 demoPath = helpers.ManifestGet(kubectl.BasePath(), "demo.yaml") 126 127 ciliumFilename = helpers.TimestampFilename("cilium.yaml") 128 DeployCiliumOptionsAndDNS(kubectl, ciliumFilename, map[string]string{ 129 "hubble.metrics.enabled": `"{dns:query;ignoreAAAA,drop,tcp,flow,port-distribution,icmp,http}"`, 130 "hubble.relay.enabled": "true", 131 "bpf.monitorAggregation": "none", 132 }) 133 134 var err error 135 ciliumPodK8s1, err = kubectl.GetCiliumPodOnNode(helpers.K8s1) 136 Expect(err).Should(BeNil(), "unable to find hubble-cli pod on %s", helpers.K8s1) 137 138 ExpectHubbleRelayReady(kubectl, hubbleRelayNamespace) 139 hubbleRelayIP, hubbleRelayPort, err := kubectl.GetServiceHostPort(hubbleRelayNamespace, hubbleRelayService) 140 Expect(err).Should(BeNil(), "Cannot get service %s", hubbleRelayService) 141 Expect(net.ParseIP(hubbleRelayIP) != nil).Should(BeTrue(), "hubbleRelayIP is not an IP") 142 hubbleRelayAddress = net.JoinHostPort(hubbleRelayIP, strconv.Itoa(hubbleRelayPort)) 143 144 namespaceForTest = helpers.GenerateNamespaceForTest("") 145 kubectl.NamespaceDelete(namespaceForTest) 146 res := kubectl.NamespaceCreate(namespaceForTest) 147 res.ExpectSuccess("could not create namespace") 148 149 res = kubectl.Apply(helpers.ApplyOptions{FilePath: demoPath, Namespace: namespaceForTest}) 150 res.ExpectSuccess("could not create resource") 151 152 err = kubectl.WaitforPods(namespaceForTest, "-l zgroup=testapp", helpers.HelperTimeout) 153 Expect(err).Should(BeNil(), "test pods are not ready after timeout") 154 155 appPods = helpers.GetAppPods(apps, namespaceForTest, kubectl, "id") 156 app1ClusterIP, app1Port, err = kubectl.GetServiceHostPort(namespaceForTest, app1Service) 157 Expect(err).To(BeNil(), "unable to find service in %q namespace", namespaceForTest) 158 }) 159 160 AfterFailed(func() { 161 kubectl.CiliumReport("cilium-dbg endpoint list") 162 }) 163 164 JustAfterEach(func() { 165 kubectl.ValidateNoErrorsInLogs(CurrentGinkgoTestDescription().Duration) 166 }) 167 168 AfterAll(func() { 169 kubectl.Delete(demoPath) 170 kubectl.NamespaceDelete(namespaceForTest) 171 ExpectAllPodsTerminated(kubectl) 172 173 kubectl.DeleteHubbleRelay(hubbleRelayNamespace) 174 UninstallCiliumFromManifest(kubectl, ciliumFilename) 175 kubectl.CloseSSHClient() 176 }) 177 178 It("Test L3/L4 Flow", func() { 179 ctx, cancel := context.WithTimeout(context.Background(), helpers.MidCommandTimeout) 180 defer cancel() 181 follow, err := kubectl.HubbleObserveFollow(ctx, ciliumPodK8s1, fmt.Sprintf( 182 "--last 1 --type trace --from-pod %s/%s --to-namespace %s --to-label %s --to-port %d", 183 namespaceForTest, appPods[helpers.App2], namespaceForTest, app1Labels, app1Port)) 184 Expect(err).To(BeNil(), "Failed to start hubble observe") 185 186 res := kubectl.ExecPodCmd(namespaceForTest, appPods[helpers.App2], 187 helpers.CurlFail(fmt.Sprintf("http://%s/public", app1ClusterIP))) 188 res.ExpectSuccess("%q cannot curl clusterIP %q", appPods[helpers.App2], app1ClusterIP) 189 190 err = follow.WaitUntilMatchFilterLineTimeout(`{$.flow.Type}`, "L3_L4", helpers.ShortCommandTimeout) 191 Expect(err).To(BeNil(), fmt.Sprintf("hubble observe query timed out on %q", follow.OutputPrettyPrint())) 192 193 // Basic check for L4 Prometheus metrics. 194 _, nodeIP := kubectl.GetNodeInfo(helpers.K8s1) 195 metricsUrl := fmt.Sprintf("%s/metrics", net.JoinHostPort(nodeIP, prometheusPort)) 196 res = kubectl.ExecInHostNetNS(ctx, k8s1NodeName, helpers.CurlFail(metricsUrl)) 197 res.ExpectSuccess("%s/%s cannot curl metrics %q", helpers.CiliumNamespace, ciliumPodK8s1, app1ClusterIP) 198 res.ExpectContains(`hubble_flows_processed_total{protocol="TCP",subtype="to-endpoint",type="Trace",verdict="FORWARDED"}`) 199 }) 200 201 It("Test TLS certificate", func() { 202 certpath := "/var/lib/cilium/tls/hubble/server.crt" 203 res := kubectl.ExecPodCmd(helpers.CiliumNamespace, ciliumPodK8s1, helpers.ReadFile(certpath)) 204 res.ExpectSuccess("Cilium pod cannot read the hubble server TLS certificate") 205 expected := string(res.GetStdOut().Bytes()) 206 207 serverName := fmt.Sprintf("%s.default.hubble-grpc.cilium.io", helpers.K8s1) 208 cmd := helpers.OpenSSLShowCerts("localhost", defaults.ServerPort, serverName) 209 res = kubectl.ExecPodCmd(helpers.CiliumNamespace, ciliumPodK8s1, cmd) 210 res.ExpectSuccess("Cilium pod cannot initiate TLS handshake to Hubble") 211 cert := string(res.GetStdOut().Bytes()) 212 213 Expect(cert).To(Equal(expected)) 214 }) 215 216 It("Test L3/L4 Flow with hubble-relay", func() { 217 res := kubectl.ExecPodCmd(namespaceForTest, appPods[helpers.App2], 218 helpers.CurlFail(fmt.Sprintf("http://%s/public", app1ClusterIP))) 219 res.ExpectSuccess("%q cannot curl clusterIP %q", appPods[helpers.App2], app1ClusterIP) 220 221 flows := getFlowsFromRelay(fmt.Sprintf( 222 "--last 1 --type trace --from-pod %s/%s --to-namespace %s --to-label %s --to-port %d", 223 namespaceForTest, appPods[helpers.App2], namespaceForTest, app1Labels, app1Port)) 224 Expect(flows).NotTo(BeEmpty()) 225 }) 226 227 It("Test L7 Flow", func() { 228 defer removeVisibilityAnnotation(namespaceForTest, app1Labels) 229 addVisibilityAnnotation(namespaceForTest, app1Labels, "Ingress", "80", "TCP", "HTTP") 230 231 ctx, cancel := context.WithTimeout(context.Background(), helpers.MidCommandTimeout) 232 defer cancel() 233 follow, err := kubectl.HubbleObserveFollow(ctx, ciliumPodK8s1, fmt.Sprintf( 234 "--last 1 --type l7 --from-pod %s/%s --to-namespace %s --to-label %s --protocol http", 235 namespaceForTest, appPods[helpers.App2], namespaceForTest, app1Labels)) 236 Expect(err).To(BeNil(), "Failed to start hubble observe") 237 238 res := kubectl.ExecPodCmd(namespaceForTest, appPods[helpers.App2], 239 helpers.CurlFail(fmt.Sprintf("http://%s/public", app1ClusterIP))) 240 res.ExpectSuccess("%q cannot curl clusterIP %q", appPods[helpers.App2], app1ClusterIP) 241 242 err = follow.WaitUntilMatchFilterLineTimeout(`{$.flow.Type}`, "L7", helpers.ShortCommandTimeout) 243 Expect(err).To(BeNil(), fmt.Sprintf("hubble observe query timed out on %q", follow.OutputPrettyPrint())) 244 245 // Basic check for L7 Prometheus metrics. 246 _, nodeIP := kubectl.GetNodeInfo(helpers.K8s1) 247 metricsUrl := fmt.Sprintf("%s/metrics", net.JoinHostPort(nodeIP, prometheusPort)) 248 res = kubectl.ExecInHostNetNS(ctx, k8s1NodeName, helpers.CurlFail(metricsUrl)) 249 res.ExpectSuccess("%s/%s cannot curl metrics %q", helpers.CiliumNamespace, ciliumPodK8s1, app1ClusterIP) 250 res.ExpectContains(`hubble_flows_processed_total{protocol="HTTP",subtype="HTTP",type="L7",verdict="FORWARDED"}`) 251 }) 252 253 It("Test L7 Flow with hubble-relay", func() { 254 defer removeVisibilityAnnotation(namespaceForTest, app1Labels) 255 addVisibilityAnnotation(namespaceForTest, app1Labels, "Ingress", "80", "TCP", "HTTP") 256 257 res := kubectl.ExecPodCmd(namespaceForTest, appPods[helpers.App2], 258 helpers.CurlFail(fmt.Sprintf("http://%s/public", app1ClusterIP))) 259 res.ExpectSuccess("%q cannot curl clusterIP %q", appPods[helpers.App2], app1ClusterIP) 260 261 flows := getFlowsFromRelay(fmt.Sprintf( 262 "--last 1 --type l7 --from-pod %s/%s --to-namespace %s --to-label %s --protocol http", 263 namespaceForTest, appPods[helpers.App2], namespaceForTest, app1Labels)) 264 Expect(flows).NotTo(BeEmpty()) 265 }) 266 267 It("Test FQDN Policy with Relay", func() { 268 fqdnProxyPolicy := helpers.ManifestGet(kubectl.BasePath(), "fqdn-proxy-policy.yaml") 269 fqdnTarget := "vagrant-cache.ci.cilium.io" 270 271 _, err := kubectl.CiliumPolicyAction( 272 namespaceForTest, fqdnProxyPolicy, 273 helpers.KubectlApply, helpers.HelperTimeout) 274 Expect(err).To(BeNil(), "Cannot install fqdn proxy policy") 275 defer kubectl.CiliumPolicyAction(namespaceForTest, fqdnProxyPolicy, 276 helpers.KubectlDelete, helpers.HelperTimeout) 277 278 res := kubectl.ExecPodCmd(namespaceForTest, appPods[helpers.App2], 279 helpers.CurlFail(fmt.Sprintf("http://%s", fqdnTarget))) 280 res.ExpectSuccess("%q cannot curl fqdn target %q", appPods[helpers.App2], fqdnTarget) 281 282 flows := getFlowsFromRelay(fmt.Sprintf( 283 "--last 1 --type trace:from-endpoint --from-pod %s/%s --to-fqdn %s", 284 namespaceForTest, appPods[helpers.App2], fqdnTarget)) 285 Expect(flows).To(HaveLen(1)) 286 Expect(flows[0].GetFlow().Destination.Identity).To(BeNumerically(">=", identity.MinimalNumericIdentity)) 287 }) 288 }) 289 })