github.com/cilium/cilium@v1.16.2/test/k8s/hubble.go (about)

     1  // SPDX-License-Identifier: Apache-2.0
     2  // Copyright Authors of Cilium
     3  
     4  package k8sTest
     5  
     6  import (
     7  	"context"
     8  	"fmt"
     9  	"net"
    10  	"strconv"
    11  	"strings"
    12  
    13  	. "github.com/onsi/gomega"
    14  	"google.golang.org/protobuf/encoding/protojson"
    15  
    16  	observerpb "github.com/cilium/cilium/api/v1/observer"
    17  	"github.com/cilium/cilium/pkg/annotation"
    18  	"github.com/cilium/cilium/pkg/hubble/defaults"
    19  	"github.com/cilium/cilium/pkg/identity"
    20  	. "github.com/cilium/cilium/test/ginkgo-ext"
    21  	"github.com/cilium/cilium/test/helpers"
    22  )
    23  
    24  var _ = Describe("K8sAgentHubbleTest", func() {
    25  	// We want to run Hubble tests both with and without our kube-proxy
    26  	// replacement, as the trace events depend on it. We thus run the tests
    27  	// on GKE and our 4.19 pipeline.
    28  	SkipContextIf(func() bool {
    29  		return helpers.RunsOnNetNextKernel() || helpers.RunsOnAKS()
    30  	}, "Hubble Observe", func() {
    31  		var (
    32  			kubectl        *helpers.Kubectl
    33  			ciliumFilename string
    34  			k8s1NodeName   string
    35  			ciliumPodK8s1  string
    36  
    37  			hubbleRelayNamespace = helpers.CiliumNamespace
    38  			hubbleRelayService   = "hubble-relay"
    39  			hubbleRelayAddress   string
    40  
    41  			demoPath string
    42  
    43  			app1Service    = "app1-service"
    44  			app1Labels     = "id=app1,zgroup=testapp"
    45  			apps           = []string{helpers.App1, helpers.App2, helpers.App3}
    46  			prometheusPort = "9965"
    47  
    48  			namespaceForTest string
    49  			appPods          map[string]string
    50  			app1ClusterIP    string
    51  			app1Port         int
    52  		)
    53  
    54  		addVisibilityAnnotation := func(ns, podLabels, direction, port, l4proto, l7proto string) {
    55  			visibilityAnnotation := fmt.Sprintf("<%s/%s/%s/%s>", direction, port, l4proto, l7proto)
    56  			By("Adding visibility annotation %s on pod with labels %s", visibilityAnnotation, podLabels)
    57  
    58  			// Prints <node>=<ns>/<podname> for each pod the annotation was applied to
    59  			res := kubectl.Exec(fmt.Sprintf("%s annotate pod -n %s -l %s %s=%q"+
    60  				" -o 'jsonpath={.spec.nodeName}={.metadata.namespace}/{.metadata.name}{\"\\n\"}'",
    61  				helpers.KubectlCmd,
    62  				ns, app1Labels,
    63  				annotation.ProxyVisibility, visibilityAnnotation))
    64  			res.ExpectSuccess("adding proxy visibility annotation failed")
    65  
    66  			// For each pod, check that the Cilium proxy-statistics contain the new annotation
    67  			expectedProxyState := strings.ToLower(visibilityAnnotation)
    68  			for node, podName := range res.KVOutput() {
    69  				ciliumPod, err := kubectl.GetCiliumPodOnNodeByName(node)
    70  				Expect(err).To(BeNil())
    71  
    72  				// Extract annotation from endpoint model of pod. It does not have the l4proto, so we insert it manually.
    73  				cmd := fmt.Sprintf("cilium-dbg endpoint get pod-name:%s"+
    74  					" -o jsonpath='{range [*].status.policy.proxy-statistics[*]}<{.location}/{.port}/%s/{.protocol}>{\"\\n\"}{end}'",
    75  					podName, strings.ToLower(l4proto))
    76  				err = kubectl.CiliumExecUntilMatch(ciliumPod, cmd, expectedProxyState)
    77  				Expect(err).To(BeNil(), "timed out waiting for endpoint to regenerate for visibility annotation")
    78  			}
    79  		}
    80  
    81  		removeVisibilityAnnotation := func(ns, podLabels string) {
    82  			By("Removing visibility annotation on pod with labels %s", app1Labels)
    83  			res := kubectl.Exec(fmt.Sprintf("%s annotate pod -n %s -l %s %s-", helpers.KubectlCmd, ns, podLabels, annotation.ProxyVisibility))
    84  			res.ExpectSuccess("removing proxy visibility annotation failed")
    85  		}
    86  
    87  		getFlowsFromRelay := func(args string) []*observerpb.GetFlowsResponse {
    88  			args = fmt.Sprintf("--server %s %s", hubbleRelayAddress, args)
    89  
    90  			var result []*observerpb.GetFlowsResponse
    91  			hubbleObserve := func() error {
    92  				res := kubectl.HubbleObserve(ciliumPodK8s1, args)
    93  				res.ExpectSuccess("hubble observe invocation failed: %q", res.OutputPrettyPrint())
    94  
    95  				lines := res.ByLines()
    96  				flows := make([]*observerpb.GetFlowsResponse, 0, len(lines))
    97  				for _, line := range lines {
    98  					if len(line) == 0 {
    99  						continue
   100  					}
   101  
   102  					f := &observerpb.GetFlowsResponse{}
   103  					if err := protojson.Unmarshal([]byte(line), f); err != nil {
   104  						return fmt.Errorf("failed to decode in %q: %w", lines, err)
   105  					}
   106  					flows = append(flows, f)
   107  				}
   108  
   109  				if len(flows) == 0 {
   110  					return fmt.Errorf("no flows returned for query %q", args)
   111  				}
   112  
   113  				result = flows
   114  				return nil
   115  			}
   116  
   117  			Eventually(hubbleObserve, helpers.MidCommandTimeout).Should(BeNil())
   118  			return result
   119  		}
   120  
   121  		BeforeAll(func() {
   122  			kubectl = helpers.CreateKubectl(helpers.K8s1VMName(), logger)
   123  			k8s1NodeName, _ = kubectl.GetNodeInfo(helpers.K8s1)
   124  
   125  			demoPath = helpers.ManifestGet(kubectl.BasePath(), "demo.yaml")
   126  
   127  			ciliumFilename = helpers.TimestampFilename("cilium.yaml")
   128  			DeployCiliumOptionsAndDNS(kubectl, ciliumFilename, map[string]string{
   129  				"hubble.metrics.enabled": `"{dns:query;ignoreAAAA,drop,tcp,flow,port-distribution,icmp,http}"`,
   130  				"hubble.relay.enabled":   "true",
   131  				"bpf.monitorAggregation": "none",
   132  			})
   133  
   134  			var err error
   135  			ciliumPodK8s1, err = kubectl.GetCiliumPodOnNode(helpers.K8s1)
   136  			Expect(err).Should(BeNil(), "unable to find hubble-cli pod on %s", helpers.K8s1)
   137  
   138  			ExpectHubbleRelayReady(kubectl, hubbleRelayNamespace)
   139  			hubbleRelayIP, hubbleRelayPort, err := kubectl.GetServiceHostPort(hubbleRelayNamespace, hubbleRelayService)
   140  			Expect(err).Should(BeNil(), "Cannot get service %s", hubbleRelayService)
   141  			Expect(net.ParseIP(hubbleRelayIP) != nil).Should(BeTrue(), "hubbleRelayIP is not an IP")
   142  			hubbleRelayAddress = net.JoinHostPort(hubbleRelayIP, strconv.Itoa(hubbleRelayPort))
   143  
   144  			namespaceForTest = helpers.GenerateNamespaceForTest("")
   145  			kubectl.NamespaceDelete(namespaceForTest)
   146  			res := kubectl.NamespaceCreate(namespaceForTest)
   147  			res.ExpectSuccess("could not create namespace")
   148  
   149  			res = kubectl.Apply(helpers.ApplyOptions{FilePath: demoPath, Namespace: namespaceForTest})
   150  			res.ExpectSuccess("could not create resource")
   151  
   152  			err = kubectl.WaitforPods(namespaceForTest, "-l zgroup=testapp", helpers.HelperTimeout)
   153  			Expect(err).Should(BeNil(), "test pods are not ready after timeout")
   154  
   155  			appPods = helpers.GetAppPods(apps, namespaceForTest, kubectl, "id")
   156  			app1ClusterIP, app1Port, err = kubectl.GetServiceHostPort(namespaceForTest, app1Service)
   157  			Expect(err).To(BeNil(), "unable to find service in %q namespace", namespaceForTest)
   158  		})
   159  
   160  		AfterFailed(func() {
   161  			kubectl.CiliumReport("cilium-dbg endpoint list")
   162  		})
   163  
   164  		JustAfterEach(func() {
   165  			kubectl.ValidateNoErrorsInLogs(CurrentGinkgoTestDescription().Duration)
   166  		})
   167  
   168  		AfterAll(func() {
   169  			kubectl.Delete(demoPath)
   170  			kubectl.NamespaceDelete(namespaceForTest)
   171  			ExpectAllPodsTerminated(kubectl)
   172  
   173  			kubectl.DeleteHubbleRelay(hubbleRelayNamespace)
   174  			UninstallCiliumFromManifest(kubectl, ciliumFilename)
   175  			kubectl.CloseSSHClient()
   176  		})
   177  
   178  		It("Test L3/L4 Flow", func() {
   179  			ctx, cancel := context.WithTimeout(context.Background(), helpers.MidCommandTimeout)
   180  			defer cancel()
   181  			follow, err := kubectl.HubbleObserveFollow(ctx, ciliumPodK8s1, fmt.Sprintf(
   182  				"--last 1 --type trace --from-pod %s/%s --to-namespace %s --to-label %s --to-port %d",
   183  				namespaceForTest, appPods[helpers.App2], namespaceForTest, app1Labels, app1Port))
   184  			Expect(err).To(BeNil(), "Failed to start hubble observe")
   185  
   186  			res := kubectl.ExecPodCmd(namespaceForTest, appPods[helpers.App2],
   187  				helpers.CurlFail(fmt.Sprintf("http://%s/public", app1ClusterIP)))
   188  			res.ExpectSuccess("%q cannot curl clusterIP %q", appPods[helpers.App2], app1ClusterIP)
   189  
   190  			err = follow.WaitUntilMatchFilterLineTimeout(`{$.flow.Type}`, "L3_L4", helpers.ShortCommandTimeout)
   191  			Expect(err).To(BeNil(), fmt.Sprintf("hubble observe query timed out on %q", follow.OutputPrettyPrint()))
   192  
   193  			// Basic check for L4 Prometheus metrics.
   194  			_, nodeIP := kubectl.GetNodeInfo(helpers.K8s1)
   195  			metricsUrl := fmt.Sprintf("%s/metrics", net.JoinHostPort(nodeIP, prometheusPort))
   196  			res = kubectl.ExecInHostNetNS(ctx, k8s1NodeName, helpers.CurlFail(metricsUrl))
   197  			res.ExpectSuccess("%s/%s cannot curl metrics %q", helpers.CiliumNamespace, ciliumPodK8s1, app1ClusterIP)
   198  			res.ExpectContains(`hubble_flows_processed_total{protocol="TCP",subtype="to-endpoint",type="Trace",verdict="FORWARDED"}`)
   199  		})
   200  
   201  		It("Test TLS certificate", func() {
   202  			certpath := "/var/lib/cilium/tls/hubble/server.crt"
   203  			res := kubectl.ExecPodCmd(helpers.CiliumNamespace, ciliumPodK8s1, helpers.ReadFile(certpath))
   204  			res.ExpectSuccess("Cilium pod cannot read the hubble server TLS certificate")
   205  			expected := string(res.GetStdOut().Bytes())
   206  
   207  			serverName := fmt.Sprintf("%s.default.hubble-grpc.cilium.io", helpers.K8s1)
   208  			cmd := helpers.OpenSSLShowCerts("localhost", defaults.ServerPort, serverName)
   209  			res = kubectl.ExecPodCmd(helpers.CiliumNamespace, ciliumPodK8s1, cmd)
   210  			res.ExpectSuccess("Cilium pod cannot initiate TLS handshake to Hubble")
   211  			cert := string(res.GetStdOut().Bytes())
   212  
   213  			Expect(cert).To(Equal(expected))
   214  		})
   215  
   216  		It("Test L3/L4 Flow with hubble-relay", func() {
   217  			res := kubectl.ExecPodCmd(namespaceForTest, appPods[helpers.App2],
   218  				helpers.CurlFail(fmt.Sprintf("http://%s/public", app1ClusterIP)))
   219  			res.ExpectSuccess("%q cannot curl clusterIP %q", appPods[helpers.App2], app1ClusterIP)
   220  
   221  			flows := getFlowsFromRelay(fmt.Sprintf(
   222  				"--last 1 --type trace --from-pod %s/%s --to-namespace %s --to-label %s --to-port %d",
   223  				namespaceForTest, appPods[helpers.App2], namespaceForTest, app1Labels, app1Port))
   224  			Expect(flows).NotTo(BeEmpty())
   225  		})
   226  
   227  		It("Test L7 Flow", func() {
   228  			defer removeVisibilityAnnotation(namespaceForTest, app1Labels)
   229  			addVisibilityAnnotation(namespaceForTest, app1Labels, "Ingress", "80", "TCP", "HTTP")
   230  
   231  			ctx, cancel := context.WithTimeout(context.Background(), helpers.MidCommandTimeout)
   232  			defer cancel()
   233  			follow, err := kubectl.HubbleObserveFollow(ctx, ciliumPodK8s1, fmt.Sprintf(
   234  				"--last 1 --type l7 --from-pod %s/%s --to-namespace %s --to-label %s --protocol http",
   235  				namespaceForTest, appPods[helpers.App2], namespaceForTest, app1Labels))
   236  			Expect(err).To(BeNil(), "Failed to start hubble observe")
   237  
   238  			res := kubectl.ExecPodCmd(namespaceForTest, appPods[helpers.App2],
   239  				helpers.CurlFail(fmt.Sprintf("http://%s/public", app1ClusterIP)))
   240  			res.ExpectSuccess("%q cannot curl clusterIP %q", appPods[helpers.App2], app1ClusterIP)
   241  
   242  			err = follow.WaitUntilMatchFilterLineTimeout(`{$.flow.Type}`, "L7", helpers.ShortCommandTimeout)
   243  			Expect(err).To(BeNil(), fmt.Sprintf("hubble observe query timed out on %q", follow.OutputPrettyPrint()))
   244  
   245  			// Basic check for L7 Prometheus metrics.
   246  			_, nodeIP := kubectl.GetNodeInfo(helpers.K8s1)
   247  			metricsUrl := fmt.Sprintf("%s/metrics", net.JoinHostPort(nodeIP, prometheusPort))
   248  			res = kubectl.ExecInHostNetNS(ctx, k8s1NodeName, helpers.CurlFail(metricsUrl))
   249  			res.ExpectSuccess("%s/%s cannot curl metrics %q", helpers.CiliumNamespace, ciliumPodK8s1, app1ClusterIP)
   250  			res.ExpectContains(`hubble_flows_processed_total{protocol="HTTP",subtype="HTTP",type="L7",verdict="FORWARDED"}`)
   251  		})
   252  
   253  		It("Test L7 Flow with hubble-relay", func() {
   254  			defer removeVisibilityAnnotation(namespaceForTest, app1Labels)
   255  			addVisibilityAnnotation(namespaceForTest, app1Labels, "Ingress", "80", "TCP", "HTTP")
   256  
   257  			res := kubectl.ExecPodCmd(namespaceForTest, appPods[helpers.App2],
   258  				helpers.CurlFail(fmt.Sprintf("http://%s/public", app1ClusterIP)))
   259  			res.ExpectSuccess("%q cannot curl clusterIP %q", appPods[helpers.App2], app1ClusterIP)
   260  
   261  			flows := getFlowsFromRelay(fmt.Sprintf(
   262  				"--last 1 --type l7 --from-pod %s/%s --to-namespace %s --to-label %s --protocol http",
   263  				namespaceForTest, appPods[helpers.App2], namespaceForTest, app1Labels))
   264  			Expect(flows).NotTo(BeEmpty())
   265  		})
   266  
   267  		It("Test FQDN Policy with Relay", func() {
   268  			fqdnProxyPolicy := helpers.ManifestGet(kubectl.BasePath(), "fqdn-proxy-policy.yaml")
   269  			fqdnTarget := "vagrant-cache.ci.cilium.io"
   270  
   271  			_, err := kubectl.CiliumPolicyAction(
   272  				namespaceForTest, fqdnProxyPolicy,
   273  				helpers.KubectlApply, helpers.HelperTimeout)
   274  			Expect(err).To(BeNil(), "Cannot install fqdn proxy policy")
   275  			defer kubectl.CiliumPolicyAction(namespaceForTest, fqdnProxyPolicy,
   276  				helpers.KubectlDelete, helpers.HelperTimeout)
   277  
   278  			res := kubectl.ExecPodCmd(namespaceForTest, appPods[helpers.App2],
   279  				helpers.CurlFail(fmt.Sprintf("http://%s", fqdnTarget)))
   280  			res.ExpectSuccess("%q cannot curl fqdn target %q", appPods[helpers.App2], fqdnTarget)
   281  
   282  			flows := getFlowsFromRelay(fmt.Sprintf(
   283  				"--last 1 --type trace:from-endpoint --from-pod %s/%s --to-fqdn %s",
   284  				namespaceForTest, appPods[helpers.App2], fqdnTarget))
   285  			Expect(flows).To(HaveLen(1))
   286  			Expect(flows[0].GetFlow().Destination.Identity).To(BeNumerically(">=", identity.MinimalNumericIdentity))
   287  		})
   288  	})
   289  })