github.com/cilium/cilium@v1.16.2/test/k8s/net_policies.go (about) 1 // SPDX-License-Identifier: Apache-2.0 2 // Copyright Authors of Cilium 3 4 package k8sTest 5 6 import ( 7 "context" 8 "fmt" 9 "regexp" 10 "strings" 11 "sync" 12 "time" 13 14 "github.com/google/uuid" 15 . "github.com/onsi/gomega" 16 "github.com/onsi/gomega/types" 17 "github.com/sirupsen/logrus" 18 v1 "k8s.io/api/core/v1" 19 20 "github.com/cilium/cilium/api/v1/models" 21 "github.com/cilium/cilium/pkg/annotation" 22 "github.com/cilium/cilium/pkg/policy" 23 . "github.com/cilium/cilium/test/ginkgo-ext" 24 "github.com/cilium/cilium/test/helpers" 25 ) 26 27 var _ = SkipDescribeIf(func() bool { 28 // We only need to run on 4.19 with kube-proxy and net-next with KPR 29 // and the third node. Other CI jobs are not expected to increase 30 // code coverage. 31 // 32 // For GKE coverage, see the K8sPolicyTestExtended Describe block below. 33 return helpers.RunsOnGKE() || helpers.RunsOn54Kernel() || helpers.RunsOnAKS() 34 }, "K8sAgentPolicyTest", func() { 35 36 var ( 37 kubectl *helpers.Kubectl 38 39 // these are set in BeforeAll() 40 ciliumFilename string 41 demoPath string 42 l3Policy string 43 l7Policy string 44 l7PolicyDefAllow string 45 connectivityCheckYml string 46 47 app1Service = "app1-service" 48 apps = []string{helpers.App1, helpers.App2, helpers.App3} 49 daemonCfg map[string]string 50 ) 51 52 BeforeAll(func() { 53 kubectl = helpers.CreateKubectl(helpers.K8s1VMName(), logger) 54 55 demoPath = helpers.ManifestGet(kubectl.BasePath(), "demo-named-port.yaml") 56 l3Policy = helpers.ManifestGet(kubectl.BasePath(), "l3-l4-policy.yaml") 57 l7Policy = helpers.ManifestGet(kubectl.BasePath(), "l7-policy.yaml") 58 l7PolicyDefAllow = helpers.ManifestGet(kubectl.BasePath(), "l7-policy-allow.yaml") 59 connectivityCheckYml = kubectl.GetFilePath("../examples/kubernetes/connectivity-check/connectivity-check-proxy.yaml") 60 61 daemonCfg = map[string]string{ 62 "tls.secretsBackend": "k8s", 63 "debug.verbose": "envoy", 64 } 65 ciliumFilename = helpers.TimestampFilename("cilium.yaml") 66 DeployCiliumOptionsAndDNS(kubectl, ciliumFilename, daemonCfg) 67 }) 68 69 AfterFailed(func() { 70 kubectl.CiliumReport("cilium-dbg service list", "cilium-dbg endpoint list") 71 }) 72 73 AfterAll(func() { 74 UninstallCiliumFromManifest(kubectl, ciliumFilename) 75 kubectl.CloseSSHClient() 76 }) 77 78 JustAfterEach(func() { 79 kubectl.ValidateNoErrorsInLogs(CurrentGinkgoTestDescription().Duration) 80 }) 81 82 Context("Basic Test", func() { 83 var ( 84 ciliumPod string 85 clusterIP string 86 namespaceForTest string 87 ) 88 89 BeforeAll(func() { 90 namespaceForTest = helpers.GenerateNamespaceForTest("") 91 kubectl.NamespaceDelete(namespaceForTest) 92 kubectl.NamespaceCreate(namespaceForTest).ExpectSuccess("could not create namespace") 93 kubectl.Apply(helpers.ApplyOptions{FilePath: demoPath, Namespace: namespaceForTest}).ExpectSuccess("could not create resource") 94 95 err := kubectl.WaitforPods(namespaceForTest, "-l zgroup=testapp", helpers.HelperTimeout) 96 Expect(err).Should(BeNil(), "Test pods are not ready after timeout") 97 98 ciliumPod, err = kubectl.GetCiliumPodOnNode(helpers.K8s1) 99 Expect(err).Should(BeNil(), "cannot get CiliumPod") 100 101 clusterIP, _, err = kubectl.GetServiceHostPort(namespaceForTest, app1Service) 102 Expect(err).To(BeNil(), "Cannot get service in %q namespace", namespaceForTest) 103 logger.WithFields(logrus.Fields{ 104 "ciliumPod": ciliumPod, 105 "clusterIP": clusterIP}).Info("Initial data") 106 107 }) 108 109 AfterAll(func() { 110 kubectl.NamespaceDelete(namespaceForTest) 111 kubectl.Delete(demoPath) 112 ExpectAllPodsTerminated(kubectl) 113 }) 114 115 BeforeEach(func() { 116 kubectl.CiliumExecMustSucceed(context.TODO(), 117 ciliumPod, fmt.Sprintf("cilium-dbg config %s=%s", 118 helpers.PolicyEnforcement, helpers.PolicyEnforcementDefault)) 119 120 err := kubectl.CiliumEndpointWaitReady() 121 Expect(err).To(BeNil(), "Endpoints are not ready after timeout") 122 123 err = kubectl.WaitforPods(namespaceForTest, "-l zgroup=testapp", helpers.HelperTimeout) 124 Expect(err).Should(BeNil()) 125 126 }) 127 128 AfterEach(func() { 129 cmd := fmt.Sprintf("%s delete --all cnp,ccnp,netpol -n %s", helpers.KubectlCmd, namespaceForTest) 130 _ = kubectl.Exec(cmd) 131 }) 132 133 // Tests involving the L7 proxy do not work when built with -race, see issue #13757. 134 SkipContextIf(helpers.SkipRaceDetectorEnabled, "Traffic redirections to proxy", func() { 135 var ( 136 // track which app1 pod we care about, and its corresponding 137 // cilium pod. 138 app1Pod string 139 app2Pod string 140 ciliumPod string 141 nodeName string 142 appPods map[string]string 143 app1PodIP string 144 worldTarget = "http://vagrant-cache.ci.cilium.io" 145 ) 146 147 BeforeAll(func() { 148 appPods = helpers.GetAppPods(apps, namespaceForTest, kubectl, "id") 149 podsNodes, err := kubectl.GetPodsNodes(namespaceForTest, "id=app1") 150 Expect(err).To(BeNil(), "error getting pod->node mapping") 151 Expect(len(podsNodes)).To(Equal(2)) 152 // Just grab the first one. 153 for k, v := range podsNodes { 154 app1Pod = k 155 nodeName = v 156 break 157 } 158 159 podsNodes, err = kubectl.GetPodsNodes(namespaceForTest, "id=app2") 160 Expect(err).To(BeNil(), "error getting pod->node mapping") 161 Expect(len(podsNodes)).To(Equal(1)) 162 for k := range podsNodes { 163 app2Pod = k 164 break 165 } 166 167 Expect(kubectl.WaitforPods(namespaceForTest, "-l zgroup=testapp", helpers.HelperTimeout)).To(BeNil()) 168 var podList v1.PodList 169 err = kubectl.GetPods(namespaceForTest, fmt.Sprintf("-n %s -l k8s-app=cilium --field-selector spec.nodeName=%s", helpers.CiliumNamespace, nodeName)).Unmarshal(&podList) 170 Expect(err).To(BeNil()) 171 172 var app1PodModel v1.Pod 173 Expect(kubectl.Exec(fmt.Sprintf("%s get pod -n %s %s -o json", helpers.KubectlCmd, namespaceForTest, app1Pod)).Unmarshal(&app1PodModel)).To(BeNil()) 174 Expect(app1PodModel).ToNot(BeNil()) 175 Expect(len(podList.Items)).To(Equal(1)) 176 ciliumPod = podList.Items[0].Name 177 app1PodIP = app1PodModel.Status.PodIP 178 //var app1Ep *models.Endpoint 179 var endpoints []*models.Endpoint 180 err = kubectl.ExecPodCmd(helpers.CiliumNamespace, ciliumPod, "cilium-dbg endpoint list -o json").Unmarshal(&endpoints) 181 Expect(err).To(BeNil()) 182 for _, ep := range endpoints { 183 if ep.Status.Networking.Addressing[0].IPV4 == app1PodIP { 184 break 185 } 186 } 187 }) 188 189 AfterEach(func() { 190 // Remove the proxy visibility annotation - this is done by specifying the annotation followed by a '-'. 191 kubectl.Exec(fmt.Sprintf("%s annotate pod %s -n %s %s-", helpers.KubectlCmd, appPods[helpers.App1], namespaceForTest, annotation.ProxyVisibility)) 192 kubectl.Exec(fmt.Sprintf("%s annotate pod %s -n %s %s-", helpers.KubectlCmd, appPods[helpers.App2], namespaceForTest, annotation.ProxyVisibility)) 193 cmd := fmt.Sprintf("%s delete --all cnp,ccnp,netpol -n %s", helpers.KubectlCmd, namespaceForTest) 194 _ = kubectl.Exec(cmd) 195 }) 196 197 checkProxyRedirection := func(resource string, redirected bool, parser policy.L7ParserType, retryCurl bool) { 198 var ( 199 not = " " 200 filter string // jsonpath filter 201 expect string // expected result 202 curlCmd string 203 hubbleTimeout = 10 * time.Second 204 ) 205 206 if !redirected { 207 not = " not " 208 } 209 210 switch parser { 211 case policy.ParserTypeDNS: 212 // response DNS L7 flow 213 filter = "{.flow.destination.namespace} {.flow.l7.type} {.flow.l7.dns.query}" 214 expect = fmt.Sprintf( 215 "%s RESPONSE %s", 216 namespaceForTest, 217 "vagrant-cache.ci.cilium.io.", 218 ) 219 if retryCurl { 220 curlCmd = helpers.CurlWithRetries(resource, 5, true) 221 } else { 222 curlCmd = helpers.CurlFail(resource) 223 } 224 case policy.ParserTypeHTTP: 225 filter = "{.flow.destination.namespace} {.flow.l7.type} {.flow.l7.http.url} {.flow.l7.http.code} {.flow.l7.http.method}" 226 expect = fmt.Sprintf( 227 "%s RESPONSE %s 200 GET", 228 namespaceForTest, 229 fmt.Sprintf("http://%s/public", resource), 230 ) 231 232 if retryCurl { 233 curlCmd = helpers.CurlWithRetries(fmt.Sprintf("http://%s/public", resource), 5, true) 234 } else { 235 curlCmd = helpers.CurlFail(fmt.Sprintf("http://%s/public", resource)) 236 } 237 default: 238 Fail(fmt.Sprintf("invalid parser type for proxy visibility: %s", parser)) 239 } 240 241 observeFile := fmt.Sprintf("hubble-observe-%s", uuid.New().String()) 242 243 // curl commands are issued from the first k8s worker where all 244 // the app instances are running 245 By("Starting hubble observe and generating traffic which should%s redirect to proxy", not) 246 ctx, cancel := context.WithCancel(context.Background()) 247 hubbleRes, err := kubectl.HubbleObserveFollow( 248 ctx, ciliumPod, 249 // since 0s is important here so no historic events from the 250 // buffer are shown, only follow from the current time 251 "--type l7 --since 0s", 252 ) 253 Expect(err).To(BeNil(), "Failed to start hubble observe") 254 255 // clean up at the end of the test 256 defer func() { 257 cancel() 258 hubbleRes.WaitUntilFinish() 259 helpers.WriteToReportFile(hubbleRes.CombineOutput().Bytes(), observeFile) 260 }() 261 262 // Let the monitor get started since it is started in the background. 263 res := kubectl.ExecPodCmd( 264 namespaceForTest, appPods[helpers.App2], 265 curlCmd) 266 // Give time for the monitor to be notified of the proxy flow. 267 time.Sleep(2 * time.Second) 268 res.ExpectSuccess("%q cannot curl %q", appPods[helpers.App2], resource) 269 270 By("Checking that aforementioned traffic was%sredirected to the proxy", not) 271 err = hubbleRes.WaitUntilMatchFilterLineTimeout(filter, expect, hubbleTimeout) 272 if redirected { 273 ExpectWithOffset(1, err).To(BeNil(), "traffic was not redirected to the proxy when it should have been") 274 } else { 275 ExpectWithOffset(1, err).ToNot(BeNil(), "traffic was redirected to the proxy when it should have not been redirected") 276 } 277 278 if parser == policy.ParserTypeDNS && redirected { 279 By("Checking that Hubble is correctly annotating the DNS names") 280 res := kubectl.HubbleObserve(ciliumPod, 281 fmt.Sprintf("--last 1 --from-pod %s/%s --to-fqdn %q", 282 namespaceForTest, appPods[helpers.App2], "*.cilium.io")) 283 res.ExpectContainsFilterLine("{.flow.destination_names[0]}", "vagrant-cache.ci.cilium.io") 284 } 285 } 286 287 proxyVisibilityTest := func(resource, podToAnnotate, anno string, parserType policy.L7ParserType, retryCurl bool) { 288 checkProxyRedirection(resource, false, parserType, retryCurl) 289 290 By("Annotating %s with %s", podToAnnotate, anno) 291 res := kubectl.Exec(fmt.Sprintf("%s annotate pod %s -n %s %s=\"%s\"", helpers.KubectlCmd, podToAnnotate, namespaceForTest, annotation.ProxyVisibility, anno)) 292 res.ExpectSuccess("annotating pod with proxy visibility annotation failed") 293 Expect(kubectl.CiliumEndpointWaitReady()).To(BeNil()) 294 295 checkProxyRedirection(resource, true, parserType, retryCurl) 296 297 By("Removing proxy visibility annotation on %s", podToAnnotate) 298 kubectl.Exec(fmt.Sprintf("%s annotate pod %s -n %s %s-", helpers.KubectlCmd, podToAnnotate, namespaceForTest, annotation.ProxyVisibility)).ExpectSuccess() 299 Expect(kubectl.CiliumEndpointWaitReady()).To(BeNil()) 300 301 checkProxyRedirection(resource, false, parserType, retryCurl) 302 } 303 304 It("Tests HTTP proxy visibility without policy", func() { 305 proxyVisibilityTest(app1PodIP, app1Pod, "<Ingress/80/TCP/HTTP>", policy.ParserTypeHTTP, false) 306 }) 307 308 It("Tests DNS proxy visibility without policy", func() { 309 proxyVisibilityTest(worldTarget, app2Pod, "<Egress/53/UDP/DNS>", policy.ParserTypeDNS, true) 310 }) 311 312 It("Tests proxy visibility interactions with policy lifecycle operations", func() { 313 checkProxyRedirection(app1PodIP, false, policy.ParserTypeHTTP, false) 314 315 By("Annotating %s with <Ingress/80/TCP/HTTP>", app1Pod) 316 res := kubectl.Exec(fmt.Sprintf("%s annotate pod %s -n %s %s=\"<Ingress/80/TCP/HTTP>\"", helpers.KubectlCmd, app1Pod, namespaceForTest, annotation.ProxyVisibility)) 317 res.ExpectSuccess("annotating pod with proxy visibility annotation failed") 318 Expect(kubectl.CiliumEndpointWaitReady()).To(BeNil()) 319 320 checkProxyRedirection(app1PodIP, true, policy.ParserTypeHTTP, false) 321 322 By("Importing policy which selects app1") 323 324 _, err := kubectl.CiliumPolicyAction( 325 namespaceForTest, l3Policy, helpers.KubectlApply, helpers.HelperTimeout) 326 Expect(err).Should(BeNil(), 327 "policy %s cannot be applied in %q namespace", l3Policy, namespaceForTest) 328 329 By("Checking that proxy visibility annotation is still applied even while a policy was imported") 330 checkProxyRedirection(app1PodIP, true, policy.ParserTypeHTTP, false) 331 332 _, err = kubectl.CiliumPolicyAction( 333 namespaceForTest, l3Policy, helpers.KubectlDelete, helpers.HelperTimeout) 334 Expect(err).Should(BeNil(), 335 "policy %s cannot be deleted in %q namespace", l3Policy, namespaceForTest) 336 337 By("Checking that proxy visibility annotation is still applied after policy is removed") 338 checkProxyRedirection(app1PodIP, true, policy.ParserTypeHTTP, false) 339 340 By("Importing policy using named ports which selects app1; proxy-visibility annotation should remain") 341 }) 342 343 It("Tests proxy visibility with L7 rules", func() { 344 By("Creating a l7 policy for the pod") 345 _, err := kubectl.CiliumPolicyAction( 346 namespaceForTest, l7Policy, helpers.KubectlApply, helpers.HelperTimeout) 347 Expect(err).Should(BeNil(), 348 "policy %s cannot be applied in %q namespace", l3Policy, namespaceForTest) 349 350 By("Checking that traffic is proxied") 351 checkProxyRedirection(app1PodIP, true, policy.ParserTypeHTTP, false) 352 353 By("Checking that ping is blocked") 354 res := kubectl.ExecPodCmd( 355 namespaceForTest, appPods[helpers.App2], 356 helpers.Ping(app1PodIP)) 357 res.ExpectFail("Ingrress ping connectivity should be denied for pod %q", helpers.App2) 358 }) 359 360 It("Tests proxy visibility with L7 default-allow rules", func() { 361 By("Creating a l7 policy with default-allow for the pod") 362 _, err := kubectl.CiliumPolicyAction( 363 namespaceForTest, l7PolicyDefAllow, helpers.KubectlApply, helpers.HelperTimeout) 364 Expect(err).Should(BeNil(), 365 "policy %s cannot be applied in %q namespace", l3Policy, namespaceForTest) 366 367 By("Checking that traffic is proxied") 368 checkProxyRedirection(app1PodIP, true, policy.ParserTypeHTTP, false) 369 370 By("Checking that ping is allowed") 371 res := kubectl.ExecPodCmd( 372 namespaceForTest, appPods[helpers.App2], 373 helpers.Ping(app1PodIP)) 374 res.ExpectSuccess("Ingrress ping connectivity should be allowed for pod %q", helpers.App2) 375 376 }) 377 }) 378 }) 379 380 Context("Multi-node policy test", func() { 381 const ( 382 testDS = "zgroup=testDS" 383 384 // This currently matches GetPodOnNodeWithOffset(). 385 testNamespace = helpers.DefaultNamespace 386 ) 387 var demoYAML string 388 389 BeforeAll(func() { 390 By("Deploying demo daemonset") 391 demoYAML = helpers.ManifestGet(kubectl.BasePath(), "demo_ds.yaml") 392 res := kubectl.ApplyDefault(demoYAML) 393 res.ExpectSuccess("Unable to apply %s", demoYAML) 394 395 err := kubectl.WaitforPods(testNamespace, fmt.Sprintf("-l %s", testDS), helpers.HelperTimeout) 396 Expect(err).Should(BeNil()) 397 }) 398 399 AfterAll(func() { 400 // Explicitly ignore result of deletion of resources to 401 // avoid incomplete teardown if any step fails. 402 _ = kubectl.Delete(demoYAML) 403 ExpectAllPodsTerminated(kubectl) 404 }) 405 406 AfterEach(func() { 407 By("Cleaning up after the test") 408 cmd := fmt.Sprintf("%s delete --all cnp,ccnp,netpol -n %s", helpers.KubectlCmd, testNamespace) 409 _ = kubectl.Exec(cmd) 410 411 By("Checking for pending maps") 412 kubectl.CiliumExecMustSucceedOnAll(context.Background(), "sh -c '! ls /sys/fs/bpf/tc/globals/*:pending'") 413 }) 414 415 SkipContextIf(helpers.DoesNotExistNodeWithoutCilium, "validates ingress CIDR-dependent L4", func() { 416 var ( 417 outsideNodeName, outsideIP string // k8s3 node (doesn't have agent running) 418 419 backendPod v1.Pod // The pod that k8s3 node is hitting 420 backendPodIP string 421 422 hostNodeName string // Node that backendPod ends up on 423 hostIPOfBackendPod string 424 425 policyVerdictAllowRegex, policyVerdictDenyRegex *regexp.Regexp 426 ) 427 428 BeforeAll(func() { 429 RedeployCiliumWithMerge(kubectl, ciliumFilename, daemonCfg, 430 map[string]string{ 431 "routingMode": "native", 432 "autoDirectNodeRoutes": "true", 433 434 "hostFirewall.enabled": "true", 435 }) 436 437 By("Retrieving backend pod and outside node IP addresses") 438 outsideNodeName, outsideIP = kubectl.GetNodeInfo(kubectl.GetFirstNodeWithoutCiliumLabel()) 439 440 var demoPods v1.PodList 441 kubectl.GetPods(testNamespace, fmt.Sprintf("-l %s", testDS)).Unmarshal(&demoPods) 442 Expect(demoPods.Items).To(HaveLen(2)) 443 444 backendPod = demoPods.Items[0] // We'll take the first one; doesn't matter 445 backendPodIP = backendPod.Status.PodIP 446 hostIPOfBackendPod = backendPod.Status.HostIP 447 hostNodeName = backendPod.Spec.NodeName // Save the name of node backend pod is on 448 449 By("Adding a static route to %s on the %s node (outside)", 450 backendPodIP, outsideNodeName) 451 // Add the route on the outside node to the backend pod IP 452 // directly. The reason for this is to avoid NATing when using 453 // K8s Services, for the sake of simplicity. Making the backend 454 // pod IP directly routable on the "outside" node is sufficient 455 // to validate the policy under test. 456 res := kubectl.AddIPRoute(outsideNodeName, backendPodIP, hostIPOfBackendPod, false) 457 Expect(res).To(getMatcher(true)) 458 459 policyVerdictAllowRegex = regexp.MustCompile( 460 fmt.Sprintf("Policy verdict log: .+action allow.+%s:[0-9]+ -> %s:80 tcp SYN", 461 outsideIP, backendPodIP)) 462 policyVerdictDenyRegex = regexp.MustCompile( 463 fmt.Sprintf("Policy verdict log: .+action deny.+%s:[0-9]+ -> %s:80 tcp SYN", 464 outsideIP, backendPodIP)) 465 }) 466 467 AfterAll(func() { 468 // Remove the route on the outside node. 469 kubectl.DelIPRoute(outsideNodeName, backendPodIP, hostIPOfBackendPod) 470 471 // Revert Cilium installation back to before this Context. 472 By("Redeploying Cilium with default configuration") 473 RedeployCilium(kubectl, ciliumFilename, daemonCfg) 474 }) 475 476 testConnectivity := func(dstIP string, expectSuccess bool) int { 477 action := "allowed" 478 if !expectSuccess { 479 action = "denied" 480 } 481 By("Testing that connectivity from outside node is %s", action) 482 483 var count int 484 ConsistentlyWithOffset(1, func() bool { 485 res := kubectl.ExecInHostNetNS( 486 context.TODO(), 487 outsideNodeName, 488 helpers.CurlFail("http://%s:%d", dstIP, 80), 489 ) 490 // We want to count the number of attempts that achieved 491 // their expected result, so we can assert on how many 492 // policy verdict logs we should expect from `cilium 493 // monitor`. 494 if res.WasSuccessful() == expectSuccess { 495 count++ 496 } 497 return res.WasSuccessful() 498 }, helpers.ShortCommandTimeout).Should(Equal(expectSuccess), 499 "Connectivity was expected to be %s consistently", action) 500 501 return count 502 } 503 504 It("connectivity works from the outside before any policies", func() { 505 // Ignore the return because we don't care about `cilium 506 // monitor` output in this test. 507 _ = testConnectivity(backendPodIP, true) 508 }) 509 510 It("connectivity is blocked after denying ingress", func() { 511 By("Running cilium-dbg monitor in the background") 512 ciliumPod, err := kubectl.GetCiliumPodOnNodeByName(hostNodeName) 513 Expect(ciliumPod).ToNot(BeEmpty()) 514 Expect(err).ToNot(HaveOccurred()) 515 516 ep, err := kubectl.GetCiliumEndpoint(testNamespace, backendPod.GetName()) 517 Expect(ep).ToNot(BeNil()) 518 Expect(err).ToNot(HaveOccurred()) 519 520 monitor, monitorCancel := kubectl.MonitorEndpointStart(ciliumPod, ep.ID) 521 522 By("Importing a default deny policy on ingress") 523 cnpDenyIngress := helpers.ManifestGet(kubectl.BasePath(), 524 "cnp-default-deny-ingress.yaml") 525 importPolicy(kubectl, testNamespace, cnpDenyIngress, "default-deny-ingress") 526 527 count := testConnectivity(backendPodIP, false) 528 defer monitorCancel() 529 530 By("Asserting that the expected policy verdict logs are in the monitor output") 531 Eventually(func() int { 532 return len(policyVerdictDenyRegex.FindAll(monitor.CombineOutput().Bytes(), -1)) 533 }).Should(BeNumerically(">=", count), "Monitor output is missing verdicts: %s\n%s", 534 policyVerdictDenyRegex, monitor.CombineOutput().Bytes()) 535 }) 536 537 It("connectivity is restored after importing ingress policy", func() { 538 By("Importing a default deny policy on ingress") 539 cnpDenyIngress := helpers.ManifestGet(kubectl.BasePath(), 540 "cnp-default-deny-ingress.yaml") 541 importPolicy(kubectl, testNamespace, cnpDenyIngress, "default-deny-ingress") 542 543 By("Running cilium-dbg monitor in the background") 544 ciliumPod, err := kubectl.GetCiliumPodOnNodeByName(hostNodeName) 545 Expect(ciliumPod).ToNot(BeEmpty()) 546 Expect(err).ToNot(HaveOccurred()) 547 548 ep, err := kubectl.GetCiliumEndpoint(testNamespace, backendPod.GetName()) 549 Expect(ep).ToNot(BeNil()) 550 Expect(err).ToNot(HaveOccurred()) 551 552 monitor, monitorCancel := kubectl.MonitorEndpointStart(ciliumPod, ep.ID) 553 554 By("Importing fromCIDR+toPorts policy on ingress") 555 556 originalAssignIPYAML := helpers.ManifestGet(kubectl.BasePath(), "cnp-ingress-from-cidr-to-ports.yaml") 557 res := kubectl.ExecMiddle("mktemp") 558 res.ExpectSuccess() 559 cnpAllowIngressWithIP := strings.Trim(res.Stdout(), "\n") 560 nodeIP, err := kubectl.GetNodeIPByLabel(kubectl.GetFirstNodeWithoutCiliumLabel(), false) 561 Expect(err).Should(BeNil()) 562 kubectl.ExecMiddle(fmt.Sprintf("sed 's/NODE_WITHOUT_CILIUM_IP/%s/' %s > %s", 563 nodeIP, originalAssignIPYAML, cnpAllowIngressWithIP)).ExpectSuccess() 564 565 importPolicy(kubectl, testNamespace, cnpAllowIngressWithIP, "ingress-from-cidr-to-ports") 566 count := testConnectivity(backendPodIP, true) 567 defer monitorCancel() 568 569 By("Asserting that the expected policy verdict logs are in the monitor output") 570 Eventually(func() int { 571 return len(policyVerdictAllowRegex.FindAll(monitor.CombineOutput().Bytes(), -1)) 572 }).Should(BeNumerically(">=", count), "Monitor output is missing verdicts: %s\n%s", 573 policyVerdictAllowRegex, monitor.CombineOutput().Bytes()) 574 }) 575 576 Context("With host policy", func() { 577 BeforeAll(func() { 578 // Deploy echoserver pods in host namespace. 579 echoPodPath := helpers.ManifestGet(kubectl.BasePath(), "echoserver-cilium-hostnetns.yaml") 580 kubectl.ApplyDefault(echoPodPath).ExpectSuccess("Cannot install echoserver application") 581 Expect(kubectl.WaitforPods(helpers.DefaultNamespace, "-l name=echoserver-hostnetns", 582 helpers.HelperTimeout)).Should(BeNil()) 583 584 policyVerdictAllowRegex = regexp.MustCompile( 585 fmt.Sprintf("Policy verdict log: .+action allow.+%s:[0-9]+ -> %s:80 tcp SYN", 586 outsideIP, hostIPOfBackendPod)) 587 policyVerdictDenyRegex = regexp.MustCompile( 588 fmt.Sprintf("Policy verdict log: .+action deny.+%s:[0-9]+ -> %s:80 tcp SYN", 589 outsideIP, hostIPOfBackendPod)) 590 }) 591 592 AfterAll(func() { 593 // Remove echoserver pods from host namespace. 594 echoPodPath := helpers.ManifestGet(kubectl.BasePath(), "echoserver-cilium-hostnetns.yaml") 595 kubectl.Delete(echoPodPath).ExpectSuccess("Cannot remove echoserver application") 596 ExpectAllPodsTerminated(kubectl) 597 }) 598 599 It("Connectivity to hostns is blocked after denying ingress", func() { 600 By("Running cilium-dbg monitor in the background") 601 ciliumPod, err := kubectl.GetCiliumPodOnNodeByName(hostNodeName) 602 Expect(ciliumPod).ToNot(BeEmpty()) 603 Expect(err).ToNot(HaveOccurred()) 604 605 hostEpID, err := kubectl.GetCiliumHostEndpointID(ciliumPod) 606 Expect(err).ToNot(HaveOccurred()) 607 608 monitor, monitorCancel := kubectl.MonitorEndpointStart(ciliumPod, hostEpID) 609 610 By("Importing a default-deny host policy on ingress") 611 ccnpDenyHostIngress := helpers.ManifestGet(kubectl.BasePath(), "ccnp-default-deny-host-ingress.yaml") 612 importPolicy(kubectl, testNamespace, ccnpDenyHostIngress, "default-deny-host-ingress") 613 614 testConnectivity(backendPodIP, true) 615 count := testConnectivity(hostIPOfBackendPod, false) 616 defer monitorCancel() 617 618 By("Asserting that the expected policy verdict logs are in the monitor output") 619 Eventually(func() int { 620 return len(policyVerdictDenyRegex.FindAll(monitor.CombineOutput().Bytes(), -1)) 621 }).Should(BeNumerically(">=", count), "Monitor output is missing verdicts: %s\n%s", 622 policyVerdictDenyRegex, monitor.CombineOutput().Bytes()) 623 }) 624 625 It("Connectivity is restored after importing ingress policy", func() { 626 By("Importing a default-deny host policy on ingress") 627 ccnpDenyHostIngress := helpers.ManifestGet(kubectl.BasePath(), "ccnp-default-deny-host-ingress.yaml") 628 importPolicy(kubectl, testNamespace, ccnpDenyHostIngress, "default-deny-host-ingress") 629 630 By("Running cilium-dbg monitor in the background") 631 ciliumPod, err := kubectl.GetCiliumPodOnNodeByName(hostNodeName) 632 Expect(ciliumPod).ToNot(BeEmpty()) 633 Expect(err).ToNot(HaveOccurred()) 634 635 hostEpID, err := kubectl.GetCiliumHostEndpointID(ciliumPod) 636 Expect(err).ToNot(HaveOccurred()) 637 638 monitor, monitorCancel := kubectl.MonitorEndpointStart(ciliumPod, hostEpID) 639 640 By("Importing fromCIDR+toPorts host policy on ingress") 641 originalCCNPAllowHostIngress := helpers.ManifestGet(kubectl.BasePath(), "ccnp-host-ingress-from-cidr-to-ports.yaml") 642 res := kubectl.ExecMiddle("mktemp") 643 res.ExpectSuccess() 644 ccnpAllowIngressWithIP := strings.Trim(res.Stdout(), "\n") 645 nodeIP, err := kubectl.GetNodeIPByLabel(kubectl.GetFirstNodeWithoutCiliumLabel(), false) 646 Expect(err).Should(BeNil()) 647 kubectl.ExecMiddle(fmt.Sprintf("sed 's/NODE_WITHOUT_CILIUM_IP/%s/' %s > %s", 648 nodeIP, originalCCNPAllowHostIngress, ccnpAllowIngressWithIP)).ExpectSuccess() 649 650 importPolicy(kubectl, testNamespace, ccnpAllowIngressWithIP, "host-ingress-from-cidr-to-ports") 651 652 testConnectivity(backendPodIP, true) 653 count := testConnectivity(hostIPOfBackendPod, true) 654 defer monitorCancel() 655 656 By("Asserting that the expected policy verdict logs are in the monitor output") 657 Eventually(func() int { 658 return len(policyVerdictAllowRegex.FindAll(monitor.CombineOutput().Bytes(), -1)) 659 }).Should(BeNumerically(">=", count), "Monitor output is missing verdicts: %s\n%s", 660 policyVerdictAllowRegex, monitor.CombineOutput().Bytes()) 661 662 By("Removing the fromCIDR+toPorts ingress host policy") 663 // This is to ensure this policy is always removed before the default-deny one. 664 // Otherwise, connection to the nodes may be disrupted. 665 cmd := fmt.Sprintf("%s -n %s delete ccnp host-ingress-from-cidr-to-ports", helpers.KubectlCmd, testNamespace) 666 kubectl.Exec(cmd).ExpectSuccess("Failed to delete ccnp/host-ingress-from-cidr-to-ports") 667 }) 668 }) 669 }) 670 671 Context("validates fromEntities policies", func() { 672 const ( 673 HostConnectivityDeny = false 674 HostConnectivityAllow = true 675 RemoteNodeConnectivityDeny = false 676 RemoteNodeConnectivityAllow = true 677 PodConnectivityDeny = false 678 PodConnectivityAllow = true 679 WorldConnectivityDeny = false 680 WorldConnectivityAllow = true 681 ) 682 683 var ( 684 cnpFromEntitiesRemoteNode string 685 cnpFromEntitiesCluster string 686 cnpFromEntitiesAll string 687 688 k8s1Name string 689 k8s1IP string 690 k8s1PodName string 691 k8s1PodIP, k8s2PodIP string 692 693 outsideNodeName string 694 ) 695 696 BeforeAll(func() { 697 cnpFromEntitiesRemoteNode = helpers.ManifestGet(kubectl.BasePath(), "cnp-from-entities-remote-node.yaml") 698 cnpFromEntitiesCluster = helpers.ManifestGet(kubectl.BasePath(), "cnp-from-entities-cluster.yaml") 699 cnpFromEntitiesAll = helpers.ManifestGet(kubectl.BasePath(), "cnp-from-entities-all.yaml") 700 701 k8s1Name, k8s1IP = kubectl.GetNodeInfo(helpers.K8s1) 702 k8s1PodName, k8s1PodIP = kubectl.GetPodOnNodeLabeledWithOffset(helpers.K8s1, testDS, 0) 703 _, k8s2PodIP = kubectl.GetPodOnNodeLabeledWithOffset(helpers.K8s2, testDS, 0) 704 705 if helpers.ExistNodeWithoutCilium() { 706 outsideNodeName, _ = kubectl.GetNodeInfo(kubectl.GetFirstNodeWithoutCiliumLabel()) 707 } 708 709 // Masquerade function should be disabled 710 // because the request will fail if the reply packet's source address is rewritten 711 // when sending a request directly to the Pod from outside the cluster. 712 By("Reconfiguring Cilium to disable masquerade") 713 RedeployCiliumWithMerge(kubectl, ciliumFilename, daemonCfg, 714 map[string]string{ 715 "enableIPv4Masquerade": "false", 716 "enableIPv6Masquerade": "false", 717 "bpf.masquerade": "false", 718 }) 719 720 }) 721 722 AfterAll(func() { 723 By("Redeploying Cilium with default configuration") 724 RedeployCilium(kubectl, ciliumFilename, daemonCfg) 725 }) 726 727 validateConnectivity := func(expectHostSuccess, expectRemoteNodeSuccess, expectPodSuccess, expectWorldSuccess bool) { 728 var wg sync.WaitGroup 729 wg.Add(1) 730 go func() { 731 defer GinkgoRecover() 732 defer wg.Done() 733 By("Checking ingress connectivity from k8s1 node to k8s1 pod (host)") 734 res := kubectl.ExecInHostNetNS(context.TODO(), k8s1Name, 735 helpers.CurlFail(k8s1PodIP)) 736 ExpectWithOffset(1, res).To(getMatcher(expectHostSuccess), 737 "HTTP ingress connectivity to pod %q from local host", k8s1PodIP) 738 }() 739 740 wg.Add(1) 741 go func() { 742 defer GinkgoRecover() 743 defer wg.Done() 744 By("Checking ingress connectivity from k8s1 node to k8s2 pod (remote-node)") 745 res := kubectl.ExecInHostNetNS(context.TODO(), k8s1Name, 746 helpers.CurlFail(k8s2PodIP)) 747 ExpectWithOffset(1, res).To(getMatcher(expectRemoteNodeSuccess), 748 "HTTP ingress connectivity to pod %q from remote node", k8s2PodIP) 749 }() 750 751 wg.Add(1) 752 go func() { 753 defer GinkgoRecover() 754 defer wg.Done() 755 By("Checking ingress connectivity from k8s1 pod to k8s2 pod") 756 res := kubectl.ExecPodCmd(testNamespace, k8s1PodName, helpers.CurlFail(k8s2PodIP)) 757 ExpectWithOffset(1, res).To(getMatcher(expectPodSuccess), 758 "HTTP ingress connectivity to pod %q from pod %q", k8s2PodIP, k8s1PodIP) 759 }() 760 761 if helpers.ExistNodeWithoutCilium() { 762 wg.Add(1) 763 go func() { 764 defer GinkgoRecover() 765 defer wg.Done() 766 By("Checking ingress connectivity from world to k8s1 pod") 767 By("Adding a static route to %s via %s on the %s node (outside)", k8s1PodIP, k8s1IP, outsideNodeName) 768 res := kubectl.AddIPRoute(outsideNodeName, k8s1PodIP, k8s1IP, false) 769 Expect(res).To(getMatcher(true)) 770 defer func() { 771 kubectl.DelIPRoute(outsideNodeName, k8s1PodIP, k8s1IP).ExpectSuccess("Failed to del ip route") 772 }() 773 774 if expectWorldSuccess { 775 testCurlFromOutside(kubectl, &helpers.NodesInfo{ 776 OutsideNodeName: outsideNodeName, 777 }, k8s1PodIP, 1, false) 778 } else { 779 testCurlFailFromOutside(kubectl, &helpers.NodesInfo{ 780 OutsideNodeName: outsideNodeName, 781 }, k8s1PodIP, 1) 782 } 783 }() 784 } 785 wg.Wait() 786 } 787 788 Context("with remote-node identity enabled", func() { 789 BeforeAll(func() { 790 By("Reconfiguring Cilium to enable remote-node identity") 791 RedeployCiliumWithMerge(kubectl, ciliumFilename, daemonCfg, 792 map[string]string{ 793 "enableIPv4Masquerade": "false", 794 "enableIPv6Masquerade": "false", 795 "bpf.masquerade": "false", 796 }) 797 }) 798 799 It("Validates fromEntities remote-node policy", func() { 800 installDefaultDenyIngressPolicy(kubectl, testNamespace, validateConnectivity) 801 802 By("Installing fromEntities remote-node policy") 803 importPolicy(kubectl, testNamespace, cnpFromEntitiesRemoteNode, "from-entities-remote-node") 804 805 By("Checking policy correctness") 806 validateConnectivity(HostConnectivityAllow, RemoteNodeConnectivityAllow, PodConnectivityDeny, WorldConnectivityDeny) 807 }) 808 }) 809 810 It("Validates fromEntities cluster policy", func() { 811 installDefaultDenyIngressPolicy(kubectl, testNamespace, validateConnectivity) 812 813 By("Installing fromEntities cluster policy") 814 importPolicy(kubectl, testNamespace, cnpFromEntitiesCluster, "from-entities-cluster") 815 816 By("Checking policy correctness") 817 validateConnectivity(HostConnectivityAllow, RemoteNodeConnectivityAllow, PodConnectivityAllow, WorldConnectivityDeny) 818 }) 819 820 It("Validates fromEntities all policy", func() { 821 installDefaultDenyIngressPolicy(kubectl, testNamespace, validateConnectivity) 822 823 By("Installing fromEntities all policy") 824 importPolicy(kubectl, testNamespace, cnpFromEntitiesAll, "from-entities-all") 825 826 By("Checking policy correctness") 827 validateConnectivity(HostConnectivityAllow, RemoteNodeConnectivityAllow, PodConnectivityAllow, WorldConnectivityAllow) 828 }) 829 }) 830 831 Context("with L7 policy", func() { 832 BeforeAll(func() { 833 if helpers.RunsOnNetNextKernel() { 834 By("Reconfiguring Cilium to enable BPF TProxy") 835 RedeployCiliumWithMerge(kubectl, ciliumFilename, daemonCfg, 836 map[string]string{ 837 "bpf.tproxy": "true", 838 }) 839 } 840 }) 841 842 AfterEach(func() { 843 kubectl.Delete(connectivityCheckYml) 844 ExpectAllPodsTerminated(kubectl) 845 }) 846 847 It("using connectivity-check to check datapath", func() { 848 kubectl.ApplyDefault(connectivityCheckYml).ExpectSuccess("cannot install connectivity-check") 849 850 err := kubectl.WaitforPods(helpers.DefaultNamespace, "", helpers.HelperTimeout) 851 Expect(err).Should(BeNil(), "connectivity-check pods are not ready after timeout") 852 }) 853 }) 854 }) 855 856 Context("Namespaces policies", func() { 857 858 var ( 859 err error 860 secondNS string 861 appPods map[string]string 862 appPodsNS map[string]string 863 clusterIP string 864 secondNSclusterIP string 865 nsLabel = "second" 866 867 demoPath string 868 l3L4Policy string 869 cnpSecondNS string 870 netpolNsSelector string 871 l3l4PolicySecondNS string 872 demoManifest string 873 ) 874 875 BeforeAll(func() { 876 secondNS = helpers.GenerateNamespaceForTest("2") 877 878 cnpSecondNSChart := helpers.ManifestGet(kubectl.BasePath(), "cnp-second-namespaces") 879 cnpSecondNS = helpers.ManifestGet(kubectl.BasePath(), "cnp-second-namespaces.yaml") 880 res := kubectl.HelmTemplate(cnpSecondNSChart, "", cnpSecondNS, map[string]string{ 881 "Namespace": secondNS, 882 }) 883 res.ExpectSuccess("Unable to render cnp-second-namespace chart") 884 885 demoPath = helpers.ManifestGet(kubectl.BasePath(), "demo.yaml") 886 l3L4Policy = helpers.ManifestGet(kubectl.BasePath(), "l3-l4-policy.yaml") 887 netpolNsSelector = fmt.Sprintf("%s -n %s", helpers.ManifestGet(kubectl.BasePath(), "netpol-namespace-selector.yaml"), secondNS) 888 l3l4PolicySecondNS = fmt.Sprintf("%s -n %s", l3L4Policy, secondNS) 889 demoManifest = fmt.Sprintf("%s -n %s", demoPath, secondNS) 890 891 kubectl.NamespaceDelete(secondNS) 892 res = kubectl.NamespaceCreate(secondNS) 893 res.ExpectSuccess("unable to create namespace %q", secondNS) 894 895 res = kubectl.Exec(fmt.Sprintf("kubectl label namespaces/%s nslabel=%s", secondNS, nsLabel)) 896 res.ExpectSuccess("cannot create namespace labels") 897 898 res = kubectl.ApplyDefault(demoManifest) 899 res.ExpectSuccess("unable to apply manifest") 900 901 res = kubectl.ApplyDefault(demoPath) 902 res.ExpectSuccess("unable to apply manifest") 903 904 err := kubectl.WaitforPods(secondNS, "-l zgroup=testapp", helpers.HelperTimeout) 905 Expect(err).To(BeNil(), 906 "testapp pods are not ready after timeout in namspace %q", secondNS) 907 908 err = kubectl.WaitforPods(helpers.DefaultNamespace, "-l zgroup=testapp", helpers.HelperTimeout) 909 Expect(err).To(BeNil(), 910 "testapp pods are not ready after timeout in %q namespace", helpers.DefaultNamespace) 911 912 appPods = helpers.GetAppPods(apps, helpers.DefaultNamespace, kubectl, "id") 913 appPodsNS = helpers.GetAppPods(apps, secondNS, kubectl, "id") 914 915 clusterIP, _, err = kubectl.GetServiceHostPort(helpers.DefaultNamespace, app1Service) 916 Expect(err).To(BeNil(), "Cannot get service on %q namespace", helpers.DefaultNamespace) 917 918 secondNSclusterIP, _, err = kubectl.GetServiceHostPort(secondNS, app1Service) 919 Expect(err).To(BeNil(), "Cannot get service on %q namespace", secondNS) 920 921 }) 922 923 AfterEach(func() { 924 // Explicitly do not check results to avoid incomplete teardown of test. 925 _ = kubectl.Delete(l3l4PolicySecondNS) 926 _ = kubectl.Delete(l3L4Policy) 927 _ = kubectl.Delete(netpolNsSelector) 928 929 }) 930 931 AfterAll(func() { 932 _ = kubectl.Delete(demoPath) 933 _ = kubectl.Delete(demoManifest) 934 _ = kubectl.Delete(cnpSecondNS) 935 _ = kubectl.NamespaceDelete(secondNS) 936 ExpectAllPodsTerminated(kubectl) 937 }) 938 939 It("Tests the same Policy in different namespaces", func() { 940 // Tests that the same policy(name,labels) can enforce based on the 941 // namespace and all works as expected. 942 By("Applying Policy in %q namespace", secondNS) 943 _, err = kubectl.CiliumPolicyAction( 944 secondNS, l3l4PolicySecondNS, helpers.KubectlApply, helpers.HelperTimeout) 945 Expect(err).Should(BeNil(), 946 "%q Policy cannot be applied in %q namespace", l3l4PolicySecondNS, secondNS) 947 948 By("Applying Policy in default namespace") 949 _, err = kubectl.CiliumPolicyAction( 950 helpers.DefaultNamespace, l3L4Policy, helpers.KubectlApply, helpers.HelperTimeout) 951 Expect(err).Should(BeNil(), 952 "%q Policy cannot be applied in %q namespace", l3L4Policy, helpers.DefaultNamespace) 953 954 By("Testing connectivity in %q namespace", secondNS) 955 956 res := kubectl.ExecPodCmd( 957 secondNS, appPodsNS[helpers.App2], 958 helpers.CurlFail(fmt.Sprintf("http://%s/public", secondNSclusterIP))) 959 res.ExpectSuccess("%q cannot curl service", appPods[helpers.App2]) 960 961 res = kubectl.ExecPodCmd( 962 secondNS, appPodsNS[helpers.App3], 963 helpers.CurlFail(fmt.Sprintf("http://%s/public", secondNSclusterIP))) 964 res.ExpectFail("%q can curl to service", appPods[helpers.App3]) 965 966 By("Testing connectivity in 'default' namespace") 967 968 res = kubectl.ExecPodCmd( 969 helpers.DefaultNamespace, appPods[helpers.App2], 970 helpers.CurlFail(fmt.Sprintf("http://%s/public", clusterIP))) 971 res.ExpectSuccess("%q cannot curl clusterIP %q", appPods[helpers.App2], clusterIP) 972 973 res = kubectl.ExecPodCmd( 974 helpers.DefaultNamespace, appPods[helpers.App3], 975 helpers.CurlFail(fmt.Sprintf("http://%s/public", clusterIP))) 976 res.ExpectFail("%q can curl to %q", appPods[helpers.App3], clusterIP) 977 }) 978 979 It("Kubernetes Network Policy by namespace selector", func() { 980 // Use namespace selector using Kubernetes Network Policy to make 981 // sure that it is translated correctly to Cilium and applies the 982 // policies to the right endpoints. 983 // KNP reference: 984 // https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.10/#networkpolicyspec-v1-networking-k8s-io 985 By("Testing connectivity across Namespaces without policy") 986 for _, pod := range []string{helpers.App2, helpers.App3} { 987 res := kubectl.ExecPodCmd( 988 helpers.DefaultNamespace, appPods[pod], 989 helpers.CurlFail(fmt.Sprintf("http://%s/public", secondNSclusterIP))) 990 res.ExpectSuccess("%q cannot curl service", appPods[pod]) 991 992 res = kubectl.ExecPodCmd( 993 secondNS, appPodsNS[pod], 994 helpers.CurlFail(fmt.Sprintf("http://%s/public", secondNSclusterIP))) 995 res.ExpectSuccess("%q cannot curl service", appPodsNS[pod]) 996 } 997 998 By("Applying Policy in %q namespace", secondNS) 999 _, err = kubectl.CiliumPolicyAction( 1000 secondNS, netpolNsSelector, helpers.KubectlApply, helpers.HelperTimeout) 1001 Expect(err).Should(BeNil(), "Policy cannot be applied") 1002 1003 for _, pod := range []string{helpers.App2, helpers.App3} { 1004 // Make sure that the Default namespace can NOT connect to 1005 // second namespace. 1006 res := kubectl.ExecPodCmd( 1007 helpers.DefaultNamespace, appPods[pod], 1008 helpers.CurlFail(fmt.Sprintf("http://%s/public", secondNSclusterIP))) 1009 res.ExpectFail("%q can curl to service, policy is not blocking"+ 1010 "communication to %q namespace", appPods[pod], secondNS) 1011 1012 // Second namespace pods can connect to the same namespace based on policy. 1013 res = kubectl.ExecPodCmd( 1014 secondNS, appPodsNS[pod], 1015 helpers.CurlFail(fmt.Sprintf("http://%s/public", secondNSclusterIP))) 1016 res.ExpectSuccess("%q cannot curl service", appPodsNS[pod]) 1017 } 1018 1019 By("Delete Kubernetes Network Policies in %q namespace", secondNS) 1020 _, err = kubectl.CiliumPolicyAction( 1021 secondNS, netpolNsSelector, helpers.KubectlDelete, helpers.HelperTimeout) 1022 Expect(err).Should(BeNil(), "Policy %q cannot be deleted", netpolNsSelector) 1023 1024 for _, pod := range []string{helpers.App2, helpers.App3} { 1025 res := kubectl.ExecPodCmd( 1026 helpers.DefaultNamespace, appPods[pod], 1027 helpers.CurlFail(fmt.Sprintf("http://%s/public", secondNSclusterIP))) 1028 res.ExpectSuccess("%q cannot curl service", appPods[pod]) 1029 1030 res = kubectl.ExecPodCmd( 1031 secondNS, appPodsNS[pod], 1032 helpers.CurlFail(fmt.Sprintf("http://%s/public", secondNSclusterIP))) 1033 res.ExpectSuccess("%q cannot curl service", appPodsNS[pod]) 1034 } 1035 }) 1036 1037 It("Cilium Network policy using namespace label and L7", func() { 1038 1039 _, err := kubectl.CiliumPolicyAction( 1040 helpers.DefaultNamespace, cnpSecondNS, helpers.KubectlApply, helpers.HelperTimeout) 1041 Expect(err).Should(BeNil(), "%q Policy cannot be applied", cnpSecondNS) 1042 1043 By("Testing connectivity in %q namespace", secondNS) 1044 res := kubectl.ExecPodCmd( 1045 secondNS, appPodsNS[helpers.App2], 1046 helpers.CurlFail(fmt.Sprintf("http://%s/public", clusterIP))) 1047 res.ExpectSuccess("Cannot curl service in %s ns", helpers.DefaultNamespace) 1048 1049 res = kubectl.ExecPodCmd( 1050 secondNS, appPodsNS[helpers.App3], 1051 helpers.CurlFail(fmt.Sprintf("http://%s/public", clusterIP))) 1052 res.ExpectSuccess("Cannot curl service in %s ns", helpers.DefaultNamespace) 1053 1054 By("Testing connectivity from 'default' namespace") 1055 1056 res = kubectl.ExecPodCmd( 1057 helpers.DefaultNamespace, appPods[helpers.App2], 1058 helpers.CurlFail(fmt.Sprintf("http://%s/public", clusterIP))) 1059 res.ExpectFail("Can connect when it should not") 1060 1061 res = kubectl.ExecPodCmd( 1062 helpers.DefaultNamespace, appPods[helpers.App3], 1063 helpers.CurlFail(fmt.Sprintf("http://%s/public", clusterIP))) 1064 res.ExpectFail("Can connect when it should not") 1065 }) 1066 1067 }) 1068 1069 Context("Clusterwide policies", func() { 1070 var ( 1071 demoPath string 1072 demoManifestNS1 string 1073 demoManifestNS2 string 1074 firstNS string 1075 secondNS string 1076 1077 appPodsFirstNS map[string]string 1078 appPodsSecondNS map[string]string 1079 1080 firstNSclusterIP string 1081 secondNSclusterIP string 1082 1083 ingressDenyAllPolicy string 1084 egressDenyAllPolicy string 1085 allowIngressPolicy string 1086 allowAllPolicy string 1087 1088 // non-default-deny policies 1089 egressAllowApiDefaultAllow string 1090 ) 1091 1092 BeforeAll(func() { 1093 firstNS = helpers.GenerateNamespaceForTest("1") 1094 secondNS = helpers.GenerateNamespaceForTest("2") 1095 demoPath = helpers.ManifestGet(kubectl.BasePath(), "demo.yaml") 1096 egressDenyAllPolicy = helpers.ManifestGet(kubectl.BasePath(), "ccnp-default-deny-egress.yaml") 1097 ingressDenyAllPolicy = helpers.ManifestGet(kubectl.BasePath(), "ccnp-default-deny-ingress.yaml") 1098 allowIngressPolicy = helpers.ManifestGet(kubectl.BasePath(), "ccnp-update-allow-ingress.yaml") 1099 allowAllPolicy = helpers.ManifestGet(kubectl.BasePath(), "ccnp-update-allow-all.yaml") 1100 egressAllowApiDefaultAllow = helpers.ManifestGet(kubectl.BasePath(), "ccnp-allow-apiserver-default-allow.yaml") 1101 1102 demoManifestNS1 = fmt.Sprintf("%s -n %s", demoPath, firstNS) 1103 demoManifestNS2 = fmt.Sprintf("%s -n %s", demoPath, secondNS) 1104 1105 kubectl.NamespaceDelete(firstNS) 1106 res := kubectl.NamespaceCreate(firstNS) 1107 res.ExpectSuccess("unable to create namespace %q", firstNS) 1108 1109 res = kubectl.Exec(fmt.Sprintf("kubectl label namespaces/%[1]s nslabel=%[1]s", firstNS)) 1110 res.ExpectSuccess("cannot create namespace labels") 1111 1112 kubectl.NamespaceDelete(secondNS) 1113 res = kubectl.NamespaceCreate(secondNS) 1114 res.ExpectSuccess("unable to create namespace %q", secondNS) 1115 1116 res = kubectl.Exec(fmt.Sprintf("kubectl label namespaces/%[1]s nslabel=%[1]s", secondNS)) 1117 res.ExpectSuccess("cannot create namespace labels") 1118 1119 res = kubectl.ApplyDefault(demoManifestNS1) 1120 res.ExpectSuccess("unable to apply demo manifest") 1121 1122 // Check if the Pods are ready in each namespace before the default configured 1123 // timeout. 1124 err := kubectl.WaitforPods(firstNS, "-l zgroup=testapp", helpers.HelperTimeout) 1125 Expect(err).To(BeNil(), 1126 "testapp pods are not ready after timeout in namspace %q", firstNS) 1127 1128 res = kubectl.ApplyDefault(demoManifestNS2) 1129 res.ExpectSuccess("unable to apply demo manifest") 1130 1131 err = kubectl.WaitforPods(secondNS, "-l zgroup=testapp", helpers.HelperTimeout) 1132 Expect(err).To(BeNil(), 1133 "testapp pods are not ready after timeout in namspace %q", secondNS) 1134 1135 appPodsFirstNS = helpers.GetAppPods(apps, firstNS, kubectl, "id") 1136 appPodsSecondNS = helpers.GetAppPods(apps, secondNS, kubectl, "id") 1137 1138 firstNSclusterIP, _, err = kubectl.GetServiceHostPort(firstNS, app1Service) 1139 Expect(err).To(BeNil(), "Cannot get service on %q namespace", helpers.DefaultNamespace) 1140 1141 secondNSclusterIP, _, err = kubectl.GetServiceHostPort(secondNS, app1Service) 1142 Expect(err).To(BeNil(), "Cannot get service on %q namespace", secondNS) 1143 1144 }) 1145 1146 AfterAll(func() { 1147 _ = kubectl.Delete(demoManifestNS1) 1148 _ = kubectl.Delete(demoManifestNS2) 1149 _ = kubectl.NamespaceDelete(firstNS) 1150 _ = kubectl.NamespaceDelete(secondNS) 1151 ExpectAllPodsTerminated(kubectl) 1152 }) 1153 1154 It("Test clusterwide connectivity with policies", func() { 1155 By("Applying Egress deny all clusterwide policy") 1156 _, err := kubectl.CiliumClusterwidePolicyAction( 1157 egressDenyAllPolicy, helpers.KubectlApply, helpers.HelperTimeout) 1158 Expect(err).Should(BeNil(), 1159 "%q Clusterwide Policy cannot be applied", egressDenyAllPolicy) 1160 1161 res := kubectl.ExecPodCmd( 1162 firstNS, appPodsFirstNS[helpers.App2], 1163 helpers.CurlFail("http://1.1.1.1/")) 1164 res.ExpectFail("Egress connectivity should be denied for pod %q", helpers.App2) 1165 1166 res = kubectl.ExecPodCmd( 1167 secondNS, appPodsSecondNS[helpers.App2], 1168 helpers.CurlFail("http://1.1.1.1/")) 1169 res.ExpectFail("Egress connectivity should be denied for pod %q in %q namespace", helpers.App2, secondNS) 1170 1171 res = kubectl.ExecPodCmd( 1172 firstNS, appPodsFirstNS[helpers.App3], 1173 helpers.Ping("8.8.8.8")) 1174 res.ExpectFail("Egress ping connectivity should be denied for pod %q", helpers.App3) 1175 1176 res = kubectl.ExecPodCmd( 1177 secondNS, appPodsSecondNS[helpers.App3], 1178 "host kubernetes.default.svc.cluster.local") 1179 res.ExpectFail("Egress DNS connectivity should be denied for pod %q", helpers.App3) 1180 1181 By("Deleting Egress deny all clusterwide policy") 1182 _, err = kubectl.CiliumClusterwidePolicyAction( 1183 egressDenyAllPolicy, helpers.KubectlDelete, helpers.HelperTimeout) 1184 Expect(err).Should(BeNil(), 1185 "%q Clusterwide Policy cannot be deleted", egressDenyAllPolicy) 1186 1187 By("Applying Ingress deny all clusterwide policy") 1188 _, err = kubectl.CiliumClusterwidePolicyAction( 1189 ingressDenyAllPolicy, helpers.KubectlApply, helpers.HelperTimeout) 1190 Expect(err).Should(BeNil(), 1191 "%q Clusterwide Policy cannot be applied", egressDenyAllPolicy) 1192 1193 // Validate ingress Deny All policy. 1194 By("Testing ingress connectivity from %q namespace", firstNS) 1195 res = kubectl.ExecPodCmd( 1196 firstNS, appPodsFirstNS[helpers.App2], 1197 helpers.CurlFail(fmt.Sprintf("http://%s/public", firstNSclusterIP))) 1198 res.ExpectFail("Ingress connectivity should be denied for service in %s namespace", firstNS) 1199 1200 By("Testing ingress connectivity from %q namespace", secondNS) 1201 res = kubectl.ExecPodCmd( 1202 secondNS, appPodsSecondNS[helpers.App2], 1203 helpers.CurlFail(fmt.Sprintf("http://%s/public", secondNSclusterIP))) 1204 res.ExpectFail("Ingress connectivity should be denied for service in %s namespace", secondNS) 1205 1206 By("Testing cross namespace connectivity from %q to %q namespace", firstNS, secondNS) 1207 res = kubectl.ExecPodCmd( 1208 firstNS, appPodsFirstNS[helpers.App2], 1209 helpers.CurlFail(fmt.Sprintf("http://%s/public", secondNSclusterIP))) 1210 res.ExpectFail("Ingress connectivity should be denied for service in %s namespace", secondNS) 1211 1212 By("Testing cross namespace connectivity from %q to %q namespace", secondNS, firstNS) 1213 res = kubectl.ExecPodCmd( 1214 secondNS, appPodsSecondNS[helpers.App2], 1215 helpers.CurlFail(fmt.Sprintf("http://%s/public", firstNSclusterIP))) 1216 res.ExpectFail("Ingress connectivity should be denied for service in %s namespace", firstNS) 1217 1218 // Apply both ingress deny and egress deny all policies and override the policies with 1219 // global allow all policy. 1220 By("Applying Egress deny all clusterwide policy") 1221 _, err = kubectl.CiliumClusterwidePolicyAction( 1222 egressDenyAllPolicy, helpers.KubectlApply, helpers.HelperTimeout) 1223 Expect(err).Should(BeNil(), 1224 "%q Clusterwide Policy cannot be applied", egressDenyAllPolicy) 1225 1226 By("Applying Allow all clusterwide policy over ingress deny all and egress deny all") 1227 _, err = kubectl.CiliumClusterwidePolicyAction( 1228 allowAllPolicy, helpers.KubectlApply, helpers.HelperTimeout) 1229 Expect(err).Should(BeNil(), 1230 "%q Clusterwide Policy cannot be applied", allowAllPolicy) 1231 1232 By("Testing ingress connectivity from %q namespace", firstNS) 1233 res = kubectl.ExecPodCmd( 1234 firstNS, appPodsFirstNS[helpers.App2], 1235 helpers.CurlFail(fmt.Sprintf("http://%s/public", firstNSclusterIP))) 1236 res.ExpectSuccess("Ingress connectivity should be allowed for service in %s namespace", firstNS) 1237 1238 By("Testing ingress connectivity from %q namespace", secondNS) 1239 res = kubectl.ExecPodCmd( 1240 secondNS, appPodsSecondNS[helpers.App2], 1241 helpers.CurlFail(fmt.Sprintf("http://%s/public", secondNSclusterIP))) 1242 res.ExpectSuccess("Ingress connectivity should be allowed for service in %s namespace", secondNS) 1243 1244 By("Testing cross namespace connectivity from %q to %q namespace", firstNS, secondNS) 1245 res = kubectl.ExecPodCmd( 1246 firstNS, appPodsFirstNS[helpers.App2], 1247 helpers.CurlFail(fmt.Sprintf("http://%s/public", secondNSclusterIP))) 1248 res.ExpectSuccess("Ingress connectivity should be allowed for service in %s namespace", secondNS) 1249 1250 By("Testing cross namespace connectivity from %q to %q namespace", secondNS, firstNS) 1251 res = kubectl.ExecPodCmd( 1252 secondNS, appPodsSecondNS[helpers.App2], 1253 helpers.CurlFail(fmt.Sprintf("http://%s/public", firstNSclusterIP))) 1254 res.ExpectSuccess("Ingress connectivity should be allowed for service in %s namespace", firstNS) 1255 1256 // Update the ccnp-update policy from allow-all to allow-ingress from app2 to app1. 1257 // Checks in this ingress allow policy are 1258 // 1. Check allowed ingress from app2.firstNS to app1.firstNS 1259 // 2. Check allowed ingress from app2.secondNS to app1.firstNS 1260 // 3. Check denied ingress from app3.firstNS to app1.firstNS 1261 // 4. Check denied ingress from app3.secondNS to app1.firstNS 1262 By("Update allow all policy to allow ingress from a particular app only.") 1263 _, err = kubectl.CiliumClusterwidePolicyAction( 1264 allowIngressPolicy, helpers.KubectlApply, helpers.HelperTimeout) 1265 Expect(err).Should(BeNil(), 1266 "%q Clusterwide Policy cannot be applied", allowIngressPolicy) 1267 1268 By("Deleting Egress deny all clusterwide policy") 1269 _, err = kubectl.CiliumClusterwidePolicyAction( 1270 egressDenyAllPolicy, helpers.KubectlDelete, helpers.HelperTimeout) 1271 Expect(err).Should(BeNil(), 1272 "%q Clusterwide Policy cannot be deleted", egressDenyAllPolicy) 1273 1274 By("Testing ingress connectivity from %q to %q in %q namespace", helpers.App2, helpers.App1, firstNS) 1275 res = kubectl.ExecPodCmd( 1276 firstNS, appPodsFirstNS[helpers.App2], 1277 helpers.CurlFail(fmt.Sprintf("http://%s/public", firstNSclusterIP))) 1278 res.ExpectSuccess("Ingress connectivity should be allowed for service in %s namespace", firstNS) 1279 1280 By("Testing ingress connectivity from %q to %q across two namespaces", helpers.App2, helpers.App1) 1281 res = kubectl.ExecPodCmd( 1282 secondNS, appPodsSecondNS[helpers.App2], 1283 helpers.CurlFail(fmt.Sprintf("http://%s/public", firstNSclusterIP))) 1284 res.ExpectSuccess("Ingress connectivity should be allowed for service in %s namespace", firstNS) 1285 1286 By("Testing ingress connectivity from %q to %q in %q namespace", helpers.App3, helpers.App1, firstNS) 1287 res = kubectl.ExecPodCmd( 1288 firstNS, appPodsFirstNS[helpers.App3], 1289 helpers.CurlFail(fmt.Sprintf("http://%s/public", firstNSclusterIP))) 1290 res.ExpectFail("Ingress connectivity should be denied for service in %s namespace", firstNS) 1291 1292 By("Testing ingress connectivity from %q to %q across two namespacess", helpers.App3, helpers.App1) 1293 res = kubectl.ExecPodCmd( 1294 secondNS, appPodsSecondNS[helpers.App3], 1295 helpers.CurlFail(fmt.Sprintf("http://%s/public", firstNSclusterIP))) 1296 res.ExpectFail("Ingress connectivity should be denied for service in %s namespace", firstNS) 1297 1298 // Cleanup all tested policies 1299 By("Delete allow ingress from particular app policy") 1300 _, err = kubectl.CiliumClusterwidePolicyAction( 1301 allowIngressPolicy, helpers.KubectlDelete, helpers.HelperTimeout) 1302 Expect(err).Should(BeNil(), 1303 "%q Clusterwide Policy cannot be deleted", allowIngressPolicy) 1304 1305 By("Deleting Ingress deny all clusterwide policy") 1306 _, err = kubectl.CiliumClusterwidePolicyAction( 1307 ingressDenyAllPolicy, helpers.KubectlDelete, helpers.HelperTimeout) 1308 Expect(err).Should(BeNil(), 1309 "%q Clusterwide Policy cannot be deleted", ingressDenyAllPolicy) 1310 }) 1311 1312 It("Tests connectivity with default-allow policies", func() { 1313 1314 // Cases: 1315 // 1: default-allow policy allows all traffic 1316 // 2: creating a default-deny policy flips this 1317 // 3: Without default deny, explicit Deny rules take precedence 1318 1319 // case 1: default-allow policy 1320 By("Creating a default-allow policy that allows apiserver access") 1321 _, err := kubectl.CiliumClusterwidePolicyAction( 1322 egressAllowApiDefaultAllow, helpers.KubectlApply, helpers.HelperTimeout) 1323 Expect(err).Should(BeNil(), 1324 "%q Clusterwide Policy cannot be applied", egressDenyAllPolicy) 1325 1326 app2 := appPodsFirstNS[helpers.App2] 1327 app1 := appPodsFirstNS[helpers.App1] 1328 1329 By("Testing that app2 can connect to the apiserver") 1330 res := kubectl.ExecPodCmd( 1331 firstNS, appPodsFirstNS[helpers.App2], 1332 helpers.CurlFail("https://kubernetes.default.svc.cluster.local/healthz")) 1333 res.ExpectSuccess("Connectivity should be allowed from %s to apiserver in %s", app2, firstNS) 1334 1335 By("Testing connectivity from %q to %q in %q namespace without explicit allow", app2, app1, firstNS) 1336 res = kubectl.ExecPodCmd( 1337 firstNS, appPodsFirstNS[helpers.App2], 1338 helpers.CurlFail(fmt.Sprintf("http://%s/public", firstNSclusterIP))) 1339 res.ExpectSuccess("Connectivity should be allowed from %s to %s in ns %s", app2, app1, firstNS) 1340 1341 // case 2: default-allow policy allows apiserver, default-deny policy allows nothing 1342 // so only apiserver should be allowed 1343 By("Creating a default-deny policy in namespace %s", firstNS) 1344 denyEgress := helpers.ManifestGet(kubectl.BasePath(), "cnp-default-deny-egress.yaml") 1345 _, err = kubectl.CiliumPolicyAction( 1346 firstNS, denyEgress, helpers.KubectlApply, helpers.HelperTimeout) 1347 Expect(err).Should(BeNil(), "%q Policy cannot be applied", firstNS) 1348 1349 By("Testing that %s can still connect to the apiserver", helpers.App2) 1350 res = kubectl.ExecPodCmd( 1351 firstNS, appPodsFirstNS[helpers.App2], 1352 helpers.CurlFail("https://kubernetes.default.svc.cluster.local/healthz")) 1353 res.ExpectSuccess("Connectivity should be allowed from %s to apiserver in %s", app2, firstNS) 1354 1355 By("Testing connectivity from %q to %q in %q namespace is blocked", helpers.App2, helpers.App1, firstNS) 1356 res = kubectl.ExecPodCmd( 1357 firstNS, appPodsFirstNS[helpers.App2], 1358 helpers.CurlFail(fmt.Sprintf("http://%s/public", firstNSclusterIP))) 1359 res.ExpectFail("Connectivity should be denied from %s to %s in ns %s", app2, app1, firstNS) 1360 1361 // Back to case 1 -- delete default-deny 1362 By("Deleting the default-deny egress policy") 1363 _, err = kubectl.CiliumPolicyAction( 1364 firstNS, denyEgress, helpers.KubectlDelete, helpers.HelperTimeout) 1365 Expect(err).Should(BeNil(), "%q Policy cannot be deleted", firstNS) 1366 1367 By("Testing connectivity again from %q to %q in %q namespace without explicit allow", app2, app1, firstNS) 1368 res = kubectl.ExecPodCmd( 1369 firstNS, appPodsFirstNS[helpers.App2], 1370 helpers.CurlFail(fmt.Sprintf("http://%s/public", firstNSclusterIP))) 1371 res.ExpectSuccess("Connectivity should be allowed from %s to %s in ns %s", app2, app1, firstNS) 1372 1373 // case 3: only default-allow policy, one has EgressDeny rule. 1374 // ensure that all connectivity is allowed except the explicit deny 1375 By("Creating a default-allow policy that denies access to app1") 1376 denyApp1 := helpers.ManifestGet(kubectl.BasePath(), "cnp-deny-to-app1-default-allow.yaml") 1377 _, err = kubectl.CiliumPolicyAction( 1378 firstNS, denyApp1, helpers.KubectlApply, helpers.HelperTimeout) 1379 Expect(err).Should(BeNil(), "%q Policy cannot be applied", firstNS) 1380 1381 By("Testing connectivity again from %q to %q in %q namespace with explicit deny", app2, app1, firstNS) 1382 res = kubectl.ExecPodCmd( 1383 firstNS, appPodsFirstNS[helpers.App2], 1384 helpers.CurlFail(fmt.Sprintf("http://%s/public", firstNSclusterIP))) 1385 res.ExpectFail("Connectivity should be denied from %s to %s in ns %s", app2, app1, firstNS) 1386 1387 By("Testing cross namespace connectivity from %q to %q namespace", firstNS, secondNS) 1388 res = kubectl.ExecPodCmd( 1389 firstNS, appPodsFirstNS[helpers.App2], 1390 helpers.CurlFail(fmt.Sprintf("http://%s/public", secondNSclusterIP))) 1391 res.ExpectSuccess("Connectivity should be allowed from %s to %s in ns %s", app2, secondNSclusterIP, firstNS) 1392 }) 1393 }) 1394 1395 //TODO: Check service with IPV6 1396 1397 Context("External services", func() { 1398 var ( 1399 expectedCIDR = "198.49.23.144/32" 1400 1401 endpointPath string 1402 podPath string 1403 policyPath string 1404 policyLabeledPath string 1405 servicePath string 1406 ) 1407 1408 BeforeAll(func() { 1409 endpointPath = helpers.ManifestGet(kubectl.BasePath(), "external_endpoint.yaml") 1410 podPath = helpers.ManifestGet(kubectl.BasePath(), "external_pod.yaml") 1411 policyPath = helpers.ManifestGet(kubectl.BasePath(), "external-policy.yaml") 1412 policyLabeledPath = helpers.ManifestGet(kubectl.BasePath(), "external-policy-labeled.yaml") 1413 servicePath = helpers.ManifestGet(kubectl.BasePath(), "external_service.yaml") 1414 1415 kubectl.ApplyDefault(servicePath).ExpectSuccess("cannot install external service") 1416 kubectl.ApplyDefault(podPath).ExpectSuccess("cannot install pod path") 1417 1418 err := kubectl.WaitforPods(helpers.DefaultNamespace, "", helpers.HelperTimeout) 1419 Expect(err).To(BeNil(), "Pods are not ready after timeout") 1420 1421 err = kubectl.CiliumEndpointWaitReady() 1422 Expect(err).To(BeNil(), "Endpoints are not ready after timeout") 1423 }) 1424 1425 AfterAll(func() { 1426 _ = kubectl.Delete(servicePath) 1427 _ = kubectl.Delete(podPath) 1428 1429 ExpectAllPodsTerminated(kubectl) 1430 }) 1431 1432 AfterEach(func() { 1433 _ = kubectl.Delete(policyLabeledPath) 1434 _ = kubectl.Delete(policyPath) 1435 _ = kubectl.Delete(endpointPath) 1436 }) 1437 1438 validateEgress := func(kubectl *helpers.Kubectl) { 1439 By("Checking that toServices CIDR is plumbed into the policy") 1440 Eventually(func() string { 1441 output, err := kubectl.LoadedPolicyInFirstAgent() 1442 ExpectWithOffset(1, err).To(BeNil(), "unable to retrieve policy") 1443 return output 1444 }, 2*time.Minute, 2*time.Second).Should(ContainSubstring(expectedCIDR)) 1445 } 1446 1447 validateEgressAfterDeletion := func(kubectl *helpers.Kubectl) { 1448 By("Checking that toServices CIDR is no longer plumbed into the policy") 1449 Eventually(func() string { 1450 output, err := kubectl.LoadedPolicyInFirstAgent() 1451 ExpectWithOffset(1, err).To(BeNil(), "unable to retrieve policy") 1452 return output 1453 }, 2*time.Minute, 2*time.Second).ShouldNot(ContainSubstring(expectedCIDR)) 1454 } 1455 1456 It("To Services first endpoint creation", func() { 1457 res := kubectl.ApplyDefault(endpointPath) 1458 res.ExpectSuccess() 1459 1460 applyPolicy(kubectl, policyPath) 1461 validateEgress(kubectl) 1462 1463 kubectl.Delete(policyPath) 1464 kubectl.Delete(endpointPath) 1465 validateEgressAfterDeletion(kubectl) 1466 }) 1467 1468 It("To Services first policy", func() { 1469 applyPolicy(kubectl, policyPath) 1470 res := kubectl.ApplyDefault(endpointPath) 1471 res.ExpectSuccess() 1472 1473 validateEgress(kubectl) 1474 1475 kubectl.Delete(policyPath) 1476 kubectl.Delete(endpointPath) 1477 validateEgressAfterDeletion(kubectl) 1478 }) 1479 1480 It("To Services first endpoint creation match service by labels", func() { 1481 By("Creating Kubernetes Endpoint") 1482 res := kubectl.ApplyDefault(endpointPath) 1483 res.ExpectSuccess() 1484 1485 applyPolicy(kubectl, policyLabeledPath) 1486 1487 validateEgress(kubectl) 1488 1489 kubectl.Delete(policyLabeledPath) 1490 kubectl.Delete(endpointPath) 1491 validateEgressAfterDeletion(kubectl) 1492 }) 1493 1494 It("To Services first policy, match service by labels", func() { 1495 applyPolicy(kubectl, policyLabeledPath) 1496 1497 By("Creating Kubernetes Endpoint") 1498 res := kubectl.ApplyDefault(endpointPath) 1499 res.ExpectSuccess() 1500 1501 validateEgress(kubectl) 1502 1503 kubectl.Delete(policyLabeledPath) 1504 kubectl.Delete(endpointPath) 1505 validateEgressAfterDeletion(kubectl) 1506 }) 1507 }) 1508 }) 1509 1510 // This Describe block is needed to run some tests in GKE. For example, the 1511 // kube-apiserver policy matching feature needs coverage on GKE as there are 1512 // two cases for that feature: 1513 // - kube-apiserver running within the cluster (Vagrant VMs) 1514 // - kube-apiserver running outside of the cluster (GKE) 1515 var _ = SkipDescribeIf(helpers.DoesNotRunOn54OrLaterKernel, 1516 "K8sPolicyTestExtended", func() { 1517 var ( 1518 kubectl *helpers.Kubectl 1519 1520 // these are set in BeforeAll() 1521 ciliumFilename string 1522 daemonCfg map[string]string 1523 ) 1524 1525 BeforeAll(func() { 1526 kubectl = helpers.CreateKubectl(helpers.K8s1VMName(), logger) 1527 daemonCfg = map[string]string{ 1528 "tls.secretsBackend": "k8s", 1529 } 1530 ciliumFilename = helpers.TimestampFilename("cilium.yaml") 1531 }) 1532 1533 AfterAll(func() { 1534 UninstallCiliumFromManifest(kubectl, ciliumFilename) 1535 kubectl.CloseSSHClient() 1536 }) 1537 1538 AfterFailed(func() { 1539 kubectl.CiliumReport("cilium-dbg service list", "cilium-dbg endpoint list") 1540 }) 1541 1542 AfterEach(func() { 1543 ExpectAllPodsTerminated(kubectl) 1544 }) 1545 1546 JustAfterEach(func() { 1547 kubectl.ValidateNoErrorsInLogs(CurrentGinkgoTestDescription().Duration) 1548 }) 1549 1550 // Test must run with KPR enabled, see below comments. 1551 Context("Validate toEntities KubeAPIServer", func() { 1552 var ( 1553 k8s1Name, k8s1IP string 1554 k8s1PodName, k8s2PodName string 1555 k8s1PodIP, k8s2PodIP string 1556 outsideNodeName string 1557 1558 demoLocalYAML string 1559 cnpToEntitiesKubeAPIServer string 1560 cnpToEntitiesKubeAPIServerDeny string 1561 1562 kubeAPIServerService *v1.Service 1563 1564 testNamespace = helpers.DefaultNamespace 1565 ) 1566 1567 BeforeAll(func() { 1568 cnpToEntitiesKubeAPIServer = helpers.ManifestGet( 1569 kubectl.BasePath(), "cnp-to-entities-kube-apiserver.yaml", 1570 ) 1571 cnpToEntitiesKubeAPIServerDeny = helpers.ManifestGet( 1572 kubectl.BasePath(), "cnp-to-entities-kube-apiserver-deny.yaml", 1573 ) 1574 1575 By("Redeploying Cilium with tunnel disabled and KPR enabled") 1576 RedeployCiliumWithMerge(kubectl, ciliumFilename, daemonCfg, map[string]string{ 1577 // The following are needed because of 1578 // https://github.com/cilium/cilium/issues/17962 && 1579 // https://github.com/cilium/cilium/issues/16197. 1580 "routingMode": "native", 1581 "autoDirectNodeRoutes": "true", 1582 "kubeProxyReplacement": "true", 1583 }) 1584 1585 By("Deploying demo local daemonset") 1586 demoLocalYAML = helpers.ManifestGet(kubectl.BasePath(), "demo_ds_local.yaml") 1587 kubectl.ApplyDefault(demoLocalYAML).ExpectSuccess("Unable to apply %s", demoLocalYAML) 1588 Expect(kubectl.WaitforPods( 1589 testNamespace, 1590 fmt.Sprintf("-l %s", testDS), helpers.HelperTimeout), 1591 ).Should(BeNil()) 1592 k8s1Name, k8s1IP = kubectl.GetNodeInfo(helpers.K8s1) 1593 k8s1PodName, k8s1PodIP = kubectl.GetPodOnNodeLabeledWithOffset(helpers.K8s1, testDS, 0) 1594 k8s2PodName, k8s2PodIP = kubectl.GetPodOnNodeLabeledWithOffset(helpers.K8s2, testDS, 0) 1595 if helpers.ExistNodeWithoutCilium() { 1596 outsideNodeName, _ = kubectl.GetNodeInfo(kubectl.GetFirstNodeWithoutCiliumLabel()) 1597 } 1598 1599 var err error 1600 kubeAPIServerService, err = kubectl.GetService(helpers.DefaultNamespace, "kubernetes") 1601 Expect(err).ToNot(HaveOccurred()) 1602 Expect(kubeAPIServerService).ToNot(BeNil()) 1603 }) 1604 1605 AfterAll(func() { 1606 // Explicitly ignore result of deletion of resources to 1607 // avoid incomplete teardown if any step fails. 1608 _ = kubectl.Delete(demoLocalYAML) 1609 ExpectAllPodsTerminated(kubectl) 1610 }) 1611 1612 AfterEach(func() { 1613 cmd := fmt.Sprintf("%s delete --all cnp,ccnp,netpol -n %s", helpers.KubectlCmd, testNamespace) 1614 _ = kubectl.Exec(cmd) 1615 }) 1616 1617 validateConnectivity := func( 1618 expectHostSuccess, expectRemoteNodeSuccess, expectPodSuccess, expectWorldSuccess bool, 1619 ) { 1620 var wg sync.WaitGroup 1621 wg.Add(1) 1622 go func() { 1623 defer GinkgoRecover() 1624 defer wg.Done() 1625 switch helpers.GetCurrentIntegration() { 1626 case helpers.CIIntegrationEKS, helpers.CIIntegrationEKSChaining, helpers.CIIntegrationGKE: 1627 By("Checking ingress connectivity from k8s1 node to k8s1 pod (host)") 1628 default: 1629 // We need to bypass this check as in a non-managed 1630 // environment like Vagrant, the kube-apiserver is 1631 // running locally on K8s1. This means that local host 1632 // traffic cannot be disambiguated from kube-apiserver 1633 // traffic. 1634 By("Bypassing check for ingress connectivity for host, which cannot be done in non-managed environments") 1635 return 1636 } 1637 res := kubectl.ExecInHostNetNS(context.TODO(), k8s1Name, 1638 helpers.CurlFail(k8s1PodIP)) 1639 ExpectWithOffset(1, res).To(getMatcher(expectHostSuccess), 1640 "HTTP ingress connectivity to pod %q from local host", k8s1PodIP) 1641 }() 1642 1643 wg.Add(1) 1644 go func() { 1645 defer GinkgoRecover() 1646 defer wg.Done() 1647 switch helpers.GetCurrentIntegration() { 1648 case helpers.CIIntegrationEKS, helpers.CIIntegrationEKSChaining, helpers.CIIntegrationGKE: 1649 By("Checking ingress connectivity from k8s1 node to k8s2 pod (remote-node)") 1650 default: 1651 // We need to bypass this check as in a two node 1652 // cluster, the kube-apiserver will be running on at 1653 // least one of the two nodes, which means that any 1654 // traffic to or from will be considered to / from 1655 // kube-apiserver, and not remote-node. If we had a 1656 // third node with Cilium installed, then we wouldn't 1657 // need to bypass this check. 1658 By("Bypassing check for ingress connectivity for remote-node, which cannot be done in a two-node cluster") 1659 return 1660 } 1661 res := kubectl.ExecInHostNetNS(context.TODO(), k8s1Name, 1662 helpers.CurlFail(k8s2PodIP)) 1663 ExpectWithOffset(1, res).To(getMatcher(expectRemoteNodeSuccess), 1664 "HTTP ingress connectivity to pod %q from remote node", k8s2PodIP) 1665 }() 1666 1667 wg.Add(1) 1668 go func() { 1669 defer GinkgoRecover() 1670 defer wg.Done() 1671 By("Checking ingress connectivity from k8s1 pod to k8s2 pod") 1672 res := kubectl.ExecPodCmd(testNamespace, k8s1PodName, helpers.CurlFail(k8s2PodIP)) 1673 ExpectWithOffset(1, res).To(getMatcher(expectPodSuccess), 1674 "HTTP ingress connectivity to pod %q from pod %q", k8s2PodIP, k8s1PodIP) 1675 }() 1676 1677 if helpers.ExistNodeWithoutCilium() { 1678 wg.Add(1) 1679 go func() { 1680 defer GinkgoRecover() 1681 defer wg.Done() 1682 By("Checking ingress connectivity from world to k8s1 pod") 1683 By("Adding a static route to %s via %s on the %s node (outside)", k8s1PodIP, k8s1IP, outsideNodeName) 1684 res := kubectl.AddIPRoute(outsideNodeName, k8s1PodIP, k8s1IP, false) 1685 Expect(res).To(getMatcher(true)) 1686 defer func() { 1687 kubectl.DelIPRoute(outsideNodeName, k8s1PodIP, k8s1IP).ExpectSuccess("Failed to del ip route") 1688 }() 1689 1690 if expectWorldSuccess { 1691 testCurlFromOutside(kubectl, &helpers.NodesInfo{ 1692 OutsideNodeName: outsideNodeName, 1693 }, k8s1PodIP, 1, false) 1694 } else { 1695 testCurlFailFromOutside(kubectl, &helpers.NodesInfo{ 1696 OutsideNodeName: outsideNodeName, 1697 }, k8s1PodIP, 1) 1698 } 1699 }() 1700 } 1701 wg.Wait() 1702 } 1703 1704 It("Allows connection to KubeAPIServer", func() { 1705 installDefaultDenyIngressPolicy( 1706 kubectl, 1707 testNamespace, 1708 validateConnectivity, 1709 ) 1710 installDefaultDenyEgressPolicy( 1711 kubectl, 1712 testNamespace, 1713 validateConnectivity, 1714 ) 1715 1716 By("Verifying KubeAPIServer connectivity is not yet allowed") 1717 Expect( 1718 kubectl.ExecPodCmd( 1719 testNamespace, k8s2PodName, helpers.CurlWithHTTPCode( 1720 "https://%s %s", 1721 kubeAPIServerService.Spec.ClusterIP, 1722 "--insecure", // kube-apiserver needs cert, skip verification 1723 ), 1724 ), 1725 ).To(getMatcher(false), 1726 "HTTP egress connectivity should have been denied to pod %q to kube-apiserver %q", 1727 k8s2PodName, kubeAPIServerService.Spec.ClusterIP, 1728 ) 1729 1730 By("Installing toEntities KubeAPIServer") 1731 importPolicy( 1732 kubectl, 1733 testNamespace, 1734 cnpToEntitiesKubeAPIServer, 1735 "to-entities-kube-apiserver", 1736 ) 1737 1738 By("Verifying policy correctness") 1739 validateConnectivity( 1740 true, /*HostConnectivityAllow*/ 1741 false, /*RemoteNodeConnectivityDeny*/ 1742 false, /*PodConnectivityDeny*/ 1743 false, /*WorldConnectivityDeny*/ 1744 ) 1745 1746 By("Verifying KubeAPIServer connectivity") 1747 // A 403 is a sign of success in this test due to lack of HTTP 1748 // egress policy. We expect to get back 403 because we 1749 // purposefully didn't provide the auth token to fully talk to 1750 // the kube-apiserver. 1751 Expect( 1752 kubectl.ExecPodCmd( 1753 testNamespace, k8s2PodName, helpers.CurlWithHTTPCode( 1754 "https://%s %s", 1755 kubeAPIServerService.Spec.ClusterIP, 1756 "--insecure", // kube-apiserver needs cert, skip verification 1757 ), 1758 ).Stdout(), 1759 ).To(Equal("403"), 1760 "HTTP egress connectivity to pod %q to kube-apiserver %q", 1761 k8s2PodName, kubeAPIServerService.Spec.ClusterIP, 1762 ) 1763 }) 1764 1765 It("Still allows connection to KubeAPIServer with a duplicate policy", func() { 1766 installDefaultDenyIngressPolicy( 1767 kubectl, 1768 testNamespace, 1769 validateConnectivity, 1770 ) 1771 installDefaultDenyEgressPolicy( 1772 kubectl, 1773 testNamespace, 1774 validateConnectivity, 1775 ) 1776 By("Installing toEntities KubeAPIServer") 1777 importPolicy( 1778 kubectl, 1779 testNamespace, 1780 cnpToEntitiesKubeAPIServer, 1781 "to-entities-kube-apiserver", 1782 ) 1783 1784 By("Installing duplicate toEntities KubeAPIServer") 1785 importPolicy( 1786 kubectl, 1787 testNamespace, 1788 helpers.ManifestGet( 1789 kubectl.BasePath(), "cnp-to-entities-kube-apiserver-2.yaml", 1790 ), 1791 "to-entities-kube-apiserver-2", 1792 ) 1793 1794 By("Removing the previous toEntities KubeAPIServer policy") 1795 _, err := kubectl.CiliumPolicyAction( 1796 testNamespace, cnpToEntitiesKubeAPIServer, helpers.KubectlDelete, helpers.HelperTimeout, 1797 ) 1798 Expect(err).Should( 1799 BeNil(), 1800 "policy %s cannot be deleted in %q namespace", cnpToEntitiesKubeAPIServer, testNamespace, 1801 ) 1802 1803 By("Verifying KubeAPIServer connectivity is still allowed") 1804 // See previous It() about the assertion on 403 HTTP code. 1805 Expect( 1806 kubectl.ExecPodCmd( 1807 testNamespace, k8s2PodName, helpers.CurlWithHTTPCode( 1808 "https://%s %s", 1809 kubeAPIServerService.Spec.ClusterIP, 1810 "--insecure", // kube-apiserver needs cert, skip verification 1811 ), 1812 ).Stdout(), 1813 ).To(Equal("403"), 1814 "HTTP egress connectivity to pod %q to kube-apiserver %q", 1815 k8s2PodName, kubeAPIServerService.Spec.ClusterIP, 1816 ) 1817 }) 1818 1819 It("Denies connection to KubeAPIServer", func() { 1820 By("Installing allow-all egress policy") 1821 importPolicy( 1822 kubectl, 1823 testNamespace, 1824 helpers.ManifestGet(kubectl.BasePath(), "cnp-to-entities-all.yaml"), 1825 "allow-all-egress", 1826 ) 1827 1828 By("Installing toEntities KubeAPIServer") 1829 importPolicy( 1830 kubectl, 1831 testNamespace, 1832 cnpToEntitiesKubeAPIServerDeny, 1833 "to-entities-kube-apiserver-deny", 1834 ) 1835 1836 By("Verifying policy correctness") 1837 validateConnectivity( 1838 true, /*HostConnectivityAllow*/ 1839 true, /*RemoteNodeConnectivityAllow*/ 1840 true, /*PodConnectivityAllow*/ 1841 true, /*WorldConnectivityAllow*/ 1842 ) 1843 1844 By("Verifying KubeAPIServer connectivity is denied") 1845 Expect( 1846 kubectl.ExecPodCmd( 1847 testNamespace, k8s2PodName, helpers.CurlWithHTTPCode( 1848 "https://%s %s", 1849 kubeAPIServerService.Spec.ClusterIP, 1850 "--insecure", // kube-apiserver needs cert, skip verification 1851 ), 1852 ), 1853 ).To(getMatcher(false), 1854 "HTTP egress connectivity should have been denied to pod %q to kube-apiserver %q", 1855 k8s2PodName, kubeAPIServerService.Spec.ClusterIP, 1856 ) 1857 }) 1858 }) 1859 }) 1860 1861 func importPolicy(kubectl *helpers.Kubectl, namespace, file, name string) { 1862 _, err := kubectl.CiliumPolicyAction(namespace, 1863 file, 1864 helpers.KubectlApply, 1865 helpers.HelperTimeout) 1866 ExpectWithOffset(1, err).Should(BeNil(), 1867 "policy %s cannot be applied in %q namespace", file, namespace) 1868 } 1869 1870 func installDefaultDenyIngressPolicy( 1871 kubectl *helpers.Kubectl, 1872 ns string, 1873 f func(bool, bool, bool, bool), 1874 ) { 1875 denyIngress := helpers.ManifestGet(kubectl.BasePath(), "cnp-default-deny-ingress.yaml") 1876 1877 By("Installing default-deny ingress policy") 1878 importPolicy(kubectl, ns, denyIngress, "default-deny-ingress") 1879 1880 By("Checking that remote-node is disallowed by default") 1881 f( 1882 true, /*HostConnectivityAllow*/ 1883 false, /*RemoteNodeConnectivityDeny*/ 1884 false, /*PodConnectivityDeny*/ 1885 false, /*WorldConnectivityDeny*/ 1886 ) 1887 } 1888 1889 func installDefaultDenyEgressPolicy( 1890 kubectl *helpers.Kubectl, 1891 ns string, 1892 f func(bool, bool, bool, bool), 1893 ) { 1894 denyEgress := helpers.ManifestGet(kubectl.BasePath(), "cnp-default-deny-egress.yaml") 1895 1896 By("Installing default-deny egress policy") 1897 importPolicy(kubectl, ns, denyEgress, "default-deny-egress") 1898 1899 By("Checking that remote-node is disallowed by default") 1900 f( 1901 true, /*HostConnectivityAllow*/ 1902 false, /*RemoteNodeConnectivityDeny*/ 1903 false, /*PodConnectivityDeny*/ 1904 false, /*WorldConnectivityDeny*/ 1905 ) 1906 } 1907 1908 // getMatcher returns a helper.CMDSucess() matcher for success or failure 1909 // situations. 1910 func getMatcher(val bool) types.GomegaMatcher { 1911 if val { 1912 return helpers.CMDSuccess() 1913 } 1914 return Not(helpers.CMDSuccess()) 1915 }