github.com/cilium/cilium@v1.16.2/test/k8s/config.go (about)

     1  // SPDX-License-Identifier: Apache-2.0
     2  // Copyright Authors of Cilium
     3  
     4  package k8sTest
     5  
     6  import (
     7  	"context"
     8  	"fmt"
     9  	"os"
    10  
    11  	. "github.com/onsi/gomega"
    12  
    13  	. "github.com/cilium/cilium/test/ginkgo-ext"
    14  	"github.com/cilium/cilium/test/helpers"
    15  )
    16  
    17  // Some basic checks of the CiliumNodeConfig plumbing.
    18  //
    19  // This bit is an end-to-end test to ensure that the Helm
    20  // chart is working as intended. There is more complicated
    21  // tests of the logic in unit tests.
    22  //
    23  // It creates a CiliumNodeConfig, ensure it doesn't apply,
    24  // then labels one node and ensure that, yes, it did apply
    25  var _ = SkipDescribeIf(func() bool {
    26  	// right now, the 5.4 job is the fastest, so use only that
    27  	// but only if we're on jenkins
    28  	return helpers.DoesNotRunOn54Kernel() && helpers.RunsOnJenkins()
    29  }, "K8sAgentPerNodeConfigTest", func() {
    30  	var (
    31  		kubectl *helpers.Kubectl
    32  
    33  		ciliumFilename string
    34  	)
    35  	BeforeEach(func() {
    36  		kubectl = helpers.CreateKubectl(helpers.K8s1VMName(), logger)
    37  
    38  		res := kubectl.ExecShort("kubectl label --overwrite --all node io.cilium.testing-")
    39  		Expect(res.GetErr("unlabel node")).To(BeNil())
    40  
    41  		ciliumFilename = helpers.TimestampFilename("cilium.yaml")
    42  		RedeployCilium(kubectl, ciliumFilename, map[string]string{})
    43  	})
    44  
    45  	AfterEach(func() {
    46  		res := kubectl.ExecShort("kubectl label --overwrite --all node io.cilium.testing-")
    47  		Expect(res.GetErr("unlabel node")).To(BeNil())
    48  		UninstallCiliumFromManifest(kubectl, ciliumFilename)
    49  		kubectl.CloseSSHClient()
    50  	})
    51  
    52  	It("Correctly computes config overrides with CNC v2alpha1", func() {
    53  		pods, err := kubectl.GetPodsNodes(helpers.CiliumNamespace, helpers.CiliumSelector)
    54  		Expect(err).To(BeNil(), "error finding cilium pods")
    55  
    56  		// pick a node
    57  		var nodeName string
    58  		for _, v := range pods {
    59  			nodeName = v
    60  			break
    61  		}
    62  		if nodeName == "" {
    63  			Fail("No cilium pods were found")
    64  		}
    65  
    66  		recreatePods := func() {
    67  			By("Deleting all cilium pods")
    68  			kubectl.DeleteResource("-n kube-system pod", "-l "+helpers.CiliumAgentLabel)
    69  			// Wait for pods to be up and running
    70  			kubectl.WaitForCiliumReadiness(0, "not all cilium pods returned")
    71  		}
    72  
    73  		// returns a map of nodename -> config key
    74  		getNodeConfigKeys := func(key string) map[string]string {
    75  			podtonode, err := kubectl.GetPodsNodes(helpers.CiliumNamespace, helpers.CiliumSelector)
    76  			Expect(err).To(BeNil(), "error finding cilium pods")
    77  
    78  			ress, err := kubectl.ExecInPods(context.TODO(), helpers.CiliumNamespace, helpers.CiliumSelector,
    79  				fmt.Sprintf("/bin/sh -c 'cat /tmp/cilium/config-map/%s || true'", key))
    80  			Expect(err).NotTo(HaveOccurred())
    81  			out := map[string]string{}
    82  			for p, res := range ress {
    83  				Expect(res.GetErr(p)).NotTo(HaveOccurred())
    84  				out[podtonode[p]] = res.Stdout()
    85  			}
    86  			return out
    87  		}
    88  
    89  		By("Creating a CiliumNodeConfig v2alpha1")
    90  		// Create a CiliumNodeConfig that does not apply to any nodes
    91  		cnc := `
    92  apiVersion: cilium.io/v2alpha1
    93  kind: CiliumNodeConfig
    94  metadata:
    95    namespace: kube-system
    96    name: testing
    97  spec:
    98    nodeSelector:
    99      matchLabels:
   100        io.cilium.testing: foo
   101    defaults:
   102      test-key: foo
   103  `
   104  		f, err := os.CreateTemp("", "pernodeconfig-")
   105  		Expect(err).Should(BeNil())
   106  		defer os.Remove(f.Name())
   107  
   108  		_, err = f.WriteString(cnc)
   109  		Expect(err).Should(BeNil())
   110  
   111  		res := kubectl.Apply(helpers.ApplyOptions{
   112  			FilePath:  f.Name(),
   113  			Namespace: "kube-system",
   114  		})
   115  		Expect(res.GetErr("apply")).To(BeNil())
   116  
   117  		// ensure no pods have this key
   118  		recreatePods()
   119  		nodeConfigKeys := getNodeConfigKeys("test-key")
   120  		Expect(nodeConfigKeys).To(HaveKey(nodeName))
   121  		Expect(nodeConfigKeys).To(HaveEach("")) //ensure that all elements are ""
   122  
   123  		// Now, label 1 node and check the only it gets this config key
   124  		By("Labeling node %s with io.cilium.testing=foo", nodeName)
   125  		res = kubectl.ExecShort(fmt.Sprintf("kubectl label node %s io.cilium.testing=foo", nodeName))
   126  		Expect(res.GetErr("label node")).To(BeNil())
   127  		recreatePods()
   128  		By("Ensuring only node %s has the config override", nodeName)
   129  		nodeConfigKeys = getNodeConfigKeys("test-key")
   130  		// ensure it is zero
   131  		Expect(nodeConfigKeys).To(HaveKeyWithValue(nodeName, "foo"))
   132  		// If there are other nodes, make sure it doesn't have the config key
   133  		if len(nodeConfigKeys) > 1 {
   134  			delete(nodeConfigKeys, nodeName)
   135  			Expect(nodeConfigKeys).To(HaveEach(""))
   136  		}
   137  	})
   138  
   139  	It("Correctly computes config overrides with CNC v2", func() {
   140  		pods, err := kubectl.GetPodsNodes(helpers.CiliumNamespace, helpers.CiliumSelector)
   141  		Expect(err).To(BeNil(), "error finding cilium pods")
   142  
   143  		// pick a node
   144  		var nodeName string
   145  		for _, v := range pods {
   146  			nodeName = v
   147  			break
   148  		}
   149  		if nodeName == "" {
   150  			Fail("No cilium pods were found")
   151  		}
   152  
   153  		recreatePods := func() {
   154  			By("Deleting all cilium pods")
   155  			kubectl.DeleteResource("-n kube-system pod", "-l "+helpers.CiliumAgentLabel)
   156  			// Wait for pods to be up and running
   157  			kubectl.WaitForCiliumReadiness(0, "not all cilium pods returned")
   158  		}
   159  
   160  		// returns a map of nodename -> config key
   161  		getNodeConfigKeys := func(key string) map[string]string {
   162  			podtonode, err := kubectl.GetPodsNodes(helpers.CiliumNamespace, helpers.CiliumSelector)
   163  			Expect(err).To(BeNil(), "error finding cilium pods")
   164  
   165  			ress, err := kubectl.ExecInPods(context.TODO(), helpers.CiliumNamespace, helpers.CiliumSelector,
   166  				fmt.Sprintf("/bin/sh -c 'cat /tmp/cilium/config-map/%s || true'", key))
   167  			Expect(err).NotTo(HaveOccurred())
   168  			out := map[string]string{}
   169  			for p, res := range ress {
   170  				Expect(res.GetErr(p)).NotTo((HaveOccurred()))
   171  				out[podtonode[p]] = res.Stdout()
   172  			}
   173  			return out
   174  		}
   175  
   176  		By("Creating a CiliumNodeConfig v2")
   177  		// Create a CiliumNodeConfig that does not apply to any nodes
   178  		cnc := `
   179  apiVersion: cilium.io/v2
   180  kind: CiliumNodeConfig
   181  metadata:
   182    namespace: kube-system
   183    name: testing-v2
   184  spec:
   185    nodeSelector:
   186      matchLabels:
   187        io.cilium.testing: bar
   188    defaults:
   189      test-key: bar
   190  `
   191  		f, err := os.CreateTemp("", "pernodeconfig-")
   192  		Expect(err).Should(BeNil())
   193  		defer os.Remove(f.Name())
   194  
   195  		_, err = f.WriteString(cnc)
   196  		Expect(err).Should(BeNil())
   197  
   198  		res := kubectl.Apply(helpers.ApplyOptions{
   199  			FilePath:  f.Name(),
   200  			Namespace: "kube-system",
   201  		})
   202  		Expect(res.GetErr("apply")).To(BeNil())
   203  
   204  		// ensure no pods have this key
   205  		recreatePods()
   206  		nodeConfigKeys := getNodeConfigKeys("test-key")
   207  		Expect(nodeConfigKeys).To(HaveKey(nodeName))
   208  		Expect(nodeConfigKeys).To(HaveEach("")) //ensure that all elements are ""
   209  
   210  		// Now, label 1 node and check the only it gets this config key
   211  		By("Labeling node %s with io.cilium.testing=bar", nodeName)
   212  		res = kubectl.ExecShort(fmt.Sprintf("kubectl label node %s io.cilium.testing=bar", nodeName))
   213  		Expect(res.GetErr("label node")).To(BeNil())
   214  		recreatePods()
   215  		By("Ensuring only node %s has the config override", nodeName)
   216  		nodeConfigKeys = getNodeConfigKeys("test-key")
   217  		// ensure it is zero
   218  		Expect(nodeConfigKeys).To(HaveKeyWithValue(nodeName, "bar"))
   219  		// If there are other nodes, make sure it doesn't have the config key
   220  		if len(nodeConfigKeys) > 1 {
   221  			delete(nodeConfigKeys, nodeName)
   222  			Expect(nodeConfigKeys).To(HaveEach(""))
   223  		}
   224  	})
   225  })