github.com/k8snetworkplumbingwg/sriov-network-operator@v1.2.1-0.20240408194816-2d2e5a45d453/controllers/drain_controller_test.go (about)

     1  package controllers
     2  
     3  import (
     4  	"context"
     5  	"sync"
     6  
     7  	. "github.com/onsi/ginkgo/v2"
     8  	. "github.com/onsi/gomega"
     9  
    10  	"github.com/golang/mock/gomock"
    11  	corev1 "k8s.io/api/core/v1"
    12  	"k8s.io/apimachinery/pkg/api/errors"
    13  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    14  	"k8s.io/apimachinery/pkg/types"
    15  	"k8s.io/apimachinery/pkg/util/intstr"
    16  	"k8s.io/client-go/kubernetes/scheme"
    17  	"k8s.io/utils/pointer"
    18  	"sigs.k8s.io/controller-runtime/pkg/client"
    19  
    20  	mcfgv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1"
    21  
    22  	sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1"
    23  	constants "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts"
    24  	mock_platforms "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/platforms/mock"
    25  	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/platforms/openshift"
    26  	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils"
    27  	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars"
    28  )
    29  
    30  var _ = Describe("Drain Controller", Ordered, func() {
    31  
    32  	var cancel context.CancelFunc
    33  	var ctx context.Context
    34  
    35  	BeforeAll(func() {
    36  		By("Setup controller manager")
    37  		k8sManager, err := setupK8sManagerForTest()
    38  		Expect(err).ToNot(HaveOccurred())
    39  
    40  		t := GinkgoT()
    41  		mockCtrl := gomock.NewController(t)
    42  		platformHelper := mock_platforms.NewMockInterface(mockCtrl)
    43  		platformHelper.EXPECT().GetFlavor().Return(openshift.OpenshiftFlavorDefault).AnyTimes()
    44  		platformHelper.EXPECT().IsOpenshiftCluster().Return(false).AnyTimes()
    45  		platformHelper.EXPECT().IsHypershift().Return(false).AnyTimes()
    46  		platformHelper.EXPECT().OpenshiftBeforeDrainNode(gomock.Any(), gomock.Any()).Return(true, nil).AnyTimes()
    47  		platformHelper.EXPECT().OpenshiftAfterCompleteDrainNode(gomock.Any(), gomock.Any()).Return(true, nil).AnyTimes()
    48  
    49  		// we need a client that doesn't use the local cache for the objects
    50  		drainKClient, err := client.New(cfg, client.Options{
    51  			Scheme: scheme.Scheme,
    52  			Cache: &client.CacheOptions{
    53  				DisableFor: []client.Object{
    54  					&sriovnetworkv1.SriovNetworkNodeState{},
    55  					&corev1.Node{},
    56  					&mcfgv1.MachineConfigPool{},
    57  				},
    58  			},
    59  		})
    60  		Expect(err).ToNot(HaveOccurred())
    61  
    62  		drainController, err := NewDrainReconcileController(drainKClient,
    63  			k8sManager.GetScheme(),
    64  			k8sManager.GetEventRecorderFor("operator"),
    65  			platformHelper)
    66  		Expect(err).ToNot(HaveOccurred())
    67  		err = drainController.SetupWithManager(k8sManager)
    68  		Expect(err).ToNot(HaveOccurred())
    69  
    70  		ctx, cancel = context.WithCancel(context.Background())
    71  
    72  		wg := sync.WaitGroup{}
    73  		wg.Add(1)
    74  		go func() {
    75  			defer wg.Done()
    76  			defer GinkgoRecover()
    77  			By("Start controller manager")
    78  			err := k8sManager.Start(ctx)
    79  			Expect(err).ToNot(HaveOccurred())
    80  		}()
    81  
    82  		DeferCleanup(func() {
    83  			By("Shutdown controller manager")
    84  			cancel()
    85  			wg.Wait()
    86  		})
    87  
    88  		err = k8sClient.Create(ctx, &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "default", Namespace: "default"}})
    89  		Expect(err).ToNot(HaveOccurred())
    90  	})
    91  
    92  	BeforeEach(func() {
    93  		Expect(k8sClient.DeleteAllOf(context.Background(), &corev1.Node{})).ToNot(HaveOccurred())
    94  		Expect(k8sClient.DeleteAllOf(context.Background(), &sriovnetworkv1.SriovNetworkNodeState{}, client.InNamespace(vars.Namespace))).ToNot(HaveOccurred())
    95  
    96  		poolConfig := &sriovnetworkv1.SriovNetworkPoolConfig{}
    97  		poolConfig.SetNamespace(testNamespace)
    98  		poolConfig.SetName("test-workers")
    99  		err := k8sClient.Delete(context.Background(), poolConfig)
   100  		if err != nil {
   101  			Expect(errors.IsNotFound(err)).To(BeTrue())
   102  		}
   103  
   104  		podList := &corev1.PodList{}
   105  		err = k8sClient.List(context.Background(), podList, &client.ListOptions{Namespace: "default"})
   106  		Expect(err).ToNot(HaveOccurred())
   107  		for _, podObj := range podList.Items {
   108  			err = k8sClient.Delete(context.Background(), &podObj, &client.DeleteOptions{GracePeriodSeconds: pointer.Int64(0)})
   109  			Expect(err).ToNot(HaveOccurred())
   110  		}
   111  
   112  	})
   113  
   114  	Context("when there is only one node", func() {
   115  
   116  		It("should drain", func(ctx context.Context) {
   117  			node, nodeState := createNode(ctx, "node1")
   118  
   119  			simulateDaemonSetAnnotation(node, constants.DrainRequired)
   120  
   121  			expectNodeStateAnnotation(nodeState, constants.DrainComplete)
   122  			expectNodeIsNotSchedulable(node)
   123  
   124  			simulateDaemonSetAnnotation(node, constants.DrainIdle)
   125  
   126  			expectNodeStateAnnotation(nodeState, constants.DrainIdle)
   127  			expectNodeIsSchedulable(node)
   128  		})
   129  	})
   130  
   131  	Context("when there are multiple nodes", func() {
   132  
   133  		It("should drain nodes serially with default pool selector", func(ctx context.Context) {
   134  			node1, nodeState1 := createNode(ctx, "node1")
   135  			node2, nodeState2 := createNode(ctx, "node2")
   136  			node3, nodeState3 := createNode(ctx, "node3")
   137  
   138  			// Two nodes require to drain at the same time
   139  			simulateDaemonSetAnnotation(node1, constants.DrainRequired)
   140  			simulateDaemonSetAnnotation(node2, constants.DrainRequired)
   141  
   142  			// Only the first node drains
   143  			expectNodeStateAnnotation(nodeState1, constants.DrainComplete)
   144  			expectNodeStateAnnotation(nodeState2, constants.DrainIdle)
   145  			expectNodeStateAnnotation(nodeState3, constants.DrainIdle)
   146  			expectNodeIsNotSchedulable(node1)
   147  			expectNodeIsSchedulable(node2)
   148  			expectNodeIsSchedulable(node3)
   149  
   150  			simulateDaemonSetAnnotation(node1, constants.DrainIdle)
   151  
   152  			expectNodeStateAnnotation(nodeState1, constants.DrainIdle)
   153  			expectNodeIsSchedulable(node1)
   154  
   155  			// Second node starts draining
   156  			expectNodeStateAnnotation(nodeState1, constants.DrainIdle)
   157  			expectNodeStateAnnotation(nodeState2, constants.DrainComplete)
   158  			expectNodeStateAnnotation(nodeState3, constants.DrainIdle)
   159  			expectNodeIsSchedulable(node1)
   160  			expectNodeIsNotSchedulable(node2)
   161  			expectNodeIsSchedulable(node3)
   162  
   163  			simulateDaemonSetAnnotation(node2, constants.DrainIdle)
   164  
   165  			expectNodeStateAnnotation(nodeState1, constants.DrainIdle)
   166  			expectNodeStateAnnotation(nodeState2, constants.DrainIdle)
   167  			expectNodeStateAnnotation(nodeState3, constants.DrainIdle)
   168  			expectNodeIsSchedulable(node1)
   169  			expectNodeIsSchedulable(node2)
   170  			expectNodeIsSchedulable(node3)
   171  		})
   172  
   173  		It("should drain nodes in parallel with a custom pool selector", func(ctx context.Context) {
   174  			node1, nodeState1 := createNode(ctx, "node1")
   175  			node2, nodeState2 := createNode(ctx, "node2")
   176  			node3, nodeState3 := createNode(ctx, "node3")
   177  
   178  			maxun := intstr.Parse("2")
   179  			poolConfig := &sriovnetworkv1.SriovNetworkPoolConfig{}
   180  			poolConfig.SetNamespace(testNamespace)
   181  			poolConfig.SetName("test-workers")
   182  			poolConfig.Spec = sriovnetworkv1.SriovNetworkPoolConfigSpec{MaxUnavailable: &maxun, NodeSelector: &metav1.LabelSelector{
   183  				MatchLabels: map[string]string{
   184  					"test": "",
   185  				},
   186  			}}
   187  			Expect(k8sClient.Create(context.TODO(), poolConfig)).Should(Succeed())
   188  
   189  			// Two nodes require to drain at the same time
   190  			simulateDaemonSetAnnotation(node1, constants.DrainRequired)
   191  			simulateDaemonSetAnnotation(node2, constants.DrainRequired)
   192  
   193  			// Both nodes drain
   194  			expectNodeStateAnnotation(nodeState1, constants.DrainComplete)
   195  			expectNodeStateAnnotation(nodeState2, constants.DrainComplete)
   196  			expectNodeStateAnnotation(nodeState3, constants.DrainIdle)
   197  			expectNodeIsNotSchedulable(node1)
   198  			expectNodeIsNotSchedulable(node2)
   199  			expectNodeIsSchedulable(node3)
   200  
   201  			simulateDaemonSetAnnotation(node1, constants.DrainIdle)
   202  
   203  			expectNodeStateAnnotation(nodeState1, constants.DrainIdle)
   204  			expectNodeIsSchedulable(node1)
   205  
   206  			// Second node starts draining
   207  			expectNodeStateAnnotation(nodeState1, constants.DrainIdle)
   208  			expectNodeStateAnnotation(nodeState2, constants.DrainComplete)
   209  			expectNodeStateAnnotation(nodeState3, constants.DrainIdle)
   210  			expectNodeIsSchedulable(node1)
   211  			expectNodeIsNotSchedulable(node2)
   212  			expectNodeIsSchedulable(node3)
   213  
   214  			simulateDaemonSetAnnotation(node2, constants.DrainIdle)
   215  
   216  			expectNodeStateAnnotation(nodeState1, constants.DrainIdle)
   217  			expectNodeStateAnnotation(nodeState2, constants.DrainIdle)
   218  			expectNodeStateAnnotation(nodeState3, constants.DrainIdle)
   219  			expectNodeIsSchedulable(node1)
   220  			expectNodeIsSchedulable(node2)
   221  			expectNodeIsSchedulable(node3)
   222  		})
   223  
   224  		It("should drain nodes in parallel with a custom pool selector and honor MaxUnavailable", func(ctx context.Context) {
   225  			node1, nodeState1 := createNode(ctx, "node1")
   226  			node2, nodeState2 := createNode(ctx, "node2")
   227  			node3, nodeState3 := createNode(ctx, "node3")
   228  
   229  			maxun := intstr.Parse("2")
   230  			poolConfig := &sriovnetworkv1.SriovNetworkPoolConfig{}
   231  			poolConfig.SetNamespace(testNamespace)
   232  			poolConfig.SetName("test-workers")
   233  			poolConfig.Spec = sriovnetworkv1.SriovNetworkPoolConfigSpec{MaxUnavailable: &maxun, NodeSelector: &metav1.LabelSelector{
   234  				MatchLabels: map[string]string{
   235  					"test": "",
   236  				},
   237  			}}
   238  			Expect(k8sClient.Create(context.TODO(), poolConfig)).Should(Succeed())
   239  
   240  			// Two nodes require to drain at the same time
   241  			simulateDaemonSetAnnotation(node1, constants.DrainRequired)
   242  			simulateDaemonSetAnnotation(node2, constants.DrainRequired)
   243  			simulateDaemonSetAnnotation(node3, constants.DrainRequired)
   244  
   245  			expectNumberOfDrainingNodes(2, nodeState1, nodeState2, nodeState3)
   246  			ExpectDrainCompleteNodesHaveIsNotSchedule(nodeState1, nodeState2, nodeState3)
   247  		})
   248  
   249  		It("should drain all nodes in parallel with a custom pool using nil in max unavailable", func(ctx context.Context) {
   250  			node1, nodeState1 := createNode(ctx, "node1")
   251  			node2, nodeState2 := createNode(ctx, "node2")
   252  			node3, nodeState3 := createNode(ctx, "node3")
   253  
   254  			poolConfig := &sriovnetworkv1.SriovNetworkPoolConfig{}
   255  			poolConfig.SetNamespace(testNamespace)
   256  			poolConfig.SetName("test-workers")
   257  			poolConfig.Spec = sriovnetworkv1.SriovNetworkPoolConfigSpec{MaxUnavailable: nil, NodeSelector: &metav1.LabelSelector{
   258  				MatchLabels: map[string]string{
   259  					"test": "",
   260  				},
   261  			}}
   262  			Expect(k8sClient.Create(context.TODO(), poolConfig)).Should(Succeed())
   263  
   264  			// Two nodes require to drain at the same time
   265  			simulateDaemonSetAnnotation(node1, constants.DrainRequired)
   266  			simulateDaemonSetAnnotation(node2, constants.DrainRequired)
   267  			simulateDaemonSetAnnotation(node3, constants.DrainRequired)
   268  
   269  			expectNodeStateAnnotation(nodeState1, constants.DrainComplete)
   270  			expectNodeStateAnnotation(nodeState2, constants.DrainComplete)
   271  			expectNodeStateAnnotation(nodeState3, constants.DrainComplete)
   272  			expectNodeIsNotSchedulable(node1)
   273  			expectNodeIsNotSchedulable(node2)
   274  			expectNodeIsNotSchedulable(node3)
   275  		})
   276  
   277  		It("should drain in parallel nodes from two different pools, one custom and one default", func() {
   278  			node1, nodeState1 := createNode(ctx, "node1")
   279  			node2, nodeState2 := createNodeWithLabel(ctx, "node2", "pool")
   280  			createPodOnNode(ctx, "test-node-2", "node2")
   281  
   282  			maxun := intstr.Parse("1")
   283  			poolConfig := &sriovnetworkv1.SriovNetworkPoolConfig{}
   284  			poolConfig.SetNamespace(testNamespace)
   285  			poolConfig.SetName("test-workers")
   286  			poolConfig.Spec = sriovnetworkv1.SriovNetworkPoolConfigSpec{MaxUnavailable: &maxun, NodeSelector: &metav1.LabelSelector{
   287  				MatchLabels: map[string]string{
   288  					"pool": "",
   289  				},
   290  			}}
   291  			Expect(k8sClient.Create(context.TODO(), poolConfig)).Should(Succeed())
   292  
   293  			simulateDaemonSetAnnotation(node2, constants.RebootRequired)
   294  			expectNodeStateAnnotation(nodeState2, constants.Draining)
   295  
   296  			simulateDaemonSetAnnotation(node1, constants.DrainRequired)
   297  			expectNodeStateAnnotation(nodeState1, constants.DrainComplete)
   298  		})
   299  
   300  		It("should select all the nodes to drain in parallel when the selector is empty", func() {
   301  			node1, nodeState1 := createNode(ctx, "node3")
   302  			node2, nodeState2 := createNodeWithLabel(ctx, "node4", "pool")
   303  			createPodOnNode(ctx, "test-empty-1", "node3")
   304  			createPodOnNode(ctx, "test-empty-2", "node4")
   305  
   306  			maxun := intstr.Parse("10")
   307  			poolConfig := &sriovnetworkv1.SriovNetworkPoolConfig{}
   308  			poolConfig.SetNamespace(testNamespace)
   309  			poolConfig.SetName("test-workers")
   310  			poolConfig.Spec = sriovnetworkv1.SriovNetworkPoolConfigSpec{MaxUnavailable: &maxun}
   311  			Expect(k8sClient.Create(context.TODO(), poolConfig)).Should(Succeed())
   312  
   313  			simulateDaemonSetAnnotation(node2, constants.RebootRequired)
   314  			simulateDaemonSetAnnotation(node1, constants.RebootRequired)
   315  			expectNodeStateAnnotation(nodeState2, constants.Draining)
   316  			expectNodeStateAnnotation(nodeState1, constants.Draining)
   317  		})
   318  	})
   319  })
   320  
   321  func expectNodeStateAnnotation(nodeState *sriovnetworkv1.SriovNetworkNodeState, expectedAnnotationValue string) {
   322  	EventuallyWithOffset(1, func(g Gomega) {
   323  		g.Expect(k8sClient.Get(context.Background(), types.NamespacedName{Namespace: nodeState.Namespace, Name: nodeState.Name}, nodeState)).
   324  			ToNot(HaveOccurred())
   325  
   326  		g.Expect(utils.ObjectHasAnnotation(nodeState, constants.NodeStateDrainAnnotationCurrent, expectedAnnotationValue)).
   327  			To(BeTrue(),
   328  				"Node[%s] annotation[%s] == '%s'. Expected '%s'", nodeState.Name, constants.NodeDrainAnnotation, nodeState.GetLabels()[constants.NodeStateDrainAnnotationCurrent], expectedAnnotationValue)
   329  	}, "20s", "1s").Should(Succeed())
   330  }
   331  
   332  func expectNumberOfDrainingNodes(numbOfDrain int, nodesState ...*sriovnetworkv1.SriovNetworkNodeState) {
   333  	EventuallyWithOffset(1, func(g Gomega) {
   334  		drainingNodes := 0
   335  		for _, nodeState := range nodesState {
   336  			g.Expect(k8sClient.Get(context.Background(), types.NamespacedName{Namespace: nodeState.Namespace, Name: nodeState.Name}, nodeState)).
   337  				ToNot(HaveOccurred())
   338  
   339  			if utils.ObjectHasAnnotation(nodeState, constants.NodeStateDrainAnnotationCurrent, constants.DrainComplete) {
   340  				drainingNodes++
   341  			}
   342  		}
   343  
   344  		g.Expect(drainingNodes).To(Equal(numbOfDrain))
   345  	}, "20s", "1s").Should(Succeed())
   346  }
   347  
   348  func ExpectDrainCompleteNodesHaveIsNotSchedule(nodesState ...*sriovnetworkv1.SriovNetworkNodeState) {
   349  	for _, nodeState := range nodesState {
   350  		if utils.ObjectHasAnnotation(nodeState, constants.NodeStateDrainAnnotationCurrent, constants.DrainComplete) {
   351  			node := &corev1.Node{}
   352  			Expect(k8sClient.Get(context.Background(), types.NamespacedName{Name: nodeState.Name}, node)).
   353  				ToNot(HaveOccurred())
   354  			expectNodeIsNotSchedulable(node)
   355  		}
   356  	}
   357  }
   358  
   359  func expectNodeIsNotSchedulable(node *corev1.Node) {
   360  	EventuallyWithOffset(1, func(g Gomega) {
   361  		g.Expect(k8sClient.Get(context.Background(), types.NamespacedName{Name: node.Name}, node)).
   362  			ToNot(HaveOccurred())
   363  
   364  		g.Expect(node.Spec.Unschedulable).To(BeTrue())
   365  	}, "20s", "1s").Should(Succeed())
   366  }
   367  
   368  func expectNodeIsSchedulable(node *corev1.Node) {
   369  	EventuallyWithOffset(1, func(g Gomega) {
   370  		g.Expect(k8sClient.Get(context.Background(), types.NamespacedName{Name: node.Name}, node)).
   371  			ToNot(HaveOccurred())
   372  
   373  		g.Expect(node.Spec.Unschedulable).To(BeFalse())
   374  	}, "20s", "1s").Should(Succeed())
   375  }
   376  
   377  func simulateDaemonSetAnnotation(node *corev1.Node, drainAnnotationValue string) {
   378  	ExpectWithOffset(1,
   379  		utils.AnnotateObject(context.Background(), node, constants.NodeDrainAnnotation, drainAnnotationValue, k8sClient)).
   380  		ToNot(HaveOccurred())
   381  }
   382  
   383  func createNode(ctx context.Context, nodeName string) (*corev1.Node, *sriovnetworkv1.SriovNetworkNodeState) {
   384  	node := corev1.Node{
   385  		ObjectMeta: metav1.ObjectMeta{
   386  			Name: nodeName,
   387  			Annotations: map[string]string{
   388  				constants.NodeDrainAnnotation:                     constants.DrainIdle,
   389  				"machineconfiguration.openshift.io/desiredConfig": "worker-1",
   390  			},
   391  			Labels: map[string]string{
   392  				"test": "",
   393  			},
   394  		},
   395  	}
   396  
   397  	nodeState := sriovnetworkv1.SriovNetworkNodeState{
   398  		ObjectMeta: metav1.ObjectMeta{
   399  			Name:      nodeName,
   400  			Namespace: vars.Namespace,
   401  			Labels: map[string]string{
   402  				constants.NodeStateDrainAnnotationCurrent: constants.DrainIdle,
   403  			},
   404  		},
   405  	}
   406  
   407  	Expect(k8sClient.Create(ctx, &node)).ToNot(HaveOccurred())
   408  	Expect(k8sClient.Create(ctx, &nodeState)).ToNot(HaveOccurred())
   409  
   410  	return &node, &nodeState
   411  }
   412  
   413  func createNodeWithLabel(ctx context.Context, nodeName string, label string) (*corev1.Node, *sriovnetworkv1.SriovNetworkNodeState) {
   414  	node := corev1.Node{
   415  		ObjectMeta: metav1.ObjectMeta{
   416  			Name: nodeName,
   417  			Annotations: map[string]string{
   418  				constants.NodeDrainAnnotation:                     constants.DrainIdle,
   419  				"machineconfiguration.openshift.io/desiredConfig": "worker-1",
   420  			},
   421  			Labels: map[string]string{
   422  				label: "",
   423  			},
   424  		},
   425  	}
   426  
   427  	nodeState := sriovnetworkv1.SriovNetworkNodeState{
   428  		ObjectMeta: metav1.ObjectMeta{
   429  			Name:      nodeName,
   430  			Namespace: vars.Namespace,
   431  			Labels: map[string]string{
   432  				constants.NodeStateDrainAnnotationCurrent: constants.DrainIdle,
   433  			},
   434  		},
   435  	}
   436  
   437  	Expect(k8sClient.Create(ctx, &node)).ToNot(HaveOccurred())
   438  	Expect(k8sClient.Create(ctx, &nodeState)).ToNot(HaveOccurred())
   439  
   440  	return &node, &nodeState
   441  }
   442  
   443  func createPodOnNode(ctx context.Context, podName, nodeName string) {
   444  	pod := corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: podName, Namespace: "default"},
   445  		Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "test", Image: "test", Command: []string{"test"}}},
   446  			NodeName: nodeName, TerminationGracePeriodSeconds: pointer.Int64(60)}}
   447  	Expect(k8sClient.Create(ctx, &pod)).ToNot(HaveOccurred())
   448  }