github.com/k8snetworkplumbingwg/sriov-network-operator@v1.2.1-0.20240408194816-2d2e5a45d453/test/e2e/sriovoperatornodepolicy_test.go (about)

     1  package e2e
     2  
     3  import (
     4  	goctx "context"
     5  	"strconv"
     6  	"time"
     7  
     8  	appsv1 "k8s.io/api/apps/v1"
     9  	corev1 "k8s.io/api/core/v1"
    10  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    11  
    12  	sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1"
    13  
    14  	. "github.com/onsi/ginkgo/v2"
    15  	. "github.com/onsi/gomega"
    16  
    17  	. "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util"
    18  	testclient "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/client"
    19  	"github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/cluster"
    20  	"github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/execute"
    21  )
    22  
    23  var _ = Describe("Operator", func() {
    24  	var sriovInfos *cluster.EnabledNodes
    25  	var err error
    26  	var nodeState *sriovnetworkv1.SriovNetworkNodeState
    27  	var name string
    28  
    29  	execute.BeforeAll(func() {
    30  		clients := testclient.New("")
    31  		Expect(clients).ToNot(BeNil())
    32  
    33  		Eventually(func() *cluster.EnabledNodes {
    34  			sriovInfos, _ = cluster.DiscoverSriov(clients, testNamespace)
    35  			return sriovInfos
    36  		}, timeout, interval).ShouldNot(BeNil())
    37  		Expect(len(sriovInfos.Nodes)).To(BeNumerically(">", 0))
    38  
    39  		sriovIface, err = sriovInfos.FindOneSriovDevice(sriovInfos.Nodes[0])
    40  		Expect(err).ToNot(HaveOccurred())
    41  		Expect(sriovIface).ToNot(BeNil())
    42  	})
    43  
    44  	Context("with single policy", func() {
    45  		policy1 := &sriovnetworkv1.SriovNetworkNodePolicy{
    46  			TypeMeta: metav1.TypeMeta{
    47  				Kind:       "SriovNetworkNodePolicy",
    48  				APIVersion: "sriovnetwork.openshift.io/v1",
    49  			},
    50  			ObjectMeta: metav1.ObjectMeta{
    51  				Name:      "policy-1",
    52  				Namespace: testNamespace,
    53  			},
    54  			Spec: sriovnetworkv1.SriovNetworkNodePolicySpec{
    55  				ResourceName: "resource_1",
    56  				NodeSelector: map[string]string{
    57  					"feature.node.kubernetes.io/network-sriov.capable": "true",
    58  				},
    59  				Priority:    99,
    60  				Mtu:         9000,
    61  				NumVfs:      6,
    62  				NicSelector: sriovnetworkv1.SriovNetworkNicSelector{},
    63  				DeviceType:  "vfio-pci",
    64  			},
    65  		}
    66  		policy2 := &sriovnetworkv1.SriovNetworkNodePolicy{
    67  			TypeMeta: metav1.TypeMeta{
    68  				Kind:       "SriovNetworkNodePolicy",
    69  				APIVersion: "sriovnetwork.openshift.io/v1",
    70  			},
    71  			ObjectMeta: metav1.ObjectMeta{
    72  				Name:      "policy-2",
    73  				Namespace: testNamespace,
    74  			},
    75  			Spec: sriovnetworkv1.SriovNetworkNodePolicySpec{
    76  				ResourceName: "resource_2",
    77  				NodeSelector: map[string]string{
    78  					"feature.node.kubernetes.io/network-sriov.capable": "true",
    79  				},
    80  				Priority:    99,
    81  				Mtu:         9000,
    82  				NumVfs:      6,
    83  				NicSelector: sriovnetworkv1.SriovNetworkNicSelector{},
    84  			},
    85  		}
    86  
    87  		JustBeforeEach(func() {
    88  			By("wait for the node state ready")
    89  			name = sriovInfos.Nodes[0]
    90  			nodeState = &sriovnetworkv1.SriovNetworkNodeState{}
    91  			err = WaitForSriovNetworkNodeStateReady(nodeState, k8sClient, testNamespace, name, RetryInterval, Timeout*15)
    92  			Expect(err).NotTo(HaveOccurred())
    93  		})
    94  
    95  		DescribeTable("should config sriov",
    96  			func(policy *sriovnetworkv1.SriovNetworkNodePolicy) {
    97  				policy.Spec.NicSelector.RootDevices = []string{sriovIface.PciAddress}
    98  				policy.Spec.NicSelector.PfNames = []string{sriovIface.Name}
    99  
   100  				By("apply node policy CR")
   101  				err = k8sClient.Create(goctx.TODO(), policy)
   102  				Expect(err).NotTo(HaveOccurred())
   103  				defer func() {
   104  					// Cleanup the test resource
   105  					Expect(k8sClient.Delete(goctx.TODO(), policy)).To(Succeed())
   106  				}()
   107  
   108  				By("generate the config for device plugin")
   109  				time.Sleep(30 * time.Second)
   110  				config := &corev1.ConfigMap{}
   111  				err = WaitForNamespacedObject(config, k8sClient, testNamespace, "device-plugin-config", RetryInterval, Timeout)
   112  				Expect(err).NotTo(HaveOccurred())
   113  				err = ValidateDevicePluginConfig([]*sriovnetworkv1.SriovNetworkNodePolicy{policy}, config.Data["config.json"])
   114  
   115  				By("wait for the node state ready")
   116  				err = WaitForSriovNetworkNodeStateReady(nodeState, k8sClient, testNamespace, name, RetryInterval, Timeout*15)
   117  				Expect(err).NotTo(HaveOccurred())
   118  
   119  				By("provision the cni and device plugin daemonsets")
   120  				cniDaemonSet := &appsv1.DaemonSet{}
   121  				err = WaitForDaemonSetReady(cniDaemonSet, k8sClient, testNamespace, "sriov-network-config-daemon", RetryInterval, Timeout)
   122  				Expect(err).NotTo(HaveOccurred())
   123  
   124  				dpDaemonSet := &appsv1.DaemonSet{}
   125  				err = WaitForDaemonSetReady(dpDaemonSet, k8sClient, testNamespace, "sriov-device-plugin", RetryInterval, Timeout)
   126  				Expect(err).NotTo(HaveOccurred())
   127  
   128  				By("update the spec of SriovNetworkNodeState CR")
   129  				found := false
   130  				for _, address := range policy.Spec.NicSelector.RootDevices {
   131  					for _, iface := range nodeState.Spec.Interfaces {
   132  						if iface.PciAddress == address {
   133  							found = true
   134  							Expect(iface.NumVfs).To(Equal(policy.Spec.NumVfs))
   135  							Expect(iface.Mtu).To(Equal(policy.Spec.Mtu))
   136  							Expect(iface.VfGroups[0].DeviceType).To(Equal(policy.Spec.DeviceType))
   137  							Expect(iface.VfGroups[0].ResourceName).To(Equal(policy.Spec.ResourceName))
   138  							Expect(iface.VfGroups[0].VfRange).To(Equal("0-" + strconv.Itoa(policy.Spec.NumVfs-1)))
   139  						}
   140  					}
   141  				}
   142  				Expect(found).To(BeTrue())
   143  
   144  				By("update the status of SriovNetworkNodeState CR")
   145  				found = false
   146  				for _, address := range policy.Spec.NicSelector.RootDevices {
   147  					for _, iface := range nodeState.Status.Interfaces {
   148  						if iface.PciAddress == address {
   149  							found = true
   150  							Expect(iface.NumVfs).To(Equal(policy.Spec.NumVfs))
   151  							Expect(iface.Mtu).To(Equal(policy.Spec.Mtu))
   152  							Expect(len(iface.VFs)).To(Equal(policy.Spec.NumVfs))
   153  							for _, vf := range iface.VFs {
   154  								if policy.Spec.DeviceType == "netdevice" || policy.Spec.DeviceType == "" {
   155  									Expect(vf.Mtu).To(Equal(policy.Spec.Mtu))
   156  								}
   157  								if policy.Spec.DeviceType == "vfio" {
   158  									Expect(vf.Driver).To(Equal(policy.Spec.DeviceType))
   159  								}
   160  							}
   161  							break
   162  						}
   163  					}
   164  				}
   165  				Expect(found).To(BeTrue())
   166  			},
   167  			Entry("Set MTU and vfio driver", policy1),
   168  			Entry("Set MTU and default driver", policy2),
   169  		)
   170  	})
   171  
   172  	Context("with single vf index policy", func() {
   173  		policy1 := &sriovnetworkv1.SriovNetworkNodePolicy{
   174  			TypeMeta: metav1.TypeMeta{
   175  				Kind:       "SriovNetworkNodePolicy",
   176  				APIVersion: "sriovnetwork.openshift.io/v1",
   177  			},
   178  			ObjectMeta: metav1.ObjectMeta{
   179  				Name:      "policy-1",
   180  				Namespace: testNamespace,
   181  			},
   182  			Spec: sriovnetworkv1.SriovNetworkNodePolicySpec{
   183  				ResourceName: "resource_1",
   184  				NodeSelector: map[string]string{
   185  					"feature.node.kubernetes.io/network-sriov.capable": "true",
   186  				},
   187  				Priority: 99,
   188  				Mtu:      9000,
   189  				NumVfs:   6,
   190  				NicSelector: sriovnetworkv1.SriovNetworkNicSelector{
   191  					PfNames: []string{"#0-5"},
   192  				},
   193  				DeviceType: "vfio-pci",
   194  			},
   195  		}
   196  		policy2 := &sriovnetworkv1.SriovNetworkNodePolicy{
   197  			TypeMeta: metav1.TypeMeta{
   198  				Kind:       "SriovNetworkNodePolicy",
   199  				APIVersion: "sriovnetwork.openshift.io/v1",
   200  			},
   201  			ObjectMeta: metav1.ObjectMeta{
   202  				Name:      "policy-1",
   203  				Namespace: testNamespace,
   204  			},
   205  			Spec: sriovnetworkv1.SriovNetworkNodePolicySpec{
   206  				ResourceName: "resource_1",
   207  				NodeSelector: map[string]string{
   208  					"feature.node.kubernetes.io/network-sriov.capable": "true",
   209  				},
   210  				Priority: 99,
   211  				Mtu:      9000,
   212  				NumVfs:   6,
   213  				NicSelector: sriovnetworkv1.SriovNetworkNicSelector{
   214  					PfNames: []string{"#0-0"},
   215  				},
   216  			},
   217  		}
   218  
   219  		JustBeforeEach(func() {
   220  			By("wait for the node state ready")
   221  			name = sriovInfos.Nodes[0]
   222  			nodeState = &sriovnetworkv1.SriovNetworkNodeState{}
   223  			err = WaitForSriovNetworkNodeStateReady(nodeState, k8sClient, testNamespace, name, RetryInterval, Timeout*15)
   224  			Expect(err).NotTo(HaveOccurred())
   225  		})
   226  
   227  		DescribeTable("should config sriov",
   228  			func(policy *sriovnetworkv1.SriovNetworkNodePolicy) {
   229  				policy.Spec.NicSelector.RootDevices = []string{sriovIface.PciAddress}
   230  				policy.Spec.NicSelector.PfNames[0] = sriovIface.Name + policy.Spec.NicSelector.PfNames[0]
   231  
   232  				By("apply node policy CR")
   233  				err = k8sClient.Create(goctx.TODO(), policy)
   234  				Expect(err).NotTo(HaveOccurred())
   235  				defer func() {
   236  					// Cleanup the test resource
   237  					Expect(k8sClient.Delete(goctx.TODO(), policy)).To(Succeed())
   238  				}()
   239  
   240  				By("generate the config for device plugin")
   241  				time.Sleep(30 * time.Second)
   242  				config := &corev1.ConfigMap{}
   243  				err = WaitForNamespacedObject(config, k8sClient, testNamespace, "device-plugin-config", RetryInterval, Timeout)
   244  				Expect(err).NotTo(HaveOccurred())
   245  				err = ValidateDevicePluginConfig([]*sriovnetworkv1.SriovNetworkNodePolicy{policy}, config.Data["config.json"])
   246  
   247  				By("wait for the node state ready")
   248  				err = WaitForSriovNetworkNodeStateReady(nodeState, k8sClient, testNamespace, name, RetryInterval, Timeout*15)
   249  				Expect(err).NotTo(HaveOccurred())
   250  
   251  				By("provision the cni and device plugin daemonsets")
   252  				cniDaemonSet := &appsv1.DaemonSet{}
   253  				err = WaitForDaemonSetReady(cniDaemonSet, k8sClient, testNamespace, "sriov-network-config-daemon", RetryInterval, Timeout)
   254  				Expect(err).NotTo(HaveOccurred())
   255  
   256  				dpDaemonSet := &appsv1.DaemonSet{}
   257  				err = WaitForDaemonSetReady(dpDaemonSet, k8sClient, testNamespace, "sriov-device-plugin", RetryInterval, Timeout)
   258  				Expect(err).NotTo(HaveOccurred())
   259  
   260  				By("update the spec of SriovNetworkNodeState CR")
   261  				found := false
   262  				for _, address := range policy.Spec.NicSelector.RootDevices {
   263  					for _, iface := range nodeState.Spec.Interfaces {
   264  						if iface.PciAddress == address {
   265  							found = true
   266  							Expect(iface.NumVfs).To(Equal(policy.Spec.NumVfs))
   267  							Expect(iface.Mtu).To(Equal(policy.Spec.Mtu))
   268  							Expect(iface.VfGroups[0].DeviceType).To(Equal(policy.Spec.DeviceType))
   269  							Expect(iface.VfGroups[0].ResourceName).To(Equal(policy.Spec.ResourceName))
   270  
   271  							pfName, rngStart, rngEnd, err := sriovnetworkv1.ParsePFName(policy.Spec.NicSelector.PfNames[0])
   272  							Expect(err).NotTo(HaveOccurred())
   273  							rng := strconv.Itoa(rngStart) + "-" + strconv.Itoa(rngEnd)
   274  							Expect(iface.Name).To(Equal(pfName))
   275  							Expect(iface.VfGroups[0].VfRange).To(Equal(rng))
   276  						}
   277  					}
   278  				}
   279  				Expect(found).To(BeTrue())
   280  
   281  				By("update the status of SriovNetworkNodeState CR")
   282  				found = false
   283  				for _, address := range policy.Spec.NicSelector.RootDevices {
   284  					for _, iface := range nodeState.Status.Interfaces {
   285  						if iface.PciAddress == address {
   286  							found = true
   287  							Expect(iface.NumVfs).To(Equal(policy.Spec.NumVfs))
   288  							Expect(iface.Mtu).To(Equal(policy.Spec.Mtu))
   289  							Expect(len(iface.VFs)).To(Equal(policy.Spec.NumVfs))
   290  							for _, vf := range iface.VFs {
   291  								if policy.Spec.DeviceType == "netdevice" || policy.Spec.DeviceType == "" {
   292  									Expect(vf.Mtu).To(Equal(policy.Spec.Mtu))
   293  								}
   294  								if policy.Spec.DeviceType == "vfio" {
   295  									Expect(vf.Driver).To(Equal(policy.Spec.DeviceType))
   296  								}
   297  							}
   298  							break
   299  						}
   300  					}
   301  				}
   302  				Expect(found).To(BeTrue())
   303  			},
   304  			Entry("Set one PF with VF range", policy1),
   305  			Entry("Set one PF with VF range", policy2),
   306  		)
   307  	})
   308  
   309  	Context("with multiple vf index policies", func() {
   310  		policy1 := &sriovnetworkv1.SriovNetworkNodePolicy{
   311  			TypeMeta: metav1.TypeMeta{
   312  				Kind:       "SriovNetworkNodePolicy",
   313  				APIVersion: "sriovnetwork.openshift.io/v1",
   314  			},
   315  			ObjectMeta: metav1.ObjectMeta{
   316  				Name:      "policy1",
   317  				Namespace: testNamespace,
   318  			},
   319  			Spec: sriovnetworkv1.SriovNetworkNodePolicySpec{
   320  				ResourceName: "resource1",
   321  				NodeSelector: map[string]string{
   322  					"feature.node.kubernetes.io/network-sriov.capable": "true",
   323  				},
   324  				NumVfs:     6,
   325  				DeviceType: "netdevice",
   326  				NicSelector: sriovnetworkv1.SriovNetworkNicSelector{
   327  					PfNames: []string{"#0-1"},
   328  				},
   329  			},
   330  		}
   331  		policy2 := &sriovnetworkv1.SriovNetworkNodePolicy{
   332  			TypeMeta: metav1.TypeMeta{
   333  				Kind:       "SriovNetworkNodePolicy",
   334  				APIVersion: "sriovnetwork.openshift.io/v1",
   335  			},
   336  			ObjectMeta: metav1.ObjectMeta{
   337  				Name:      "policy2",
   338  				Namespace: testNamespace,
   339  			},
   340  			Spec: sriovnetworkv1.SriovNetworkNodePolicySpec{
   341  				ResourceName: "resource2",
   342  				NodeSelector: map[string]string{
   343  					"feature.node.kubernetes.io/network-sriov.capable": "true",
   344  				},
   345  				NumVfs:     6,
   346  				DeviceType: "netdevice",
   347  				NicSelector: sriovnetworkv1.SriovNetworkNicSelector{
   348  					PfNames: []string{"#2-3"},
   349  				},
   350  			},
   351  		}
   352  
   353  		JustBeforeEach(func() {
   354  			By("wait for the node state ready")
   355  			name = sriovInfos.Nodes[0]
   356  			nodeState = &sriovnetworkv1.SriovNetworkNodeState{}
   357  			err = WaitForSriovNetworkNodeStateReady(nodeState, k8sClient, testNamespace, name, RetryInterval, Timeout*15)
   358  			Expect(err).NotTo(HaveOccurred())
   359  		})
   360  
   361  		It("should config sriov",
   362  			func() {
   363  				policy1.Spec.NicSelector.RootDevices = []string{sriovIface.PciAddress}
   364  				policy1.Spec.NicSelector.PfNames[0] = sriovIface.Name + policy1.Spec.NicSelector.PfNames[0]
   365  				policy2.Spec.NicSelector.RootDevices = []string{sriovIface.PciAddress}
   366  				policy2.Spec.NicSelector.PfNames[0] = sriovIface.Name + policy2.Spec.NicSelector.PfNames[0]
   367  				policies := []*sriovnetworkv1.SriovNetworkNodePolicy{policy1, policy2}
   368  
   369  				By("apply node policy CRs")
   370  				err = k8sClient.Create(goctx.TODO(), policy1)
   371  				Expect(err).NotTo(HaveOccurred())
   372  				defer func() {
   373  					// Cleanup the test resource
   374  					Expect(k8sClient.Delete(goctx.TODO(), policy1)).To(Succeed())
   375  				}()
   376  
   377  				err = k8sClient.Create(goctx.TODO(), policy2)
   378  				Expect(err).NotTo(HaveOccurred())
   379  				defer func() {
   380  					// Cleanup the test resource
   381  					Expect(k8sClient.Delete(goctx.TODO(), policy2)).To(Succeed())
   382  				}()
   383  
   384  				By("generate the config for device plugin")
   385  				time.Sleep(30 * time.Second)
   386  				config := &corev1.ConfigMap{}
   387  				err = WaitForNamespacedObject(config, k8sClient, testNamespace, "device-plugin-config", RetryInterval, Timeout)
   388  				Expect(err).NotTo(HaveOccurred())
   389  				err = ValidateDevicePluginConfig(policies, config.Data["config.json"])
   390  
   391  				By("wait for the node state ready")
   392  				err = WaitForSriovNetworkNodeStateReady(nodeState, k8sClient, testNamespace, name, RetryInterval, Timeout*15)
   393  				Expect(err).NotTo(HaveOccurred())
   394  
   395  				By("provision the cni and device plugin daemonsets")
   396  				cniDaemonSet := &appsv1.DaemonSet{}
   397  				err = WaitForDaemonSetReady(cniDaemonSet, k8sClient, testNamespace, "sriov-network-config-daemon", RetryInterval, Timeout)
   398  				Expect(err).NotTo(HaveOccurred())
   399  
   400  				dpDaemonSet := &appsv1.DaemonSet{}
   401  				err = WaitForDaemonSetReady(dpDaemonSet, k8sClient, testNamespace, "sriov-device-plugin", RetryInterval, Timeout)
   402  				Expect(err).NotTo(HaveOccurred())
   403  
   404  				By("update the spec of SriovNetworkNodeState CR")
   405  				found := false
   406  				for _, address := range policy1.Spec.NicSelector.RootDevices {
   407  					for _, iface := range nodeState.Spec.Interfaces {
   408  						if iface.PciAddress == address {
   409  							found = true
   410  							Expect(iface.NumVfs).To(Equal(policy1.Spec.NumVfs))
   411  							Expect(len(iface.VfGroups)).To(Equal(2))
   412  							vg1 := sriovnetworkv1.VfGroup{
   413  								ResourceName: policy1.Spec.ResourceName,
   414  								DeviceType:   "netdevice",
   415  								VfRange:      "0-1",
   416  								PolicyName:   policy1.GetName(),
   417  							}
   418  							Expect(vg1).To(BeElementOf(iface.VfGroups))
   419  							vg2 := sriovnetworkv1.VfGroup{
   420  								ResourceName: policy2.Spec.ResourceName,
   421  								DeviceType:   "netdevice",
   422  								VfRange:      "2-3",
   423  								PolicyName:   policy2.GetName(),
   424  							}
   425  							Expect(vg2).To(BeElementOf(iface.VfGroups))
   426  						}
   427  					}
   428  				}
   429  				Expect(found).To(BeTrue())
   430  
   431  				By("update the status of SriovNetworkNodeState CR")
   432  				found = false
   433  				for _, address := range policy1.Spec.NicSelector.RootDevices {
   434  					for _, iface := range nodeState.Status.Interfaces {
   435  						if iface.PciAddress == address {
   436  							found = true
   437  							Expect(iface.NumVfs).To(Equal(policy1.Spec.NumVfs))
   438  							Expect(len(iface.VFs)).To(Equal(policy1.Spec.NumVfs))
   439  							break
   440  						}
   441  					}
   442  				}
   443  				Expect(found).To(BeTrue())
   444  			})
   445  	})
   446  })