github.com/k8snetworkplumbingwg/sriov-network-operator@v1.2.1-0.20240408194816-2d2e5a45d453/test/e2e/sriovoperatornodepolicy_test.go (about) 1 package e2e 2 3 import ( 4 goctx "context" 5 "strconv" 6 "time" 7 8 appsv1 "k8s.io/api/apps/v1" 9 corev1 "k8s.io/api/core/v1" 10 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 11 12 sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" 13 14 . "github.com/onsi/ginkgo/v2" 15 . "github.com/onsi/gomega" 16 17 . "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util" 18 testclient "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/client" 19 "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/cluster" 20 "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/execute" 21 ) 22 23 var _ = Describe("Operator", func() { 24 var sriovInfos *cluster.EnabledNodes 25 var err error 26 var nodeState *sriovnetworkv1.SriovNetworkNodeState 27 var name string 28 29 execute.BeforeAll(func() { 30 clients := testclient.New("") 31 Expect(clients).ToNot(BeNil()) 32 33 Eventually(func() *cluster.EnabledNodes { 34 sriovInfos, _ = cluster.DiscoverSriov(clients, testNamespace) 35 return sriovInfos 36 }, timeout, interval).ShouldNot(BeNil()) 37 Expect(len(sriovInfos.Nodes)).To(BeNumerically(">", 0)) 38 39 sriovIface, err = sriovInfos.FindOneSriovDevice(sriovInfos.Nodes[0]) 40 Expect(err).ToNot(HaveOccurred()) 41 Expect(sriovIface).ToNot(BeNil()) 42 }) 43 44 Context("with single policy", func() { 45 policy1 := &sriovnetworkv1.SriovNetworkNodePolicy{ 46 TypeMeta: metav1.TypeMeta{ 47 Kind: "SriovNetworkNodePolicy", 48 APIVersion: "sriovnetwork.openshift.io/v1", 49 }, 50 ObjectMeta: metav1.ObjectMeta{ 51 Name: "policy-1", 52 Namespace: testNamespace, 53 }, 54 Spec: sriovnetworkv1.SriovNetworkNodePolicySpec{ 55 ResourceName: "resource_1", 56 NodeSelector: map[string]string{ 57 "feature.node.kubernetes.io/network-sriov.capable": "true", 58 }, 59 Priority: 99, 60 Mtu: 9000, 61 NumVfs: 6, 62 NicSelector: sriovnetworkv1.SriovNetworkNicSelector{}, 63 DeviceType: "vfio-pci", 64 }, 65 } 66 policy2 := &sriovnetworkv1.SriovNetworkNodePolicy{ 67 TypeMeta: metav1.TypeMeta{ 68 Kind: "SriovNetworkNodePolicy", 69 APIVersion: "sriovnetwork.openshift.io/v1", 70 }, 71 ObjectMeta: metav1.ObjectMeta{ 72 Name: "policy-2", 73 Namespace: testNamespace, 74 }, 75 Spec: sriovnetworkv1.SriovNetworkNodePolicySpec{ 76 ResourceName: "resource_2", 77 NodeSelector: map[string]string{ 78 "feature.node.kubernetes.io/network-sriov.capable": "true", 79 }, 80 Priority: 99, 81 Mtu: 9000, 82 NumVfs: 6, 83 NicSelector: sriovnetworkv1.SriovNetworkNicSelector{}, 84 }, 85 } 86 87 JustBeforeEach(func() { 88 By("wait for the node state ready") 89 name = sriovInfos.Nodes[0] 90 nodeState = &sriovnetworkv1.SriovNetworkNodeState{} 91 err = WaitForSriovNetworkNodeStateReady(nodeState, k8sClient, testNamespace, name, RetryInterval, Timeout*15) 92 Expect(err).NotTo(HaveOccurred()) 93 }) 94 95 DescribeTable("should config sriov", 96 func(policy *sriovnetworkv1.SriovNetworkNodePolicy) { 97 policy.Spec.NicSelector.RootDevices = []string{sriovIface.PciAddress} 98 policy.Spec.NicSelector.PfNames = []string{sriovIface.Name} 99 100 By("apply node policy CR") 101 err = k8sClient.Create(goctx.TODO(), policy) 102 Expect(err).NotTo(HaveOccurred()) 103 defer func() { 104 // Cleanup the test resource 105 Expect(k8sClient.Delete(goctx.TODO(), policy)).To(Succeed()) 106 }() 107 108 By("generate the config for device plugin") 109 time.Sleep(30 * time.Second) 110 config := &corev1.ConfigMap{} 111 err = WaitForNamespacedObject(config, k8sClient, testNamespace, "device-plugin-config", RetryInterval, Timeout) 112 Expect(err).NotTo(HaveOccurred()) 113 err = ValidateDevicePluginConfig([]*sriovnetworkv1.SriovNetworkNodePolicy{policy}, config.Data["config.json"]) 114 115 By("wait for the node state ready") 116 err = WaitForSriovNetworkNodeStateReady(nodeState, k8sClient, testNamespace, name, RetryInterval, Timeout*15) 117 Expect(err).NotTo(HaveOccurred()) 118 119 By("provision the cni and device plugin daemonsets") 120 cniDaemonSet := &appsv1.DaemonSet{} 121 err = WaitForDaemonSetReady(cniDaemonSet, k8sClient, testNamespace, "sriov-network-config-daemon", RetryInterval, Timeout) 122 Expect(err).NotTo(HaveOccurred()) 123 124 dpDaemonSet := &appsv1.DaemonSet{} 125 err = WaitForDaemonSetReady(dpDaemonSet, k8sClient, testNamespace, "sriov-device-plugin", RetryInterval, Timeout) 126 Expect(err).NotTo(HaveOccurred()) 127 128 By("update the spec of SriovNetworkNodeState CR") 129 found := false 130 for _, address := range policy.Spec.NicSelector.RootDevices { 131 for _, iface := range nodeState.Spec.Interfaces { 132 if iface.PciAddress == address { 133 found = true 134 Expect(iface.NumVfs).To(Equal(policy.Spec.NumVfs)) 135 Expect(iface.Mtu).To(Equal(policy.Spec.Mtu)) 136 Expect(iface.VfGroups[0].DeviceType).To(Equal(policy.Spec.DeviceType)) 137 Expect(iface.VfGroups[0].ResourceName).To(Equal(policy.Spec.ResourceName)) 138 Expect(iface.VfGroups[0].VfRange).To(Equal("0-" + strconv.Itoa(policy.Spec.NumVfs-1))) 139 } 140 } 141 } 142 Expect(found).To(BeTrue()) 143 144 By("update the status of SriovNetworkNodeState CR") 145 found = false 146 for _, address := range policy.Spec.NicSelector.RootDevices { 147 for _, iface := range nodeState.Status.Interfaces { 148 if iface.PciAddress == address { 149 found = true 150 Expect(iface.NumVfs).To(Equal(policy.Spec.NumVfs)) 151 Expect(iface.Mtu).To(Equal(policy.Spec.Mtu)) 152 Expect(len(iface.VFs)).To(Equal(policy.Spec.NumVfs)) 153 for _, vf := range iface.VFs { 154 if policy.Spec.DeviceType == "netdevice" || policy.Spec.DeviceType == "" { 155 Expect(vf.Mtu).To(Equal(policy.Spec.Mtu)) 156 } 157 if policy.Spec.DeviceType == "vfio" { 158 Expect(vf.Driver).To(Equal(policy.Spec.DeviceType)) 159 } 160 } 161 break 162 } 163 } 164 } 165 Expect(found).To(BeTrue()) 166 }, 167 Entry("Set MTU and vfio driver", policy1), 168 Entry("Set MTU and default driver", policy2), 169 ) 170 }) 171 172 Context("with single vf index policy", func() { 173 policy1 := &sriovnetworkv1.SriovNetworkNodePolicy{ 174 TypeMeta: metav1.TypeMeta{ 175 Kind: "SriovNetworkNodePolicy", 176 APIVersion: "sriovnetwork.openshift.io/v1", 177 }, 178 ObjectMeta: metav1.ObjectMeta{ 179 Name: "policy-1", 180 Namespace: testNamespace, 181 }, 182 Spec: sriovnetworkv1.SriovNetworkNodePolicySpec{ 183 ResourceName: "resource_1", 184 NodeSelector: map[string]string{ 185 "feature.node.kubernetes.io/network-sriov.capable": "true", 186 }, 187 Priority: 99, 188 Mtu: 9000, 189 NumVfs: 6, 190 NicSelector: sriovnetworkv1.SriovNetworkNicSelector{ 191 PfNames: []string{"#0-5"}, 192 }, 193 DeviceType: "vfio-pci", 194 }, 195 } 196 policy2 := &sriovnetworkv1.SriovNetworkNodePolicy{ 197 TypeMeta: metav1.TypeMeta{ 198 Kind: "SriovNetworkNodePolicy", 199 APIVersion: "sriovnetwork.openshift.io/v1", 200 }, 201 ObjectMeta: metav1.ObjectMeta{ 202 Name: "policy-1", 203 Namespace: testNamespace, 204 }, 205 Spec: sriovnetworkv1.SriovNetworkNodePolicySpec{ 206 ResourceName: "resource_1", 207 NodeSelector: map[string]string{ 208 "feature.node.kubernetes.io/network-sriov.capable": "true", 209 }, 210 Priority: 99, 211 Mtu: 9000, 212 NumVfs: 6, 213 NicSelector: sriovnetworkv1.SriovNetworkNicSelector{ 214 PfNames: []string{"#0-0"}, 215 }, 216 }, 217 } 218 219 JustBeforeEach(func() { 220 By("wait for the node state ready") 221 name = sriovInfos.Nodes[0] 222 nodeState = &sriovnetworkv1.SriovNetworkNodeState{} 223 err = WaitForSriovNetworkNodeStateReady(nodeState, k8sClient, testNamespace, name, RetryInterval, Timeout*15) 224 Expect(err).NotTo(HaveOccurred()) 225 }) 226 227 DescribeTable("should config sriov", 228 func(policy *sriovnetworkv1.SriovNetworkNodePolicy) { 229 policy.Spec.NicSelector.RootDevices = []string{sriovIface.PciAddress} 230 policy.Spec.NicSelector.PfNames[0] = sriovIface.Name + policy.Spec.NicSelector.PfNames[0] 231 232 By("apply node policy CR") 233 err = k8sClient.Create(goctx.TODO(), policy) 234 Expect(err).NotTo(HaveOccurred()) 235 defer func() { 236 // Cleanup the test resource 237 Expect(k8sClient.Delete(goctx.TODO(), policy)).To(Succeed()) 238 }() 239 240 By("generate the config for device plugin") 241 time.Sleep(30 * time.Second) 242 config := &corev1.ConfigMap{} 243 err = WaitForNamespacedObject(config, k8sClient, testNamespace, "device-plugin-config", RetryInterval, Timeout) 244 Expect(err).NotTo(HaveOccurred()) 245 err = ValidateDevicePluginConfig([]*sriovnetworkv1.SriovNetworkNodePolicy{policy}, config.Data["config.json"]) 246 247 By("wait for the node state ready") 248 err = WaitForSriovNetworkNodeStateReady(nodeState, k8sClient, testNamespace, name, RetryInterval, Timeout*15) 249 Expect(err).NotTo(HaveOccurred()) 250 251 By("provision the cni and device plugin daemonsets") 252 cniDaemonSet := &appsv1.DaemonSet{} 253 err = WaitForDaemonSetReady(cniDaemonSet, k8sClient, testNamespace, "sriov-network-config-daemon", RetryInterval, Timeout) 254 Expect(err).NotTo(HaveOccurred()) 255 256 dpDaemonSet := &appsv1.DaemonSet{} 257 err = WaitForDaemonSetReady(dpDaemonSet, k8sClient, testNamespace, "sriov-device-plugin", RetryInterval, Timeout) 258 Expect(err).NotTo(HaveOccurred()) 259 260 By("update the spec of SriovNetworkNodeState CR") 261 found := false 262 for _, address := range policy.Spec.NicSelector.RootDevices { 263 for _, iface := range nodeState.Spec.Interfaces { 264 if iface.PciAddress == address { 265 found = true 266 Expect(iface.NumVfs).To(Equal(policy.Spec.NumVfs)) 267 Expect(iface.Mtu).To(Equal(policy.Spec.Mtu)) 268 Expect(iface.VfGroups[0].DeviceType).To(Equal(policy.Spec.DeviceType)) 269 Expect(iface.VfGroups[0].ResourceName).To(Equal(policy.Spec.ResourceName)) 270 271 pfName, rngStart, rngEnd, err := sriovnetworkv1.ParsePFName(policy.Spec.NicSelector.PfNames[0]) 272 Expect(err).NotTo(HaveOccurred()) 273 rng := strconv.Itoa(rngStart) + "-" + strconv.Itoa(rngEnd) 274 Expect(iface.Name).To(Equal(pfName)) 275 Expect(iface.VfGroups[0].VfRange).To(Equal(rng)) 276 } 277 } 278 } 279 Expect(found).To(BeTrue()) 280 281 By("update the status of SriovNetworkNodeState CR") 282 found = false 283 for _, address := range policy.Spec.NicSelector.RootDevices { 284 for _, iface := range nodeState.Status.Interfaces { 285 if iface.PciAddress == address { 286 found = true 287 Expect(iface.NumVfs).To(Equal(policy.Spec.NumVfs)) 288 Expect(iface.Mtu).To(Equal(policy.Spec.Mtu)) 289 Expect(len(iface.VFs)).To(Equal(policy.Spec.NumVfs)) 290 for _, vf := range iface.VFs { 291 if policy.Spec.DeviceType == "netdevice" || policy.Spec.DeviceType == "" { 292 Expect(vf.Mtu).To(Equal(policy.Spec.Mtu)) 293 } 294 if policy.Spec.DeviceType == "vfio" { 295 Expect(vf.Driver).To(Equal(policy.Spec.DeviceType)) 296 } 297 } 298 break 299 } 300 } 301 } 302 Expect(found).To(BeTrue()) 303 }, 304 Entry("Set one PF with VF range", policy1), 305 Entry("Set one PF with VF range", policy2), 306 ) 307 }) 308 309 Context("with multiple vf index policies", func() { 310 policy1 := &sriovnetworkv1.SriovNetworkNodePolicy{ 311 TypeMeta: metav1.TypeMeta{ 312 Kind: "SriovNetworkNodePolicy", 313 APIVersion: "sriovnetwork.openshift.io/v1", 314 }, 315 ObjectMeta: metav1.ObjectMeta{ 316 Name: "policy1", 317 Namespace: testNamespace, 318 }, 319 Spec: sriovnetworkv1.SriovNetworkNodePolicySpec{ 320 ResourceName: "resource1", 321 NodeSelector: map[string]string{ 322 "feature.node.kubernetes.io/network-sriov.capable": "true", 323 }, 324 NumVfs: 6, 325 DeviceType: "netdevice", 326 NicSelector: sriovnetworkv1.SriovNetworkNicSelector{ 327 PfNames: []string{"#0-1"}, 328 }, 329 }, 330 } 331 policy2 := &sriovnetworkv1.SriovNetworkNodePolicy{ 332 TypeMeta: metav1.TypeMeta{ 333 Kind: "SriovNetworkNodePolicy", 334 APIVersion: "sriovnetwork.openshift.io/v1", 335 }, 336 ObjectMeta: metav1.ObjectMeta{ 337 Name: "policy2", 338 Namespace: testNamespace, 339 }, 340 Spec: sriovnetworkv1.SriovNetworkNodePolicySpec{ 341 ResourceName: "resource2", 342 NodeSelector: map[string]string{ 343 "feature.node.kubernetes.io/network-sriov.capable": "true", 344 }, 345 NumVfs: 6, 346 DeviceType: "netdevice", 347 NicSelector: sriovnetworkv1.SriovNetworkNicSelector{ 348 PfNames: []string{"#2-3"}, 349 }, 350 }, 351 } 352 353 JustBeforeEach(func() { 354 By("wait for the node state ready") 355 name = sriovInfos.Nodes[0] 356 nodeState = &sriovnetworkv1.SriovNetworkNodeState{} 357 err = WaitForSriovNetworkNodeStateReady(nodeState, k8sClient, testNamespace, name, RetryInterval, Timeout*15) 358 Expect(err).NotTo(HaveOccurred()) 359 }) 360 361 It("should config sriov", 362 func() { 363 policy1.Spec.NicSelector.RootDevices = []string{sriovIface.PciAddress} 364 policy1.Spec.NicSelector.PfNames[0] = sriovIface.Name + policy1.Spec.NicSelector.PfNames[0] 365 policy2.Spec.NicSelector.RootDevices = []string{sriovIface.PciAddress} 366 policy2.Spec.NicSelector.PfNames[0] = sriovIface.Name + policy2.Spec.NicSelector.PfNames[0] 367 policies := []*sriovnetworkv1.SriovNetworkNodePolicy{policy1, policy2} 368 369 By("apply node policy CRs") 370 err = k8sClient.Create(goctx.TODO(), policy1) 371 Expect(err).NotTo(HaveOccurred()) 372 defer func() { 373 // Cleanup the test resource 374 Expect(k8sClient.Delete(goctx.TODO(), policy1)).To(Succeed()) 375 }() 376 377 err = k8sClient.Create(goctx.TODO(), policy2) 378 Expect(err).NotTo(HaveOccurred()) 379 defer func() { 380 // Cleanup the test resource 381 Expect(k8sClient.Delete(goctx.TODO(), policy2)).To(Succeed()) 382 }() 383 384 By("generate the config for device plugin") 385 time.Sleep(30 * time.Second) 386 config := &corev1.ConfigMap{} 387 err = WaitForNamespacedObject(config, k8sClient, testNamespace, "device-plugin-config", RetryInterval, Timeout) 388 Expect(err).NotTo(HaveOccurred()) 389 err = ValidateDevicePluginConfig(policies, config.Data["config.json"]) 390 391 By("wait for the node state ready") 392 err = WaitForSriovNetworkNodeStateReady(nodeState, k8sClient, testNamespace, name, RetryInterval, Timeout*15) 393 Expect(err).NotTo(HaveOccurred()) 394 395 By("provision the cni and device plugin daemonsets") 396 cniDaemonSet := &appsv1.DaemonSet{} 397 err = WaitForDaemonSetReady(cniDaemonSet, k8sClient, testNamespace, "sriov-network-config-daemon", RetryInterval, Timeout) 398 Expect(err).NotTo(HaveOccurred()) 399 400 dpDaemonSet := &appsv1.DaemonSet{} 401 err = WaitForDaemonSetReady(dpDaemonSet, k8sClient, testNamespace, "sriov-device-plugin", RetryInterval, Timeout) 402 Expect(err).NotTo(HaveOccurred()) 403 404 By("update the spec of SriovNetworkNodeState CR") 405 found := false 406 for _, address := range policy1.Spec.NicSelector.RootDevices { 407 for _, iface := range nodeState.Spec.Interfaces { 408 if iface.PciAddress == address { 409 found = true 410 Expect(iface.NumVfs).To(Equal(policy1.Spec.NumVfs)) 411 Expect(len(iface.VfGroups)).To(Equal(2)) 412 vg1 := sriovnetworkv1.VfGroup{ 413 ResourceName: policy1.Spec.ResourceName, 414 DeviceType: "netdevice", 415 VfRange: "0-1", 416 PolicyName: policy1.GetName(), 417 } 418 Expect(vg1).To(BeElementOf(iface.VfGroups)) 419 vg2 := sriovnetworkv1.VfGroup{ 420 ResourceName: policy2.Spec.ResourceName, 421 DeviceType: "netdevice", 422 VfRange: "2-3", 423 PolicyName: policy2.GetName(), 424 } 425 Expect(vg2).To(BeElementOf(iface.VfGroups)) 426 } 427 } 428 } 429 Expect(found).To(BeTrue()) 430 431 By("update the status of SriovNetworkNodeState CR") 432 found = false 433 for _, address := range policy1.Spec.NicSelector.RootDevices { 434 for _, iface := range nodeState.Status.Interfaces { 435 if iface.PciAddress == address { 436 found = true 437 Expect(iface.NumVfs).To(Equal(policy1.Spec.NumVfs)) 438 Expect(len(iface.VFs)).To(Equal(policy1.Spec.NumVfs)) 439 break 440 } 441 } 442 } 443 Expect(found).To(BeTrue()) 444 }) 445 }) 446 })