k8s.io/kubernetes@v1.29.3/test/integration/volumescheduling/volume_binding_test.go (about) 1 /* 2 Copyright 2017 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package volumescheduling 18 19 // This file tests the VolumeScheduling feature. 20 21 import ( 22 "context" 23 "fmt" 24 "strconv" 25 "strings" 26 "testing" 27 "time" 28 29 "k8s.io/klog/v2" 30 "k8s.io/klog/v2/ktesting" 31 32 v1 "k8s.io/api/core/v1" 33 storagev1 "k8s.io/api/storage/v1" 34 "k8s.io/apimachinery/pkg/api/resource" 35 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 36 "k8s.io/apimachinery/pkg/util/rand" 37 "k8s.io/apimachinery/pkg/util/sets" 38 "k8s.io/apimachinery/pkg/util/wait" 39 "k8s.io/client-go/informers" 40 clientset "k8s.io/client-go/kubernetes" 41 "k8s.io/client-go/util/workqueue" 42 "k8s.io/kubernetes/pkg/controller/volume/persistentvolume" 43 "k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodevolumelimits" 44 "k8s.io/kubernetes/pkg/volume" 45 volumetest "k8s.io/kubernetes/pkg/volume/testing" 46 testutil "k8s.io/kubernetes/test/integration/util" 47 imageutils "k8s.io/kubernetes/test/utils/image" 48 ) 49 50 type testConfig struct { 51 client clientset.Interface 52 ns string 53 stop <-chan struct{} 54 teardown func() 55 } 56 57 var ( 58 // Delete API objects immediately 59 deletePeriod = int64(0) 60 deleteOption = metav1.DeleteOptions{GracePeriodSeconds: &deletePeriod} 61 62 modeWait = storagev1.VolumeBindingWaitForFirstConsumer 63 modeImmediate = storagev1.VolumeBindingImmediate 64 65 classWait = "wait" 66 classImmediate = "immediate" 67 classDynamic = "dynamic" 68 classTopoMismatch = "topomismatch" 69 70 sharedClasses = map[string]*storagev1.StorageClass{ 71 classImmediate: makeStorageClass(classImmediate, &modeImmediate), 72 classWait: makeStorageClass(classWait, &modeWait), 73 } 74 ) 75 76 const ( 77 node1 = "node-1" 78 node2 = "node-2" 79 podLimit = 50 80 volsPerPod = 3 81 nodeAffinityLabelKey = "kubernetes.io/hostname" 82 provisionerPluginName = "mock-provisioner.kubernetes.io" 83 ) 84 85 type testPV struct { 86 name string 87 scName string 88 preboundPVC string 89 node string 90 } 91 92 type testPVC struct { 93 name string 94 scName string 95 preboundPV string 96 } 97 98 func TestVolumeBinding(t *testing.T) { 99 config := setupCluster(t, "volume-scheduling-", 2, 0, 0) 100 defer config.teardown() 101 102 cases := map[string]struct { 103 pod *v1.Pod 104 pvs []*testPV 105 pvcs []*testPVC 106 // Create these, but they should not be bound in the end 107 unboundPvcs []*testPVC 108 unboundPvs []*testPV 109 shouldFail bool 110 }{ 111 "immediate can bind": { 112 pod: makePod("pod-i-canbind", config.ns, []string{"pvc-i-canbind"}), 113 pvs: []*testPV{{"pv-i-canbind", classImmediate, "", node1}}, 114 pvcs: []*testPVC{{"pvc-i-canbind", classImmediate, ""}}, 115 }, 116 "immediate cannot bind": { 117 pod: makePod("pod-i-cannotbind", config.ns, []string{"pvc-i-cannotbind"}), 118 unboundPvcs: []*testPVC{{"pvc-i-cannotbind", classImmediate, ""}}, 119 shouldFail: true, 120 }, 121 "immediate pvc prebound": { 122 pod: makePod("pod-i-pvc-prebound", config.ns, []string{"pvc-i-prebound"}), 123 pvs: []*testPV{{"pv-i-pvc-prebound", classImmediate, "", node1}}, 124 pvcs: []*testPVC{{"pvc-i-prebound", classImmediate, "pv-i-pvc-prebound"}}, 125 }, 126 "immediate pv prebound": { 127 pod: makePod("pod-i-pv-prebound", config.ns, []string{"pvc-i-pv-prebound"}), 128 pvs: []*testPV{{"pv-i-prebound", classImmediate, "pvc-i-pv-prebound", node1}}, 129 pvcs: []*testPVC{{"pvc-i-pv-prebound", classImmediate, ""}}, 130 }, 131 "wait can bind": { 132 pod: makePod("pod-w-canbind", config.ns, []string{"pvc-w-canbind"}), 133 pvs: []*testPV{{"pv-w-canbind", classWait, "", node1}}, 134 pvcs: []*testPVC{{"pvc-w-canbind", classWait, ""}}, 135 }, 136 "wait cannot bind": { 137 pod: makePod("pod-w-cannotbind", config.ns, []string{"pvc-w-cannotbind"}), 138 unboundPvcs: []*testPVC{{"pvc-w-cannotbind", classWait, ""}}, 139 shouldFail: true, 140 }, 141 "wait pvc prebound": { 142 pod: makePod("pod-w-pvc-prebound", config.ns, []string{"pvc-w-prebound"}), 143 pvs: []*testPV{{"pv-w-pvc-prebound", classWait, "", node1}}, 144 pvcs: []*testPVC{{"pvc-w-prebound", classWait, "pv-w-pvc-prebound"}}, 145 }, 146 "wait pv prebound": { 147 pod: makePod("pod-w-pv-prebound", config.ns, []string{"pvc-w-pv-prebound"}), 148 pvs: []*testPV{{"pv-w-prebound", classWait, "pvc-w-pv-prebound", node1}}, 149 pvcs: []*testPVC{{"pvc-w-pv-prebound", classWait, ""}}, 150 }, 151 "wait can bind two": { 152 pod: makePod("pod-w-canbind-2", config.ns, []string{"pvc-w-canbind-2", "pvc-w-canbind-3"}), 153 pvs: []*testPV{ 154 {"pv-w-canbind-2", classWait, "", node2}, 155 {"pv-w-canbind-3", classWait, "", node2}, 156 }, 157 pvcs: []*testPVC{ 158 {"pvc-w-canbind-2", classWait, ""}, 159 {"pvc-w-canbind-3", classWait, ""}, 160 }, 161 unboundPvs: []*testPV{ 162 {"pv-w-canbind-5", classWait, "", node1}, 163 }, 164 }, 165 "wait cannot bind two": { 166 pod: makePod("pod-w-cannotbind-2", config.ns, []string{"pvc-w-cannotbind-1", "pvc-w-cannotbind-2"}), 167 unboundPvcs: []*testPVC{ 168 {"pvc-w-cannotbind-1", classWait, ""}, 169 {"pvc-w-cannotbind-2", classWait, ""}, 170 }, 171 unboundPvs: []*testPV{ 172 {"pv-w-cannotbind-1", classWait, "", node2}, 173 {"pv-w-cannotbind-2", classWait, "", node1}, 174 }, 175 shouldFail: true, 176 }, 177 "mix immediate and wait": { 178 pod: makePod("pod-mix-bound", config.ns, []string{"pvc-w-canbind-4", "pvc-i-canbind-2"}), 179 pvs: []*testPV{ 180 {"pv-w-canbind-4", classWait, "", node1}, 181 {"pv-i-canbind-2", classImmediate, "", node1}, 182 }, 183 pvcs: []*testPVC{ 184 {"pvc-w-canbind-4", classWait, ""}, 185 {"pvc-i-canbind-2", classImmediate, ""}, 186 }, 187 }, 188 } 189 190 for name, test := range cases { 191 klog.Infof("Running test %v", name) 192 193 // Create two StorageClasses 194 suffix := rand.String(4) 195 classes := map[string]*storagev1.StorageClass{} 196 classes[classImmediate] = makeStorageClass(fmt.Sprintf("immediate-%v", suffix), &modeImmediate) 197 classes[classWait] = makeStorageClass(fmt.Sprintf("wait-%v", suffix), &modeWait) 198 for _, sc := range classes { 199 if _, err := config.client.StorageV1().StorageClasses().Create(context.TODO(), sc, metav1.CreateOptions{}); err != nil { 200 t.Fatalf("Failed to create StorageClass %q: %v", sc.Name, err) 201 } 202 } 203 204 // Create PVs 205 for _, pvConfig := range test.pvs { 206 pv := makePV(pvConfig.name, classes[pvConfig.scName].Name, pvConfig.preboundPVC, config.ns, pvConfig.node) 207 if _, err := config.client.CoreV1().PersistentVolumes().Create(context.TODO(), pv, metav1.CreateOptions{}); err != nil { 208 t.Fatalf("Failed to create PersistentVolume %q: %v", pv.Name, err) 209 } 210 } 211 212 for _, pvConfig := range test.unboundPvs { 213 pv := makePV(pvConfig.name, classes[pvConfig.scName].Name, pvConfig.preboundPVC, config.ns, pvConfig.node) 214 if _, err := config.client.CoreV1().PersistentVolumes().Create(context.TODO(), pv, metav1.CreateOptions{}); err != nil { 215 t.Fatalf("Failed to create PersistentVolume %q: %v", pv.Name, err) 216 } 217 } 218 219 // Wait for PVs to become available to avoid race condition in PV controller 220 // https://github.com/kubernetes/kubernetes/issues/85320 221 for _, pvConfig := range test.pvs { 222 if err := waitForPVPhase(config.client, pvConfig.name, v1.VolumeAvailable); err != nil { 223 t.Fatalf("PersistentVolume %q failed to become available: %v", pvConfig.name, err) 224 } 225 } 226 227 for _, pvConfig := range test.unboundPvs { 228 if err := waitForPVPhase(config.client, pvConfig.name, v1.VolumeAvailable); err != nil { 229 t.Fatalf("PersistentVolume %q failed to become available: %v", pvConfig.name, err) 230 } 231 } 232 233 // Create PVCs 234 for _, pvcConfig := range test.pvcs { 235 pvc := makePVC(pvcConfig.name, config.ns, &classes[pvcConfig.scName].Name, pvcConfig.preboundPV) 236 if _, err := config.client.CoreV1().PersistentVolumeClaims(config.ns).Create(context.TODO(), pvc, metav1.CreateOptions{}); err != nil { 237 t.Fatalf("Failed to create PersistentVolumeClaim %q: %v", pvc.Name, err) 238 } 239 } 240 for _, pvcConfig := range test.unboundPvcs { 241 pvc := makePVC(pvcConfig.name, config.ns, &classes[pvcConfig.scName].Name, pvcConfig.preboundPV) 242 if _, err := config.client.CoreV1().PersistentVolumeClaims(config.ns).Create(context.TODO(), pvc, metav1.CreateOptions{}); err != nil { 243 t.Fatalf("Failed to create PersistentVolumeClaim %q: %v", pvc.Name, err) 244 } 245 } 246 247 // Create Pod 248 if _, err := config.client.CoreV1().Pods(config.ns).Create(context.TODO(), test.pod, metav1.CreateOptions{}); err != nil { 249 t.Fatalf("Failed to create Pod %q: %v", test.pod.Name, err) 250 } 251 if test.shouldFail { 252 if err := waitForPodUnschedulable(config.client, test.pod); err != nil { 253 t.Errorf("Pod %q was not unschedulable: %v", test.pod.Name, err) 254 } 255 } else { 256 if err := waitForPodToSchedule(config.client, test.pod); err != nil { 257 t.Errorf("Failed to schedule Pod %q: %v", test.pod.Name, err) 258 } 259 } 260 261 // Validate PVC/PV binding 262 for _, pvc := range test.pvcs { 263 validatePVCPhase(t, config.client, pvc.name, config.ns, v1.ClaimBound, false) 264 } 265 for _, pvc := range test.unboundPvcs { 266 validatePVCPhase(t, config.client, pvc.name, config.ns, v1.ClaimPending, false) 267 } 268 for _, pv := range test.pvs { 269 validatePVPhase(t, config.client, pv.name, v1.VolumeBound) 270 } 271 for _, pv := range test.unboundPvs { 272 validatePVPhase(t, config.client, pv.name, v1.VolumeAvailable) 273 } 274 275 // Force delete objects, but they still may not be immediately removed 276 deleteTestObjects(config.client, config.ns, deleteOption) 277 } 278 } 279 280 // TestVolumeBindingRescheduling tests scheduler will retry scheduling when needed. 281 func TestVolumeBindingRescheduling(t *testing.T) { 282 config := setupCluster(t, "volume-scheduling-", 2, 0, 0) 283 defer config.teardown() 284 285 storageClassName := "local-storage" 286 287 cases := map[string]struct { 288 pod *v1.Pod 289 pvcs []*testPVC 290 pvs []*testPV 291 trigger func(config *testConfig) 292 shouldFail bool 293 }{ 294 "reschedule on WaitForFirstConsumer dynamic storage class add": { 295 pod: makePod("pod-reschedule-onclassadd-dynamic", config.ns, []string{"pvc-reschedule-onclassadd-dynamic"}), 296 pvcs: []*testPVC{ 297 {"pvc-reschedule-onclassadd-dynamic", "", ""}, 298 }, 299 trigger: func(config *testConfig) { 300 sc := makeDynamicProvisionerStorageClass(storageClassName, &modeWait, nil) 301 if _, err := config.client.StorageV1().StorageClasses().Create(context.TODO(), sc, metav1.CreateOptions{}); err != nil { 302 t.Fatalf("Failed to create StorageClass %q: %v", sc.Name, err) 303 } 304 }, 305 shouldFail: false, 306 }, 307 "reschedule on WaitForFirstConsumer static storage class add": { 308 pod: makePod("pod-reschedule-onclassadd-static", config.ns, []string{"pvc-reschedule-onclassadd-static"}), 309 pvcs: []*testPVC{ 310 {"pvc-reschedule-onclassadd-static", "", ""}, 311 }, 312 trigger: func(config *testConfig) { 313 sc := makeStorageClass(storageClassName, &modeWait) 314 if _, err := config.client.StorageV1().StorageClasses().Create(context.TODO(), sc, metav1.CreateOptions{}); err != nil { 315 t.Fatalf("Failed to create StorageClass %q: %v", sc.Name, err) 316 } 317 // Create pv for this class to mock static provisioner behavior. 318 pv := makePV("pv-reschedule-onclassadd-static", storageClassName, "", "", node1) 319 if pv, err := config.client.CoreV1().PersistentVolumes().Create(context.TODO(), pv, metav1.CreateOptions{}); err != nil { 320 t.Fatalf("Failed to create PersistentVolume %q: %v", pv.Name, err) 321 } 322 }, 323 shouldFail: false, 324 }, 325 "reschedule on delay binding PVC add": { 326 pod: makePod("pod-reschedule-onpvcadd", config.ns, []string{"pvc-reschedule-onpvcadd"}), 327 pvs: []*testPV{ 328 { 329 name: "pv-reschedule-onpvcadd", 330 scName: classWait, 331 node: node1, 332 }, 333 }, 334 trigger: func(config *testConfig) { 335 pvc := makePVC("pvc-reschedule-onpvcadd", config.ns, &classWait, "") 336 if _, err := config.client.CoreV1().PersistentVolumeClaims(config.ns).Create(context.TODO(), pvc, metav1.CreateOptions{}); err != nil { 337 t.Fatalf("Failed to create PersistentVolumeClaim %q: %v", pvc.Name, err) 338 } 339 }, 340 shouldFail: false, 341 }, 342 } 343 344 for name, test := range cases { 345 klog.Infof("Running test %v", name) 346 347 if test.pod == nil { 348 t.Fatal("pod is required for this test") 349 } 350 351 // Create unbound pvc 352 for _, pvcConfig := range test.pvcs { 353 pvc := makePVC(pvcConfig.name, config.ns, &storageClassName, "") 354 if _, err := config.client.CoreV1().PersistentVolumeClaims(config.ns).Create(context.TODO(), pvc, metav1.CreateOptions{}); err != nil { 355 t.Fatalf("Failed to create PersistentVolumeClaim %q: %v", pvc.Name, err) 356 } 357 } 358 359 // Create PVs 360 for _, pvConfig := range test.pvs { 361 pv := makePV(pvConfig.name, sharedClasses[pvConfig.scName].Name, pvConfig.preboundPVC, config.ns, pvConfig.node) 362 if _, err := config.client.CoreV1().PersistentVolumes().Create(context.TODO(), pv, metav1.CreateOptions{}); err != nil { 363 t.Fatalf("Failed to create PersistentVolume %q: %v", pv.Name, err) 364 } 365 } 366 367 // Create pod 368 if _, err := config.client.CoreV1().Pods(config.ns).Create(context.TODO(), test.pod, metav1.CreateOptions{}); err != nil { 369 t.Fatalf("Failed to create Pod %q: %v", test.pod.Name, err) 370 } 371 372 // Wait for pod is unschedulable. 373 klog.Infof("Waiting for pod is unschedulable") 374 if err := waitForPodUnschedulable(config.client, test.pod); err != nil { 375 t.Errorf("Failed as Pod %s was not unschedulable: %v", test.pod.Name, err) 376 } 377 378 // Trigger 379 test.trigger(config) 380 381 // Wait for pod is scheduled or unschedulable. 382 if !test.shouldFail { 383 klog.Infof("Waiting for pod is scheduled") 384 if err := waitForPodToSchedule(config.client, test.pod); err != nil { 385 t.Errorf("Failed to schedule Pod %q: %v", test.pod.Name, err) 386 } 387 } else { 388 klog.Infof("Waiting for pod is unschedulable") 389 if err := waitForPodUnschedulable(config.client, test.pod); err != nil { 390 t.Errorf("Failed as Pod %s was not unschedulable: %v", test.pod.Name, err) 391 } 392 } 393 394 // Force delete objects, but they still may not be immediately removed 395 deleteTestObjects(config.client, config.ns, deleteOption) 396 } 397 } 398 399 // TestVolumeBindingStress creates <podLimit> pods, each with <volsPerPod> unbound or prebound PVCs. 400 // PVs are precreated. 401 func TestVolumeBindingStress(t *testing.T) { 402 testVolumeBindingStress(t, 0, false, 0) 403 } 404 405 // Like TestVolumeBindingStress but with scheduler resync. In real cluster, 406 // scheduler will schedule failed pod frequently due to various events, e.g. 407 // service/node update events. 408 // This is useful to detect possible race conditions. 409 func TestVolumeBindingStressWithSchedulerResync(t *testing.T) { 410 testVolumeBindingStress(t, time.Second, false, 0) 411 } 412 413 // Like TestVolumeBindingStress but with fast dynamic provisioning 414 func TestVolumeBindingDynamicStressFast(t *testing.T) { 415 testVolumeBindingStress(t, 0, true, 0) 416 } 417 418 // Like TestVolumeBindingStress but with slow dynamic provisioning 419 func TestVolumeBindingDynamicStressSlow(t *testing.T) { 420 testVolumeBindingStress(t, 0, true, 10) 421 } 422 423 func testVolumeBindingStress(t *testing.T, schedulerResyncPeriod time.Duration, dynamic bool, provisionDelaySeconds int) { 424 config := setupCluster(t, "volume-binding-stress-", 1, schedulerResyncPeriod, provisionDelaySeconds) 425 defer config.teardown() 426 427 // Set max volume limit to the number of PVCs the test will create 428 // TODO: remove when max volume limit allows setting through storageclass 429 t.Setenv(nodevolumelimits.KubeMaxPDVols, fmt.Sprintf("%v", podLimit*volsPerPod)) 430 431 scName := &classWait 432 if dynamic { 433 scName = &classDynamic 434 sc := makeDynamicProvisionerStorageClass(*scName, &modeWait, nil) 435 if _, err := config.client.StorageV1().StorageClasses().Create(context.TODO(), sc, metav1.CreateOptions{}); err != nil { 436 t.Fatalf("Failed to create StorageClass %q: %v", sc.Name, err) 437 } 438 } 439 440 klog.Infof("Start creating PVs and PVCs") 441 // Create enough PVs and PVCs for all the pods 442 podVolumesCount := podLimit * volsPerPod 443 pvs := make([]*v1.PersistentVolume, podVolumesCount) 444 pvcs := make([]*v1.PersistentVolumeClaim, podVolumesCount) 445 workqueue.ParallelizeUntil(context.TODO(), 16, podVolumesCount, func(i int) { 446 var ( 447 pv *v1.PersistentVolume 448 pvc *v1.PersistentVolumeClaim 449 pvName = fmt.Sprintf("pv-stress-%v", i) 450 pvcName = fmt.Sprintf("pvc-stress-%v", i) 451 ) 452 // Don't create pvs for dynamic provisioning test 453 if !dynamic { 454 if rand.Int()%2 == 0 { 455 // static unbound pvs 456 pv = makePV(pvName, *scName, "", "", node1) 457 } else { 458 // static prebound pvs 459 pv = makePV(pvName, classImmediate, pvcName, config.ns, node1) 460 } 461 if pv, err := config.client.CoreV1().PersistentVolumes().Create(context.TODO(), pv, metav1.CreateOptions{}); err != nil { 462 t.Fatalf("Failed to create PersistentVolume %q: %v", pv.Name, err) 463 } 464 pvs[i] = pv 465 } 466 if pv != nil && pv.Spec.ClaimRef != nil && pv.Spec.ClaimRef.Name == pvcName { 467 pvc = makePVC(pvcName, config.ns, &classImmediate, pv.Name) 468 } else { 469 pvc = makePVC(pvcName, config.ns, scName, "") 470 } 471 if pvc, err := config.client.CoreV1().PersistentVolumeClaims(config.ns).Create(context.TODO(), pvc, metav1.CreateOptions{}); err != nil { 472 t.Fatalf("Failed to create PersistentVolumeClaim %q: %v", pvc.Name, err) 473 } 474 pvcs[i] = pvc 475 }) 476 477 klog.Infof("Start creating Pods") 478 pods := make([]*v1.Pod, podLimit) 479 workqueue.ParallelizeUntil(context.TODO(), 16, podLimit, func(i int) { 480 // Generate string of all the PVCs for the pod 481 podPvcs := []string{} 482 for j := i * volsPerPod; j < (i+1)*volsPerPod; j++ { 483 podPvcs = append(podPvcs, pvcs[j].Name) 484 } 485 486 pod := makePod(fmt.Sprintf("pod%03d", i), config.ns, podPvcs) 487 if pod, err := config.client.CoreV1().Pods(config.ns).Create(context.TODO(), pod, metav1.CreateOptions{}); err != nil { 488 t.Fatalf("Failed to create Pod %q: %v", pod.Name, err) 489 } 490 pods[i] = pod 491 }) 492 493 klog.Infof("Start validating pod scheduled") 494 // Validate Pods scheduled 495 workqueue.ParallelizeUntil(context.TODO(), 16, len(pods), func(i int) { 496 pod := pods[i] 497 // Use increased timeout for stress test because there is a higher chance of 498 // PV sync error 499 if err := waitForPodToScheduleWithTimeout(config.client, pod, 2*time.Minute); err != nil { 500 t.Errorf("Failed to schedule Pod %q: %v", pod.Name, err) 501 } 502 }) 503 504 klog.Infof("Start validating PVCs scheduled") 505 // Validate PVC/PV binding 506 workqueue.ParallelizeUntil(context.TODO(), 16, len(pvcs), func(i int) { 507 validatePVCPhase(t, config.client, pvcs[i].Name, config.ns, v1.ClaimBound, dynamic) 508 }) 509 510 // Don't validate pv for dynamic provisioning test 511 if !dynamic { 512 klog.Infof("Start validating PVs scheduled") 513 workqueue.ParallelizeUntil(context.TODO(), 16, len(pvs), func(i int) { 514 validatePVPhase(t, config.client, pvs[i].Name, v1.VolumeBound) 515 }) 516 } 517 } 518 519 func testVolumeBindingWithAffinity(t *testing.T, anti bool, numNodes, numPods, numPVsFirstNode int) { 520 config := setupCluster(t, "volume-pod-affinity-", numNodes, 0, 0) 521 defer config.teardown() 522 523 pods := []*v1.Pod{} 524 pvcs := []*v1.PersistentVolumeClaim{} 525 526 // Create PVs for the first node 527 for i := 0; i < numPVsFirstNode; i++ { 528 pv := makePV(fmt.Sprintf("pv-node1-%v", i), classWait, "", "", node1) 529 if pv, err := config.client.CoreV1().PersistentVolumes().Create(context.TODO(), pv, metav1.CreateOptions{}); err != nil { 530 t.Fatalf("Failed to create PersistentVolume %q: %v", pv.Name, err) 531 } 532 } 533 534 // Create 1 PV per Node for the remaining nodes 535 for i := 2; i <= numNodes; i++ { 536 pv := makePV(fmt.Sprintf("pv-node%v-0", i), classWait, "", "", fmt.Sprintf("node-%v", i)) 537 if pv, err := config.client.CoreV1().PersistentVolumes().Create(context.TODO(), pv, metav1.CreateOptions{}); err != nil { 538 t.Fatalf("Failed to create PersistentVolume %q: %v", pv.Name, err) 539 } 540 } 541 542 // Create pods 543 for i := 0; i < numPods; i++ { 544 // Create one pvc per pod 545 pvc := makePVC(fmt.Sprintf("pvc-%v", i), config.ns, &classWait, "") 546 if pvc, err := config.client.CoreV1().PersistentVolumeClaims(config.ns).Create(context.TODO(), pvc, metav1.CreateOptions{}); err != nil { 547 t.Fatalf("Failed to create PersistentVolumeClaim %q: %v", pvc.Name, err) 548 } 549 pvcs = append(pvcs, pvc) 550 551 // Create pod with pod affinity 552 pod := makePod(fmt.Sprintf("pod%03d", i), config.ns, []string{pvc.Name}) 553 pod.Spec.Affinity = &v1.Affinity{} 554 affinityTerms := []v1.PodAffinityTerm{ 555 { 556 LabelSelector: &metav1.LabelSelector{ 557 MatchExpressions: []metav1.LabelSelectorRequirement{ 558 { 559 Key: "app", 560 Operator: metav1.LabelSelectorOpIn, 561 Values: []string{"volume-binding-test"}, 562 }, 563 }, 564 }, 565 TopologyKey: nodeAffinityLabelKey, 566 }, 567 } 568 if anti { 569 pod.Spec.Affinity.PodAntiAffinity = &v1.PodAntiAffinity{ 570 RequiredDuringSchedulingIgnoredDuringExecution: affinityTerms, 571 } 572 } else { 573 pod.Spec.Affinity.PodAffinity = &v1.PodAffinity{ 574 RequiredDuringSchedulingIgnoredDuringExecution: affinityTerms, 575 } 576 } 577 578 if pod, err := config.client.CoreV1().Pods(config.ns).Create(context.TODO(), pod, metav1.CreateOptions{}); err != nil { 579 t.Fatalf("Failed to create Pod %q: %v", pod.Name, err) 580 } 581 pods = append(pods, pod) 582 } 583 584 // Validate Pods scheduled 585 scheduledNodes := sets.NewString() 586 for _, pod := range pods { 587 if err := waitForPodToSchedule(config.client, pod); err != nil { 588 t.Errorf("Failed to schedule Pod %q: %v", pod.Name, err) 589 } else { 590 // Keep track of all the nodes that the Pods were scheduled on 591 pod, err = config.client.CoreV1().Pods(config.ns).Get(context.TODO(), pod.Name, metav1.GetOptions{}) 592 if err != nil { 593 t.Fatalf("Failed to get Pod %q: %v", pod.Name, err) 594 } 595 if pod.Spec.NodeName == "" { 596 t.Fatalf("Pod %q node name unset after scheduling", pod.Name) 597 } 598 scheduledNodes.Insert(pod.Spec.NodeName) 599 } 600 } 601 602 // Validate the affinity policy 603 if anti { 604 // The pods should have been spread across different nodes 605 if scheduledNodes.Len() != numPods { 606 t.Errorf("Pods were scheduled across %v nodes instead of %v", scheduledNodes.Len(), numPods) 607 } 608 } else { 609 // The pods should have been scheduled on 1 node 610 if scheduledNodes.Len() != 1 { 611 t.Errorf("Pods were scheduled across %v nodes instead of %v", scheduledNodes.Len(), 1) 612 } 613 } 614 615 // Validate PVC binding 616 for _, pvc := range pvcs { 617 validatePVCPhase(t, config.client, pvc.Name, config.ns, v1.ClaimBound, false) 618 } 619 } 620 621 func TestVolumeBindingWithAntiAffinity(t *testing.T) { 622 numNodes := 10 623 // Create as many pods as number of nodes 624 numPods := numNodes 625 // Create many more PVs on node1 to increase chance of selecting node1 626 numPVsFirstNode := 10 * numNodes 627 628 testVolumeBindingWithAffinity(t, true, numNodes, numPods, numPVsFirstNode) 629 } 630 631 func TestVolumeBindingWithAffinity(t *testing.T) { 632 numPods := 10 633 // Create many more nodes to increase chance of selecting a PV on a different node than node1 634 numNodes := 10 * numPods 635 // Create numPods PVs on the first node 636 numPVsFirstNode := numPods 637 638 testVolumeBindingWithAffinity(t, true, numNodes, numPods, numPVsFirstNode) 639 } 640 641 func TestPVAffinityConflict(t *testing.T) { 642 config := setupCluster(t, "volume-scheduling-", 3, 0, 0) 643 defer config.teardown() 644 645 pv := makePV("local-pv", classImmediate, "", "", node1) 646 pvc := makePVC("local-pvc", config.ns, &classImmediate, "") 647 648 // Create PV 649 if _, err := config.client.CoreV1().PersistentVolumes().Create(context.TODO(), pv, metav1.CreateOptions{}); err != nil { 650 t.Fatalf("Failed to create PersistentVolume %q: %v", pv.Name, err) 651 } 652 653 // Create PVC 654 if _, err := config.client.CoreV1().PersistentVolumeClaims(config.ns).Create(context.TODO(), pvc, metav1.CreateOptions{}); err != nil { 655 t.Fatalf("Failed to create PersistentVolumeClaim %q: %v", pvc.Name, err) 656 } 657 658 // Wait for PVC bound 659 if err := waitForPVCBound(config.client, pvc); err != nil { 660 t.Fatalf("PVC %q failed to bind: %v", pvc.Name, err) 661 } 662 663 nodeMarkers := []interface{}{ 664 markNodeAffinity, 665 markNodeSelector, 666 } 667 for i := 0; i < len(nodeMarkers); i++ { 668 podName := "local-pod-" + strconv.Itoa(i+1) 669 pod := makePod(podName, config.ns, []string{"local-pvc"}) 670 nodeMarkers[i].(func(*v1.Pod, string))(pod, "node-2") 671 // Create Pod 672 if _, err := config.client.CoreV1().Pods(config.ns).Create(context.TODO(), pod, metav1.CreateOptions{}); err != nil { 673 t.Fatalf("Failed to create Pod %q: %v", pod.Name, err) 674 } 675 // Give time to scheduler to attempt to schedule pod 676 if err := waitForPodUnschedulable(config.client, pod); err != nil { 677 t.Errorf("Failed as Pod %s was not unschedulable: %v", pod.Name, err) 678 } 679 // Check pod conditions 680 p, err := config.client.CoreV1().Pods(config.ns).Get(context.TODO(), podName, metav1.GetOptions{}) 681 if err != nil { 682 t.Fatalf("Failed to access Pod %s status: %v", podName, err) 683 } 684 if strings.Compare(string(p.Status.Phase), "Pending") != 0 { 685 t.Fatalf("Failed as Pod %s was in: %s state and not in expected: Pending state", podName, p.Status.Phase) 686 } 687 if strings.Compare(p.Status.Conditions[0].Reason, "Unschedulable") != 0 { 688 t.Fatalf("Failed as Pod %s reason was: %s but expected: Unschedulable", podName, p.Status.Conditions[0].Reason) 689 } 690 if !strings.Contains(p.Status.Conditions[0].Message, "node(s) didn't match Pod's node affinity") { 691 t.Fatalf("Failed as Pod's %s failure message does not contain expected message: node(s) didn't match Pod's node affinity. Got message %q", podName, p.Status.Conditions[0].Message) 692 } 693 // Deleting test pod 694 if err := config.client.CoreV1().Pods(config.ns).Delete(context.TODO(), podName, metav1.DeleteOptions{}); err != nil { 695 t.Fatalf("Failed to delete Pod %s: %v", podName, err) 696 } 697 } 698 } 699 700 func TestVolumeProvision(t *testing.T) { 701 config := setupCluster(t, "volume-scheduling", 1, 0, 0) 702 defer config.teardown() 703 704 type testcaseType struct { 705 pod *v1.Pod 706 pvs []*testPV 707 boundPvcs []*testPVC 708 provisionedPvcs []*testPVC 709 // Create these, but they should not be bound in the end 710 unboundPvcs []*testPVC 711 shouldFail bool 712 } 713 714 cases := map[string]testcaseType{ 715 "wait provisioned": { 716 pod: makePod("pod-pvc-canprovision", config.ns, []string{"pvc-canprovision"}), 717 provisionedPvcs: []*testPVC{{"pvc-canprovision", classWait, ""}}, 718 }, 719 "topolgy unsatisfied": { 720 pod: makePod("pod-pvc-topomismatch", config.ns, []string{"pvc-topomismatch"}), 721 unboundPvcs: []*testPVC{{"pvc-topomismatch", classTopoMismatch, ""}}, 722 shouldFail: true, 723 }, 724 "wait one bound, one provisioned": { 725 pod: makePod("pod-pvc-canbind-or-provision", config.ns, []string{"pvc-w-canbind", "pvc-canprovision"}), 726 pvs: []*testPV{{"pv-w-canbind", classWait, "", node1}}, 727 boundPvcs: []*testPVC{{"pvc-w-canbind", classWait, ""}}, 728 provisionedPvcs: []*testPVC{{"pvc-canprovision", classWait, ""}}, 729 }, 730 "one immediate pv prebound, one wait provisioned": { 731 pod: makePod("pod-i-pv-prebound-w-provisioned", config.ns, []string{"pvc-i-pv-prebound", "pvc-canprovision"}), 732 pvs: []*testPV{{"pv-i-prebound", classImmediate, "pvc-i-pv-prebound", node1}}, 733 boundPvcs: []*testPVC{{"pvc-i-pv-prebound", classImmediate, ""}}, 734 provisionedPvcs: []*testPVC{{"pvc-canprovision", classWait, ""}}, 735 }, 736 "wait one pv prebound, one provisioned": { 737 pod: makePod("pod-w-pv-prebound-w-provisioned", config.ns, []string{"pvc-w-pv-prebound", "pvc-canprovision"}), 738 pvs: []*testPV{{"pv-w-prebound", classWait, "pvc-w-pv-prebound", node1}}, 739 boundPvcs: []*testPVC{{"pvc-w-pv-prebound", classWait, ""}}, 740 provisionedPvcs: []*testPVC{{"pvc-canprovision", classWait, ""}}, 741 }, 742 "immediate provisioned by controller": { 743 pod: makePod("pod-i-unbound", config.ns, []string{"pvc-controller-provisioned"}), 744 // A pvc of immediate binding mode is expected to be provisioned by controller, 745 // we treat it as "bound" here because it is supposed to be in same state 746 // with bound claims, i.e. in bound status and has no selectedNode annotation. 747 boundPvcs: []*testPVC{{"pvc-controller-provisioned", classImmediate, ""}}, 748 }, 749 } 750 751 run := func(t *testing.T, test testcaseType) { 752 t.Log("Creating StorageClass") 753 suffix := rand.String(4) 754 classes := map[string]*storagev1.StorageClass{} 755 classes[classImmediate] = makeDynamicProvisionerStorageClass(fmt.Sprintf("immediate-%v", suffix), &modeImmediate, nil) 756 classes[classWait] = makeDynamicProvisionerStorageClass(fmt.Sprintf("wait-%v", suffix), &modeWait, nil) 757 topo := []v1.TopologySelectorTerm{ 758 { 759 MatchLabelExpressions: []v1.TopologySelectorLabelRequirement{ 760 { 761 Key: nodeAffinityLabelKey, 762 Values: []string{node2}, 763 }, 764 }, 765 }, 766 } 767 classes[classTopoMismatch] = makeDynamicProvisionerStorageClass(fmt.Sprintf("topomismatch-%v", suffix), &modeWait, topo) 768 for _, sc := range classes { 769 if _, err := config.client.StorageV1().StorageClasses().Create(context.TODO(), sc, metav1.CreateOptions{}); err != nil { 770 t.Fatalf("Failed to create StorageClass %q: %v", sc.Name, err) 771 } 772 } 773 774 t.Log("Creating PVs") 775 for _, pvConfig := range test.pvs { 776 pv := makePV(pvConfig.name, classes[pvConfig.scName].Name, pvConfig.preboundPVC, config.ns, pvConfig.node) 777 if _, err := config.client.CoreV1().PersistentVolumes().Create(context.TODO(), pv, metav1.CreateOptions{}); err != nil { 778 t.Fatalf("Failed to create PersistentVolume %q: %v", pv.Name, err) 779 } 780 if err := waitForPVPhase(config.client, pvConfig.name, v1.VolumeAvailable); err != nil { 781 t.Fatalf("PersistentVolume %q failed to become available: %v", pvConfig.name, err) 782 } 783 } 784 785 t.Log("Creating PVCs") 786 for _, pvcConfig := range test.boundPvcs { 787 pvc := makePVC(pvcConfig.name, config.ns, &classes[pvcConfig.scName].Name, pvcConfig.preboundPV) 788 if _, err := config.client.CoreV1().PersistentVolumeClaims(config.ns).Create(context.TODO(), pvc, metav1.CreateOptions{}); err != nil { 789 t.Fatalf("Failed to create PersistentVolumeClaim %q: %v", pvc.Name, err) 790 } 791 } 792 793 t.Log("Creating unbound PVCs") 794 for _, pvcConfig := range test.unboundPvcs { 795 pvc := makePVC(pvcConfig.name, config.ns, &classes[pvcConfig.scName].Name, pvcConfig.preboundPV) 796 if _, err := config.client.CoreV1().PersistentVolumeClaims(config.ns).Create(context.TODO(), pvc, metav1.CreateOptions{}); err != nil { 797 t.Fatalf("Failed to create PersistentVolumeClaim %q: %v", pvc.Name, err) 798 } 799 } 800 801 t.Log("Creating unbound PVCs which should be dynamically provisioned") 802 for _, pvcConfig := range test.provisionedPvcs { 803 pvc := makePVC(pvcConfig.name, config.ns, &classes[pvcConfig.scName].Name, pvcConfig.preboundPV) 804 if _, err := config.client.CoreV1().PersistentVolumeClaims(config.ns).Create(context.TODO(), pvc, metav1.CreateOptions{}); err != nil { 805 t.Fatalf("Failed to create PersistentVolumeClaim %q: %v", pvc.Name, err) 806 } 807 } 808 809 t.Log("Creating the pod to schedule") 810 if _, err := config.client.CoreV1().Pods(config.ns).Create(context.TODO(), test.pod, metav1.CreateOptions{}); err != nil { 811 t.Fatalf("Failed to create Pod %q: %v", test.pod.Name, err) 812 } 813 if test.shouldFail { 814 if err := waitForPodUnschedulable(config.client, test.pod); err != nil { 815 t.Errorf("Pod %q was not unschedulable: %v", test.pod.Name, err) 816 } 817 } else { 818 if err := waitForPodToSchedule(config.client, test.pod); err != nil { 819 t.Errorf("Failed to schedule Pod %q: %v", test.pod.Name, err) 820 } 821 } 822 823 t.Log("Validating PVC/PV binding") 824 for _, pvc := range test.boundPvcs { 825 validatePVCPhase(t, config.client, pvc.name, config.ns, v1.ClaimBound, false) 826 } 827 for _, pvc := range test.unboundPvcs { 828 validatePVCPhase(t, config.client, pvc.name, config.ns, v1.ClaimPending, false) 829 } 830 for _, pvc := range test.provisionedPvcs { 831 validatePVCPhase(t, config.client, pvc.name, config.ns, v1.ClaimBound, true) 832 } 833 for _, pv := range test.pvs { 834 validatePVPhase(t, config.client, pv.name, v1.VolumeBound) 835 } 836 837 // Force delete objects, but they still may not be immediately removed 838 t.Log("Deleting test objects") 839 deleteTestObjects(config.client, config.ns, deleteOption) 840 } 841 842 for name, test := range cases { 843 t.Run(name, func(t *testing.T) { run(t, test) }) 844 } 845 } 846 847 // TestCapacity covers different scenarios involving CSIStorageCapacity objects. 848 func TestCapacity(t *testing.T) { 849 config := setupCluster(t, "volume-scheduling", 1, 0, 0) 850 defer config.teardown() 851 852 type testcaseType struct { 853 pod *v1.Pod 854 pvcs []*testPVC 855 haveCapacity bool 856 capacitySupported bool 857 } 858 859 cases := map[string]testcaseType{ 860 "baseline": { 861 pod: makePod("pod-pvc-canprovision", config.ns, []string{"pvc-canprovision"}), 862 pvcs: []*testPVC{{"pvc-canprovision", classWait, ""}}, 863 }, 864 "out of space": { 865 pod: makePod("pod-pvc-canprovision", config.ns, []string{"pvc-canprovision"}), 866 pvcs: []*testPVC{{"pvc-canprovision", classWait, ""}}, 867 capacitySupported: true, 868 }, 869 "with space": { 870 pod: makePod("pod-pvc-canprovision", config.ns, []string{"pvc-canprovision"}), 871 pvcs: []*testPVC{{"pvc-canprovision", classWait, ""}}, 872 capacitySupported: true, 873 haveCapacity: true, 874 }, 875 "ignored": { 876 pod: makePod("pod-pvc-canprovision", config.ns, []string{"pvc-canprovision"}), 877 pvcs: []*testPVC{{"pvc-canprovision", classWait, ""}}, 878 haveCapacity: true, 879 }, 880 } 881 882 run := func(t *testing.T, test testcaseType) { 883 // Create StorageClasses 884 suffix := rand.String(4) 885 classes := map[string]*storagev1.StorageClass{} 886 classes[classImmediate] = makeDynamicProvisionerStorageClass(fmt.Sprintf("immediate-%v", suffix), &modeImmediate, nil) 887 classes[classWait] = makeDynamicProvisionerStorageClass(fmt.Sprintf("wait-%v", suffix), &modeWait, nil) 888 topo := []v1.TopologySelectorTerm{ 889 { 890 MatchLabelExpressions: []v1.TopologySelectorLabelRequirement{ 891 { 892 Key: nodeAffinityLabelKey, 893 Values: []string{node2}, 894 }, 895 }, 896 }, 897 } 898 classes[classTopoMismatch] = makeDynamicProvisionerStorageClass(fmt.Sprintf("topomismatch-%v", suffix), &modeWait, topo) 899 for _, sc := range classes { 900 if _, err := config.client.StorageV1().StorageClasses().Create(context.TODO(), sc, metav1.CreateOptions{}); err != nil { 901 t.Fatalf("Failed to create StorageClass %q: %v", sc.Name, err) 902 } 903 } 904 905 // The provisioner isn't actually a CSI driver, but 906 // that doesn't matter here. 907 if test.capacitySupported { 908 if _, err := config.client.StorageV1().CSIDrivers().Create(context.TODO(), 909 &storagev1.CSIDriver{ 910 ObjectMeta: metav1.ObjectMeta{ 911 Name: provisionerPluginName, 912 }, 913 Spec: storagev1.CSIDriverSpec{ 914 StorageCapacity: &test.capacitySupported, 915 }, 916 }, 917 metav1.CreateOptions{}); err != nil { 918 t.Fatalf("Failed to create CSIDriver: %v", err) 919 } 920 921 // kube-scheduler may need some time before it gets the CSIDriver object. 922 // Without it, scheduling will happen without considering capacity, which 923 // is not what we want to test. 924 time.Sleep(5 * time.Second) 925 } 926 927 // Create CSIStorageCapacity 928 if test.haveCapacity { 929 if _, err := config.client.StorageV1().CSIStorageCapacities("default").Create(context.TODO(), 930 &storagev1.CSIStorageCapacity{ 931 ObjectMeta: metav1.ObjectMeta{ 932 GenerateName: "foo-", 933 }, 934 StorageClassName: classes[classWait].Name, 935 NodeTopology: &metav1.LabelSelector{}, 936 // More than the 5Gi used in makePVC. 937 Capacity: resource.NewQuantity(6*1024*1024*1024, resource.BinarySI), 938 }, 939 metav1.CreateOptions{}); err != nil { 940 t.Fatalf("Failed to create CSIStorageCapacity: %v", err) 941 } 942 } 943 944 // Create PVCs 945 for _, pvcConfig := range test.pvcs { 946 pvc := makePVC(pvcConfig.name, config.ns, &classes[pvcConfig.scName].Name, pvcConfig.preboundPV) 947 if _, err := config.client.CoreV1().PersistentVolumeClaims(config.ns).Create(context.TODO(), pvc, metav1.CreateOptions{}); err != nil { 948 t.Fatalf("Failed to create PersistentVolumeClaim %q: %v", pvc.Name, err) 949 } 950 } 951 952 // Create Pod 953 if _, err := config.client.CoreV1().Pods(config.ns).Create(context.TODO(), test.pod, metav1.CreateOptions{}); err != nil { 954 t.Fatalf("Failed to create Pod %q: %v", test.pod.Name, err) 955 } 956 957 // Lack of capacity prevents pod scheduling and binding. 958 shouldFail := test.capacitySupported && !test.haveCapacity 959 if shouldFail { 960 if err := waitForPodUnschedulable(config.client, test.pod); err != nil { 961 t.Errorf("Pod %q was not unschedulable: %v", test.pod.Name, err) 962 } 963 } else { 964 if err := waitForPodToSchedule(config.client, test.pod); err != nil { 965 t.Errorf("Failed to schedule Pod %q: %v", test.pod.Name, err) 966 } 967 } 968 969 // Validate 970 for _, pvc := range test.pvcs { 971 if shouldFail { 972 validatePVCPhase(t, config.client, pvc.name, config.ns, v1.ClaimPending, false) 973 } else { 974 validatePVCPhase(t, config.client, pvc.name, config.ns, v1.ClaimBound, true) 975 } 976 } 977 978 // Force delete objects, but they still may not be immediately removed 979 deleteTestObjects(config.client, config.ns, deleteOption) 980 } 981 982 for name, test := range cases { 983 t.Run(name, func(t *testing.T) { run(t, test) }) 984 } 985 } 986 987 // TestRescheduleProvisioning validate that PV controller will remove 988 // selectedNode annotation from a claim to reschedule volume provision 989 // on provision failure. 990 func TestRescheduleProvisioning(t *testing.T) { 991 testCtx := testutil.InitTestAPIServer(t, "reschedule-volume-provision", nil) 992 993 clientset := testCtx.ClientSet 994 ns := testCtx.NS.Name 995 996 defer func() { 997 deleteTestObjects(clientset, ns, metav1.DeleteOptions{}) 998 }() 999 1000 ctrl, informerFactory, err := initPVController(t, testCtx, 0) 1001 if err != nil { 1002 t.Fatalf("Failed to create PV controller: %v", err) 1003 } 1004 1005 // Prepare node and storage class. 1006 testNode := makeNode(1) 1007 if _, err := clientset.CoreV1().Nodes().Create(context.TODO(), testNode, metav1.CreateOptions{}); err != nil { 1008 t.Fatalf("Failed to create Node %q: %v", testNode.Name, err) 1009 } 1010 scName := "fail-provision" 1011 sc := makeDynamicProvisionerStorageClass(scName, &modeWait, nil) 1012 // Expect the storage class fail to provision. 1013 sc.Parameters[volumetest.ExpectProvisionFailureKey] = "" 1014 if _, err := clientset.StorageV1().StorageClasses().Create(context.TODO(), sc, metav1.CreateOptions{}); err != nil { 1015 t.Fatalf("Failed to create StorageClass %q: %v", sc.Name, err) 1016 } 1017 1018 // Create a pvc with selected node annotation. 1019 pvcName := "pvc-fail-to-provision" 1020 pvc := makePVC(pvcName, ns, &scName, "") 1021 pvc.Annotations = map[string]string{"volume.kubernetes.io/selected-node": node1} 1022 pvc, err = clientset.CoreV1().PersistentVolumeClaims(ns).Create(context.TODO(), pvc, metav1.CreateOptions{}) 1023 if err != nil { 1024 t.Fatalf("Failed to create PersistentVolumeClaim %q: %v", pvc.Name, err) 1025 } 1026 // Validate selectedNode annotation exists on created claim. 1027 selectedNodeAnn, exist := pvc.Annotations["volume.kubernetes.io/selected-node"] 1028 if !exist || selectedNodeAnn != node1 { 1029 t.Fatalf("Created pvc is not annotated as expected") 1030 } 1031 1032 // Start controller. 1033 go ctrl.Run(testCtx.Ctx) 1034 informerFactory.Start(testCtx.Ctx.Done()) 1035 informerFactory.WaitForCacheSync(testCtx.Ctx.Done()) 1036 1037 // Validate that the annotation is removed by controller for provision reschedule. 1038 if err := waitForProvisionAnn(clientset, pvc, false); err != nil { 1039 t.Errorf("Expect to reschedule provision for PVC %v/%v, but still found selected-node annotation on it", ns, pvcName) 1040 } 1041 } 1042 1043 func setupCluster(t *testing.T, nsName string, numberOfNodes int, resyncPeriod time.Duration, provisionDelaySeconds int) *testConfig { 1044 testCtx := testutil.InitTestSchedulerWithOptions(t, testutil.InitTestAPIServer(t, nsName, nil), resyncPeriod) 1045 testutil.SyncSchedulerInformerFactory(testCtx) 1046 go testCtx.Scheduler.Run(testCtx.Ctx) 1047 1048 clientset := testCtx.ClientSet 1049 ns := testCtx.NS.Name 1050 1051 ctrl, informerFactory, err := initPVController(t, testCtx, provisionDelaySeconds) 1052 if err != nil { 1053 t.Fatalf("Failed to create PV controller: %v", err) 1054 } 1055 go ctrl.Run(testCtx.Ctx) 1056 // Start informer factory after all controllers are configured and running. 1057 informerFactory.Start(testCtx.Ctx.Done()) 1058 informerFactory.WaitForCacheSync(testCtx.Ctx.Done()) 1059 1060 // Create shared objects 1061 // Create nodes 1062 for i := 0; i < numberOfNodes; i++ { 1063 testNode := makeNode(i + 1) 1064 if _, err := clientset.CoreV1().Nodes().Create(context.TODO(), testNode, metav1.CreateOptions{}); err != nil { 1065 t.Fatalf("Failed to create Node %q: %v", testNode.Name, err) 1066 } 1067 } 1068 1069 // Create SCs 1070 for _, sc := range sharedClasses { 1071 if _, err := clientset.StorageV1().StorageClasses().Create(context.TODO(), sc, metav1.CreateOptions{}); err != nil { 1072 t.Fatalf("Failed to create StorageClass %q: %v", sc.Name, err) 1073 } 1074 } 1075 1076 return &testConfig{ 1077 client: clientset, 1078 ns: ns, 1079 stop: testCtx.Ctx.Done(), 1080 teardown: func() { 1081 klog.Infof("test cluster %q start to tear down", ns) 1082 deleteTestObjects(clientset, ns, metav1.DeleteOptions{}) 1083 }, 1084 } 1085 } 1086 1087 func initPVController(t *testing.T, testCtx *testutil.TestContext, provisionDelaySeconds int) (*persistentvolume.PersistentVolumeController, informers.SharedInformerFactory, error) { 1088 clientset := testCtx.ClientSet 1089 // Informers factory for controllers 1090 informerFactory := informers.NewSharedInformerFactory(clientset, 0) 1091 1092 // Start PV controller for volume binding. 1093 host := volumetest.NewFakeVolumeHost(t, "/tmp/fake", nil, nil) 1094 plugin := &volumetest.FakeVolumePlugin{ 1095 PluginName: provisionerPluginName, 1096 Host: host, 1097 Config: volume.VolumeConfig{}, 1098 LastProvisionerOptions: volume.VolumeOptions{}, 1099 ProvisionDelaySeconds: provisionDelaySeconds, 1100 NewAttacherCallCount: 0, 1101 NewDetacherCallCount: 0, 1102 Mounters: nil, 1103 Unmounters: nil, 1104 Attachers: nil, 1105 Detachers: nil, 1106 } 1107 plugins := []volume.VolumePlugin{plugin} 1108 1109 params := persistentvolume.ControllerParameters{ 1110 KubeClient: clientset, 1111 // Use a frequent resync period to retry API update conflicts due to 1112 // https://github.com/kubernetes/kubernetes/issues/85320 1113 SyncPeriod: 5 * time.Second, 1114 VolumePlugins: plugins, 1115 Cloud: nil, 1116 ClusterName: "volume-test-cluster", 1117 VolumeInformer: informerFactory.Core().V1().PersistentVolumes(), 1118 ClaimInformer: informerFactory.Core().V1().PersistentVolumeClaims(), 1119 ClassInformer: informerFactory.Storage().V1().StorageClasses(), 1120 PodInformer: informerFactory.Core().V1().Pods(), 1121 NodeInformer: informerFactory.Core().V1().Nodes(), 1122 EnableDynamicProvisioning: true, 1123 } 1124 _, ctx := ktesting.NewTestContext(t) 1125 ctrl, err := persistentvolume.NewController(ctx, params) 1126 if err != nil { 1127 return nil, nil, err 1128 } 1129 1130 return ctrl, informerFactory, nil 1131 } 1132 1133 func deleteTestObjects(client clientset.Interface, ns string, option metav1.DeleteOptions) { 1134 client.CoreV1().Pods(ns).DeleteCollection(context.TODO(), option, metav1.ListOptions{}) 1135 client.CoreV1().PersistentVolumeClaims(ns).DeleteCollection(context.TODO(), option, metav1.ListOptions{}) 1136 client.CoreV1().PersistentVolumes().DeleteCollection(context.TODO(), option, metav1.ListOptions{}) 1137 client.StorageV1().StorageClasses().DeleteCollection(context.TODO(), option, metav1.ListOptions{}) 1138 client.StorageV1().CSIDrivers().DeleteCollection(context.TODO(), option, metav1.ListOptions{}) 1139 client.StorageV1().CSIStorageCapacities("default").DeleteCollection(context.TODO(), option, metav1.ListOptions{}) 1140 } 1141 1142 func makeStorageClass(name string, mode *storagev1.VolumeBindingMode) *storagev1.StorageClass { 1143 return &storagev1.StorageClass{ 1144 ObjectMeta: metav1.ObjectMeta{ 1145 Name: name, 1146 }, 1147 Provisioner: "kubernetes.io/no-provisioner", 1148 VolumeBindingMode: mode, 1149 } 1150 } 1151 1152 func makeDynamicProvisionerStorageClass(name string, mode *storagev1.VolumeBindingMode, allowedTopologies []v1.TopologySelectorTerm) *storagev1.StorageClass { 1153 return &storagev1.StorageClass{ 1154 ObjectMeta: metav1.ObjectMeta{ 1155 Name: name, 1156 }, 1157 Provisioner: provisionerPluginName, 1158 VolumeBindingMode: mode, 1159 AllowedTopologies: allowedTopologies, 1160 Parameters: map[string]string{}, 1161 } 1162 } 1163 1164 func makePV(name, scName, pvcName, ns, node string) *v1.PersistentVolume { 1165 pv := &v1.PersistentVolume{ 1166 ObjectMeta: metav1.ObjectMeta{ 1167 Name: name, 1168 Annotations: map[string]string{}, 1169 }, 1170 Spec: v1.PersistentVolumeSpec{ 1171 Capacity: v1.ResourceList{ 1172 v1.ResourceName(v1.ResourceStorage): resource.MustParse("5Gi"), 1173 }, 1174 AccessModes: []v1.PersistentVolumeAccessMode{ 1175 v1.ReadWriteOnce, 1176 }, 1177 StorageClassName: scName, 1178 PersistentVolumeSource: v1.PersistentVolumeSource{ 1179 Local: &v1.LocalVolumeSource{ 1180 Path: "/test-path", 1181 }, 1182 }, 1183 }, 1184 } 1185 1186 if pvcName != "" { 1187 pv.Spec.ClaimRef = &v1.ObjectReference{Name: pvcName, Namespace: ns} 1188 } 1189 1190 if node != "" { 1191 pv.Spec.NodeAffinity = &v1.VolumeNodeAffinity{ 1192 Required: &v1.NodeSelector{ 1193 NodeSelectorTerms: []v1.NodeSelectorTerm{ 1194 { 1195 MatchExpressions: []v1.NodeSelectorRequirement{ 1196 { 1197 Key: nodeAffinityLabelKey, 1198 Operator: v1.NodeSelectorOpIn, 1199 Values: []string{node}, 1200 }, 1201 }, 1202 }, 1203 }, 1204 }, 1205 } 1206 } 1207 1208 return pv 1209 } 1210 1211 func makePVC(name, ns string, scName *string, volumeName string) *v1.PersistentVolumeClaim { 1212 return &v1.PersistentVolumeClaim{ 1213 ObjectMeta: metav1.ObjectMeta{ 1214 Name: name, 1215 Namespace: ns, 1216 }, 1217 Spec: v1.PersistentVolumeClaimSpec{ 1218 AccessModes: []v1.PersistentVolumeAccessMode{ 1219 v1.ReadWriteOnce, 1220 }, 1221 Resources: v1.VolumeResourceRequirements{ 1222 Requests: v1.ResourceList{ 1223 v1.ResourceName(v1.ResourceStorage): resource.MustParse("5Gi"), 1224 }, 1225 }, 1226 StorageClassName: scName, 1227 VolumeName: volumeName, 1228 }, 1229 } 1230 } 1231 1232 func makePod(name, ns string, pvcs []string) *v1.Pod { 1233 volumes := []v1.Volume{} 1234 for i, pvc := range pvcs { 1235 volumes = append(volumes, v1.Volume{ 1236 Name: fmt.Sprintf("vol%v", i), 1237 VolumeSource: v1.VolumeSource{ 1238 PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ 1239 ClaimName: pvc, 1240 }, 1241 }, 1242 }) 1243 } 1244 1245 return &v1.Pod{ 1246 ObjectMeta: metav1.ObjectMeta{ 1247 Name: name, 1248 Namespace: ns, 1249 Labels: map[string]string{ 1250 "app": "volume-binding-test", 1251 }, 1252 }, 1253 Spec: v1.PodSpec{ 1254 Containers: []v1.Container{ 1255 { 1256 Name: "write-pod", 1257 Image: imageutils.GetE2EImage(imageutils.BusyBox), 1258 Command: []string{"/bin/sh"}, 1259 Args: []string{"-c", "while true; do sleep 1; done"}, 1260 }, 1261 }, 1262 Volumes: volumes, 1263 }, 1264 } 1265 } 1266 1267 // makeNode creates a node with the name "node-<index>" 1268 func makeNode(index int) *v1.Node { 1269 name := fmt.Sprintf("node-%d", index) 1270 return &v1.Node{ 1271 ObjectMeta: metav1.ObjectMeta{ 1272 Name: name, 1273 Labels: map[string]string{nodeAffinityLabelKey: name}, 1274 }, 1275 Spec: v1.NodeSpec{Unschedulable: false}, 1276 Status: v1.NodeStatus{ 1277 Capacity: v1.ResourceList{ 1278 v1.ResourcePods: *resource.NewQuantity(podLimit, resource.DecimalSI), 1279 }, 1280 Conditions: []v1.NodeCondition{ 1281 { 1282 Type: v1.NodeReady, 1283 Status: v1.ConditionTrue, 1284 Reason: fmt.Sprintf("schedulable condition"), 1285 LastHeartbeatTime: metav1.Time{Time: time.Now()}, 1286 }, 1287 }, 1288 }, 1289 } 1290 } 1291 1292 func validatePVCPhase(t *testing.T, client clientset.Interface, pvcName string, ns string, phase v1.PersistentVolumeClaimPhase, isProvisioned bool) { 1293 claim, err := client.CoreV1().PersistentVolumeClaims(ns).Get(context.TODO(), pvcName, metav1.GetOptions{}) 1294 if err != nil { 1295 t.Errorf("Failed to get PVC %v/%v: %v", ns, pvcName, err) 1296 } 1297 1298 if claim.Status.Phase != phase { 1299 t.Errorf("PVC %v/%v phase not %v, got %v", ns, pvcName, phase, claim.Status.Phase) 1300 } 1301 1302 // Check whether the bound claim is provisioned/bound as expect. 1303 if phase == v1.ClaimBound { 1304 if err := validateProvisionAnn(claim, isProvisioned); err != nil { 1305 t.Errorf("Provisoning annotation on PVC %v/%v not as expected: %v", ns, pvcName, err) 1306 } 1307 } 1308 } 1309 1310 func validateProvisionAnn(claim *v1.PersistentVolumeClaim, volIsProvisioned bool) error { 1311 selectedNode, provisionAnnoExist := claim.Annotations["volume.kubernetes.io/selected-node"] 1312 if volIsProvisioned { 1313 if !provisionAnnoExist { 1314 return fmt.Errorf("PVC %v/%v expected to be provisioned, but no selected-node annotation found", claim.Namespace, claim.Name) 1315 } 1316 if selectedNode != node1 { 1317 return fmt.Errorf("PVC %v/%v expected to be annotated as %v, but got %v", claim.Namespace, claim.Name, node1, selectedNode) 1318 } 1319 } 1320 if !volIsProvisioned && provisionAnnoExist { 1321 return fmt.Errorf("PVC %v/%v not expected to be provisioned, but found selected-node annotation", claim.Namespace, claim.Name) 1322 } 1323 1324 return nil 1325 } 1326 1327 func waitForProvisionAnn(client clientset.Interface, pvc *v1.PersistentVolumeClaim, annShouldExist bool) error { 1328 return wait.Poll(time.Second, 30*time.Second, func() (bool, error) { 1329 claim, err := client.CoreV1().PersistentVolumeClaims(pvc.Namespace).Get(context.TODO(), pvc.Name, metav1.GetOptions{}) 1330 if err != nil { 1331 return false, err 1332 } 1333 if err := validateProvisionAnn(claim, annShouldExist); err == nil { 1334 return true, nil 1335 } 1336 return false, nil 1337 }) 1338 } 1339 1340 func validatePVPhase(t *testing.T, client clientset.Interface, pvName string, phase v1.PersistentVolumePhase) { 1341 pv, err := client.CoreV1().PersistentVolumes().Get(context.TODO(), pvName, metav1.GetOptions{}) 1342 if err != nil { 1343 t.Errorf("Failed to get PV %v: %v", pvName, err) 1344 } 1345 1346 if pv.Status.Phase != phase { 1347 t.Errorf("PV %v phase not %v, got %v", pvName, phase, pv.Status.Phase) 1348 } 1349 } 1350 1351 func waitForPVPhase(client clientset.Interface, pvName string, phase v1.PersistentVolumePhase) error { 1352 return wait.PollImmediate(time.Second, 30*time.Second, func() (bool, error) { 1353 pv, err := client.CoreV1().PersistentVolumes().Get(context.TODO(), pvName, metav1.GetOptions{}) 1354 if err != nil { 1355 return false, err 1356 } 1357 1358 if pv.Status.Phase == phase { 1359 return true, nil 1360 } 1361 return false, nil 1362 }) 1363 } 1364 1365 func waitForPVCBound(client clientset.Interface, pvc *v1.PersistentVolumeClaim) error { 1366 return wait.Poll(time.Second, 30*time.Second, func() (bool, error) { 1367 claim, err := client.CoreV1().PersistentVolumeClaims(pvc.Namespace).Get(context.TODO(), pvc.Name, metav1.GetOptions{}) 1368 if err != nil { 1369 return false, err 1370 } 1371 if claim.Status.Phase == v1.ClaimBound { 1372 return true, nil 1373 } 1374 return false, nil 1375 }) 1376 } 1377 1378 func markNodeAffinity(pod *v1.Pod, node string) { 1379 affinity := &v1.Affinity{ 1380 NodeAffinity: &v1.NodeAffinity{ 1381 RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ 1382 NodeSelectorTerms: []v1.NodeSelectorTerm{ 1383 { 1384 MatchExpressions: []v1.NodeSelectorRequirement{ 1385 { 1386 Key: nodeAffinityLabelKey, 1387 Operator: v1.NodeSelectorOpIn, 1388 Values: []string{node}, 1389 }, 1390 }, 1391 }, 1392 }, 1393 }, 1394 }, 1395 } 1396 pod.Spec.Affinity = affinity 1397 } 1398 1399 func markNodeSelector(pod *v1.Pod, node string) { 1400 ns := map[string]string{ 1401 nodeAffinityLabelKey: node, 1402 } 1403 pod.Spec.NodeSelector = ns 1404 }