k8s.io/kubernetes@v1.29.3/test/integration/volume/persistent_volumes_test.go (about) 1 /* 2 Copyright 2014 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package volume 18 19 import ( 20 "context" 21 "fmt" 22 "math/rand" 23 "os" 24 "strconv" 25 "testing" 26 "time" 27 28 v1 "k8s.io/api/core/v1" 29 storage "k8s.io/api/storage/v1" 30 "k8s.io/apimachinery/pkg/api/resource" 31 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 32 "k8s.io/apimachinery/pkg/watch" 33 "k8s.io/client-go/informers" 34 clientset "k8s.io/client-go/kubernetes" 35 restclient "k8s.io/client-go/rest" 36 ref "k8s.io/client-go/tools/reference" 37 fakecloud "k8s.io/cloud-provider/fake" 38 kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing" 39 "k8s.io/kubernetes/pkg/api/legacyscheme" 40 persistentvolumecontroller "k8s.io/kubernetes/pkg/controller/volume/persistentvolume" 41 "k8s.io/kubernetes/pkg/volume" 42 volumetest "k8s.io/kubernetes/pkg/volume/testing" 43 "k8s.io/kubernetes/pkg/volume/util" 44 "k8s.io/kubernetes/test/integration/framework" 45 46 "k8s.io/klog/v2" 47 "k8s.io/klog/v2/ktesting" 48 ) 49 50 // Several tests in this file are configurable by environment variables: 51 // KUBE_INTEGRATION_PV_OBJECTS - nr. of PVs/PVCs to be created 52 // 53 // (100 by default) 54 // 55 // KUBE_INTEGRATION_PV_SYNC_PERIOD - volume controller sync period 56 // 57 // (1s by default) 58 // 59 // KUBE_INTEGRATION_PV_END_SLEEP - for how long should 60 // 61 // TestPersistentVolumeMultiPVsPVCs sleep when it's finished (0s by 62 // default). This is useful to test how long does it take for periodic sync 63 // to process bound PVs/PVCs. 64 const defaultObjectCount = 100 65 const defaultSyncPeriod = 1 * time.Second 66 67 const provisionerPluginName = "kubernetes.io/mock-provisioner" 68 69 func getObjectCount() int { 70 objectCount := defaultObjectCount 71 if s := os.Getenv("KUBE_INTEGRATION_PV_OBJECTS"); s != "" { 72 var err error 73 objectCount, err = strconv.Atoi(s) 74 if err != nil { 75 klog.Fatalf("cannot parse value of KUBE_INTEGRATION_PV_OBJECTS: %v", err) 76 } 77 } 78 klog.V(2).Infof("using KUBE_INTEGRATION_PV_OBJECTS=%d", objectCount) 79 return objectCount 80 } 81 82 func getSyncPeriod(syncPeriod time.Duration) time.Duration { 83 period := syncPeriod 84 if s := os.Getenv("KUBE_INTEGRATION_PV_SYNC_PERIOD"); s != "" { 85 var err error 86 period, err = time.ParseDuration(s) 87 if err != nil { 88 klog.Fatalf("cannot parse value of KUBE_INTEGRATION_PV_SYNC_PERIOD: %v", err) 89 } 90 } 91 klog.V(2).Infof("using KUBE_INTEGRATION_PV_SYNC_PERIOD=%v", period) 92 return period 93 } 94 95 func testSleep() { 96 var period time.Duration 97 if s := os.Getenv("KUBE_INTEGRATION_PV_END_SLEEP"); s != "" { 98 var err error 99 period, err = time.ParseDuration(s) 100 if err != nil { 101 klog.Fatalf("cannot parse value of KUBE_INTEGRATION_PV_END_SLEEP: %v", err) 102 } 103 } 104 klog.V(2).Infof("using KUBE_INTEGRATION_PV_END_SLEEP=%v", period) 105 if period != 0 { 106 time.Sleep(period) 107 klog.V(2).Infof("sleep finished") 108 } 109 } 110 111 func TestPersistentVolumeRecycler(t *testing.T) { 112 klog.V(2).Infof("TestPersistentVolumeRecycler started") 113 s := kubeapiservertesting.StartTestServerOrDie(t, nil, []string{"--disable-admission-plugins=ServiceAccount,StorageObjectInUseProtection"}, framework.SharedEtcd()) 114 defer s.TearDownFn() 115 namespaceName := "pv-recycler" 116 117 testClient, ctrl, informers, watchPV, watchPVC := createClients(namespaceName, t, s, defaultSyncPeriod) 118 defer watchPV.Stop() 119 defer watchPVC.Stop() 120 121 ns := framework.CreateNamespaceOrDie(testClient, namespaceName, t) 122 defer framework.DeleteNamespaceOrDie(testClient, ns, t) 123 124 // NOTE: This test cannot run in parallel, because it is creating and deleting 125 // non-namespaced objects (PersistenceVolumes). 126 defer testClient.CoreV1().PersistentVolumes().DeleteCollection(context.TODO(), metav1.DeleteOptions{}, metav1.ListOptions{}) 127 128 ctx, cancel := context.WithCancel(context.TODO()) 129 informers.Start(ctx.Done()) 130 go ctrl.Run(ctx) 131 defer cancel() 132 133 // This PV will be claimed, released, and recycled. 134 pv := createPV("fake-pv-recycler", "/tmp/foo", "10G", []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, v1.PersistentVolumeReclaimRecycle) 135 pvc := createPVC("fake-pvc-recycler", ns.Name, "5G", []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, "") 136 137 _, err := testClient.CoreV1().PersistentVolumes().Create(context.TODO(), pv, metav1.CreateOptions{}) 138 if err != nil { 139 t.Errorf("Failed to create PersistentVolume: %v", err) 140 } 141 klog.V(2).Infof("TestPersistentVolumeRecycler pvc created") 142 143 _, err = testClient.CoreV1().PersistentVolumeClaims(ns.Name).Create(context.TODO(), pvc, metav1.CreateOptions{}) 144 if err != nil { 145 t.Errorf("Failed to create PersistentVolumeClaim: %v", err) 146 } 147 klog.V(2).Infof("TestPersistentVolumeRecycler pvc created") 148 149 // wait until the controller pairs the volume and claim 150 waitForPersistentVolumePhase(testClient, pv.Name, watchPV, v1.VolumeBound) 151 klog.V(2).Infof("TestPersistentVolumeRecycler pv bound") 152 waitForPersistentVolumeClaimPhase(testClient, pvc.Name, ns.Name, watchPVC, v1.ClaimBound) 153 klog.V(2).Infof("TestPersistentVolumeRecycler pvc bound") 154 155 // deleting a claim releases the volume, after which it can be recycled 156 if err := testClient.CoreV1().PersistentVolumeClaims(ns.Name).Delete(context.TODO(), pvc.Name, metav1.DeleteOptions{}); err != nil { 157 t.Errorf("error deleting claim %s", pvc.Name) 158 } 159 klog.V(2).Infof("TestPersistentVolumeRecycler pvc deleted") 160 161 waitForPersistentVolumePhase(testClient, pv.Name, watchPV, v1.VolumeReleased) 162 klog.V(2).Infof("TestPersistentVolumeRecycler pv released") 163 waitForPersistentVolumePhase(testClient, pv.Name, watchPV, v1.VolumeAvailable) 164 klog.V(2).Infof("TestPersistentVolumeRecycler pv available") 165 } 166 167 func TestPersistentVolumeDeleter(t *testing.T) { 168 klog.V(2).Infof("TestPersistentVolumeDeleter started") 169 s := kubeapiservertesting.StartTestServerOrDie(t, nil, []string{"--disable-admission-plugins=ServiceAccount,StorageObjectInUseProtection"}, framework.SharedEtcd()) 170 defer s.TearDownFn() 171 namespaceName := "pv-deleter" 172 173 testClient, ctrl, informers, watchPV, watchPVC := createClients(namespaceName, t, s, defaultSyncPeriod) 174 defer watchPV.Stop() 175 defer watchPVC.Stop() 176 177 ns := framework.CreateNamespaceOrDie(testClient, namespaceName, t) 178 defer framework.DeleteNamespaceOrDie(testClient, ns, t) 179 180 // NOTE: This test cannot run in parallel, because it is creating and deleting 181 // non-namespaced objects (PersistenceVolumes). 182 defer testClient.CoreV1().PersistentVolumes().DeleteCollection(context.TODO(), metav1.DeleteOptions{}, metav1.ListOptions{}) 183 184 ctx, cancel := context.WithCancel(context.TODO()) 185 informers.Start(ctx.Done()) 186 go ctrl.Run(ctx) 187 defer cancel() 188 189 // This PV will be claimed, released, and deleted. 190 pv := createPV("fake-pv-deleter", "/tmp/foo", "10G", []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, v1.PersistentVolumeReclaimDelete) 191 pvc := createPVC("fake-pvc-deleter", ns.Name, "5G", []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, "") 192 193 _, err := testClient.CoreV1().PersistentVolumes().Create(context.TODO(), pv, metav1.CreateOptions{}) 194 if err != nil { 195 t.Errorf("Failed to create PersistentVolume: %v", err) 196 } 197 klog.V(2).Infof("TestPersistentVolumeDeleter pv created") 198 _, err = testClient.CoreV1().PersistentVolumeClaims(ns.Name).Create(context.TODO(), pvc, metav1.CreateOptions{}) 199 if err != nil { 200 t.Errorf("Failed to create PersistentVolumeClaim: %v", err) 201 } 202 klog.V(2).Infof("TestPersistentVolumeDeleter pvc created") 203 waitForPersistentVolumePhase(testClient, pv.Name, watchPV, v1.VolumeBound) 204 klog.V(2).Infof("TestPersistentVolumeDeleter pv bound") 205 waitForPersistentVolumeClaimPhase(testClient, pvc.Name, ns.Name, watchPVC, v1.ClaimBound) 206 klog.V(2).Infof("TestPersistentVolumeDeleter pvc bound") 207 208 // deleting a claim releases the volume, after which it can be recycled 209 if err := testClient.CoreV1().PersistentVolumeClaims(ns.Name).Delete(context.TODO(), pvc.Name, metav1.DeleteOptions{}); err != nil { 210 t.Errorf("error deleting claim %s", pvc.Name) 211 } 212 klog.V(2).Infof("TestPersistentVolumeDeleter pvc deleted") 213 214 waitForPersistentVolumePhase(testClient, pv.Name, watchPV, v1.VolumeReleased) 215 klog.V(2).Infof("TestPersistentVolumeDeleter pv released") 216 217 for { 218 event := <-watchPV.ResultChan() 219 if event.Type == watch.Deleted { 220 break 221 } 222 } 223 klog.V(2).Infof("TestPersistentVolumeDeleter pv deleted") 224 } 225 226 func TestPersistentVolumeBindRace(t *testing.T) { 227 // Test a race binding many claims to a PV that is pre-bound to a specific 228 // PVC. Only this specific PVC should get bound. 229 klog.V(2).Infof("TestPersistentVolumeBindRace started") 230 s := kubeapiservertesting.StartTestServerOrDie(t, nil, []string{"--disable-admission-plugins=ServiceAccount,StorageObjectInUseProtection"}, framework.SharedEtcd()) 231 defer s.TearDownFn() 232 namespaceName := "pv-bind-race" 233 234 testClient, ctrl, informers, watchPV, watchPVC := createClients(namespaceName, t, s, defaultSyncPeriod) 235 defer watchPV.Stop() 236 defer watchPVC.Stop() 237 238 ns := framework.CreateNamespaceOrDie(testClient, namespaceName, t) 239 defer framework.DeleteNamespaceOrDie(testClient, ns, t) 240 241 // NOTE: This test cannot run in parallel, because it is creating and deleting 242 // non-namespaced objects (PersistenceVolumes). 243 defer testClient.CoreV1().PersistentVolumes().DeleteCollection(context.TODO(), metav1.DeleteOptions{}, metav1.ListOptions{}) 244 245 ctx, cancel := context.WithCancel(context.TODO()) 246 informers.Start(ctx.Done()) 247 go ctrl.Run(ctx) 248 defer cancel() 249 250 pv := createPV("fake-pv-race", "/tmp/foo", "10G", []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, v1.PersistentVolumeReclaimRetain) 251 pvc := createPVC("fake-pvc-race", ns.Name, "5G", []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, "") 252 counter := 0 253 maxClaims := 100 254 claims := []*v1.PersistentVolumeClaim{} 255 for counter <= maxClaims { 256 counter++ 257 newPvc := pvc.DeepCopy() 258 newPvc.ObjectMeta = metav1.ObjectMeta{Name: fmt.Sprintf("fake-pvc-race-%d", counter)} 259 claim, err := testClient.CoreV1().PersistentVolumeClaims(ns.Name).Create(context.TODO(), newPvc, metav1.CreateOptions{}) 260 if err != nil { 261 t.Fatalf("Error creating newPvc: %v", err) 262 } 263 claims = append(claims, claim) 264 } 265 klog.V(2).Infof("TestPersistentVolumeBindRace claims created") 266 267 // putting a bind manually on a pv should only match the claim it is bound to 268 claim := claims[rand.Intn(maxClaims-1)] 269 claimRef, err := ref.GetReference(legacyscheme.Scheme, claim) 270 if err != nil { 271 t.Fatalf("Unexpected error getting claimRef: %v", err) 272 } 273 pv.Spec.ClaimRef = claimRef 274 pv.Spec.ClaimRef.UID = "" 275 276 pv, err = testClient.CoreV1().PersistentVolumes().Create(context.TODO(), pv, metav1.CreateOptions{}) 277 if err != nil { 278 t.Fatalf("Unexpected error creating pv: %v", err) 279 } 280 klog.V(2).Infof("TestPersistentVolumeBindRace pv created, pre-bound to %s", claim.Name) 281 282 waitForPersistentVolumePhase(testClient, pv.Name, watchPV, v1.VolumeBound) 283 klog.V(2).Infof("TestPersistentVolumeBindRace pv bound") 284 waitForAnyPersistentVolumeClaimPhase(watchPVC, v1.ClaimBound) 285 klog.V(2).Infof("TestPersistentVolumeBindRace pvc bound") 286 287 pv, err = testClient.CoreV1().PersistentVolumes().Get(context.TODO(), pv.Name, metav1.GetOptions{}) 288 if err != nil { 289 t.Fatalf("Unexpected error getting pv: %v", err) 290 } 291 if pv.Spec.ClaimRef == nil { 292 t.Fatalf("Unexpected nil claimRef") 293 } 294 if pv.Spec.ClaimRef.Namespace != claimRef.Namespace || pv.Spec.ClaimRef.Name != claimRef.Name { 295 t.Fatalf("Bind mismatch! Expected %s/%s but got %s/%s", claimRef.Namespace, claimRef.Name, pv.Spec.ClaimRef.Namespace, pv.Spec.ClaimRef.Name) 296 } 297 } 298 299 // TestPersistentVolumeClaimLabelSelector test binding using label selectors 300 func TestPersistentVolumeClaimLabelSelector(t *testing.T) { 301 s := kubeapiservertesting.StartTestServerOrDie(t, nil, []string{"--disable-admission-plugins=ServiceAccount,StorageObjectInUseProtection"}, framework.SharedEtcd()) 302 defer s.TearDownFn() 303 namespaceName := "pvc-label-selector" 304 305 testClient, controller, informers, watchPV, watchPVC := createClients(namespaceName, t, s, defaultSyncPeriod) 306 defer watchPV.Stop() 307 defer watchPVC.Stop() 308 309 ns := framework.CreateNamespaceOrDie(testClient, namespaceName, t) 310 defer framework.DeleteNamespaceOrDie(testClient, ns, t) 311 312 // NOTE: This test cannot run in parallel, because it is creating and deleting 313 // non-namespaced objects (PersistenceVolumes). 314 defer testClient.CoreV1().PersistentVolumes().DeleteCollection(context.TODO(), metav1.DeleteOptions{}, metav1.ListOptions{}) 315 316 ctx, cancel := context.WithCancel(context.TODO()) 317 informers.Start(ctx.Done()) 318 go controller.Run(ctx) 319 defer cancel() 320 321 var ( 322 err error 323 modes = []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce} 324 reclaim = v1.PersistentVolumeReclaimRetain 325 326 pvTrue = createPV("pv-true", "/tmp/foo-label", "1G", modes, reclaim) 327 pvFalse = createPV("pv-false", "/tmp/foo-label", "1G", modes, reclaim) 328 pvc = createPVC("pvc-ls-1", ns.Name, "1G", modes, "") 329 ) 330 331 pvTrue.ObjectMeta.SetLabels(map[string]string{"foo": "true"}) 332 pvFalse.ObjectMeta.SetLabels(map[string]string{"foo": "false"}) 333 334 _, err = testClient.CoreV1().PersistentVolumes().Create(context.TODO(), pvTrue, metav1.CreateOptions{}) 335 if err != nil { 336 t.Fatalf("Failed to create PersistentVolume: %v", err) 337 } 338 _, err = testClient.CoreV1().PersistentVolumes().Create(context.TODO(), pvFalse, metav1.CreateOptions{}) 339 if err != nil { 340 t.Fatalf("Failed to create PersistentVolume: %v", err) 341 } 342 t.Log("volumes created") 343 344 pvc.Spec.Selector = &metav1.LabelSelector{ 345 MatchLabels: map[string]string{ 346 "foo": "true", 347 }, 348 } 349 350 _, err = testClient.CoreV1().PersistentVolumeClaims(ns.Name).Create(context.TODO(), pvc, metav1.CreateOptions{}) 351 if err != nil { 352 t.Fatalf("Failed to create PersistentVolumeClaim: %v", err) 353 } 354 t.Log("claim created") 355 356 waitForAnyPersistentVolumePhase(watchPV, v1.VolumeBound) 357 t.Log("volume bound") 358 waitForPersistentVolumeClaimPhase(testClient, pvc.Name, ns.Name, watchPVC, v1.ClaimBound) 359 t.Log("claim bound") 360 361 pv, err := testClient.CoreV1().PersistentVolumes().Get(context.TODO(), "pv-false", metav1.GetOptions{}) 362 if err != nil { 363 t.Fatalf("Unexpected error getting pv: %v", err) 364 } 365 if pv.Spec.ClaimRef != nil { 366 t.Fatalf("False PV shouldn't be bound") 367 } 368 pv, err = testClient.CoreV1().PersistentVolumes().Get(context.TODO(), "pv-true", metav1.GetOptions{}) 369 if err != nil { 370 t.Fatalf("Unexpected error getting pv: %v", err) 371 } 372 if pv.Spec.ClaimRef == nil { 373 t.Fatalf("True PV should be bound") 374 } 375 if pv.Spec.ClaimRef.Namespace != pvc.Namespace || pv.Spec.ClaimRef.Name != pvc.Name { 376 t.Fatalf("Bind mismatch! Expected %s/%s but got %s/%s", pvc.Namespace, pvc.Name, pv.Spec.ClaimRef.Namespace, pv.Spec.ClaimRef.Name) 377 } 378 } 379 380 // TestPersistentVolumeClaimLabelSelectorMatchExpressions test binding using 381 // MatchExpressions label selectors 382 func TestPersistentVolumeClaimLabelSelectorMatchExpressions(t *testing.T) { 383 s := kubeapiservertesting.StartTestServerOrDie(t, nil, []string{"--disable-admission-plugins=ServiceAccount,StorageObjectInUseProtection"}, framework.SharedEtcd()) 384 defer s.TearDownFn() 385 namespaceName := "pvc-match-expressions" 386 387 testClient, controller, informers, watchPV, watchPVC := createClients(namespaceName, t, s, defaultSyncPeriod) 388 defer watchPV.Stop() 389 defer watchPVC.Stop() 390 391 ns := framework.CreateNamespaceOrDie(testClient, namespaceName, t) 392 defer framework.DeleteNamespaceOrDie(testClient, ns, t) 393 394 // NOTE: This test cannot run in parallel, because it is creating and deleting 395 // non-namespaced objects (PersistenceVolumes). 396 defer testClient.CoreV1().PersistentVolumes().DeleteCollection(context.TODO(), metav1.DeleteOptions{}, metav1.ListOptions{}) 397 398 ctx, cancel := context.WithCancel(context.TODO()) 399 informers.Start(ctx.Done()) 400 go controller.Run(ctx) 401 defer cancel() 402 403 var ( 404 err error 405 modes = []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce} 406 reclaim = v1.PersistentVolumeReclaimRetain 407 408 pvTrue = createPV("pv-true", "/tmp/foo-label", "1G", modes, reclaim) 409 pvFalse = createPV("pv-false", "/tmp/foo-label", "1G", modes, reclaim) 410 pvc = createPVC("pvc-ls-1", ns.Name, "1G", modes, "") 411 ) 412 413 pvTrue.ObjectMeta.SetLabels(map[string]string{"foo": "valA", "bar": ""}) 414 pvFalse.ObjectMeta.SetLabels(map[string]string{"foo": "valB", "baz": ""}) 415 416 _, err = testClient.CoreV1().PersistentVolumes().Create(context.TODO(), pvTrue, metav1.CreateOptions{}) 417 if err != nil { 418 t.Fatalf("Failed to create PersistentVolume: %v", err) 419 } 420 _, err = testClient.CoreV1().PersistentVolumes().Create(context.TODO(), pvFalse, metav1.CreateOptions{}) 421 if err != nil { 422 t.Fatalf("Failed to create PersistentVolume: %v", err) 423 } 424 t.Log("volumes created") 425 426 pvc.Spec.Selector = &metav1.LabelSelector{ 427 MatchExpressions: []metav1.LabelSelectorRequirement{ 428 { 429 Key: "foo", 430 Operator: metav1.LabelSelectorOpIn, 431 Values: []string{"valA"}, 432 }, 433 { 434 Key: "foo", 435 Operator: metav1.LabelSelectorOpNotIn, 436 Values: []string{"valB"}, 437 }, 438 { 439 Key: "bar", 440 Operator: metav1.LabelSelectorOpExists, 441 Values: []string{}, 442 }, 443 { 444 Key: "baz", 445 Operator: metav1.LabelSelectorOpDoesNotExist, 446 Values: []string{}, 447 }, 448 }, 449 } 450 451 _, err = testClient.CoreV1().PersistentVolumeClaims(ns.Name).Create(context.TODO(), pvc, metav1.CreateOptions{}) 452 if err != nil { 453 t.Fatalf("Failed to create PersistentVolumeClaim: %v", err) 454 } 455 t.Log("claim created") 456 457 waitForAnyPersistentVolumePhase(watchPV, v1.VolumeBound) 458 t.Log("volume bound") 459 waitForPersistentVolumeClaimPhase(testClient, pvc.Name, ns.Name, watchPVC, v1.ClaimBound) 460 t.Log("claim bound") 461 462 pv, err := testClient.CoreV1().PersistentVolumes().Get(context.TODO(), "pv-false", metav1.GetOptions{}) 463 if err != nil { 464 t.Fatalf("Unexpected error getting pv: %v", err) 465 } 466 if pv.Spec.ClaimRef != nil { 467 t.Fatalf("False PV shouldn't be bound") 468 } 469 pv, err = testClient.CoreV1().PersistentVolumes().Get(context.TODO(), "pv-true", metav1.GetOptions{}) 470 if err != nil { 471 t.Fatalf("Unexpected error getting pv: %v", err) 472 } 473 if pv.Spec.ClaimRef == nil { 474 t.Fatalf("True PV should be bound") 475 } 476 if pv.Spec.ClaimRef.Namespace != pvc.Namespace || pv.Spec.ClaimRef.Name != pvc.Name { 477 t.Fatalf("Bind mismatch! Expected %s/%s but got %s/%s", pvc.Namespace, pvc.Name, pv.Spec.ClaimRef.Namespace, pv.Spec.ClaimRef.Name) 478 } 479 } 480 481 // TestPersistentVolumeMultiPVs tests binding of one PVC to 100 PVs with 482 // different size. 483 func TestPersistentVolumeMultiPVs(t *testing.T) { 484 s := kubeapiservertesting.StartTestServerOrDie(t, nil, []string{"--disable-admission-plugins=ServiceAccount,StorageObjectInUseProtection"}, framework.SharedEtcd()) 485 defer s.TearDownFn() 486 namespaceName := "multi-pvs" 487 488 testClient, controller, informers, watchPV, watchPVC := createClients(namespaceName, t, s, defaultSyncPeriod) 489 defer watchPV.Stop() 490 defer watchPVC.Stop() 491 492 ns := framework.CreateNamespaceOrDie(testClient, namespaceName, t) 493 defer framework.DeleteNamespaceOrDie(testClient, ns, t) 494 495 // NOTE: This test cannot run in parallel, because it is creating and deleting 496 // non-namespaced objects (PersistenceVolumes). 497 defer testClient.CoreV1().PersistentVolumes().DeleteCollection(context.TODO(), metav1.DeleteOptions{}, metav1.ListOptions{}) 498 499 ctx, cancel := context.WithCancel(context.TODO()) 500 informers.Start(ctx.Done()) 501 go controller.Run(ctx) 502 defer cancel() 503 504 maxPVs := getObjectCount() 505 pvs := make([]*v1.PersistentVolume, maxPVs) 506 for i := 0; i < maxPVs; i++ { 507 // This PV will be claimed, released, and deleted 508 pvs[i] = createPV("pv-"+strconv.Itoa(i), "/tmp/foo"+strconv.Itoa(i), strconv.Itoa(i+1)+"G", 509 []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, v1.PersistentVolumeReclaimRetain) 510 } 511 512 pvc := createPVC("pvc-2", ns.Name, strconv.Itoa(maxPVs/2)+"G", []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, "") 513 514 for i := 0; i < maxPVs; i++ { 515 _, err := testClient.CoreV1().PersistentVolumes().Create(context.TODO(), pvs[i], metav1.CreateOptions{}) 516 if err != nil { 517 t.Errorf("Failed to create PersistentVolume %d: %v", i, err) 518 } 519 waitForPersistentVolumePhase(testClient, pvs[i].Name, watchPV, v1.VolumeAvailable) 520 } 521 t.Log("volumes created") 522 523 _, err := testClient.CoreV1().PersistentVolumeClaims(ns.Name).Create(context.TODO(), pvc, metav1.CreateOptions{}) 524 if err != nil { 525 t.Errorf("Failed to create PersistentVolumeClaim: %v", err) 526 } 527 t.Log("claim created") 528 529 // wait until the binder pairs the claim with a volume 530 waitForAnyPersistentVolumePhase(watchPV, v1.VolumeBound) 531 t.Log("volume bound") 532 waitForPersistentVolumeClaimPhase(testClient, pvc.Name, ns.Name, watchPVC, v1.ClaimBound) 533 t.Log("claim bound") 534 535 // only one PV is bound 536 bound := 0 537 for i := 0; i < maxPVs; i++ { 538 pv, err := testClient.CoreV1().PersistentVolumes().Get(context.TODO(), pvs[i].Name, metav1.GetOptions{}) 539 if err != nil { 540 t.Fatalf("Unexpected error getting pv: %v", err) 541 } 542 if pv.Spec.ClaimRef == nil { 543 continue 544 } 545 // found a bounded PV 546 p := pv.Spec.Capacity[v1.ResourceStorage] 547 pvCap := p.Value() 548 expectedCap := resource.MustParse(strconv.Itoa(maxPVs/2) + "G") 549 expectedCapVal := expectedCap.Value() 550 if pv.Spec.ClaimRef.Name != pvc.Name || pvCap != expectedCapVal { 551 t.Fatalf("Bind mismatch! Expected %s capacity %d but got %s capacity %d", pvc.Name, expectedCapVal, pv.Spec.ClaimRef.Name, pvCap) 552 } 553 t.Logf("claim bounded to %s capacity %v", pv.Name, pv.Spec.Capacity[v1.ResourceStorage]) 554 bound++ 555 } 556 t.Log("volumes checked") 557 558 if bound != 1 { 559 t.Fatalf("Only 1 PV should be bound but got %d", bound) 560 } 561 562 // deleting a claim releases the volume 563 if err := testClient.CoreV1().PersistentVolumeClaims(ns.Name).Delete(context.TODO(), pvc.Name, metav1.DeleteOptions{}); err != nil { 564 t.Errorf("error deleting claim %s", pvc.Name) 565 } 566 t.Log("claim deleted") 567 568 waitForAnyPersistentVolumePhase(watchPV, v1.VolumeReleased) 569 t.Log("volumes released") 570 } 571 572 // TestPersistentVolumeMultiPVsPVCs tests binding of 100 PVC to 100 PVs. 573 // This test is configurable by KUBE_INTEGRATION_PV_* variables. 574 func TestPersistentVolumeMultiPVsPVCs(t *testing.T) { 575 s := kubeapiservertesting.StartTestServerOrDie(t, nil, []string{"--disable-admission-plugins=ServiceAccount,StorageObjectInUseProtection"}, framework.SharedEtcd()) 576 defer s.TearDownFn() 577 namespaceName := "multi-pvs-pvcs" 578 579 testClient, binder, informers, watchPV, watchPVC := createClients(namespaceName, t, s, defaultSyncPeriod) 580 defer watchPV.Stop() 581 defer watchPVC.Stop() 582 583 ns := framework.CreateNamespaceOrDie(testClient, namespaceName, t) 584 defer framework.DeleteNamespaceOrDie(testClient, ns, t) 585 586 // NOTE: This test cannot run in parallel, because it is creating and deleting 587 // non-namespaced objects (PersistenceVolumes). 588 defer testClient.CoreV1().PersistentVolumes().DeleteCollection(context.TODO(), metav1.DeleteOptions{}, metav1.ListOptions{}) 589 590 ctx, cancel := context.WithCancel(context.TODO()) 591 informers.Start(ctx.Done()) 592 go binder.Run(ctx) 593 defer cancel() 594 595 objCount := getObjectCount() 596 pvs := make([]*v1.PersistentVolume, objCount) 597 pvcs := make([]*v1.PersistentVolumeClaim, objCount) 598 for i := 0; i < objCount; i++ { 599 // This PV will be claimed, released, and deleted 600 pvs[i] = createPV("pv-"+strconv.Itoa(i), "/tmp/foo"+strconv.Itoa(i), "1G", 601 []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, v1.PersistentVolumeReclaimRetain) 602 pvcs[i] = createPVC("pvc-"+strconv.Itoa(i), ns.Name, "1G", []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, "") 603 } 604 605 // Create PVs first 606 klog.V(2).Infof("TestPersistentVolumeMultiPVsPVCs: start") 607 608 // Create the volumes in a separate goroutine to pop events from 609 // watchPV early - it seems it has limited capacity and it gets stuck 610 // with >3000 volumes. 611 go func() { 612 for i := 0; i < objCount; i++ { 613 _, _ = testClient.CoreV1().PersistentVolumes().Create(context.TODO(), pvs[i], metav1.CreateOptions{}) 614 } 615 }() 616 // Wait for them to get Available 617 for i := 0; i < objCount; i++ { 618 waitForAnyPersistentVolumePhase(watchPV, v1.VolumeAvailable) 619 klog.V(1).Infof("%d volumes available", i+1) 620 } 621 klog.V(2).Infof("TestPersistentVolumeMultiPVsPVCs: volumes are Available") 622 623 // Start a separate goroutine that randomly modifies PVs and PVCs while the 624 // binder is working. We test that the binder can bind volumes despite 625 // users modifying objects underneath. 626 stopCh := make(chan struct{}, 0) 627 go func() { 628 for { 629 // Roll a dice and decide a PV or PVC to modify 630 if rand.Intn(2) == 0 { 631 // Modify PV 632 i := rand.Intn(objCount) 633 name := "pv-" + strconv.Itoa(i) 634 pv, err := testClient.CoreV1().PersistentVolumes().Get(context.TODO(), name, metav1.GetOptions{}) 635 if err != nil { 636 // Silently ignore error, the PV may have be already deleted 637 // or not exists yet. 638 klog.V(4).Infof("Failed to read PV %s: %v", name, err) 639 continue 640 } 641 if pv.Annotations == nil { 642 pv.Annotations = map[string]string{"TestAnnotation": fmt.Sprint(rand.Int())} 643 } else { 644 pv.Annotations["TestAnnotation"] = fmt.Sprint(rand.Int()) 645 } 646 _, err = testClient.CoreV1().PersistentVolumes().Update(context.TODO(), pv, metav1.UpdateOptions{}) 647 if err != nil { 648 // Silently ignore error, the PV may have been updated by 649 // the controller. 650 klog.V(4).Infof("Failed to update PV %s: %v", pv.Name, err) 651 continue 652 } 653 klog.V(4).Infof("Updated PV %s", pv.Name) 654 } else { 655 // Modify PVC 656 i := rand.Intn(objCount) 657 name := "pvc-" + strconv.Itoa(i) 658 pvc, err := testClient.CoreV1().PersistentVolumeClaims(metav1.NamespaceDefault).Get(context.TODO(), name, metav1.GetOptions{}) 659 if err != nil { 660 // Silently ignore error, the PVC may have be already 661 // deleted or not exists yet. 662 klog.V(4).Infof("Failed to read PVC %s: %v", name, err) 663 continue 664 } 665 if pvc.Annotations == nil { 666 pvc.Annotations = map[string]string{"TestAnnotation": fmt.Sprint(rand.Int())} 667 } else { 668 pvc.Annotations["TestAnnotation"] = fmt.Sprint(rand.Int()) 669 } 670 _, err = testClient.CoreV1().PersistentVolumeClaims(metav1.NamespaceDefault).Update(context.TODO(), pvc, metav1.UpdateOptions{}) 671 if err != nil { 672 // Silently ignore error, the PVC may have been updated by 673 // the controller. 674 klog.V(4).Infof("Failed to update PVC %s: %v", pvc.Name, err) 675 continue 676 } 677 klog.V(4).Infof("Updated PVC %s", pvc.Name) 678 } 679 680 select { 681 case <-stopCh: 682 return 683 default: 684 continue 685 } 686 } 687 }() 688 689 // Create the claims, again in a separate goroutine. 690 go func() { 691 for i := 0; i < objCount; i++ { 692 _, _ = testClient.CoreV1().PersistentVolumeClaims(ns.Name).Create(context.TODO(), pvcs[i], metav1.CreateOptions{}) 693 } 694 }() 695 696 // wait until the binder pairs all claims 697 for i := 0; i < objCount; i++ { 698 waitForAnyPersistentVolumeClaimPhase(watchPVC, v1.ClaimBound) 699 klog.V(1).Infof("%d claims bound", i+1) 700 } 701 // wait until the binder pairs all volumes 702 for i := 0; i < objCount; i++ { 703 waitForPersistentVolumePhase(testClient, pvs[i].Name, watchPV, v1.VolumeBound) 704 klog.V(1).Infof("%d claims bound", i+1) 705 } 706 707 klog.V(2).Infof("TestPersistentVolumeMultiPVsPVCs: claims are bound") 708 close(stopCh) 709 710 // check that everything is bound to something 711 for i := 0; i < objCount; i++ { 712 pv, err := testClient.CoreV1().PersistentVolumes().Get(context.TODO(), pvs[i].Name, metav1.GetOptions{}) 713 if err != nil { 714 t.Fatalf("Unexpected error getting pv: %v", err) 715 } 716 if pv.Spec.ClaimRef == nil { 717 t.Fatalf("PV %q is not bound", pv.Name) 718 } 719 klog.V(2).Infof("PV %q is bound to PVC %q", pv.Name, pv.Spec.ClaimRef.Name) 720 721 pvc, err := testClient.CoreV1().PersistentVolumeClaims(ns.Name).Get(context.TODO(), pvcs[i].Name, metav1.GetOptions{}) 722 if err != nil { 723 t.Fatalf("Unexpected error getting pvc: %v", err) 724 } 725 if pvc.Spec.VolumeName == "" { 726 t.Fatalf("PVC %q is not bound", pvc.Name) 727 } 728 klog.V(2).Infof("PVC %q is bound to PV %q", pvc.Name, pvc.Spec.VolumeName) 729 } 730 testSleep() 731 } 732 733 // TestPersistentVolumeControllerStartup tests startup of the controller. 734 // The controller should not unbind any volumes when it starts. 735 func TestPersistentVolumeControllerStartup(t *testing.T) { 736 s := kubeapiservertesting.StartTestServerOrDie(t, nil, []string{"--disable-admission-plugins=ServiceAccount,StorageObjectInUseProtection"}, framework.SharedEtcd()) 737 defer s.TearDownFn() 738 namespaceName := "controller-startup" 739 740 objCount := getObjectCount() 741 742 const shortSyncPeriod = 2 * time.Second 743 syncPeriod := getSyncPeriod(shortSyncPeriod) 744 745 testClient, binder, informers, watchPV, watchPVC := createClients(namespaceName, t, s, shortSyncPeriod) 746 defer watchPV.Stop() 747 defer watchPVC.Stop() 748 749 ns := framework.CreateNamespaceOrDie(testClient, namespaceName, t) 750 defer framework.DeleteNamespaceOrDie(testClient, ns, t) 751 752 // Create *bound* volumes and PVCs 753 pvs := make([]*v1.PersistentVolume, objCount) 754 pvcs := make([]*v1.PersistentVolumeClaim, objCount) 755 for i := 0; i < objCount; i++ { 756 pvName := "pv-startup-" + strconv.Itoa(i) 757 pvcName := "pvc-startup-" + strconv.Itoa(i) 758 759 pvc := createPVC(pvcName, ns.Name, "1G", []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, "") 760 pvc.Annotations = map[string]string{"annBindCompleted": ""} 761 pvc.Spec.VolumeName = pvName 762 newPVC, err := testClient.CoreV1().PersistentVolumeClaims(ns.Name).Create(context.TODO(), pvc, metav1.CreateOptions{}) 763 if err != nil { 764 t.Fatalf("Cannot create claim %q: %v", pvc.Name, err) 765 } 766 // Save Bound status as a separate transaction 767 newPVC.Status.Phase = v1.ClaimBound 768 newPVC, err = testClient.CoreV1().PersistentVolumeClaims(ns.Name).UpdateStatus(context.TODO(), newPVC, metav1.UpdateOptions{}) 769 if err != nil { 770 t.Fatalf("Cannot update claim status %q: %v", pvc.Name, err) 771 } 772 pvcs[i] = newPVC 773 // Drain watchPVC with all events generated by the PVC until it's bound 774 // We don't want to catch "PVC created with Status.Phase == Pending" 775 // later in this test. 776 waitForAnyPersistentVolumeClaimPhase(watchPVC, v1.ClaimBound) 777 778 pv := createPV(pvName, "/tmp/foo"+strconv.Itoa(i), "1G", 779 []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, v1.PersistentVolumeReclaimRetain) 780 claimRef, err := ref.GetReference(legacyscheme.Scheme, newPVC) 781 if err != nil { 782 klog.V(3).Infof("unexpected error getting claim reference: %v", err) 783 return 784 } 785 pv.Spec.ClaimRef = claimRef 786 newPV, err := testClient.CoreV1().PersistentVolumes().Create(context.TODO(), pv, metav1.CreateOptions{}) 787 if err != nil { 788 t.Fatalf("Cannot create volume %q: %v", pv.Name, err) 789 } 790 // Save Bound status as a separate transaction 791 newPV.Status.Phase = v1.VolumeBound 792 newPV, err = testClient.CoreV1().PersistentVolumes().UpdateStatus(context.TODO(), newPV, metav1.UpdateOptions{}) 793 if err != nil { 794 t.Fatalf("Cannot update volume status %q: %v", pv.Name, err) 795 } 796 pvs[i] = newPV 797 // Drain watchPV with all events generated by the PV until it's bound 798 // We don't want to catch "PV created with Status.Phase == Pending" 799 // later in this test. 800 waitForAnyPersistentVolumePhase(watchPV, v1.VolumeBound) 801 } 802 803 // Start the controller when all PVs and PVCs are already saved in etcd 804 ctx, cancel := context.WithCancel(context.TODO()) 805 informers.Start(ctx.Done()) 806 go binder.Run(ctx) 807 defer cancel() 808 809 // wait for at least two sync periods for changes. No volume should be 810 // Released and no claim should be Lost during this time. 811 timer := time.NewTimer(2 * syncPeriod) 812 defer timer.Stop() 813 finished := false 814 for !finished { 815 select { 816 case volumeEvent := <-watchPV.ResultChan(): 817 volume, ok := volumeEvent.Object.(*v1.PersistentVolume) 818 if !ok { 819 continue 820 } 821 if volume.Status.Phase != v1.VolumeBound { 822 t.Errorf("volume %s unexpectedly changed state to %s", volume.Name, volume.Status.Phase) 823 } 824 825 case claimEvent := <-watchPVC.ResultChan(): 826 claim, ok := claimEvent.Object.(*v1.PersistentVolumeClaim) 827 if !ok { 828 continue 829 } 830 if claim.Status.Phase != v1.ClaimBound { 831 t.Errorf("claim %s unexpectedly changed state to %s", claim.Name, claim.Status.Phase) 832 } 833 834 case <-timer.C: 835 // Wait finished 836 klog.V(2).Infof("Wait finished") 837 finished = true 838 } 839 } 840 841 // check that everything is bound to something 842 for i := 0; i < objCount; i++ { 843 pv, err := testClient.CoreV1().PersistentVolumes().Get(context.TODO(), pvs[i].Name, metav1.GetOptions{}) 844 if err != nil { 845 t.Fatalf("Unexpected error getting pv: %v", err) 846 } 847 if pv.Spec.ClaimRef == nil { 848 t.Fatalf("PV %q is not bound", pv.Name) 849 } 850 klog.V(2).Infof("PV %q is bound to PVC %q", pv.Name, pv.Spec.ClaimRef.Name) 851 852 pvc, err := testClient.CoreV1().PersistentVolumeClaims(ns.Name).Get(context.TODO(), pvcs[i].Name, metav1.GetOptions{}) 853 if err != nil { 854 t.Fatalf("Unexpected error getting pvc: %v", err) 855 } 856 if pvc.Spec.VolumeName == "" { 857 t.Fatalf("PVC %q is not bound", pvc.Name) 858 } 859 klog.V(2).Infof("PVC %q is bound to PV %q", pvc.Name, pvc.Spec.VolumeName) 860 } 861 } 862 863 // TestPersistentVolumeProvisionMultiPVCs tests provisioning of many PVCs. 864 // This test is configurable by KUBE_INTEGRATION_PV_* variables. 865 func TestPersistentVolumeProvisionMultiPVCs(t *testing.T) { 866 s := kubeapiservertesting.StartTestServerOrDie(t, nil, []string{"--disable-admission-plugins=ServiceAccount,StorageObjectInUseProtection"}, framework.SharedEtcd()) 867 defer s.TearDownFn() 868 namespaceName := "provision-multi-pvs" 869 870 testClient, binder, informers, watchPV, watchPVC := createClients(namespaceName, t, s, defaultSyncPeriod) 871 defer watchPV.Stop() 872 defer watchPVC.Stop() 873 874 ns := framework.CreateNamespaceOrDie(testClient, namespaceName, t) 875 defer framework.DeleteNamespaceOrDie(testClient, ns, t) 876 877 // NOTE: This test cannot run in parallel, because it is creating and deleting 878 // non-namespaced objects (PersistenceVolumes and StorageClasses). 879 defer testClient.CoreV1().PersistentVolumes().DeleteCollection(context.TODO(), metav1.DeleteOptions{}, metav1.ListOptions{}) 880 defer testClient.StorageV1().StorageClasses().DeleteCollection(context.TODO(), metav1.DeleteOptions{}, metav1.ListOptions{}) 881 882 storageClass := storage.StorageClass{ 883 TypeMeta: metav1.TypeMeta{ 884 Kind: "StorageClass", 885 }, 886 ObjectMeta: metav1.ObjectMeta{ 887 Name: "gold", 888 }, 889 Provisioner: provisionerPluginName, 890 } 891 testClient.StorageV1().StorageClasses().Create(context.TODO(), &storageClass, metav1.CreateOptions{}) 892 893 ctx, cancel := context.WithCancel(context.TODO()) 894 informers.Start(ctx.Done()) 895 go binder.Run(ctx) 896 defer cancel() 897 898 objCount := getObjectCount() 899 pvcs := make([]*v1.PersistentVolumeClaim, objCount) 900 for i := 0; i < objCount; i++ { 901 pvc := createPVC("pvc-provision-"+strconv.Itoa(i), ns.Name, "1G", []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, "gold") 902 pvcs[i] = pvc 903 } 904 905 klog.V(2).Infof("TestPersistentVolumeProvisionMultiPVCs: start") 906 // Create the claims in a separate goroutine to pop events from watchPVC 907 // early. It gets stuck with >3000 claims. 908 go func() { 909 for i := 0; i < objCount; i++ { 910 _, _ = testClient.CoreV1().PersistentVolumeClaims(ns.Name).Create(context.TODO(), pvcs[i], metav1.CreateOptions{}) 911 } 912 }() 913 914 // Wait until the controller provisions and binds all of them 915 for i := 0; i < objCount; i++ { 916 waitForAnyPersistentVolumeClaimPhase(watchPVC, v1.ClaimBound) 917 klog.V(1).Infof("%d claims bound", i+1) 918 } 919 klog.V(2).Infof("TestPersistentVolumeProvisionMultiPVCs: claims are bound") 920 921 // check that we have enough bound PVs 922 pvList, err := testClient.CoreV1().PersistentVolumes().List(context.TODO(), metav1.ListOptions{}) 923 if err != nil { 924 t.Fatalf("Failed to list volumes: %s", err) 925 } 926 if len(pvList.Items) != objCount { 927 t.Fatalf("Expected to get %d volumes, got %d", objCount, len(pvList.Items)) 928 } 929 for i := 0; i < objCount; i++ { 930 pv := &pvList.Items[i] 931 if pv.Status.Phase != v1.VolumeBound { 932 t.Fatalf("Expected volume %s to be bound, is %s instead", pv.Name, pv.Status.Phase) 933 } 934 klog.V(2).Infof("PV %q is bound to PVC %q", pv.Name, pv.Spec.ClaimRef.Name) 935 } 936 937 // Delete the claims 938 for i := 0; i < objCount; i++ { 939 _ = testClient.CoreV1().PersistentVolumeClaims(ns.Name).Delete(context.TODO(), pvcs[i].Name, metav1.DeleteOptions{}) 940 } 941 942 // Wait for the PVs to get deleted by listing remaining volumes 943 // (delete events were unreliable) 944 for { 945 volumes, err := testClient.CoreV1().PersistentVolumes().List(context.TODO(), metav1.ListOptions{}) 946 if err != nil { 947 t.Fatalf("Failed to list volumes: %v", err) 948 } 949 950 klog.V(1).Infof("%d volumes remaining", len(volumes.Items)) 951 if len(volumes.Items) == 0 { 952 break 953 } 954 time.Sleep(time.Second) 955 } 956 klog.V(2).Infof("TestPersistentVolumeProvisionMultiPVCs: volumes are deleted") 957 } 958 959 // TestPersistentVolumeMultiPVsDiffAccessModes tests binding of one PVC to two 960 // PVs with different access modes. 961 func TestPersistentVolumeMultiPVsDiffAccessModes(t *testing.T) { 962 s := kubeapiservertesting.StartTestServerOrDie(t, nil, []string{"--disable-admission-plugins=ServiceAccount,StorageObjectInUseProtection"}, framework.SharedEtcd()) 963 defer s.TearDownFn() 964 namespaceName := "multi-pvs-diff-access" 965 966 testClient, controller, informers, watchPV, watchPVC := createClients(namespaceName, t, s, defaultSyncPeriod) 967 defer watchPV.Stop() 968 defer watchPVC.Stop() 969 970 ns := framework.CreateNamespaceOrDie(testClient, namespaceName, t) 971 defer framework.DeleteNamespaceOrDie(testClient, ns, t) 972 973 // NOTE: This test cannot run in parallel, because it is creating and deleting 974 // non-namespaced objects (PersistenceVolumes). 975 defer testClient.CoreV1().PersistentVolumes().DeleteCollection(context.TODO(), metav1.DeleteOptions{}, metav1.ListOptions{}) 976 977 ctx, cancel := context.WithCancel(context.TODO()) 978 informers.Start(ctx.Done()) 979 go controller.Run(ctx) 980 defer cancel() 981 982 // This PV will be claimed, released, and deleted 983 pvRwo := createPV("pv-rwo", "/tmp/foo", "10G", 984 []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, v1.PersistentVolumeReclaimRetain) 985 pvRwm := createPV("pv-rwm", "/tmp/bar", "10G", 986 []v1.PersistentVolumeAccessMode{v1.ReadWriteMany}, v1.PersistentVolumeReclaimRetain) 987 988 pvc := createPVC("pvc-rwm", ns.Name, "5G", []v1.PersistentVolumeAccessMode{v1.ReadWriteMany}, "") 989 990 _, err := testClient.CoreV1().PersistentVolumes().Create(context.TODO(), pvRwm, metav1.CreateOptions{}) 991 if err != nil { 992 t.Errorf("Failed to create PersistentVolume: %v", err) 993 } 994 _, err = testClient.CoreV1().PersistentVolumes().Create(context.TODO(), pvRwo, metav1.CreateOptions{}) 995 if err != nil { 996 t.Errorf("Failed to create PersistentVolume: %v", err) 997 } 998 t.Log("volumes created") 999 1000 _, err = testClient.CoreV1().PersistentVolumeClaims(ns.Name).Create(context.TODO(), pvc, metav1.CreateOptions{}) 1001 if err != nil { 1002 t.Errorf("Failed to create PersistentVolumeClaim: %v", err) 1003 } 1004 t.Log("claim created") 1005 1006 // wait until the controller pairs the volume and claim 1007 waitForAnyPersistentVolumePhase(watchPV, v1.VolumeBound) 1008 t.Log("volume bound") 1009 waitForPersistentVolumeClaimPhase(testClient, pvc.Name, ns.Name, watchPVC, v1.ClaimBound) 1010 t.Log("claim bound") 1011 1012 // only RWM PV is bound 1013 pv, err := testClient.CoreV1().PersistentVolumes().Get(context.TODO(), "pv-rwo", metav1.GetOptions{}) 1014 if err != nil { 1015 t.Fatalf("Unexpected error getting pv: %v", err) 1016 } 1017 if pv.Spec.ClaimRef != nil { 1018 t.Fatalf("ReadWriteOnce PV shouldn't be bound") 1019 } 1020 pv, err = testClient.CoreV1().PersistentVolumes().Get(context.TODO(), "pv-rwm", metav1.GetOptions{}) 1021 if err != nil { 1022 t.Fatalf("Unexpected error getting pv: %v", err) 1023 } 1024 if pv.Spec.ClaimRef == nil { 1025 t.Fatalf("ReadWriteMany PV should be bound") 1026 } 1027 if pv.Spec.ClaimRef.Name != pvc.Name { 1028 t.Fatalf("Bind mismatch! Expected %s but got %s", pvc.Name, pv.Spec.ClaimRef.Name) 1029 } 1030 1031 // deleting a claim releases the volume 1032 if err := testClient.CoreV1().PersistentVolumeClaims(ns.Name).Delete(context.TODO(), pvc.Name, metav1.DeleteOptions{}); err != nil { 1033 t.Errorf("error deleting claim %s", pvc.Name) 1034 } 1035 t.Log("claim deleted") 1036 1037 waitForAnyPersistentVolumePhase(watchPV, v1.VolumeReleased) 1038 t.Log("volume released") 1039 } 1040 1041 // TestRetroactiveStorageClassAssignment tests PVC retroactive storage class 1042 // assignment and binding of PVCs with storage class name set to nil or "" with 1043 // and without presence of a default SC. 1044 func TestRetroactiveStorageClassAssignment(t *testing.T) { 1045 s := kubeapiservertesting.StartTestServerOrDie(t, nil, []string{"--disable-admission-plugins=DefaultStorageClass"}, framework.SharedEtcd()) 1046 defer s.TearDownFn() 1047 namespaceName := "retro-pvc-sc" 1048 defaultStorageClassName := "gold" 1049 storageClassName := "silver" 1050 1051 testClient, binder, informers, watchPV, watchPVC := createClients(namespaceName, t, s, defaultSyncPeriod) 1052 defer watchPV.Stop() 1053 defer watchPVC.Stop() 1054 1055 ns := framework.CreateNamespaceOrDie(testClient, namespaceName, t) 1056 defer framework.DeleteNamespaceOrDie(testClient, ns, t) 1057 1058 // NOTE: This test cannot run in parallel, because it is creating and deleting 1059 // non-namespaced objects (PersistenceVolumes and StorageClasses). 1060 defer testClient.CoreV1().PersistentVolumes().DeleteCollection(context.TODO(), metav1.DeleteOptions{}, metav1.ListOptions{}) 1061 defer testClient.CoreV1().PersistentVolumeClaims(namespaceName).DeleteCollection(context.TODO(), metav1.DeleteOptions{}, metav1.ListOptions{}) 1062 defer testClient.StorageV1().StorageClasses().DeleteCollection(context.TODO(), metav1.DeleteOptions{}, metav1.ListOptions{}) 1063 1064 // Create non default SC (extra SC - should not be used by any PVC in this test). 1065 nonDefaultSC := storage.StorageClass{ 1066 TypeMeta: metav1.TypeMeta{ 1067 Kind: "StorageClass", 1068 }, 1069 ObjectMeta: metav1.ObjectMeta{ 1070 Name: storageClassName, 1071 Annotations: map[string]string{ 1072 util.IsDefaultStorageClassAnnotation: "false", 1073 }, 1074 }, 1075 Provisioner: provisionerPluginName, 1076 } 1077 if _, err := testClient.StorageV1().StorageClasses().Create(context.TODO(), &nonDefaultSC, metav1.CreateOptions{}); err != nil { 1078 t.Errorf("Failed to create a storage class: %v", err) 1079 } 1080 1081 ctx, cancel := context.WithCancel(context.TODO()) 1082 informers.Start(ctx.Done()) 1083 go binder.Run(ctx) 1084 defer cancel() 1085 1086 klog.V(2).Infof("TestRetroactiveStorageClassAssignment: start") 1087 1088 // 1. Test that PV with SC set to "" binds to PVC with SC set to nil while default SC does not exist (verifies that feature enablement does not break old behavior). 1089 pv1 := createPVWithStorageClass("pv-1", "/tmp/foo", "5G", "", 1090 []v1.PersistentVolumeAccessMode{v1.ReadWriteMany}, v1.PersistentVolumeReclaimRetain) 1091 _, err := testClient.CoreV1().PersistentVolumes().Create(context.TODO(), pv1, metav1.CreateOptions{}) 1092 if err != nil { 1093 t.Errorf("Failed to create PersistentVolume: %v", err) 1094 } 1095 1096 pvc1 := createPVCWithNilStorageClass("pvc-1", ns.Name, "5G", []v1.PersistentVolumeAccessMode{v1.ReadWriteMany}) 1097 _, err = testClient.CoreV1().PersistentVolumeClaims(ns.Name).Create(context.TODO(), pvc1, metav1.CreateOptions{}) 1098 if err != nil { 1099 t.Errorf("Failed to create PersistentVolumeClaim: %v", err) 1100 } 1101 1102 // Wait until the controller pairs the volume. 1103 waitForPersistentVolumePhase(testClient, pv1.Name, watchPV, v1.VolumeBound) 1104 t.Log("volume bound") 1105 waitForPersistentVolumeClaimPhase(testClient, pvc1.Name, ns.Name, watchPVC, v1.ClaimBound) 1106 t.Log("claim bound") 1107 1108 pv, err := testClient.CoreV1().PersistentVolumes().Get(context.TODO(), "pv-1", metav1.GetOptions{}) 1109 if err != nil { 1110 t.Fatalf("Unexpected error getting pv: %v", err) 1111 } 1112 if pv.Spec.ClaimRef == nil { 1113 t.Fatalf("PV %s with \"\" storage class should have been bound to PVC %s that has nil storage class", pv1.Name, pvc1.Name) 1114 } 1115 if pv.Spec.ClaimRef.Name != pvc1.Name { 1116 t.Fatalf("Bind mismatch! Expected %s but got %s", pvc1.Name, pv.Spec.ClaimRef.Name) 1117 } 1118 1119 // 2. Test that retroactive SC assignment works - default SC is created after creation of PVC with nil SC. 1120 pvcRetro := createPVCWithNilStorageClass("pvc-provision-noclass", ns.Name, "5G", []v1.PersistentVolumeAccessMode{v1.ReadWriteMany}) 1121 if _, err := testClient.CoreV1().PersistentVolumeClaims(ns.Name).Create(context.TODO(), pvcRetro, metav1.CreateOptions{}); err != nil { 1122 t.Errorf("Failed to create PVC: %v", err) 1123 } 1124 t.Log("claim created") 1125 1126 // Create default SC. 1127 defaultSC := storage.StorageClass{ 1128 TypeMeta: metav1.TypeMeta{ 1129 Kind: "StorageClass", 1130 }, 1131 ObjectMeta: metav1.ObjectMeta{ 1132 Name: defaultStorageClassName, 1133 Annotations: map[string]string{ 1134 util.IsDefaultStorageClassAnnotation: "true", 1135 }, 1136 }, 1137 Provisioner: provisionerPluginName, 1138 } 1139 if _, err := testClient.StorageV1().StorageClasses().Create(context.TODO(), &defaultSC, metav1.CreateOptions{}); err != nil { 1140 t.Errorf("Failed to create a storage class: %v", err) 1141 } 1142 1143 // Verify SC was assigned retroactively to PVC. 1144 if _, ok := waitForPersistentVolumeClaimStorageClass(t, pvcRetro.Name, defaultStorageClassName, watchPVC, 20*time.Second); !ok { 1145 t.Errorf("Expected claim %s to get a storage class %s assigned retroactively", pvcRetro.Name, defaultStorageClassName) 1146 } 1147 1148 waitForPersistentVolumeClaimPhase(testClient, pvcRetro.Name, ns.Name, watchPVC, v1.ClaimBound) 1149 1150 // 3. Test that a new claim with nil class will still bind to PVs with SC set to "" (if available) and SC will not be assigned retroactively. 1151 pv3 := createPVWithStorageClass("pv-3", "/tmp/bar", "5G", "", 1152 []v1.PersistentVolumeAccessMode{v1.ReadWriteMany}, v1.PersistentVolumeReclaimRetain) 1153 _, err = testClient.CoreV1().PersistentVolumes().Create(context.TODO(), pv3, metav1.CreateOptions{}) 1154 if err != nil { 1155 t.Errorf("Failed to create PersistentVolume: %v", err) 1156 } 1157 waitForPersistentVolumePhase(testClient, pv3.Name, watchPV, v1.VolumeAvailable) 1158 1159 pvc3 := createPVCWithNilStorageClass("pvc-3", ns.Name, "5G", []v1.PersistentVolumeAccessMode{v1.ReadWriteMany}) 1160 if _, err := testClient.CoreV1().PersistentVolumeClaims(ns.Name).Create(context.TODO(), pvc3, metav1.CreateOptions{}); err != nil { 1161 t.Errorf("Failed to create PVC: %v", err) 1162 } 1163 t.Log("claim created") 1164 1165 waitForPersistentVolumeClaimPhase(testClient, pvc3.Name, ns.Name, watchPVC, v1.ClaimBound) 1166 1167 pvc, err := testClient.CoreV1().PersistentVolumeClaims(ns.Name).Get(context.TODO(), "pvc-3", metav1.GetOptions{}) 1168 if err != nil { 1169 t.Fatalf("Unexpected error getting pv: %v", err) 1170 } 1171 if pvc.Spec.StorageClassName != nil { 1172 t.Errorf("claim %s should still have nil storage class because it bound to existing PV", pvc.Name) 1173 } 1174 1175 // Create another PV which should remain unbound. 1176 pvUnbound := createPVWithStorageClass("pv-unbound", "/tmp/bar", "5G", "", 1177 []v1.PersistentVolumeAccessMode{v1.ReadWriteMany}, v1.PersistentVolumeReclaimRetain) 1178 _, err = testClient.CoreV1().PersistentVolumes().Create(context.TODO(), pvUnbound, metav1.CreateOptions{}) 1179 if err != nil { 1180 t.Errorf("Failed to create PersistentVolume: %v", err) 1181 } 1182 1183 waitForPersistentVolumePhase(testClient, pvUnbound.Name, watchPV, v1.VolumeAvailable) 1184 1185 pv, err = testClient.CoreV1().PersistentVolumes().Get(context.TODO(), "pv-unbound", metav1.GetOptions{}) 1186 if err != nil { 1187 t.Fatalf("Unexpected error getting pv: %v", err) 1188 } 1189 if pv.Spec.ClaimRef != nil { 1190 t.Fatalf("PV %s shouldn't be bound", pvUnbound.Name) 1191 } 1192 1193 // Remove the PV to not interfere with next test. 1194 testClient.CoreV1().PersistentVolumes().Delete(context.TODO(), pvUnbound.Name, metav1.DeleteOptions{}) 1195 1196 // 4. Test that PV with SC set to "" binds to PVC with SC set to "" while default SC exists. 1197 // This tests that the feature enablement and default SC presence does not break this binding. 1198 // If this breaks there would be no way to ever bind PVs with SC set to "". 1199 pvc4 := createPVC("pvc-4", ns.Name, "5G", []v1.PersistentVolumeAccessMode{v1.ReadWriteMany}, "") 1200 _, err = testClient.CoreV1().PersistentVolumeClaims(ns.Name).Create(context.TODO(), pvc4, metav1.CreateOptions{}) 1201 if err != nil { 1202 t.Errorf("Failed to create PersistentVolumeClaim: %v", err) 1203 } 1204 1205 pv4 := createPVWithStorageClass("pv-4", "/tmp/bar", "5G", "", 1206 []v1.PersistentVolumeAccessMode{v1.ReadWriteMany}, v1.PersistentVolumeReclaimRetain) 1207 _, err = testClient.CoreV1().PersistentVolumes().Create(context.TODO(), pv4, metav1.CreateOptions{}) 1208 if err != nil { 1209 t.Errorf("Failed to create PersistentVolume: %v", err) 1210 } 1211 1212 // Wait until the controller pairs the volume. 1213 waitForPersistentVolumePhase(testClient, pv4.Name, watchPV, v1.VolumeBound) 1214 t.Log("volume bound") 1215 waitForPersistentVolumeClaimPhase(testClient, pvc4.Name, ns.Name, watchPVC, v1.ClaimBound) 1216 t.Log("claim bound") 1217 1218 pv, err = testClient.CoreV1().PersistentVolumes().Get(context.TODO(), "pv-4", metav1.GetOptions{}) 1219 if err != nil { 1220 t.Fatalf("Unexpected error getting pv: %v", err) 1221 } 1222 if pv.Spec.ClaimRef == nil { 1223 t.Fatalf("PV %s with \"\" storage class should have been bound to PVC %s that also has \"\" storage class", pv4.Name, pvc4.Name) 1224 } 1225 if pv.Spec.ClaimRef.Name != pvc4.Name { 1226 t.Fatalf("Bind mismatch! Expected PV %s to bind to PVC %s but instead it bound to PVC %s", pv.Name, pvc4.Name, pv.Spec.ClaimRef.Name) 1227 } 1228 } 1229 1230 func waitForPersistentVolumePhase(client *clientset.Clientset, pvName string, w watch.Interface, phase v1.PersistentVolumePhase) { 1231 // Check if the volume is already in requested phase 1232 volume, err := client.CoreV1().PersistentVolumes().Get(context.TODO(), pvName, metav1.GetOptions{}) 1233 if err == nil && volume.Status.Phase == phase { 1234 return 1235 } 1236 1237 // Wait for the phase 1238 for { 1239 event := <-w.ResultChan() 1240 volume, ok := event.Object.(*v1.PersistentVolume) 1241 if !ok { 1242 continue 1243 } 1244 if volume.Status.Phase == phase && volume.Name == pvName { 1245 klog.V(2).Infof("volume %q is %s", volume.Name, phase) 1246 break 1247 } 1248 } 1249 } 1250 1251 func waitForPersistentVolumeClaimPhase(client *clientset.Clientset, claimName, namespace string, w watch.Interface, phase v1.PersistentVolumeClaimPhase) { 1252 // Check if the claim is already in requested phase 1253 claim, err := client.CoreV1().PersistentVolumeClaims(namespace).Get(context.TODO(), claimName, metav1.GetOptions{}) 1254 if err == nil && claim.Status.Phase == phase { 1255 return 1256 } 1257 1258 // Wait for the phase 1259 for { 1260 event := <-w.ResultChan() 1261 claim, ok := event.Object.(*v1.PersistentVolumeClaim) 1262 if !ok { 1263 continue 1264 } 1265 if claim.Status.Phase == phase && claim.Name == claimName { 1266 klog.V(2).Infof("claim %q is %s", claim.Name, phase) 1267 break 1268 } 1269 } 1270 } 1271 1272 func waitForAnyPersistentVolumePhase(w watch.Interface, phase v1.PersistentVolumePhase) { 1273 for { 1274 event := <-w.ResultChan() 1275 volume, ok := event.Object.(*v1.PersistentVolume) 1276 if !ok { 1277 continue 1278 } 1279 if volume.Status.Phase == phase { 1280 klog.V(2).Infof("volume %q is %s", volume.Name, phase) 1281 break 1282 } 1283 } 1284 } 1285 1286 func waitForAnyPersistentVolumeClaimPhase(w watch.Interface, phase v1.PersistentVolumeClaimPhase) { 1287 for { 1288 event := <-w.ResultChan() 1289 claim, ok := event.Object.(*v1.PersistentVolumeClaim) 1290 if !ok { 1291 continue 1292 } 1293 if claim.Status.Phase == phase { 1294 klog.V(2).Infof("claim %q is %s", claim.Name, phase) 1295 break 1296 } 1297 } 1298 } 1299 1300 func waitForPersistentVolumeClaimStorageClass(t *testing.T, claimName, scName string, w watch.Interface, duration time.Duration) (*v1.PersistentVolumeClaim, bool) { 1301 stopTimer := time.NewTimer(duration) 1302 defer stopTimer.Stop() 1303 1304 // Wait for the storage class 1305 for { 1306 select { 1307 case event := <-w.ResultChan(): 1308 claim, ok := event.Object.(*v1.PersistentVolumeClaim) 1309 if ok { 1310 t.Logf("Watching claim %s", claim.Name) 1311 } else { 1312 t.Errorf("Watch closed unexpectedly") 1313 } 1314 if claim.Spec.StorageClassName == nil { 1315 t.Logf("Claim %v does not yet have expected storage class %v", claim.Name, scName) 1316 continue 1317 } 1318 if *claim.Spec.StorageClassName == scName && claim.Name == claimName { 1319 t.Logf("Claim %s now has expected storage class %s", claim.Name, *claim.Spec.StorageClassName) 1320 return claim, true 1321 } 1322 case <-stopTimer.C: 1323 return nil, false 1324 } 1325 1326 } 1327 } 1328 1329 func createClients(namespaceName string, t *testing.T, s *kubeapiservertesting.TestServer, syncPeriod time.Duration) (*clientset.Clientset, *persistentvolumecontroller.PersistentVolumeController, informers.SharedInformerFactory, watch.Interface, watch.Interface) { 1330 // Use higher QPS and Burst, there is a test for race conditions which 1331 // creates many objects and default values were too low. 1332 binderConfig := restclient.CopyConfig(s.ClientConfig) 1333 binderConfig.QPS = 1000000 1334 binderConfig.Burst = 1000000 1335 binderClient := clientset.NewForConfigOrDie(binderConfig) 1336 testConfig := restclient.CopyConfig(s.ClientConfig) 1337 testConfig.QPS = 1000000 1338 testConfig.Burst = 1000000 1339 testClient := clientset.NewForConfigOrDie(testConfig) 1340 1341 host := volumetest.NewFakeVolumeHost(t, "/tmp/fake", nil, nil) 1342 plugin := &volumetest.FakeVolumePlugin{ 1343 PluginName: provisionerPluginName, 1344 Host: host, 1345 Config: volume.VolumeConfig{}, 1346 LastProvisionerOptions: volume.VolumeOptions{}, 1347 NewAttacherCallCount: 0, 1348 NewDetacherCallCount: 0, 1349 Mounters: nil, 1350 Unmounters: nil, 1351 Attachers: nil, 1352 Detachers: nil, 1353 } 1354 plugins := []volume.VolumePlugin{plugin} 1355 cloud := &fakecloud.Cloud{} 1356 informers := informers.NewSharedInformerFactory(testClient, getSyncPeriod(syncPeriod)) 1357 _, ctx := ktesting.NewTestContext(t) 1358 ctrl, err := persistentvolumecontroller.NewController( 1359 ctx, 1360 persistentvolumecontroller.ControllerParameters{ 1361 KubeClient: binderClient, 1362 SyncPeriod: getSyncPeriod(syncPeriod), 1363 VolumePlugins: plugins, 1364 Cloud: cloud, 1365 VolumeInformer: informers.Core().V1().PersistentVolumes(), 1366 ClaimInformer: informers.Core().V1().PersistentVolumeClaims(), 1367 ClassInformer: informers.Storage().V1().StorageClasses(), 1368 PodInformer: informers.Core().V1().Pods(), 1369 NodeInformer: informers.Core().V1().Nodes(), 1370 EnableDynamicProvisioning: true, 1371 }) 1372 if err != nil { 1373 t.Fatalf("Failed to construct PersistentVolumes: %v", err) 1374 } 1375 1376 watchPV, err := testClient.CoreV1().PersistentVolumes().Watch(context.TODO(), metav1.ListOptions{}) 1377 if err != nil { 1378 t.Fatalf("Failed to watch PersistentVolumes: %v", err) 1379 } 1380 watchPVC, err := testClient.CoreV1().PersistentVolumeClaims(namespaceName).Watch(context.TODO(), metav1.ListOptions{}) 1381 if err != nil { 1382 t.Fatalf("Failed to watch PersistentVolumeClaims: %v", err) 1383 } 1384 1385 return testClient, ctrl, informers, watchPV, watchPVC 1386 } 1387 1388 func createPV(name, path, cap string, mode []v1.PersistentVolumeAccessMode, reclaim v1.PersistentVolumeReclaimPolicy) *v1.PersistentVolume { 1389 return &v1.PersistentVolume{ 1390 ObjectMeta: metav1.ObjectMeta{Name: name}, 1391 Spec: v1.PersistentVolumeSpec{ 1392 PersistentVolumeSource: v1.PersistentVolumeSource{HostPath: &v1.HostPathVolumeSource{Path: path}}, 1393 Capacity: v1.ResourceList{v1.ResourceName(v1.ResourceStorage): resource.MustParse(cap)}, 1394 AccessModes: mode, 1395 PersistentVolumeReclaimPolicy: reclaim, 1396 }, 1397 } 1398 } 1399 1400 func createPVWithStorageClass(name, path, cap, scName string, mode []v1.PersistentVolumeAccessMode, reclaim v1.PersistentVolumeReclaimPolicy) *v1.PersistentVolume { 1401 return &v1.PersistentVolume{ 1402 ObjectMeta: metav1.ObjectMeta{Name: name}, 1403 Spec: v1.PersistentVolumeSpec{ 1404 PersistentVolumeSource: v1.PersistentVolumeSource{HostPath: &v1.HostPathVolumeSource{Path: path}}, 1405 Capacity: v1.ResourceList{v1.ResourceName(v1.ResourceStorage): resource.MustParse(cap)}, 1406 AccessModes: mode, 1407 PersistentVolumeReclaimPolicy: reclaim, 1408 StorageClassName: scName, 1409 }, 1410 } 1411 } 1412 1413 func createPVC(name, namespace, cap string, mode []v1.PersistentVolumeAccessMode, class string) *v1.PersistentVolumeClaim { 1414 return &v1.PersistentVolumeClaim{ 1415 ObjectMeta: metav1.ObjectMeta{ 1416 Name: name, 1417 Namespace: namespace, 1418 }, 1419 Spec: v1.PersistentVolumeClaimSpec{ 1420 Resources: v1.VolumeResourceRequirements{Requests: v1.ResourceList{v1.ResourceName(v1.ResourceStorage): resource.MustParse(cap)}}, 1421 AccessModes: mode, 1422 StorageClassName: &class, 1423 }, 1424 } 1425 } 1426 1427 func createPVCWithNilStorageClass(name, namespace, cap string, mode []v1.PersistentVolumeAccessMode) *v1.PersistentVolumeClaim { 1428 return &v1.PersistentVolumeClaim{ 1429 ObjectMeta: metav1.ObjectMeta{ 1430 Name: name, 1431 Namespace: namespace, 1432 }, 1433 Spec: v1.PersistentVolumeClaimSpec{ 1434 Resources: v1.VolumeResourceRequirements{Requests: v1.ResourceList{v1.ResourceName(v1.ResourceStorage): resource.MustParse(cap)}}, 1435 AccessModes: mode, 1436 }, 1437 } 1438 }