k8s.io/kubernetes@v1.29.3/test/e2e/framework/pv/pv.go (about) 1 /* 2 Copyright 2015 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package pv 18 19 import ( 20 "context" 21 "fmt" 22 "strings" 23 "time" 24 25 "k8s.io/apimachinery/pkg/util/wait" 26 27 "k8s.io/kubernetes/test/e2e/storage/utils" 28 29 "github.com/onsi/ginkgo/v2" 30 v1 "k8s.io/api/core/v1" 31 apierrors "k8s.io/apimachinery/pkg/api/errors" 32 "k8s.io/apimachinery/pkg/api/resource" 33 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 34 "k8s.io/apimachinery/pkg/labels" 35 "k8s.io/apimachinery/pkg/types" 36 clientset "k8s.io/client-go/kubernetes" 37 "k8s.io/kubernetes/pkg/volume/util" 38 "k8s.io/kubernetes/test/e2e/framework" 39 e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" 40 ) 41 42 const ( 43 pdRetryTimeout = 5 * time.Minute 44 pdRetryPollTime = 5 * time.Second 45 46 // VolumeSelectorKey is the key for volume selector. 47 VolumeSelectorKey = "e2e-pv-pool" 48 49 // volumeGidAnnotationKey is the of the annotation on the PersistentVolume 50 // object that specifies a supplemental GID. 51 // it is copied from k8s.io/kubernetes/pkg/volume/util VolumeGidAnnotationKey 52 volumeGidAnnotationKey = "pv.beta.kubernetes.io/gid" 53 ) 54 55 var ( 56 // SELinuxLabel is common selinux labels. 57 SELinuxLabel = &v1.SELinuxOptions{ 58 Level: "s0:c0,c1"} 59 ) 60 61 type pvval struct{} 62 63 // PVMap is a map of all PVs used in the multi pv-pvc tests. The key is the PV's name, which is 64 // guaranteed to be unique. The value is {} (empty struct) since we're only interested 65 // in the PV's name and if it is present. We must always Get the pv object before 66 // referencing any of its values, eg its ClaimRef. 67 type PVMap map[string]pvval 68 69 type pvcval struct{} 70 71 // PVCMap is a map of all PVCs used in the multi pv-pvc tests. The key is "namespace/pvc.Name". The 72 // value is {} (empty struct) since we're only interested in the PVC's name and if it is 73 // present. We must always Get the pvc object before referencing any of its values, eg. 74 // its VolumeName. 75 // Note: It's unsafe to add keys to a map in a loop. Their insertion in the map is 76 // 77 // unpredictable and can result in the same key being iterated over again. 78 type PVCMap map[types.NamespacedName]pvcval 79 80 // PersistentVolumeConfig is consumed by MakePersistentVolume() to generate a PV object 81 // for varying storage options (NFS, ceph, etc.). 82 // (+optional) prebind holds a pre-bound PVC 83 // Example pvSource: 84 // 85 // pvSource: api.PersistentVolumeSource{ 86 // NFS: &api.NFSVolumeSource{ 87 // ... 88 // }, 89 // } 90 type PersistentVolumeConfig struct { 91 // [Optional] NamePrefix defaults to "pv-" if unset 92 NamePrefix string 93 // [Optional] Labels contains information used to organize and categorize 94 // objects 95 Labels labels.Set 96 // PVSource contains the details of the underlying volume and must be set 97 PVSource v1.PersistentVolumeSource 98 // [Optional] Prebind lets you specify a PVC to bind this PV to before 99 // creation 100 Prebind *v1.PersistentVolumeClaim 101 // [Optiona] ReclaimPolicy defaults to "Reclaim" if unset 102 ReclaimPolicy v1.PersistentVolumeReclaimPolicy 103 StorageClassName string 104 // [Optional] NodeAffinity defines constraints that limit what nodes this 105 // volume can be accessed from. 106 NodeAffinity *v1.VolumeNodeAffinity 107 // [Optional] VolumeMode defaults to "Filesystem" if unset 108 VolumeMode *v1.PersistentVolumeMode 109 // [Optional] AccessModes defaults to RWO if unset 110 AccessModes []v1.PersistentVolumeAccessMode 111 // [Optional] Capacity is the storage capacity in Quantity format. Defaults 112 // to "2Gi" if unset 113 Capacity string 114 } 115 116 // PersistentVolumeClaimConfig is consumed by MakePersistentVolumeClaim() to 117 // generate a PVC object. 118 type PersistentVolumeClaimConfig struct { 119 // Name of the PVC. If set, overrides NamePrefix 120 Name string 121 // NamePrefix defaults to "pvc-" if unspecified 122 NamePrefix string 123 // ClaimSize must be specified in the Quantity format. Defaults to 2Gi if 124 // unspecified 125 ClaimSize string 126 // AccessModes defaults to RWO if unspecified 127 AccessModes []v1.PersistentVolumeAccessMode 128 Annotations map[string]string 129 Selector *metav1.LabelSelector 130 StorageClassName *string 131 // VolumeMode defaults to nil if unspecified or specified as the empty 132 // string 133 VolumeMode *v1.PersistentVolumeMode 134 } 135 136 // PVPVCCleanup cleans up a pv and pvc in a single pv/pvc test case. 137 // Note: delete errors are appended to []error so that we can attempt to delete both the pvc and pv. 138 func PVPVCCleanup(ctx context.Context, c clientset.Interface, ns string, pv *v1.PersistentVolume, pvc *v1.PersistentVolumeClaim) []error { 139 var errs []error 140 141 if pvc != nil { 142 err := DeletePersistentVolumeClaim(ctx, c, pvc.Name, ns) 143 if err != nil { 144 errs = append(errs, fmt.Errorf("failed to delete PVC %q: %w", pvc.Name, err)) 145 } 146 } else { 147 framework.Logf("pvc is nil") 148 } 149 if pv != nil { 150 err := DeletePersistentVolume(ctx, c, pv.Name) 151 if err != nil { 152 errs = append(errs, fmt.Errorf("failed to delete PV %q: %w", pv.Name, err)) 153 } 154 } else { 155 framework.Logf("pv is nil") 156 } 157 return errs 158 } 159 160 // PVPVCMapCleanup Cleans up pvs and pvcs in multi-pv-pvc test cases. Entries found in the pv and claim maps are 161 // deleted as long as the Delete api call succeeds. 162 // Note: delete errors are appended to []error so that as many pvcs and pvs as possible are deleted. 163 func PVPVCMapCleanup(ctx context.Context, c clientset.Interface, ns string, pvols PVMap, claims PVCMap) []error { 164 var errs []error 165 166 for pvcKey := range claims { 167 err := DeletePersistentVolumeClaim(ctx, c, pvcKey.Name, ns) 168 if err != nil { 169 errs = append(errs, fmt.Errorf("failed to delete PVC %q: %w", pvcKey.Name, err)) 170 } else { 171 delete(claims, pvcKey) 172 } 173 } 174 175 for pvKey := range pvols { 176 err := DeletePersistentVolume(ctx, c, pvKey) 177 if err != nil { 178 errs = append(errs, fmt.Errorf("failed to delete PV %q: %w", pvKey, err)) 179 } else { 180 delete(pvols, pvKey) 181 } 182 } 183 return errs 184 } 185 186 // DeletePersistentVolume deletes the PV. 187 func DeletePersistentVolume(ctx context.Context, c clientset.Interface, pvName string) error { 188 if c != nil && len(pvName) > 0 { 189 framework.Logf("Deleting PersistentVolume %q", pvName) 190 err := c.CoreV1().PersistentVolumes().Delete(ctx, pvName, metav1.DeleteOptions{}) 191 if err != nil && !apierrors.IsNotFound(err) { 192 return fmt.Errorf("PV Delete API error: %w", err) 193 } 194 } 195 return nil 196 } 197 198 // DeletePersistentVolumeClaim deletes the Claim. 199 func DeletePersistentVolumeClaim(ctx context.Context, c clientset.Interface, pvcName string, ns string) error { 200 if c != nil && len(pvcName) > 0 { 201 framework.Logf("Deleting PersistentVolumeClaim %q", pvcName) 202 err := c.CoreV1().PersistentVolumeClaims(ns).Delete(ctx, pvcName, metav1.DeleteOptions{}) 203 if err != nil && !apierrors.IsNotFound(err) { 204 return fmt.Errorf("PVC Delete API error: %w", err) 205 } 206 } 207 return nil 208 } 209 210 // DeletePVCandValidatePV deletes the PVC and waits for the PV to enter its expected phase. Validate that the PV 211 // has been reclaimed (assumption here about reclaimPolicy). Caller tells this func which 212 // phase value to expect for the pv bound to the to-be-deleted claim. 213 func DeletePVCandValidatePV(ctx context.Context, c clientset.Interface, timeouts *framework.TimeoutContext, ns string, pvc *v1.PersistentVolumeClaim, pv *v1.PersistentVolume, expectPVPhase v1.PersistentVolumePhase) error { 214 pvname := pvc.Spec.VolumeName 215 framework.Logf("Deleting PVC %v to trigger reclamation of PV %v", pvc.Name, pvname) 216 err := DeletePersistentVolumeClaim(ctx, c, pvc.Name, ns) 217 if err != nil { 218 return err 219 } 220 221 // Wait for the PV's phase to return to be `expectPVPhase` 222 framework.Logf("Waiting for reclaim process to complete.") 223 err = WaitForPersistentVolumePhase(ctx, expectPVPhase, c, pv.Name, framework.Poll, timeouts.PVReclaim) 224 if err != nil { 225 return fmt.Errorf("pv %q phase did not become %v: %w", pv.Name, expectPVPhase, err) 226 } 227 228 // examine the pv's ClaimRef and UID and compare to expected values 229 pv, err = c.CoreV1().PersistentVolumes().Get(ctx, pv.Name, metav1.GetOptions{}) 230 if err != nil { 231 return fmt.Errorf("PV Get API error: %w", err) 232 } 233 cr := pv.Spec.ClaimRef 234 if expectPVPhase == v1.VolumeAvailable { 235 if cr != nil && len(cr.UID) > 0 { 236 return fmt.Errorf("PV is 'Available' but ClaimRef.UID is not empty") 237 } 238 } else if expectPVPhase == v1.VolumeBound { 239 if cr == nil { 240 return fmt.Errorf("PV is 'Bound' but ClaimRef is nil") 241 } 242 if len(cr.UID) == 0 { 243 return fmt.Errorf("PV is 'Bound' but ClaimRef.UID is empty") 244 } 245 } 246 247 framework.Logf("PV %v now in %q phase", pv.Name, expectPVPhase) 248 return nil 249 } 250 251 // DeletePVCandValidatePVGroup wraps deletePVCandValidatePV() by calling the function in a loop over the PV map. Only bound PVs 252 // are deleted. Validates that the claim was deleted and the PV is in the expected Phase (Released, 253 // Available, Bound). 254 // Note: if there are more claims than pvs then some of the remaining claims may bind to just made 255 // 256 // available pvs. 257 func DeletePVCandValidatePVGroup(ctx context.Context, c clientset.Interface, timeouts *framework.TimeoutContext, ns string, pvols PVMap, claims PVCMap, expectPVPhase v1.PersistentVolumePhase) error { 258 var boundPVs, deletedPVCs int 259 260 for pvName := range pvols { 261 pv, err := c.CoreV1().PersistentVolumes().Get(ctx, pvName, metav1.GetOptions{}) 262 if err != nil { 263 return fmt.Errorf("PV Get API error: %w", err) 264 } 265 cr := pv.Spec.ClaimRef 266 // if pv is bound then delete the pvc it is bound to 267 if cr != nil && len(cr.Name) > 0 { 268 boundPVs++ 269 // Assert bound PVC is tracked in this test. Failing this might 270 // indicate external PVCs interfering with the test. 271 pvcKey := makePvcKey(ns, cr.Name) 272 if _, found := claims[pvcKey]; !found { 273 return fmt.Errorf("internal: claims map is missing pvc %q", pvcKey) 274 } 275 // get the pvc for the delete call below 276 pvc, err := c.CoreV1().PersistentVolumeClaims(ns).Get(ctx, cr.Name, metav1.GetOptions{}) 277 if err == nil { 278 if err = DeletePVCandValidatePV(ctx, c, timeouts, ns, pvc, pv, expectPVPhase); err != nil { 279 return err 280 } 281 } else if !apierrors.IsNotFound(err) { 282 return fmt.Errorf("PVC Get API error: %w", err) 283 } 284 // delete pvckey from map even if apierrors.IsNotFound above is true and thus the 285 // claim was not actually deleted here 286 delete(claims, pvcKey) 287 deletedPVCs++ 288 } 289 } 290 if boundPVs != deletedPVCs { 291 return fmt.Errorf("expect number of bound PVs (%v) to equal number of deleted PVCs (%v)", boundPVs, deletedPVCs) 292 } 293 return nil 294 } 295 296 // create the PV resource. Fails test on error. 297 func createPV(ctx context.Context, c clientset.Interface, timeouts *framework.TimeoutContext, pv *v1.PersistentVolume) (*v1.PersistentVolume, error) { 298 var resultPV *v1.PersistentVolume 299 var lastCreateErr error 300 err := wait.PollUntilContextTimeout(ctx, 29*time.Second, timeouts.PVCreate, true, func(ctx context.Context) (done bool, err error) { 301 resultPV, lastCreateErr = c.CoreV1().PersistentVolumes().Create(ctx, pv, metav1.CreateOptions{}) 302 if lastCreateErr != nil { 303 // If we hit a quota problem, we are not done and should retry again. This happens to be the quota failure string for GCP. 304 // If quota failure strings are found for other platforms, they can be added to improve reliability when running 305 // many parallel test jobs in a single cloud account. This corresponds to controller-like behavior and 306 // to what we would recommend for general clients. 307 if strings.Contains(lastCreateErr.Error(), `googleapi: Error 403: Quota exceeded for quota group`) { 308 return false, nil 309 } 310 311 // if it was not a quota failure, fail immediately 312 return false, lastCreateErr 313 } 314 315 return true, nil 316 }) 317 // if we have an error from creating the PV, use that instead of a timeout error 318 if lastCreateErr != nil { 319 return nil, fmt.Errorf("PV Create API error: %w", err) 320 } 321 if err != nil { 322 return nil, fmt.Errorf("PV Create API error: %w", err) 323 } 324 325 return resultPV, nil 326 } 327 328 // CreatePV creates the PV resource. Fails test on error. 329 func CreatePV(ctx context.Context, c clientset.Interface, timeouts *framework.TimeoutContext, pv *v1.PersistentVolume) (*v1.PersistentVolume, error) { 330 return createPV(ctx, c, timeouts, pv) 331 } 332 333 // CreatePVC creates the PVC resource. Fails test on error. 334 func CreatePVC(ctx context.Context, c clientset.Interface, ns string, pvc *v1.PersistentVolumeClaim) (*v1.PersistentVolumeClaim, error) { 335 pvc, err := c.CoreV1().PersistentVolumeClaims(ns).Create(ctx, pvc, metav1.CreateOptions{}) 336 if err != nil { 337 return nil, fmt.Errorf("PVC Create API error: %w", err) 338 } 339 return pvc, nil 340 } 341 342 // CreatePVCPV creates a PVC followed by the PV based on the passed in nfs-server ip and 343 // namespace. If the "preBind" bool is true then pre-bind the PV to the PVC 344 // via the PV's ClaimRef. Return the pv and pvc to reflect the created objects. 345 // Note: in the pre-bind case the real PVC name, which is generated, is not 346 // 347 // known until after the PVC is instantiated. This is why the pvc is created 348 // before the pv. 349 func CreatePVCPV(ctx context.Context, c clientset.Interface, timeouts *framework.TimeoutContext, pvConfig PersistentVolumeConfig, pvcConfig PersistentVolumeClaimConfig, ns string, preBind bool) (*v1.PersistentVolume, *v1.PersistentVolumeClaim, error) { 350 // make the pvc spec 351 pvc := MakePersistentVolumeClaim(pvcConfig, ns) 352 preBindMsg := "" 353 if preBind { 354 preBindMsg = " pre-bound" 355 pvConfig.Prebind = pvc 356 } 357 // make the pv spec 358 pv := MakePersistentVolume(pvConfig) 359 360 ginkgo.By(fmt.Sprintf("Creating a PVC followed by a%s PV", preBindMsg)) 361 pvc, err := CreatePVC(ctx, c, ns, pvc) 362 if err != nil { 363 return nil, nil, err 364 } 365 366 // instantiate the pv, handle pre-binding by ClaimRef if needed 367 if preBind { 368 pv.Spec.ClaimRef.Name = pvc.Name 369 } 370 pv, err = createPV(ctx, c, timeouts, pv) 371 if err != nil { 372 return nil, pvc, err 373 } 374 return pv, pvc, nil 375 } 376 377 // CreatePVPVC creates a PV followed by the PVC based on the passed in nfs-server ip and 378 // namespace. If the "preBind" bool is true then pre-bind the PVC to the PV 379 // via the PVC's VolumeName. Return the pv and pvc to reflect the created 380 // objects. 381 // Note: in the pre-bind case the real PV name, which is generated, is not 382 // 383 // known until after the PV is instantiated. This is why the pv is created 384 // before the pvc. 385 func CreatePVPVC(ctx context.Context, c clientset.Interface, timeouts *framework.TimeoutContext, pvConfig PersistentVolumeConfig, pvcConfig PersistentVolumeClaimConfig, ns string, preBind bool) (*v1.PersistentVolume, *v1.PersistentVolumeClaim, error) { 386 preBindMsg := "" 387 if preBind { 388 preBindMsg = " pre-bound" 389 } 390 framework.Logf("Creating a PV followed by a%s PVC", preBindMsg) 391 392 // make the pv and pvc definitions 393 pv := MakePersistentVolume(pvConfig) 394 pvc := MakePersistentVolumeClaim(pvcConfig, ns) 395 396 // instantiate the pv 397 pv, err := createPV(ctx, c, timeouts, pv) 398 if err != nil { 399 return nil, nil, err 400 } 401 // instantiate the pvc, handle pre-binding by VolumeName if needed 402 if preBind { 403 pvc.Spec.VolumeName = pv.Name 404 } 405 pvc, err = CreatePVC(ctx, c, ns, pvc) 406 if err != nil { 407 return pv, nil, err 408 } 409 return pv, pvc, nil 410 } 411 412 // CreatePVsPVCs creates the desired number of PVs and PVCs and returns them in separate maps. If the 413 // number of PVs != the number of PVCs then the min of those two counts is the number of 414 // PVs expected to bind. If a Create error occurs, the returned maps may contain pv and pvc 415 // entries for the resources that were successfully created. In other words, when the caller 416 // sees an error returned, it needs to decide what to do about entries in the maps. 417 // Note: when the test suite deletes the namespace orphaned pvcs and pods are deleted. However, 418 // 419 // orphaned pvs are not deleted and will remain after the suite completes. 420 func CreatePVsPVCs(ctx context.Context, numpvs, numpvcs int, c clientset.Interface, timeouts *framework.TimeoutContext, ns string, pvConfig PersistentVolumeConfig, pvcConfig PersistentVolumeClaimConfig) (PVMap, PVCMap, error) { 421 pvMap := make(PVMap, numpvs) 422 pvcMap := make(PVCMap, numpvcs) 423 extraPVCs := 0 424 extraPVs := numpvs - numpvcs 425 if extraPVs < 0 { 426 extraPVCs = -extraPVs 427 extraPVs = 0 428 } 429 pvsToCreate := numpvs - extraPVs // want the min(numpvs, numpvcs) 430 431 // create pvs and pvcs 432 for i := 0; i < pvsToCreate; i++ { 433 pv, pvc, err := CreatePVPVC(ctx, c, timeouts, pvConfig, pvcConfig, ns, false) 434 if err != nil { 435 return pvMap, pvcMap, err 436 } 437 pvMap[pv.Name] = pvval{} 438 pvcMap[makePvcKey(ns, pvc.Name)] = pvcval{} 439 } 440 441 // create extra pvs or pvcs as needed 442 for i := 0; i < extraPVs; i++ { 443 pv := MakePersistentVolume(pvConfig) 444 pv, err := createPV(ctx, c, timeouts, pv) 445 if err != nil { 446 return pvMap, pvcMap, err 447 } 448 pvMap[pv.Name] = pvval{} 449 } 450 for i := 0; i < extraPVCs; i++ { 451 pvc := MakePersistentVolumeClaim(pvcConfig, ns) 452 pvc, err := CreatePVC(ctx, c, ns, pvc) 453 if err != nil { 454 return pvMap, pvcMap, err 455 } 456 pvcMap[makePvcKey(ns, pvc.Name)] = pvcval{} 457 } 458 return pvMap, pvcMap, nil 459 } 460 461 // WaitOnPVandPVC waits for the pv and pvc to bind to each other. 462 func WaitOnPVandPVC(ctx context.Context, c clientset.Interface, timeouts *framework.TimeoutContext, ns string, pv *v1.PersistentVolume, pvc *v1.PersistentVolumeClaim) error { 463 // Wait for newly created PVC to bind to the PV 464 framework.Logf("Waiting for PV %v to bind to PVC %v", pv.Name, pvc.Name) 465 err := WaitForPersistentVolumeClaimPhase(ctx, v1.ClaimBound, c, ns, pvc.Name, framework.Poll, timeouts.ClaimBound) 466 if err != nil { 467 return fmt.Errorf("PVC %q did not become Bound: %w", pvc.Name, err) 468 } 469 470 // Wait for PersistentVolume.Status.Phase to be Bound, which it should be 471 // since the PVC is already bound. 472 err = WaitForPersistentVolumePhase(ctx, v1.VolumeBound, c, pv.Name, framework.Poll, timeouts.PVBound) 473 if err != nil { 474 return fmt.Errorf("PV %q did not become Bound: %w", pv.Name, err) 475 } 476 477 // Re-get the pv and pvc objects 478 pv, err = c.CoreV1().PersistentVolumes().Get(ctx, pv.Name, metav1.GetOptions{}) 479 if err != nil { 480 return fmt.Errorf("PV Get API error: %w", err) 481 } 482 pvc, err = c.CoreV1().PersistentVolumeClaims(ns).Get(ctx, pvc.Name, metav1.GetOptions{}) 483 if err != nil { 484 return fmt.Errorf("PVC Get API error: %w", err) 485 } 486 487 // The pv and pvc are both bound, but to each other? 488 // Check that the PersistentVolume.ClaimRef matches the PVC 489 if pv.Spec.ClaimRef == nil { 490 return fmt.Errorf("PV %q ClaimRef is nil", pv.Name) 491 } 492 if pv.Spec.ClaimRef.Name != pvc.Name { 493 return fmt.Errorf("PV %q ClaimRef's name (%q) should be %q", pv.Name, pv.Spec.ClaimRef.Name, pvc.Name) 494 } 495 if pvc.Spec.VolumeName != pv.Name { 496 return fmt.Errorf("PVC %q VolumeName (%q) should be %q", pvc.Name, pvc.Spec.VolumeName, pv.Name) 497 } 498 if pv.Spec.ClaimRef.UID != pvc.UID { 499 return fmt.Errorf("PV %q ClaimRef's UID (%q) should be %q", pv.Name, pv.Spec.ClaimRef.UID, pvc.UID) 500 } 501 return nil 502 } 503 504 // WaitAndVerifyBinds searches for bound PVs and PVCs by examining pvols for non-nil claimRefs. 505 // NOTE: Each iteration waits for a maximum of 3 minutes per PV and, if the PV is bound, 506 // 507 // up to 3 minutes for the PVC. When the number of PVs != number of PVCs, this can lead 508 // to situations where the maximum wait times are reached several times in succession, 509 // extending test time. Thus, it is recommended to keep the delta between PVs and PVCs 510 // small. 511 func WaitAndVerifyBinds(ctx context.Context, c clientset.Interface, timeouts *framework.TimeoutContext, ns string, pvols PVMap, claims PVCMap, testExpected bool) error { 512 var actualBinds int 513 expectedBinds := len(pvols) 514 if expectedBinds > len(claims) { // want the min of # pvs or #pvcs 515 expectedBinds = len(claims) 516 } 517 518 for pvName := range pvols { 519 err := WaitForPersistentVolumePhase(ctx, v1.VolumeBound, c, pvName, framework.Poll, timeouts.PVBound) 520 if err != nil && len(pvols) > len(claims) { 521 framework.Logf("WARN: pv %v is not bound after max wait", pvName) 522 framework.Logf(" This may be ok since there are more pvs than pvcs") 523 continue 524 } 525 if err != nil { 526 return fmt.Errorf("PV %q did not become Bound: %w", pvName, err) 527 } 528 529 pv, err := c.CoreV1().PersistentVolumes().Get(ctx, pvName, metav1.GetOptions{}) 530 if err != nil { 531 return fmt.Errorf("PV Get API error: %w", err) 532 } 533 cr := pv.Spec.ClaimRef 534 if cr != nil && len(cr.Name) > 0 { 535 // Assert bound pvc is a test resource. Failing assertion could 536 // indicate non-test PVC interference or a bug in the test 537 pvcKey := makePvcKey(ns, cr.Name) 538 if _, found := claims[pvcKey]; !found { 539 return fmt.Errorf("internal: claims map is missing pvc %q", pvcKey) 540 } 541 542 err := WaitForPersistentVolumeClaimPhase(ctx, v1.ClaimBound, c, ns, cr.Name, framework.Poll, timeouts.ClaimBound) 543 if err != nil { 544 return fmt.Errorf("PVC %q did not become Bound: %w", cr.Name, err) 545 } 546 actualBinds++ 547 } 548 } 549 550 if testExpected && actualBinds != expectedBinds { 551 return fmt.Errorf("expect number of bound PVs (%v) to equal number of claims (%v)", actualBinds, expectedBinds) 552 } 553 return nil 554 } 555 556 // Return a pvckey struct. 557 func makePvcKey(ns, name string) types.NamespacedName { 558 return types.NamespacedName{Namespace: ns, Name: name} 559 } 560 561 // MakePersistentVolume returns a PV definition based on the nfs server IP. If the PVC is not nil 562 // then the PV is defined with a ClaimRef which includes the PVC's namespace. 563 // If the PVC is nil then the PV is not defined with a ClaimRef. If no reclaimPolicy 564 // is assigned, assumes "Retain". Specs are expected to match the test's PVC. 565 // Note: the passed-in claim does not have a name until it is created and thus the PV's 566 // 567 // ClaimRef cannot be completely filled-in in this func. Therefore, the ClaimRef's name 568 // is added later in CreatePVCPV. 569 func MakePersistentVolume(pvConfig PersistentVolumeConfig) *v1.PersistentVolume { 570 var claimRef *v1.ObjectReference 571 572 if len(pvConfig.AccessModes) == 0 { 573 pvConfig.AccessModes = append(pvConfig.AccessModes, v1.ReadWriteOnce) 574 } 575 576 if len(pvConfig.NamePrefix) == 0 { 577 pvConfig.NamePrefix = "pv-" 578 } 579 580 if pvConfig.ReclaimPolicy == "" { 581 pvConfig.ReclaimPolicy = v1.PersistentVolumeReclaimRetain 582 } 583 584 if len(pvConfig.Capacity) == 0 { 585 pvConfig.Capacity = "2Gi" 586 } 587 588 if pvConfig.Prebind != nil { 589 claimRef = &v1.ObjectReference{ 590 Kind: "PersistentVolumeClaim", 591 APIVersion: "v1", 592 Name: pvConfig.Prebind.Name, 593 Namespace: pvConfig.Prebind.Namespace, 594 UID: pvConfig.Prebind.UID, 595 } 596 } 597 598 return &v1.PersistentVolume{ 599 ObjectMeta: metav1.ObjectMeta{ 600 GenerateName: pvConfig.NamePrefix, 601 Labels: pvConfig.Labels, 602 Annotations: map[string]string{ 603 volumeGidAnnotationKey: "777", 604 }, 605 }, 606 Spec: v1.PersistentVolumeSpec{ 607 PersistentVolumeReclaimPolicy: pvConfig.ReclaimPolicy, 608 Capacity: v1.ResourceList{ 609 v1.ResourceStorage: resource.MustParse(pvConfig.Capacity), 610 }, 611 PersistentVolumeSource: pvConfig.PVSource, 612 AccessModes: pvConfig.AccessModes, 613 ClaimRef: claimRef, 614 StorageClassName: pvConfig.StorageClassName, 615 NodeAffinity: pvConfig.NodeAffinity, 616 VolumeMode: pvConfig.VolumeMode, 617 }, 618 } 619 } 620 621 // MakePersistentVolumeClaim returns a PVC API Object based on the PersistentVolumeClaimConfig. 622 func MakePersistentVolumeClaim(cfg PersistentVolumeClaimConfig, ns string) *v1.PersistentVolumeClaim { 623 624 if len(cfg.AccessModes) == 0 { 625 cfg.AccessModes = append(cfg.AccessModes, v1.ReadWriteOnce) 626 } 627 628 if len(cfg.ClaimSize) == 0 { 629 cfg.ClaimSize = "2Gi" 630 } 631 632 if len(cfg.NamePrefix) == 0 { 633 cfg.NamePrefix = "pvc-" 634 } 635 636 if cfg.VolumeMode != nil && *cfg.VolumeMode == "" { 637 framework.Logf("Warning: Making PVC: VolumeMode specified as invalid empty string, treating as nil") 638 cfg.VolumeMode = nil 639 } 640 641 return &v1.PersistentVolumeClaim{ 642 ObjectMeta: metav1.ObjectMeta{ 643 Name: cfg.Name, 644 GenerateName: cfg.NamePrefix, 645 Namespace: ns, 646 Annotations: cfg.Annotations, 647 }, 648 Spec: v1.PersistentVolumeClaimSpec{ 649 Selector: cfg.Selector, 650 AccessModes: cfg.AccessModes, 651 Resources: v1.VolumeResourceRequirements{ 652 Requests: v1.ResourceList{ 653 v1.ResourceStorage: resource.MustParse(cfg.ClaimSize), 654 }, 655 }, 656 StorageClassName: cfg.StorageClassName, 657 VolumeMode: cfg.VolumeMode, 658 }, 659 } 660 } 661 662 func createPDWithRetry(ctx context.Context, zone string) (string, error) { 663 var err error 664 var newDiskName string 665 for start := time.Now(); ; time.Sleep(pdRetryPollTime) { 666 if time.Since(start) >= pdRetryTimeout || 667 ctx.Err() != nil { 668 return "", fmt.Errorf("timed out while trying to create PD in zone %q, last error: %w", zone, err) 669 } 670 671 newDiskName, err = createPD(zone) 672 if err != nil { 673 framework.Logf("Couldn't create a new PD in zone %q, sleeping 5 seconds: %v", zone, err) 674 continue 675 } 676 framework.Logf("Successfully created a new PD in zone %q: %q.", zone, newDiskName) 677 return newDiskName, nil 678 } 679 } 680 681 func CreateShare() (string, string, string, error) { 682 return framework.TestContext.CloudConfig.Provider.CreateShare() 683 } 684 685 func DeleteShare(accountName, shareName string) error { 686 return framework.TestContext.CloudConfig.Provider.DeleteShare(accountName, shareName) 687 } 688 689 // CreatePDWithRetry creates PD with retry. 690 func CreatePDWithRetry(ctx context.Context) (string, error) { 691 return createPDWithRetry(ctx, "") 692 } 693 694 // CreatePDWithRetryAndZone creates PD on zone with retry. 695 func CreatePDWithRetryAndZone(ctx context.Context, zone string) (string, error) { 696 return createPDWithRetry(ctx, zone) 697 } 698 699 // DeletePDWithRetry deletes PD with retry. 700 func DeletePDWithRetry(ctx context.Context, diskName string) error { 701 var err error 702 for start := time.Now(); ; time.Sleep(pdRetryPollTime) { 703 if time.Since(start) >= pdRetryTimeout || 704 ctx.Err() != nil { 705 return fmt.Errorf("timed out while trying to delete PD %q, last error: %w", diskName, err) 706 } 707 err = deletePD(diskName) 708 if err != nil { 709 framework.Logf("Couldn't delete PD %q, sleeping %v: %v", diskName, pdRetryPollTime, err) 710 continue 711 } 712 framework.Logf("Successfully deleted PD %q.", diskName) 713 return nil 714 } 715 } 716 717 func createPD(zone string) (string, error) { 718 if zone == "" { 719 zone = framework.TestContext.CloudConfig.Zone 720 } 721 return framework.TestContext.CloudConfig.Provider.CreatePD(zone) 722 } 723 724 func deletePD(pdName string) error { 725 return framework.TestContext.CloudConfig.Provider.DeletePD(pdName) 726 } 727 728 // WaitForPVClaimBoundPhase waits until all pvcs phase set to bound 729 func WaitForPVClaimBoundPhase(ctx context.Context, client clientset.Interface, pvclaims []*v1.PersistentVolumeClaim, timeout time.Duration) ([]*v1.PersistentVolume, error) { 730 persistentvolumes := make([]*v1.PersistentVolume, len(pvclaims)) 731 732 for index, claim := range pvclaims { 733 err := WaitForPersistentVolumeClaimPhase(ctx, v1.ClaimBound, client, claim.Namespace, claim.Name, framework.Poll, timeout) 734 if err != nil { 735 return persistentvolumes, err 736 } 737 // Get new copy of the claim 738 claim, err = client.CoreV1().PersistentVolumeClaims(claim.Namespace).Get(ctx, claim.Name, metav1.GetOptions{}) 739 if err != nil { 740 return persistentvolumes, fmt.Errorf("PVC Get API error: %w", err) 741 } 742 // Get the bounded PV 743 persistentvolumes[index], err = client.CoreV1().PersistentVolumes().Get(ctx, claim.Spec.VolumeName, metav1.GetOptions{}) 744 if err != nil { 745 return persistentvolumes, fmt.Errorf("PV Get API error: %w", err) 746 } 747 } 748 return persistentvolumes, nil 749 } 750 751 // WaitForPersistentVolumePhase waits for a PersistentVolume to be in a specific phase or until timeout occurs, whichever comes first. 752 func WaitForPersistentVolumePhase(ctx context.Context, phase v1.PersistentVolumePhase, c clientset.Interface, pvName string, poll, timeout time.Duration) error { 753 framework.Logf("Waiting up to %v for PersistentVolume %s to have phase %s", timeout, pvName, phase) 754 for start := time.Now(); time.Since(start) < timeout; time.Sleep(poll) { 755 pv, err := c.CoreV1().PersistentVolumes().Get(ctx, pvName, metav1.GetOptions{}) 756 if err != nil { 757 framework.Logf("Get persistent volume %s in failed, ignoring for %v: %v", pvName, poll, err) 758 continue 759 } 760 if pv.Status.Phase == phase { 761 framework.Logf("PersistentVolume %s found and phase=%s (%v)", pvName, phase, time.Since(start)) 762 return nil 763 } 764 framework.Logf("PersistentVolume %s found but phase is %s instead of %s.", pvName, pv.Status.Phase, phase) 765 } 766 return fmt.Errorf("PersistentVolume %s not in phase %s within %v", pvName, phase, timeout) 767 } 768 769 // WaitForPersistentVolumeClaimPhase waits for a PersistentVolumeClaim to be in a specific phase or until timeout occurs, whichever comes first. 770 func WaitForPersistentVolumeClaimPhase(ctx context.Context, phase v1.PersistentVolumeClaimPhase, c clientset.Interface, ns string, pvcName string, poll, timeout time.Duration) error { 771 return WaitForPersistentVolumeClaimsPhase(ctx, phase, c, ns, []string{pvcName}, poll, timeout, true) 772 } 773 774 // WaitForPersistentVolumeClaimsPhase waits for any (if matchAny is true) or all (if matchAny is false) PersistentVolumeClaims 775 // to be in a specific phase or until timeout occurs, whichever comes first. 776 func WaitForPersistentVolumeClaimsPhase(ctx context.Context, phase v1.PersistentVolumeClaimPhase, c clientset.Interface, ns string, pvcNames []string, poll, timeout time.Duration, matchAny bool) error { 777 if len(pvcNames) == 0 { 778 return fmt.Errorf("Incorrect parameter: Need at least one PVC to track. Found 0") 779 } 780 framework.Logf("Waiting up to timeout=%v for PersistentVolumeClaims %v to have phase %s", timeout, pvcNames, phase) 781 for start := time.Now(); time.Since(start) < timeout; time.Sleep(poll) { 782 phaseFoundInAllClaims := true 783 for _, pvcName := range pvcNames { 784 pvc, err := c.CoreV1().PersistentVolumeClaims(ns).Get(ctx, pvcName, metav1.GetOptions{}) 785 if err != nil { 786 framework.Logf("Failed to get claim %q, retrying in %v. Error: %v", pvcName, poll, err) 787 phaseFoundInAllClaims = false 788 break 789 } 790 if pvc.Status.Phase == phase { 791 framework.Logf("PersistentVolumeClaim %s found and phase=%s (%v)", pvcName, phase, time.Since(start)) 792 if matchAny { 793 return nil 794 } 795 } else { 796 framework.Logf("PersistentVolumeClaim %s found but phase is %s instead of %s.", pvcName, pvc.Status.Phase, phase) 797 phaseFoundInAllClaims = false 798 } 799 } 800 if phaseFoundInAllClaims { 801 return nil 802 } 803 } 804 return fmt.Errorf("PersistentVolumeClaims %v not all in phase %s within %v", pvcNames, phase, timeout) 805 } 806 807 // CreatePVSource creates a PV source. 808 func CreatePVSource(ctx context.Context, zone string) (*v1.PersistentVolumeSource, error) { 809 diskName, err := CreatePDWithRetryAndZone(ctx, zone) 810 if err != nil { 811 return nil, err 812 } 813 return framework.TestContext.CloudConfig.Provider.CreatePVSource(ctx, zone, diskName) 814 } 815 816 // DeletePVSource deletes a PV source. 817 func DeletePVSource(ctx context.Context, pvSource *v1.PersistentVolumeSource) error { 818 return framework.TestContext.CloudConfig.Provider.DeletePVSource(ctx, pvSource) 819 } 820 821 // GetDefaultStorageClassName returns default storageClass or return error 822 func GetDefaultStorageClassName(ctx context.Context, c clientset.Interface) (string, error) { 823 list, err := c.StorageV1().StorageClasses().List(ctx, metav1.ListOptions{}) 824 if err != nil { 825 return "", fmt.Errorf("Error listing storage classes: %w", err) 826 } 827 var scName string 828 for _, sc := range list.Items { 829 if util.IsDefaultAnnotation(sc.ObjectMeta) { 830 if len(scName) != 0 { 831 return "", fmt.Errorf("Multiple default storage classes found: %q and %q", scName, sc.Name) 832 } 833 scName = sc.Name 834 } 835 } 836 if len(scName) == 0 { 837 return "", fmt.Errorf("No default storage class found") 838 } 839 framework.Logf("Default storage class: %q", scName) 840 return scName, nil 841 } 842 843 // SkipIfNoDefaultStorageClass skips tests if no default SC can be found. 844 func SkipIfNoDefaultStorageClass(ctx context.Context, c clientset.Interface) { 845 _, err := GetDefaultStorageClassName(ctx, c) 846 if err != nil { 847 e2eskipper.Skipf("error finding default storageClass : %v", err) 848 } 849 } 850 851 // WaitForPersistentVolumeDeleted waits for a PersistentVolume to get deleted or until timeout occurs, whichever comes first. 852 func WaitForPersistentVolumeDeleted(ctx context.Context, c clientset.Interface, pvName string, poll, timeout time.Duration) error { 853 framework.Logf("Waiting up to %v for PersistentVolume %s to get deleted", timeout, pvName) 854 for start := time.Now(); time.Since(start) < timeout; time.Sleep(poll) { 855 pv, err := c.CoreV1().PersistentVolumes().Get(ctx, pvName, metav1.GetOptions{}) 856 if err == nil { 857 framework.Logf("PersistentVolume %s found and phase=%s (%v)", pvName, pv.Status.Phase, time.Since(start)) 858 continue 859 } 860 if apierrors.IsNotFound(err) { 861 framework.Logf("PersistentVolume %s was removed", pvName) 862 return nil 863 } 864 framework.Logf("Get persistent volume %s in failed, ignoring for %v: %v", pvName, poll, err) 865 } 866 return fmt.Errorf("PersistentVolume %s still exists within %v", pvName, timeout) 867 } 868 869 // WaitForPVCFinalizer waits for a finalizer to be added to a PVC in a given namespace. 870 func WaitForPVCFinalizer(ctx context.Context, cs clientset.Interface, name, namespace, finalizer string, poll, timeout time.Duration) error { 871 var ( 872 err error 873 pvc *v1.PersistentVolumeClaim 874 ) 875 framework.Logf("Waiting up to %v for PersistentVolumeClaim %s/%s to contain finalizer %s", timeout, namespace, name, finalizer) 876 if successful := utils.WaitUntil(poll, timeout, func() bool { 877 pvc, err = cs.CoreV1().PersistentVolumeClaims(namespace).Get(ctx, name, metav1.GetOptions{}) 878 if err != nil { 879 framework.Logf("Failed to get PersistentVolumeClaim %s/%s with err: %v. Will retry in %v", name, namespace, err, timeout) 880 return false 881 } 882 for _, f := range pvc.Finalizers { 883 if f == finalizer { 884 return true 885 } 886 } 887 return false 888 }); successful { 889 return nil 890 } 891 if err == nil { 892 err = fmt.Errorf("finalizer %s not added to pvc %s/%s", finalizer, namespace, name) 893 } 894 return err 895 } 896 897 // GetDefaultFSType returns the default fsType 898 func GetDefaultFSType() string { 899 if framework.NodeOSDistroIs("windows") { 900 return "ntfs" 901 } 902 return "ext4" 903 }