k8s.io/kubernetes@v1.31.0-alpha.0.0.20240520171757-56147500dadc/pkg/controller/volume/persistentvolume/delete_test.go (about)

     1  /*
     2  Copyright 2016 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package persistentvolume
    18  
    19  import (
    20  	"errors"
    21  	"testing"
    22  
    23  	v1 "k8s.io/api/core/v1"
    24  	storage "k8s.io/api/storage/v1"
    25  	utilfeature "k8s.io/apiserver/pkg/util/feature"
    26  	featuregatetesting "k8s.io/component-base/featuregate/testing"
    27  	"k8s.io/component-helpers/storage/volume"
    28  	"k8s.io/klog/v2/ktesting"
    29  	pvtesting "k8s.io/kubernetes/pkg/controller/volume/persistentvolume/testing"
    30  	"k8s.io/kubernetes/pkg/features"
    31  )
    32  
    33  // Test single call to syncVolume, expecting recycling to happen.
    34  // 1. Fill in the controller with initial data
    35  // 2. Call the syncVolume *once*.
    36  // 3. Compare resulting volumes with expected volumes.
    37  func TestDeleteSync(t *testing.T) {
    38  	const gceDriver = "pd.csi.storage.gke.io"
    39  	// Default enable the HonorPVReclaimPolicy feature gate.
    40  	featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.HonorPVReclaimPolicy, true)
    41  	_, ctx := ktesting.NewTestContext(t)
    42  	tests := []controllerTest{
    43  		{
    44  			// delete volume bound by controller
    45  			name:            "8-1 - successful delete",
    46  			initialVolumes:  newVolumeArray("volume8-1", "1Gi", "uid8-1", "claim8-1", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty, volume.AnnBoundByController),
    47  			expectedVolumes: novolumes,
    48  			initialClaims:   noclaims,
    49  			expectedClaims:  noclaims,
    50  			expectedEvents:  noevents,
    51  			errors:          noerrors,
    52  			// Inject deleter into the controller and call syncVolume. The
    53  			// deleter simulates one delete() call that succeeds.
    54  			test: wrapTestWithReclaimCalls(operationDelete, []error{nil}, testSyncVolume),
    55  		},
    56  		{
    57  			// delete volume bound by user
    58  			name:            "8-2 - successful delete with prebound volume",
    59  			initialVolumes:  newVolumeArray("volume8-2", "1Gi", "uid8-2", "claim8-2", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty),
    60  			expectedVolumes: novolumes,
    61  			initialClaims:   noclaims,
    62  			expectedClaims:  noclaims,
    63  			expectedEvents:  noevents,
    64  			errors:          noerrors,
    65  			// Inject deleter into the controller and call syncVolume. The
    66  			// deleter simulates one delete() call that succeeds.
    67  			test: wrapTestWithReclaimCalls(operationDelete, []error{nil}, testSyncVolume),
    68  		},
    69  		{
    70  			// delete failure - plugin not found
    71  			name:            "8-3 - plugin not found",
    72  			initialVolumes:  newVolumeArray("volume8-3", "1Gi", "uid8-3", "claim8-3", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty),
    73  			expectedVolumes: withMessage("error getting deleter volume plugin for volume \"volume8-3\": no volume plugin matched", newVolumeArray("volume8-3", "1Gi", "uid8-3", "claim8-3", v1.VolumeFailed, v1.PersistentVolumeReclaimDelete, classEmpty)),
    74  			initialClaims:   noclaims,
    75  			expectedClaims:  noclaims,
    76  			expectedEvents:  []string{"Warning VolumeFailedDelete"},
    77  			errors:          noerrors,
    78  			test:            testSyncVolume,
    79  		},
    80  		{
    81  			// delete failure - newDeleter returns error
    82  			name:            "8-4 - newDeleter returns error",
    83  			initialVolumes:  newVolumeArray("volume8-4", "1Gi", "uid8-4", "claim8-4", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty),
    84  			expectedVolumes: withMessage("failed to create deleter for volume \"volume8-4\": Mock plugin error: no deleteCalls configured", newVolumeArray("volume8-4", "1Gi", "uid8-4", "claim8-4", v1.VolumeFailed, v1.PersistentVolumeReclaimDelete, classEmpty)),
    85  			initialClaims:   noclaims,
    86  			expectedClaims:  noclaims,
    87  			expectedEvents:  []string{"Warning VolumeFailedDelete"},
    88  			errors:          noerrors,
    89  			test:            wrapTestWithReclaimCalls(operationDelete, []error{}, testSyncVolume),
    90  		},
    91  		{
    92  			// delete failure - delete() returns error
    93  			name:            "8-5 - delete returns error",
    94  			initialVolumes:  newVolumeArray("volume8-5", "1Gi", "uid8-5", "claim8-5", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty),
    95  			expectedVolumes: withMessage("Mock delete error", newVolumeArray("volume8-5", "1Gi", "uid8-5", "claim8-5", v1.VolumeFailed, v1.PersistentVolumeReclaimDelete, classEmpty)),
    96  			initialClaims:   noclaims,
    97  			expectedClaims:  noclaims,
    98  			expectedEvents:  []string{"Warning VolumeFailedDelete"},
    99  			errors:          noerrors,
   100  			test:            wrapTestWithReclaimCalls(operationDelete, []error{errors.New("Mock delete error")}, testSyncVolume),
   101  		},
   102  		{
   103  			// delete success(?) - volume is deleted before doDelete() starts
   104  			name:            "8-6 - volume is deleted before deleting",
   105  			initialVolumes:  newVolumeArray("volume8-6", "1Gi", "uid8-6", "claim8-6", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty),
   106  			expectedVolumes: novolumes,
   107  			initialClaims:   noclaims,
   108  			expectedClaims:  noclaims,
   109  			expectedEvents:  noevents,
   110  			errors:          noerrors,
   111  			test: wrapTestWithInjectedOperation(ctx, wrapTestWithReclaimCalls(operationDelete, []error{}, testSyncVolume), func(ctrl *PersistentVolumeController, reactor *pvtesting.VolumeReactor) {
   112  				// Delete the volume before delete operation starts
   113  				reactor.DeleteVolume("volume8-6")
   114  			}),
   115  		},
   116  		{
   117  			// delete success(?) - volume is bound just at the time doDelete()
   118  			// starts. This simulates "volume no longer needs recycling,
   119  			// skipping".
   120  			name:            "8-7 - volume is bound before deleting",
   121  			initialVolumes:  newVolumeArray("volume8-7", "1Gi", "uid8-7", "claim8-7", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty, volume.AnnBoundByController),
   122  			expectedVolumes: newVolumeArray("volume8-7", "1Gi", "uid8-7", "claim8-7", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty, volume.AnnBoundByController),
   123  			initialClaims:   noclaims,
   124  			expectedClaims:  newClaimArray("claim8-7", "uid8-7", "10Gi", "volume8-7", v1.ClaimBound, nil),
   125  			expectedEvents:  noevents,
   126  			errors:          noerrors,
   127  			test: wrapTestWithInjectedOperation(ctx, wrapTestWithReclaimCalls(operationDelete, []error{}, testSyncVolume), func(ctrl *PersistentVolumeController, reactor *pvtesting.VolumeReactor) {
   128  				// Bind the volume to resurrected claim (this should never
   129  				// happen)
   130  				claim := newClaim("claim8-7", "uid8-7", "10Gi", "volume8-7", v1.ClaimBound, nil)
   131  				reactor.AddClaimBoundToVolume(claim)
   132  				ctrl.claims.Add(claim)
   133  			}),
   134  		},
   135  		{
   136  			// delete success - volume bound by user is deleted, while a new
   137  			// claim is created with another UID.
   138  			name:            "8-9 - prebound volume is deleted while the claim exists",
   139  			initialVolumes:  newVolumeArray("volume8-9", "1Gi", "uid8-9", "claim8-9", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty),
   140  			expectedVolumes: novolumes,
   141  			initialClaims:   newClaimArray("claim8-9", "uid8-9-x", "10Gi", "", v1.ClaimPending, nil),
   142  			expectedClaims:  newClaimArray("claim8-9", "uid8-9-x", "10Gi", "", v1.ClaimPending, nil),
   143  			expectedEvents:  noevents,
   144  			errors:          noerrors,
   145  			// Inject deleter into the controller and call syncVolume. The
   146  			// deleter simulates one delete() call that succeeds.
   147  			test: wrapTestWithReclaimCalls(operationDelete, []error{nil}, testSyncVolume),
   148  		},
   149  		{
   150  			// PV requires external deleter
   151  			name:            "8-10-1 - external deleter when volume is dynamic provisioning",
   152  			initialVolumes:  []*v1.PersistentVolume{newExternalProvisionedVolume("volume8-10-1", "1Gi", "uid10-1-1", "claim10-1-1", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty, gceDriver, nil, volume.AnnBoundByController)},
   153  			expectedVolumes: []*v1.PersistentVolume{newExternalProvisionedVolume("volume8-10-1", "1Gi", "uid10-1-1", "claim10-1-1", v1.VolumeReleased, v1.PersistentVolumeReclaimDelete, classEmpty, gceDriver, nil, volume.AnnBoundByController)},
   154  			initialClaims:   noclaims,
   155  			expectedClaims:  noclaims,
   156  			expectedEvents:  noevents,
   157  			errors:          noerrors,
   158  			test: func(ctrl *PersistentVolumeController, reactor *pvtesting.VolumeReactor, test controllerTest) error {
   159  				// Inject external deleter annotation
   160  				test.initialVolumes[0].Annotations[volume.AnnDynamicallyProvisioned] = "external.io/test"
   161  				test.expectedVolumes[0].Annotations[volume.AnnDynamicallyProvisioned] = "external.io/test"
   162  				return testSyncVolume(ctrl, reactor, test)
   163  			},
   164  		},
   165  		{
   166  			// PV requires external deleter
   167  			name:            "8-10-2 - external deleter when volume is static provisioning",
   168  			initialVolumes:  []*v1.PersistentVolume{newExternalProvisionedVolume("volume8-10-2", "1Gi", "uid10-1-2", "claim10-1-2", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty, gceDriver, nil, volume.AnnBoundByController)},
   169  			expectedVolumes: []*v1.PersistentVolume{newExternalProvisionedVolume("volume8-10-2", "1Gi", "uid10-1-2", "claim10-1-2", v1.VolumeReleased, v1.PersistentVolumeReclaimDelete, classEmpty, gceDriver, nil, volume.AnnBoundByController)},
   170  			initialClaims:   noclaims,
   171  			expectedClaims:  noclaims,
   172  			expectedEvents:  noevents,
   173  			errors:          noerrors,
   174  			test:            testSyncVolume,
   175  		},
   176  		{
   177  			// PV requires external deleter
   178  			name:            "8-10-3 - external deleter when volume is migrated",
   179  			initialVolumes:  []*v1.PersistentVolume{volumeWithAnnotation(volume.AnnMigratedTo, "pd.csi.storage.gke.io", volumeWithAnnotation(volume.AnnDynamicallyProvisioned, "kubernetes.io/gce-pd", newVolume("volume8-10-3", "1Gi", "uid10-1-3", "claim10-1-3", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty, volume.AnnDynamicallyProvisioned)))},
   180  			expectedVolumes: []*v1.PersistentVolume{volumeWithAnnotation(volume.AnnMigratedTo, "pd.csi.storage.gke.io", volumeWithAnnotation(volume.AnnDynamicallyProvisioned, "kubernetes.io/gce-pd", newVolume("volume8-10-3", "1Gi", "uid10-1-3", "claim10-1-3", v1.VolumeReleased, v1.PersistentVolumeReclaimDelete, classEmpty, volume.AnnDynamicallyProvisioned)))},
   181  			initialClaims:   noclaims,
   182  			expectedClaims:  noclaims,
   183  			expectedEvents:  noevents,
   184  			errors:          noerrors,
   185  			test:            testSyncVolume,
   186  		},
   187  		{
   188  			// delete success - two PVs are provisioned for a single claim.
   189  			// One of the PVs is deleted.
   190  			name: "8-11 - two PVs provisioned for a single claim",
   191  			initialVolumes: []*v1.PersistentVolume{
   192  				newVolume("volume8-11-1", "1Gi", "uid8-11", "claim8-11", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty, volume.AnnDynamicallyProvisioned),
   193  				newVolume("volume8-11-2", "1Gi", "uid8-11", "claim8-11", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty, volume.AnnDynamicallyProvisioned),
   194  			},
   195  			expectedVolumes: []*v1.PersistentVolume{
   196  				newVolume("volume8-11-2", "1Gi", "uid8-11", "claim8-11", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty, volume.AnnDynamicallyProvisioned),
   197  			},
   198  			// the claim is bound to volume8-11-2 -> volume8-11-1 has lost the race and will be deleted
   199  			initialClaims:  newClaimArray("claim8-11", "uid8-11", "10Gi", "volume8-11-2", v1.ClaimBound, nil),
   200  			expectedClaims: newClaimArray("claim8-11", "uid8-11", "10Gi", "volume8-11-2", v1.ClaimBound, nil),
   201  			expectedEvents: noevents,
   202  			errors:         noerrors,
   203  			// Inject deleter into the controller and call syncVolume. The
   204  			// deleter simulates one delete() call that succeeds.
   205  			test: wrapTestWithReclaimCalls(operationDelete, []error{nil}, testSyncVolume),
   206  		},
   207  		{
   208  			// delete success - two PVs are externally provisioned for a single
   209  			// claim. One of the PVs is marked as Released to be deleted by the
   210  			// external provisioner.
   211  			name: "8-12 - two PVs externally provisioned for a single claim",
   212  			initialVolumes: []*v1.PersistentVolume{
   213  				newExternalProvisionedVolume("volume8-12-1", "1Gi", "uid8-12", "claim8-12", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty, gceDriver, nil, volume.AnnDynamicallyProvisioned),
   214  				newExternalProvisionedVolume("volume8-12-2", "1Gi", "uid8-12", "claim8-12", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty, gceDriver, nil, volume.AnnDynamicallyProvisioned),
   215  			},
   216  			expectedVolumes: []*v1.PersistentVolume{
   217  				newExternalProvisionedVolume("volume8-12-1", "1Gi", "uid8-12", "claim8-12", v1.VolumeReleased, v1.PersistentVolumeReclaimDelete, classEmpty, gceDriver, nil, volume.AnnDynamicallyProvisioned),
   218  				newExternalProvisionedVolume("volume8-12-2", "1Gi", "uid8-12", "claim8-12", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty, gceDriver, nil, volume.AnnDynamicallyProvisioned),
   219  			},
   220  			// the claim is bound to volume8-12-2 -> volume8-12-1 has lost the race and will be "Released"
   221  			initialClaims:  newClaimArray("claim8-12", "uid8-12", "10Gi", "volume8-12-2", v1.ClaimBound, nil),
   222  			expectedClaims: newClaimArray("claim8-12", "uid8-12", "10Gi", "volume8-12-2", v1.ClaimBound, nil),
   223  			expectedEvents: noevents,
   224  			errors:         noerrors,
   225  			test: func(ctrl *PersistentVolumeController, reactor *pvtesting.VolumeReactor, test controllerTest) error {
   226  				// Inject external deleter annotation
   227  				test.initialVolumes[0].Annotations[volume.AnnDynamicallyProvisioned] = "external.io/test"
   228  				test.expectedVolumes[0].Annotations[volume.AnnDynamicallyProvisioned] = "external.io/test"
   229  				return testSyncVolume(ctrl, reactor, test)
   230  			},
   231  		},
   232  		{
   233  			// delete success - volume has deletion timestamp before doDelete() starts
   234  			name:            "8-13 - volume has deletion timestamp and processed",
   235  			initialVolumes:  volumesWithFinalizers(withVolumeDeletionTimestamp(newVolumeArray("volume8-13", "1Gi", "uid8-13", "claim8-13", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty, volume.AnnBoundByController)), []string{volume.PVDeletionInTreeProtectionFinalizer}),
   236  			expectedVolumes: novolumes,
   237  			initialClaims:   noclaims,
   238  			expectedClaims:  noclaims,
   239  			expectedEvents:  noevents,
   240  			errors:          noerrors,
   241  			test:            wrapTestWithReclaimCalls(operationDelete, []error{nil}, testSyncVolume),
   242  		},
   243  	}
   244  	runSyncTests(t, ctx, tests, []*storage.StorageClass{}, []*v1.Pod{})
   245  }
   246  
   247  // Test multiple calls to syncClaim/syncVolume and periodic sync of all
   248  // volume/claims. The test follows this pattern:
   249  //  0. Load the controller with initial data.
   250  //  1. Call controllerTest.testCall() once as in TestSync()
   251  //  2. For all volumes/claims changed by previous syncVolume/syncClaim calls,
   252  //     call appropriate syncVolume/syncClaim (simulating "volume/claim changed"
   253  //     events). Go to 2. if these calls change anything.
   254  //  3. When all changes are processed and no new changes were made, call
   255  //     syncVolume/syncClaim on all volumes/claims (simulating "periodic sync").
   256  //  4. If some changes were done by step 3., go to 2. (simulation of
   257  //     "volume/claim updated" events, eventually performing step 3. again)
   258  //  5. When 3. does not do any changes, finish the tests and compare final set
   259  //     of volumes/claims with expected claims/volumes and report differences.
   260  //
   261  // Some limit of calls in enforced to prevent endless loops.
   262  func TestDeleteMultiSync(t *testing.T) {
   263  	tests := []controllerTest{
   264  		{
   265  			// delete failure - delete returns error. The controller should
   266  			// try again.
   267  			name:            "9-1 - delete returns error",
   268  			initialVolumes:  volumesWithFinalizers(newVolumeArray("volume9-1", "1Gi", "uid9-1", "claim9-1", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), []string{volume.PVDeletionInTreeProtectionFinalizer}),
   269  			expectedVolumes: novolumes,
   270  			initialClaims:   noclaims,
   271  			expectedClaims:  noclaims,
   272  			expectedEvents:  []string{"Warning VolumeFailedDelete"},
   273  			errors:          noerrors,
   274  			test:            wrapTestWithReclaimCalls(operationDelete, []error{errors.New("Mock delete error"), nil}, testSyncVolume),
   275  		},
   276  	}
   277  	_, ctx := ktesting.NewTestContext(t)
   278  	runMultisyncTests(t, ctx, tests, []*storage.StorageClass{}, "")
   279  }