k8s.io/kubernetes@v1.31.0-alpha.0.0.20240520171757-56147500dadc/pkg/controller/volume/persistentvolume/delete_test.go (about) 1 /* 2 Copyright 2016 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package persistentvolume 18 19 import ( 20 "errors" 21 "testing" 22 23 v1 "k8s.io/api/core/v1" 24 storage "k8s.io/api/storage/v1" 25 utilfeature "k8s.io/apiserver/pkg/util/feature" 26 featuregatetesting "k8s.io/component-base/featuregate/testing" 27 "k8s.io/component-helpers/storage/volume" 28 "k8s.io/klog/v2/ktesting" 29 pvtesting "k8s.io/kubernetes/pkg/controller/volume/persistentvolume/testing" 30 "k8s.io/kubernetes/pkg/features" 31 ) 32 33 // Test single call to syncVolume, expecting recycling to happen. 34 // 1. Fill in the controller with initial data 35 // 2. Call the syncVolume *once*. 36 // 3. Compare resulting volumes with expected volumes. 37 func TestDeleteSync(t *testing.T) { 38 const gceDriver = "pd.csi.storage.gke.io" 39 // Default enable the HonorPVReclaimPolicy feature gate. 40 featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.HonorPVReclaimPolicy, true) 41 _, ctx := ktesting.NewTestContext(t) 42 tests := []controllerTest{ 43 { 44 // delete volume bound by controller 45 name: "8-1 - successful delete", 46 initialVolumes: newVolumeArray("volume8-1", "1Gi", "uid8-1", "claim8-1", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty, volume.AnnBoundByController), 47 expectedVolumes: novolumes, 48 initialClaims: noclaims, 49 expectedClaims: noclaims, 50 expectedEvents: noevents, 51 errors: noerrors, 52 // Inject deleter into the controller and call syncVolume. The 53 // deleter simulates one delete() call that succeeds. 54 test: wrapTestWithReclaimCalls(operationDelete, []error{nil}, testSyncVolume), 55 }, 56 { 57 // delete volume bound by user 58 name: "8-2 - successful delete with prebound volume", 59 initialVolumes: newVolumeArray("volume8-2", "1Gi", "uid8-2", "claim8-2", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), 60 expectedVolumes: novolumes, 61 initialClaims: noclaims, 62 expectedClaims: noclaims, 63 expectedEvents: noevents, 64 errors: noerrors, 65 // Inject deleter into the controller and call syncVolume. The 66 // deleter simulates one delete() call that succeeds. 67 test: wrapTestWithReclaimCalls(operationDelete, []error{nil}, testSyncVolume), 68 }, 69 { 70 // delete failure - plugin not found 71 name: "8-3 - plugin not found", 72 initialVolumes: newVolumeArray("volume8-3", "1Gi", "uid8-3", "claim8-3", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), 73 expectedVolumes: withMessage("error getting deleter volume plugin for volume \"volume8-3\": no volume plugin matched", newVolumeArray("volume8-3", "1Gi", "uid8-3", "claim8-3", v1.VolumeFailed, v1.PersistentVolumeReclaimDelete, classEmpty)), 74 initialClaims: noclaims, 75 expectedClaims: noclaims, 76 expectedEvents: []string{"Warning VolumeFailedDelete"}, 77 errors: noerrors, 78 test: testSyncVolume, 79 }, 80 { 81 // delete failure - newDeleter returns error 82 name: "8-4 - newDeleter returns error", 83 initialVolumes: newVolumeArray("volume8-4", "1Gi", "uid8-4", "claim8-4", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), 84 expectedVolumes: withMessage("failed to create deleter for volume \"volume8-4\": Mock plugin error: no deleteCalls configured", newVolumeArray("volume8-4", "1Gi", "uid8-4", "claim8-4", v1.VolumeFailed, v1.PersistentVolumeReclaimDelete, classEmpty)), 85 initialClaims: noclaims, 86 expectedClaims: noclaims, 87 expectedEvents: []string{"Warning VolumeFailedDelete"}, 88 errors: noerrors, 89 test: wrapTestWithReclaimCalls(operationDelete, []error{}, testSyncVolume), 90 }, 91 { 92 // delete failure - delete() returns error 93 name: "8-5 - delete returns error", 94 initialVolumes: newVolumeArray("volume8-5", "1Gi", "uid8-5", "claim8-5", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), 95 expectedVolumes: withMessage("Mock delete error", newVolumeArray("volume8-5", "1Gi", "uid8-5", "claim8-5", v1.VolumeFailed, v1.PersistentVolumeReclaimDelete, classEmpty)), 96 initialClaims: noclaims, 97 expectedClaims: noclaims, 98 expectedEvents: []string{"Warning VolumeFailedDelete"}, 99 errors: noerrors, 100 test: wrapTestWithReclaimCalls(operationDelete, []error{errors.New("Mock delete error")}, testSyncVolume), 101 }, 102 { 103 // delete success(?) - volume is deleted before doDelete() starts 104 name: "8-6 - volume is deleted before deleting", 105 initialVolumes: newVolumeArray("volume8-6", "1Gi", "uid8-6", "claim8-6", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), 106 expectedVolumes: novolumes, 107 initialClaims: noclaims, 108 expectedClaims: noclaims, 109 expectedEvents: noevents, 110 errors: noerrors, 111 test: wrapTestWithInjectedOperation(ctx, wrapTestWithReclaimCalls(operationDelete, []error{}, testSyncVolume), func(ctrl *PersistentVolumeController, reactor *pvtesting.VolumeReactor) { 112 // Delete the volume before delete operation starts 113 reactor.DeleteVolume("volume8-6") 114 }), 115 }, 116 { 117 // delete success(?) - volume is bound just at the time doDelete() 118 // starts. This simulates "volume no longer needs recycling, 119 // skipping". 120 name: "8-7 - volume is bound before deleting", 121 initialVolumes: newVolumeArray("volume8-7", "1Gi", "uid8-7", "claim8-7", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty, volume.AnnBoundByController), 122 expectedVolumes: newVolumeArray("volume8-7", "1Gi", "uid8-7", "claim8-7", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty, volume.AnnBoundByController), 123 initialClaims: noclaims, 124 expectedClaims: newClaimArray("claim8-7", "uid8-7", "10Gi", "volume8-7", v1.ClaimBound, nil), 125 expectedEvents: noevents, 126 errors: noerrors, 127 test: wrapTestWithInjectedOperation(ctx, wrapTestWithReclaimCalls(operationDelete, []error{}, testSyncVolume), func(ctrl *PersistentVolumeController, reactor *pvtesting.VolumeReactor) { 128 // Bind the volume to resurrected claim (this should never 129 // happen) 130 claim := newClaim("claim8-7", "uid8-7", "10Gi", "volume8-7", v1.ClaimBound, nil) 131 reactor.AddClaimBoundToVolume(claim) 132 ctrl.claims.Add(claim) 133 }), 134 }, 135 { 136 // delete success - volume bound by user is deleted, while a new 137 // claim is created with another UID. 138 name: "8-9 - prebound volume is deleted while the claim exists", 139 initialVolumes: newVolumeArray("volume8-9", "1Gi", "uid8-9", "claim8-9", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), 140 expectedVolumes: novolumes, 141 initialClaims: newClaimArray("claim8-9", "uid8-9-x", "10Gi", "", v1.ClaimPending, nil), 142 expectedClaims: newClaimArray("claim8-9", "uid8-9-x", "10Gi", "", v1.ClaimPending, nil), 143 expectedEvents: noevents, 144 errors: noerrors, 145 // Inject deleter into the controller and call syncVolume. The 146 // deleter simulates one delete() call that succeeds. 147 test: wrapTestWithReclaimCalls(operationDelete, []error{nil}, testSyncVolume), 148 }, 149 { 150 // PV requires external deleter 151 name: "8-10-1 - external deleter when volume is dynamic provisioning", 152 initialVolumes: []*v1.PersistentVolume{newExternalProvisionedVolume("volume8-10-1", "1Gi", "uid10-1-1", "claim10-1-1", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty, gceDriver, nil, volume.AnnBoundByController)}, 153 expectedVolumes: []*v1.PersistentVolume{newExternalProvisionedVolume("volume8-10-1", "1Gi", "uid10-1-1", "claim10-1-1", v1.VolumeReleased, v1.PersistentVolumeReclaimDelete, classEmpty, gceDriver, nil, volume.AnnBoundByController)}, 154 initialClaims: noclaims, 155 expectedClaims: noclaims, 156 expectedEvents: noevents, 157 errors: noerrors, 158 test: func(ctrl *PersistentVolumeController, reactor *pvtesting.VolumeReactor, test controllerTest) error { 159 // Inject external deleter annotation 160 test.initialVolumes[0].Annotations[volume.AnnDynamicallyProvisioned] = "external.io/test" 161 test.expectedVolumes[0].Annotations[volume.AnnDynamicallyProvisioned] = "external.io/test" 162 return testSyncVolume(ctrl, reactor, test) 163 }, 164 }, 165 { 166 // PV requires external deleter 167 name: "8-10-2 - external deleter when volume is static provisioning", 168 initialVolumes: []*v1.PersistentVolume{newExternalProvisionedVolume("volume8-10-2", "1Gi", "uid10-1-2", "claim10-1-2", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty, gceDriver, nil, volume.AnnBoundByController)}, 169 expectedVolumes: []*v1.PersistentVolume{newExternalProvisionedVolume("volume8-10-2", "1Gi", "uid10-1-2", "claim10-1-2", v1.VolumeReleased, v1.PersistentVolumeReclaimDelete, classEmpty, gceDriver, nil, volume.AnnBoundByController)}, 170 initialClaims: noclaims, 171 expectedClaims: noclaims, 172 expectedEvents: noevents, 173 errors: noerrors, 174 test: testSyncVolume, 175 }, 176 { 177 // PV requires external deleter 178 name: "8-10-3 - external deleter when volume is migrated", 179 initialVolumes: []*v1.PersistentVolume{volumeWithAnnotation(volume.AnnMigratedTo, "pd.csi.storage.gke.io", volumeWithAnnotation(volume.AnnDynamicallyProvisioned, "kubernetes.io/gce-pd", newVolume("volume8-10-3", "1Gi", "uid10-1-3", "claim10-1-3", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty, volume.AnnDynamicallyProvisioned)))}, 180 expectedVolumes: []*v1.PersistentVolume{volumeWithAnnotation(volume.AnnMigratedTo, "pd.csi.storage.gke.io", volumeWithAnnotation(volume.AnnDynamicallyProvisioned, "kubernetes.io/gce-pd", newVolume("volume8-10-3", "1Gi", "uid10-1-3", "claim10-1-3", v1.VolumeReleased, v1.PersistentVolumeReclaimDelete, classEmpty, volume.AnnDynamicallyProvisioned)))}, 181 initialClaims: noclaims, 182 expectedClaims: noclaims, 183 expectedEvents: noevents, 184 errors: noerrors, 185 test: testSyncVolume, 186 }, 187 { 188 // delete success - two PVs are provisioned for a single claim. 189 // One of the PVs is deleted. 190 name: "8-11 - two PVs provisioned for a single claim", 191 initialVolumes: []*v1.PersistentVolume{ 192 newVolume("volume8-11-1", "1Gi", "uid8-11", "claim8-11", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty, volume.AnnDynamicallyProvisioned), 193 newVolume("volume8-11-2", "1Gi", "uid8-11", "claim8-11", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty, volume.AnnDynamicallyProvisioned), 194 }, 195 expectedVolumes: []*v1.PersistentVolume{ 196 newVolume("volume8-11-2", "1Gi", "uid8-11", "claim8-11", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty, volume.AnnDynamicallyProvisioned), 197 }, 198 // the claim is bound to volume8-11-2 -> volume8-11-1 has lost the race and will be deleted 199 initialClaims: newClaimArray("claim8-11", "uid8-11", "10Gi", "volume8-11-2", v1.ClaimBound, nil), 200 expectedClaims: newClaimArray("claim8-11", "uid8-11", "10Gi", "volume8-11-2", v1.ClaimBound, nil), 201 expectedEvents: noevents, 202 errors: noerrors, 203 // Inject deleter into the controller and call syncVolume. The 204 // deleter simulates one delete() call that succeeds. 205 test: wrapTestWithReclaimCalls(operationDelete, []error{nil}, testSyncVolume), 206 }, 207 { 208 // delete success - two PVs are externally provisioned for a single 209 // claim. One of the PVs is marked as Released to be deleted by the 210 // external provisioner. 211 name: "8-12 - two PVs externally provisioned for a single claim", 212 initialVolumes: []*v1.PersistentVolume{ 213 newExternalProvisionedVolume("volume8-12-1", "1Gi", "uid8-12", "claim8-12", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty, gceDriver, nil, volume.AnnDynamicallyProvisioned), 214 newExternalProvisionedVolume("volume8-12-2", "1Gi", "uid8-12", "claim8-12", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty, gceDriver, nil, volume.AnnDynamicallyProvisioned), 215 }, 216 expectedVolumes: []*v1.PersistentVolume{ 217 newExternalProvisionedVolume("volume8-12-1", "1Gi", "uid8-12", "claim8-12", v1.VolumeReleased, v1.PersistentVolumeReclaimDelete, classEmpty, gceDriver, nil, volume.AnnDynamicallyProvisioned), 218 newExternalProvisionedVolume("volume8-12-2", "1Gi", "uid8-12", "claim8-12", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty, gceDriver, nil, volume.AnnDynamicallyProvisioned), 219 }, 220 // the claim is bound to volume8-12-2 -> volume8-12-1 has lost the race and will be "Released" 221 initialClaims: newClaimArray("claim8-12", "uid8-12", "10Gi", "volume8-12-2", v1.ClaimBound, nil), 222 expectedClaims: newClaimArray("claim8-12", "uid8-12", "10Gi", "volume8-12-2", v1.ClaimBound, nil), 223 expectedEvents: noevents, 224 errors: noerrors, 225 test: func(ctrl *PersistentVolumeController, reactor *pvtesting.VolumeReactor, test controllerTest) error { 226 // Inject external deleter annotation 227 test.initialVolumes[0].Annotations[volume.AnnDynamicallyProvisioned] = "external.io/test" 228 test.expectedVolumes[0].Annotations[volume.AnnDynamicallyProvisioned] = "external.io/test" 229 return testSyncVolume(ctrl, reactor, test) 230 }, 231 }, 232 { 233 // delete success - volume has deletion timestamp before doDelete() starts 234 name: "8-13 - volume has deletion timestamp and processed", 235 initialVolumes: volumesWithFinalizers(withVolumeDeletionTimestamp(newVolumeArray("volume8-13", "1Gi", "uid8-13", "claim8-13", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty, volume.AnnBoundByController)), []string{volume.PVDeletionInTreeProtectionFinalizer}), 236 expectedVolumes: novolumes, 237 initialClaims: noclaims, 238 expectedClaims: noclaims, 239 expectedEvents: noevents, 240 errors: noerrors, 241 test: wrapTestWithReclaimCalls(operationDelete, []error{nil}, testSyncVolume), 242 }, 243 } 244 runSyncTests(t, ctx, tests, []*storage.StorageClass{}, []*v1.Pod{}) 245 } 246 247 // Test multiple calls to syncClaim/syncVolume and periodic sync of all 248 // volume/claims. The test follows this pattern: 249 // 0. Load the controller with initial data. 250 // 1. Call controllerTest.testCall() once as in TestSync() 251 // 2. For all volumes/claims changed by previous syncVolume/syncClaim calls, 252 // call appropriate syncVolume/syncClaim (simulating "volume/claim changed" 253 // events). Go to 2. if these calls change anything. 254 // 3. When all changes are processed and no new changes were made, call 255 // syncVolume/syncClaim on all volumes/claims (simulating "periodic sync"). 256 // 4. If some changes were done by step 3., go to 2. (simulation of 257 // "volume/claim updated" events, eventually performing step 3. again) 258 // 5. When 3. does not do any changes, finish the tests and compare final set 259 // of volumes/claims with expected claims/volumes and report differences. 260 // 261 // Some limit of calls in enforced to prevent endless loops. 262 func TestDeleteMultiSync(t *testing.T) { 263 tests := []controllerTest{ 264 { 265 // delete failure - delete returns error. The controller should 266 // try again. 267 name: "9-1 - delete returns error", 268 initialVolumes: volumesWithFinalizers(newVolumeArray("volume9-1", "1Gi", "uid9-1", "claim9-1", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), []string{volume.PVDeletionInTreeProtectionFinalizer}), 269 expectedVolumes: novolumes, 270 initialClaims: noclaims, 271 expectedClaims: noclaims, 272 expectedEvents: []string{"Warning VolumeFailedDelete"}, 273 errors: noerrors, 274 test: wrapTestWithReclaimCalls(operationDelete, []error{errors.New("Mock delete error"), nil}, testSyncVolume), 275 }, 276 } 277 _, ctx := ktesting.NewTestContext(t) 278 runMultisyncTests(t, ctx, tests, []*storage.StorageClass{}, "") 279 }