github.com/1aal/kubeblocks@v0.0.0-20231107070852-e1c03e598921/controllers/dataprotection/volumepopulator_controller_test.go (about) 1 /* 2 Copyright (C) 2022-2023 ApeCloud Co., Ltd 3 4 This file is part of KubeBlocks project 5 6 This program is free software: you can redistribute it and/or modify 7 it under the terms of the GNU Affero General Public License as published by 8 the Free Software Foundation, either version 3 of the License, or 9 (at your option) any later version. 10 11 This program is distributed in the hope that it will be useful 12 but WITHOUT ANY WARRANTY; without even the implied warranty of 13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 GNU Affero General Public License for more details. 15 16 You should have received a copy of the GNU Affero General Public License 17 along with this program. If not, see <http://www.gnu.org/licenses/>. 18 */ 19 20 package dataprotection 21 22 import ( 23 . "github.com/onsi/ginkgo/v2" 24 . "github.com/onsi/gomega" 25 batchv1 "k8s.io/api/batch/v1" 26 corev1 "k8s.io/api/core/v1" 27 storagev1 "k8s.io/api/storage/v1" 28 "k8s.io/apimachinery/pkg/api/resource" 29 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 30 "k8s.io/apimachinery/pkg/types" 31 "sigs.k8s.io/controller-runtime/pkg/client" 32 33 dpv1alpha1 "github.com/1aal/kubeblocks/apis/dataprotection/v1alpha1" 34 dprestore "github.com/1aal/kubeblocks/pkg/dataprotection/restore" 35 dptypes "github.com/1aal/kubeblocks/pkg/dataprotection/types" 36 "github.com/1aal/kubeblocks/pkg/generics" 37 testapps "github.com/1aal/kubeblocks/pkg/testutil/apps" 38 testdp "github.com/1aal/kubeblocks/pkg/testutil/dataprotection" 39 ) 40 41 var _ = Describe("Volume Populator Controller test", func() { 42 cleanEnv := func() { 43 // must wait till resources deleted and no longer existed before the testcases start, 44 // otherwise if later it needs to create some new resource objects with the same name, 45 // in race conditions, it will find the existence of old objects, resulting failure to 46 // create the new objects. 47 By("clean resources") 48 49 // delete rest mocked objects 50 inNS := client.InNamespace(testCtx.DefaultNamespace) 51 ml := client.HasLabels{testCtx.TestObjLabelKey} 52 53 // namespaced 54 testapps.ClearResources(&testCtx, generics.PodSignature, inNS, ml) 55 testapps.ClearResourcesWithRemoveFinalizerOption(&testCtx, generics.BackupSignature, true, inNS) 56 57 // wait all backup to be deleted, otherwise the controller maybe create 58 // job to delete the backup between the ClearResources function delete 59 // the job and get the job list, resulting the ClearResources panic. 60 Eventually(testapps.List(&testCtx, generics.BackupSignature, inNS)).Should(HaveLen(0)) 61 62 testapps.ClearResourcesWithRemoveFinalizerOption(&testCtx, generics.RestoreSignature, true, inNS) 63 testapps.ClearResourcesWithRemoveFinalizerOption(&testCtx, generics.JobSignature, true, inNS) 64 testapps.ClearResourcesWithRemoveFinalizerOption(&testCtx, generics.PersistentVolumeClaimSignature, true, inNS) 65 66 // non-namespaced 67 testapps.ClearResourcesWithRemoveFinalizerOption(&testCtx, generics.ActionSetSignature, true, ml) 68 testapps.ClearResourcesWithRemoveFinalizerOption(&testCtx, generics.StorageClassSignature, true, ml) 69 testapps.ClearResourcesWithRemoveFinalizerOption(&testCtx, generics.PersistentVolumeSignature, true, ml) 70 } 71 72 BeforeEach(func() { 73 cleanEnv() 74 }) 75 76 AfterEach(func() { 77 cleanEnv() 78 }) 79 80 When("volume populator controller test", func() { 81 var ( 82 actionSet *dpv1alpha1.ActionSet 83 pvcName = "data-mysql-mysql-0" 84 storageSize = "20Gi" 85 // intreeProvisioner = "kubernetes.io/no-provisioner" 86 csiProvisioner = "csi.test.io/provisioner" 87 ) 88 89 createStorageClass := func(volumeBinding storagev1.VolumeBindingMode) *storagev1.StorageClass { 90 storageClass := &storagev1.StorageClass{ 91 ObjectMeta: metav1.ObjectMeta{ 92 Name: testdp.StorageClassName, 93 }, 94 Provisioner: csiProvisioner, 95 VolumeBindingMode: &volumeBinding, 96 } 97 Expect(testCtx.CreateObj(testCtx.Ctx, storageClass)).Should(Succeed()) 98 return storageClass 99 } 100 101 BeforeEach(func() { 102 By("create actionSet") 103 actionSet = testdp.NewFakeActionSet(&testCtx) 104 }) 105 106 initResources := func(volumeBinding storagev1.VolumeBindingMode, useVolumeSnapshotBackup, mockBackupCompleted bool) *corev1.PersistentVolumeClaim { 107 By("create storageClass") 108 createStorageClass(volumeBinding) 109 110 By("create backup") 111 backup := mockBackupForRestore(actionSet.Name, "", mockBackupCompleted, useVolumeSnapshotBackup) 112 113 By("create restore ") 114 restore := testdp.NewRestoreactory(testCtx.DefaultNamespace, testdp.RestoreName). 115 SetBackup(backup.Name, testCtx.DefaultNamespace). 116 SetDataSourceRef(testdp.DataVolumeName, testdp.DataVolumeMountPath). 117 Create(&testCtx).GetObject() 118 119 By("create PVC and set spec.dataSourceRef to restore") 120 pvc := testapps.NewPersistentVolumeClaimFactory( 121 testCtx.DefaultNamespace, pvcName, testdp.ClusterName, testdp.ComponentName, testdp.DataVolumeName). 122 SetStorage(storageSize). 123 SetStorageClass(testdp.StorageClassName). 124 SetDataSourceRef(dptypes.DataprotectionAPIGroup, dptypes.RestoreKind, restore.Name). 125 Create(&testCtx).GetObject() 126 return pvc 127 } 128 129 mockPV := func(populatePVCName string) *corev1.PersistentVolume { 130 pv := &corev1.PersistentVolume{ 131 ObjectMeta: metav1.ObjectMeta{ 132 Name: "test-pv", 133 }, 134 Spec: corev1.PersistentVolumeSpec{ 135 StorageClassName: testdp.StorageClassName, 136 AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, 137 Capacity: map[corev1.ResourceName]resource.Quantity{ 138 corev1.ResourceStorage: resource.MustParse(storageSize), 139 }, 140 ClaimRef: &corev1.ObjectReference{ 141 Namespace: testCtx.DefaultNamespace, 142 Name: populatePVCName, 143 }, 144 PersistentVolumeSource: corev1.PersistentVolumeSource{ 145 CSI: &corev1.CSIPersistentVolumeSource{ 146 Driver: "kubernetes.io", 147 VolumeHandle: "test-volume-handle", 148 }, 149 }, 150 }, 151 } 152 Expect(testCtx.Create(ctx, pv)).Should(Succeed()) 153 populatePVC := &corev1.PersistentVolumeClaim{} 154 // bind pv 155 Expect(k8sClient.Get(ctx, types.NamespacedName{Name: populatePVCName, Namespace: testCtx.DefaultNamespace}, populatePVC)).Should(Succeed()) 156 Expect(testapps.ChangeObj(&testCtx, populatePVC, func(c *corev1.PersistentVolumeClaim) { 157 c.Spec.VolumeName = pv.Name 158 })).Should(Succeed()) 159 return pv 160 } 161 162 testVolumePopulate := func(volumeBinding storagev1.VolumeBindingMode, useVolumeSnapshotBackup bool) { 163 pvc := initResources(volumeBinding, useVolumeSnapshotBackup, true) 164 165 pvcKey := client.ObjectKeyFromObject(pvc) 166 if volumeBinding == storagev1.VolumeBindingWaitForFirstConsumer { 167 By("wait for pvc has selected the node") 168 Eventually(testapps.CheckObj(&testCtx, pvcKey, func(g Gomega, tmpPVC *corev1.PersistentVolumeClaim) { 169 g.Expect(len(tmpPVC.Status.Conditions)).Should(Equal(0)) 170 })).Should(Succeed()) 171 } 172 173 By("mock pvc has selected the node") 174 Expect(testapps.ChangeObj(&testCtx, pvc, func(claim *corev1.PersistentVolumeClaim) { 175 if claim.Annotations == nil { 176 claim.Annotations = map[string]string{} 177 } 178 claim.Annotations[annSelectedNode] = "test-node" 179 })).Should(Succeed()) 180 Eventually(testapps.CheckObj(&testCtx, pvcKey, func(g Gomega, tmpPVC *corev1.PersistentVolumeClaim) { 181 g.Expect(len(tmpPVC.Status.Conditions)).Should(Equal(1)) 182 g.Expect(tmpPVC.Status.Conditions[0].Type).Should(Equal(PersistentVolumeClaimPopulating)) 183 })).Should(Succeed()) 184 185 By("expect for populate pvc created") 186 populatePVCName := getPopulatePVCName(pvc.UID) 187 populatePVC := &corev1.PersistentVolumeClaim{} 188 Eventually(testapps.CheckObjExists(&testCtx, types.NamespacedName{Namespace: testCtx.DefaultNamespace, 189 Name: populatePVCName}, populatePVC, true)) 190 191 By("expect for job created") 192 Eventually(testapps.List(&testCtx, generics.JobSignature, 193 client.MatchingLabels{dprestore.DataProtectionLabelPopulatePVCKey: populatePVCName}, 194 client.InNamespace(testCtx.DefaultNamespace))).Should(HaveLen(1)) 195 196 By("mock to create pv and bind to populate pvc") 197 pv := mockPV(populatePVCName) 198 199 By("mock job to succeed") 200 jobList := &batchv1.JobList{} 201 Expect(k8sClient.List(ctx, jobList, 202 client.MatchingLabels{dprestore.DataProtectionLabelPopulatePVCKey: getPopulatePVCName(pvc.UID)}, 203 client.InNamespace(testCtx.DefaultNamespace))).Should(Succeed()) 204 testdp.ReplaceK8sJobStatus(&testCtx, client.ObjectKeyFromObject(&jobList.Items[0]), batchv1.JobComplete) 205 206 By("expect for pvc has been populated") 207 Eventually(testapps.CheckObj(&testCtx, pvcKey, func(g Gomega, tmpPVC *corev1.PersistentVolumeClaim) { 208 g.Expect(tmpPVC.Status.Conditions[0].Reason).Should(Equal(reasonPopulatingSucceed)) 209 })).Should(Succeed()) 210 211 Eventually(testapps.CheckObj(&testCtx, client.ObjectKeyFromObject(pv), func(g Gomega, tmpPV *corev1.PersistentVolume) { 212 g.Expect(tmpPV.Spec.ClaimRef.Name).Should(Equal(pvc.Name)) 213 g.Expect(tmpPV.Spec.ClaimRef.UID).Should(Equal(pvc.UID)) 214 })).Should(Succeed()) 215 216 // mock pvc.spec.volumeName 217 Expect(testapps.ChangeObj(&testCtx, pvc, func(claim *corev1.PersistentVolumeClaim) { 218 claim.Spec.VolumeName = pv.Name 219 })).Should(Succeed()) 220 221 By("expect for resources are cleaned up") 222 Eventually(testapps.List(&testCtx, generics.JobSignature, 223 client.MatchingLabels{dprestore.DataProtectionLabelPopulatePVCKey: populatePVCName}, 224 client.InNamespace(testCtx.DefaultNamespace))).Should(HaveLen(0)) 225 Eventually(testapps.CheckObjExists(&testCtx, types.NamespacedName{Namespace: testCtx.DefaultNamespace, 226 Name: populatePVCName}, populatePVC, false)) 227 } 228 229 Context("test volume populator", func() { 230 It("test VolumePopulator when volumeBinding of storageClass is WaitForFirstConsumer", func() { 231 testVolumePopulate(storagev1.VolumeBindingWaitForFirstConsumer, false) 232 }) 233 234 It("test VolumePopulator when volumeBinding of storageClass is Immediate", func() { 235 testVolumePopulate(storagev1.VolumeBindingImmediate, false) 236 }) 237 238 It("test VolumePopulator when backup uses volume snapshot", func() { 239 testVolumePopulate(storagev1.VolumeBindingWaitForFirstConsumer, true) 240 }) 241 242 It("test VolumePopulator when it fails", func() { 243 pvc := initResources(storagev1.VolumeBindingImmediate, false, false) 244 Eventually(testapps.CheckObj(&testCtx, client.ObjectKeyFromObject(pvc), func(g Gomega, tmpPVC *corev1.PersistentVolumeClaim) { 245 g.Expect(len(tmpPVC.Status.Conditions)).Should(Equal(1)) 246 g.Expect(tmpPVC.Status.Conditions[0].Reason).Should(Equal(reasonPopulatingFailed)) 247 })).Should(Succeed()) 248 249 }) 250 251 }) 252 }) 253 })