github.com/1aal/kubeblocks@v0.0.0-20231107070852-e1c03e598921/controllers/dataprotection/volumepopulator_controller_test.go (about)

     1  /*
     2  Copyright (C) 2022-2023 ApeCloud Co., Ltd
     3  
     4  This file is part of KubeBlocks project
     5  
     6  This program is free software: you can redistribute it and/or modify
     7  it under the terms of the GNU Affero General Public License as published by
     8  the Free Software Foundation, either version 3 of the License, or
     9  (at your option) any later version.
    10  
    11  This program is distributed in the hope that it will be useful
    12  but WITHOUT ANY WARRANTY; without even the implied warranty of
    13  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    14  GNU Affero General Public License for more details.
    15  
    16  You should have received a copy of the GNU Affero General Public License
    17  along with this program.  If not, see <http://www.gnu.org/licenses/>.
    18  */
    19  
    20  package dataprotection
    21  
    22  import (
    23  	. "github.com/onsi/ginkgo/v2"
    24  	. "github.com/onsi/gomega"
    25  	batchv1 "k8s.io/api/batch/v1"
    26  	corev1 "k8s.io/api/core/v1"
    27  	storagev1 "k8s.io/api/storage/v1"
    28  	"k8s.io/apimachinery/pkg/api/resource"
    29  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    30  	"k8s.io/apimachinery/pkg/types"
    31  	"sigs.k8s.io/controller-runtime/pkg/client"
    32  
    33  	dpv1alpha1 "github.com/1aal/kubeblocks/apis/dataprotection/v1alpha1"
    34  	dprestore "github.com/1aal/kubeblocks/pkg/dataprotection/restore"
    35  	dptypes "github.com/1aal/kubeblocks/pkg/dataprotection/types"
    36  	"github.com/1aal/kubeblocks/pkg/generics"
    37  	testapps "github.com/1aal/kubeblocks/pkg/testutil/apps"
    38  	testdp "github.com/1aal/kubeblocks/pkg/testutil/dataprotection"
    39  )
    40  
    41  var _ = Describe("Volume Populator Controller test", func() {
    42  	cleanEnv := func() {
    43  		// must wait till resources deleted and no longer existed before the testcases start,
    44  		// otherwise if later it needs to create some new resource objects with the same name,
    45  		// in race conditions, it will find the existence of old objects, resulting failure to
    46  		// create the new objects.
    47  		By("clean resources")
    48  
    49  		// delete rest mocked objects
    50  		inNS := client.InNamespace(testCtx.DefaultNamespace)
    51  		ml := client.HasLabels{testCtx.TestObjLabelKey}
    52  
    53  		// namespaced
    54  		testapps.ClearResources(&testCtx, generics.PodSignature, inNS, ml)
    55  		testapps.ClearResourcesWithRemoveFinalizerOption(&testCtx, generics.BackupSignature, true, inNS)
    56  
    57  		// wait all backup to be deleted, otherwise the controller maybe create
    58  		// job to delete the backup between the ClearResources function delete
    59  		// the job and get the job list, resulting the ClearResources panic.
    60  		Eventually(testapps.List(&testCtx, generics.BackupSignature, inNS)).Should(HaveLen(0))
    61  
    62  		testapps.ClearResourcesWithRemoveFinalizerOption(&testCtx, generics.RestoreSignature, true, inNS)
    63  		testapps.ClearResourcesWithRemoveFinalizerOption(&testCtx, generics.JobSignature, true, inNS)
    64  		testapps.ClearResourcesWithRemoveFinalizerOption(&testCtx, generics.PersistentVolumeClaimSignature, true, inNS)
    65  
    66  		// non-namespaced
    67  		testapps.ClearResourcesWithRemoveFinalizerOption(&testCtx, generics.ActionSetSignature, true, ml)
    68  		testapps.ClearResourcesWithRemoveFinalizerOption(&testCtx, generics.StorageClassSignature, true, ml)
    69  		testapps.ClearResourcesWithRemoveFinalizerOption(&testCtx, generics.PersistentVolumeSignature, true, ml)
    70  	}
    71  
    72  	BeforeEach(func() {
    73  		cleanEnv()
    74  	})
    75  
    76  	AfterEach(func() {
    77  		cleanEnv()
    78  	})
    79  
    80  	When("volume populator controller test", func() {
    81  		var (
    82  			actionSet   *dpv1alpha1.ActionSet
    83  			pvcName     = "data-mysql-mysql-0"
    84  			storageSize = "20Gi"
    85  			// intreeProvisioner = "kubernetes.io/no-provisioner"
    86  			csiProvisioner = "csi.test.io/provisioner"
    87  		)
    88  
    89  		createStorageClass := func(volumeBinding storagev1.VolumeBindingMode) *storagev1.StorageClass {
    90  			storageClass := &storagev1.StorageClass{
    91  				ObjectMeta: metav1.ObjectMeta{
    92  					Name: testdp.StorageClassName,
    93  				},
    94  				Provisioner:       csiProvisioner,
    95  				VolumeBindingMode: &volumeBinding,
    96  			}
    97  			Expect(testCtx.CreateObj(testCtx.Ctx, storageClass)).Should(Succeed())
    98  			return storageClass
    99  		}
   100  
   101  		BeforeEach(func() {
   102  			By("create actionSet")
   103  			actionSet = testdp.NewFakeActionSet(&testCtx)
   104  		})
   105  
   106  		initResources := func(volumeBinding storagev1.VolumeBindingMode, useVolumeSnapshotBackup, mockBackupCompleted bool) *corev1.PersistentVolumeClaim {
   107  			By("create storageClass")
   108  			createStorageClass(volumeBinding)
   109  
   110  			By("create backup")
   111  			backup := mockBackupForRestore(actionSet.Name, "", mockBackupCompleted, useVolumeSnapshotBackup)
   112  
   113  			By("create restore ")
   114  			restore := testdp.NewRestoreactory(testCtx.DefaultNamespace, testdp.RestoreName).
   115  				SetBackup(backup.Name, testCtx.DefaultNamespace).
   116  				SetDataSourceRef(testdp.DataVolumeName, testdp.DataVolumeMountPath).
   117  				Create(&testCtx).GetObject()
   118  
   119  			By("create PVC and set spec.dataSourceRef to restore")
   120  			pvc := testapps.NewPersistentVolumeClaimFactory(
   121  				testCtx.DefaultNamespace, pvcName, testdp.ClusterName, testdp.ComponentName, testdp.DataVolumeName).
   122  				SetStorage(storageSize).
   123  				SetStorageClass(testdp.StorageClassName).
   124  				SetDataSourceRef(dptypes.DataprotectionAPIGroup, dptypes.RestoreKind, restore.Name).
   125  				Create(&testCtx).GetObject()
   126  			return pvc
   127  		}
   128  
   129  		mockPV := func(populatePVCName string) *corev1.PersistentVolume {
   130  			pv := &corev1.PersistentVolume{
   131  				ObjectMeta: metav1.ObjectMeta{
   132  					Name: "test-pv",
   133  				},
   134  				Spec: corev1.PersistentVolumeSpec{
   135  					StorageClassName: testdp.StorageClassName,
   136  					AccessModes:      []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce},
   137  					Capacity: map[corev1.ResourceName]resource.Quantity{
   138  						corev1.ResourceStorage: resource.MustParse(storageSize),
   139  					},
   140  					ClaimRef: &corev1.ObjectReference{
   141  						Namespace: testCtx.DefaultNamespace,
   142  						Name:      populatePVCName,
   143  					},
   144  					PersistentVolumeSource: corev1.PersistentVolumeSource{
   145  						CSI: &corev1.CSIPersistentVolumeSource{
   146  							Driver:       "kubernetes.io",
   147  							VolumeHandle: "test-volume-handle",
   148  						},
   149  					},
   150  				},
   151  			}
   152  			Expect(testCtx.Create(ctx, pv)).Should(Succeed())
   153  			populatePVC := &corev1.PersistentVolumeClaim{}
   154  			// bind pv
   155  			Expect(k8sClient.Get(ctx, types.NamespacedName{Name: populatePVCName, Namespace: testCtx.DefaultNamespace}, populatePVC)).Should(Succeed())
   156  			Expect(testapps.ChangeObj(&testCtx, populatePVC, func(c *corev1.PersistentVolumeClaim) {
   157  				c.Spec.VolumeName = pv.Name
   158  			})).Should(Succeed())
   159  			return pv
   160  		}
   161  
   162  		testVolumePopulate := func(volumeBinding storagev1.VolumeBindingMode, useVolumeSnapshotBackup bool) {
   163  			pvc := initResources(volumeBinding, useVolumeSnapshotBackup, true)
   164  
   165  			pvcKey := client.ObjectKeyFromObject(pvc)
   166  			if volumeBinding == storagev1.VolumeBindingWaitForFirstConsumer {
   167  				By("wait for pvc has selected the node")
   168  				Eventually(testapps.CheckObj(&testCtx, pvcKey, func(g Gomega, tmpPVC *corev1.PersistentVolumeClaim) {
   169  					g.Expect(len(tmpPVC.Status.Conditions)).Should(Equal(0))
   170  				})).Should(Succeed())
   171  			}
   172  
   173  			By("mock pvc has selected the node")
   174  			Expect(testapps.ChangeObj(&testCtx, pvc, func(claim *corev1.PersistentVolumeClaim) {
   175  				if claim.Annotations == nil {
   176  					claim.Annotations = map[string]string{}
   177  				}
   178  				claim.Annotations[annSelectedNode] = "test-node"
   179  			})).Should(Succeed())
   180  			Eventually(testapps.CheckObj(&testCtx, pvcKey, func(g Gomega, tmpPVC *corev1.PersistentVolumeClaim) {
   181  				g.Expect(len(tmpPVC.Status.Conditions)).Should(Equal(1))
   182  				g.Expect(tmpPVC.Status.Conditions[0].Type).Should(Equal(PersistentVolumeClaimPopulating))
   183  			})).Should(Succeed())
   184  
   185  			By("expect for populate pvc created")
   186  			populatePVCName := getPopulatePVCName(pvc.UID)
   187  			populatePVC := &corev1.PersistentVolumeClaim{}
   188  			Eventually(testapps.CheckObjExists(&testCtx, types.NamespacedName{Namespace: testCtx.DefaultNamespace,
   189  				Name: populatePVCName}, populatePVC, true))
   190  
   191  			By("expect for job created")
   192  			Eventually(testapps.List(&testCtx, generics.JobSignature,
   193  				client.MatchingLabels{dprestore.DataProtectionLabelPopulatePVCKey: populatePVCName},
   194  				client.InNamespace(testCtx.DefaultNamespace))).Should(HaveLen(1))
   195  
   196  			By("mock to create pv and bind to populate pvc")
   197  			pv := mockPV(populatePVCName)
   198  
   199  			By("mock job to succeed")
   200  			jobList := &batchv1.JobList{}
   201  			Expect(k8sClient.List(ctx, jobList,
   202  				client.MatchingLabels{dprestore.DataProtectionLabelPopulatePVCKey: getPopulatePVCName(pvc.UID)},
   203  				client.InNamespace(testCtx.DefaultNamespace))).Should(Succeed())
   204  			testdp.ReplaceK8sJobStatus(&testCtx, client.ObjectKeyFromObject(&jobList.Items[0]), batchv1.JobComplete)
   205  
   206  			By("expect for pvc has been populated")
   207  			Eventually(testapps.CheckObj(&testCtx, pvcKey, func(g Gomega, tmpPVC *corev1.PersistentVolumeClaim) {
   208  				g.Expect(tmpPVC.Status.Conditions[0].Reason).Should(Equal(reasonPopulatingSucceed))
   209  			})).Should(Succeed())
   210  
   211  			Eventually(testapps.CheckObj(&testCtx, client.ObjectKeyFromObject(pv), func(g Gomega, tmpPV *corev1.PersistentVolume) {
   212  				g.Expect(tmpPV.Spec.ClaimRef.Name).Should(Equal(pvc.Name))
   213  				g.Expect(tmpPV.Spec.ClaimRef.UID).Should(Equal(pvc.UID))
   214  			})).Should(Succeed())
   215  
   216  			// mock pvc.spec.volumeName
   217  			Expect(testapps.ChangeObj(&testCtx, pvc, func(claim *corev1.PersistentVolumeClaim) {
   218  				claim.Spec.VolumeName = pv.Name
   219  			})).Should(Succeed())
   220  
   221  			By("expect for resources are cleaned up")
   222  			Eventually(testapps.List(&testCtx, generics.JobSignature,
   223  				client.MatchingLabels{dprestore.DataProtectionLabelPopulatePVCKey: populatePVCName},
   224  				client.InNamespace(testCtx.DefaultNamespace))).Should(HaveLen(0))
   225  			Eventually(testapps.CheckObjExists(&testCtx, types.NamespacedName{Namespace: testCtx.DefaultNamespace,
   226  				Name: populatePVCName}, populatePVC, false))
   227  		}
   228  
   229  		Context("test volume populator", func() {
   230  			It("test VolumePopulator when volumeBinding of storageClass is WaitForFirstConsumer", func() {
   231  				testVolumePopulate(storagev1.VolumeBindingWaitForFirstConsumer, false)
   232  			})
   233  
   234  			It("test VolumePopulator when volumeBinding of storageClass is Immediate", func() {
   235  				testVolumePopulate(storagev1.VolumeBindingImmediate, false)
   236  			})
   237  
   238  			It("test VolumePopulator when backup uses volume snapshot", func() {
   239  				testVolumePopulate(storagev1.VolumeBindingWaitForFirstConsumer, true)
   240  			})
   241  
   242  			It("test VolumePopulator when it fails", func() {
   243  				pvc := initResources(storagev1.VolumeBindingImmediate, false, false)
   244  				Eventually(testapps.CheckObj(&testCtx, client.ObjectKeyFromObject(pvc), func(g Gomega, tmpPVC *corev1.PersistentVolumeClaim) {
   245  					g.Expect(len(tmpPVC.Status.Conditions)).Should(Equal(1))
   246  					g.Expect(tmpPVC.Status.Conditions[0].Reason).Should(Equal(reasonPopulatingFailed))
   247  				})).Should(Succeed())
   248  
   249  			})
   250  
   251  		})
   252  	})
   253  })