github.com/verrazzano/verrazzano@v1.7.1/tests/e2e/backup/helpers/crd_helpers.go (about)

     1  // Copyright (c) 2022, Oracle and/or its affiliates.
     2  // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
     3  
     4  package helpers
     5  
     6  import (
     7  	"bytes"
     8  	"context"
     9  	"encoding/json"
    10  	"fmt"
    11  	"os"
    12  	"strings"
    13  	"text/tabwriter"
    14  	"text/template"
    15  
    16  	"github.com/verrazzano/verrazzano/pkg/k8s/resource"
    17  	"github.com/verrazzano/verrazzano/pkg/k8sutil"
    18  	"go.uber.org/zap"
    19  	k8serror "k8s.io/apimachinery/pkg/api/errors"
    20  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    21  	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
    22  	"k8s.io/apimachinery/pkg/runtime/schema"
    23  	"k8s.io/client-go/dynamic"
    24  )
    25  
    26  // getUnstructuredData common utility to fetch unstructured data
    27  func getUnstructuredData(group, version, resource, resourceName, nameSpaceName, component string, log *zap.SugaredLogger) (*unstructured.Unstructured, error) {
    28  	var dataFetched *unstructured.Unstructured
    29  	var err error
    30  	config, err := k8sutil.GetKubeConfig()
    31  	if err != nil {
    32  		log.Errorf("Unable to fetch kubeconfig %v", zap.Error(err))
    33  		return nil, err
    34  	}
    35  	dclient, err := dynamic.NewForConfig(config)
    36  	if err != nil {
    37  		log.Errorf("Unable to create dynamic client %v", zap.Error(err))
    38  		return nil, err
    39  	}
    40  
    41  	gvr := schema.GroupVersionResource{
    42  		Group:    group,
    43  		Version:  version,
    44  		Resource: resource,
    45  	}
    46  
    47  	if nameSpaceName != "" {
    48  		log.Infof("Fetching '%s' '%s' '%s' in namespace '%s'", component, resource, resourceName, nameSpaceName)
    49  		dataFetched, err = dclient.Resource(gvr).Namespace(nameSpaceName).Get(context.TODO(), resourceName, metav1.GetOptions{})
    50  	} else {
    51  		log.Infof("Fetching '%s' '%s' '%s'", component, resource, resourceName)
    52  		dataFetched, err = dclient.Resource(gvr).Get(context.TODO(), resourceName, metav1.GetOptions{})
    53  	}
    54  	if err != nil {
    55  		log.Errorf("Unable to fetch %s %s %s due to '%v'", component, resource, resourceName, zap.Error(err))
    56  		return nil, err
    57  	}
    58  	return dataFetched, nil
    59  }
    60  
    61  // getUnstructuredData common utility to fetch list of unstructured data
    62  func getUnstructuredDataList(group, version, resource, nameSpaceName, component string, log *zap.SugaredLogger) (*unstructured.UnstructuredList, error) {
    63  	config, err := k8sutil.GetKubeConfig()
    64  	if err != nil {
    65  		log.Errorf("Unable to fetch kubeconfig %v", zap.Error(err))
    66  		return nil, err
    67  	}
    68  	dclient, err := dynamic.NewForConfig(config)
    69  	if err != nil {
    70  		log.Errorf("Unable to create dynamic client %v", zap.Error(err))
    71  		return nil, err
    72  	}
    73  
    74  	log.Infof("Fetching %s %s", component, resource)
    75  	gvr := schema.GroupVersionResource{
    76  		Group:    group,
    77  		Version:  version,
    78  		Resource: resource,
    79  	}
    80  
    81  	dataFetched, err := dclient.Resource(gvr).Namespace(nameSpaceName).List(context.TODO(), metav1.ListOptions{})
    82  	if err != nil {
    83  		log.Errorf("Unable to fetch %s %s due to '%v'", component, resource, zap.Error(err))
    84  		return nil, err
    85  	}
    86  	return dataFetched, nil
    87  }
    88  
    89  // CreateVeleroBackupLocationObject creates velero backup object location
    90  func CreateVeleroBackupLocationObject(backupStorageName, backupSecretName string, log *zap.SugaredLogger) error {
    91  	var b bytes.Buffer
    92  	template, _ := template.New("velero-backup-location").Parse(VeleroBackupLocation)
    93  
    94  	data := VeleroBackupLocationObjectData{
    95  		VeleroBackupStorageName:          backupStorageName,
    96  		VeleroNamespaceName:              VeleroNameSpace,
    97  		VeleroObjectStoreBucketName:      OciBucketName,
    98  		VeleroSecretName:                 backupSecretName,
    99  		VeleroObjectStorageNamespaceName: OciNamespaceName,
   100  		VeleroBackupRegion:               BackupRegion,
   101  	}
   102  	template.Execute(&b, data)
   103  	err := resource.CreateOrUpdateResourceFromBytes(b.Bytes(), log)
   104  	if err != nil {
   105  		log.Errorf("Error creating velero backup location ", zap.Error(err))
   106  		return err
   107  	}
   108  	return nil
   109  }
   110  
   111  // GetRancherBackupFileName gets the filename backed up to object store
   112  func GetRancherBackupFileName(backupName string, log *zap.SugaredLogger) (string, error) {
   113  
   114  	log.Infof("Fetching uploaded filename from backup '%s'", backupName)
   115  	backupFetched, err := getUnstructuredData("resources.cattle.io", "v1", "backups", backupName, "", "rancher", log)
   116  	if err != nil {
   117  		log.Errorf("Unable to fetch Rancher backup '%s' due to '%v'", backupName, zap.Error(err))
   118  		return "", err
   119  	}
   120  	if backupFetched == nil {
   121  		log.Infof("No Rancher backup with name '%s'' was detected", backupName)
   122  	}
   123  
   124  	var backup RancherBackupModel
   125  	bdata, err := json.Marshal(backupFetched)
   126  	if err != nil {
   127  		log.Errorf("Json marshalling error %v", zap.Error(err))
   128  		return "", err
   129  	}
   130  	err = json.Unmarshal(bdata, &backup)
   131  	if err != nil {
   132  		log.Errorf("Json unmarshall error %v", zap.Error(err))
   133  		return "", err
   134  	}
   135  	return backup.Status.Filename, nil
   136  }
   137  
   138  // GetVeleroBackup Retrieves Velero backup object from the cluster
   139  func GetVeleroBackup(namespace, backupName string, log *zap.SugaredLogger) (*VeleroBackupModel, error) {
   140  	backupFetched, err := getUnstructuredData("velero.io", "v1", "backups", backupName, namespace, "velero", log)
   141  	if err != nil {
   142  		log.Errorf("Unable to fetch Velero backup '%s' due to '%v'", backupName, zap.Error(err))
   143  		return nil, err
   144  	}
   145  
   146  	if backupFetched == nil {
   147  		log.Infof("No Velero backup with name '%s' in namespace '%s' was detected", backupName, namespace)
   148  	}
   149  
   150  	var backup VeleroBackupModel
   151  	bdata, err := json.Marshal(backupFetched)
   152  	if err != nil {
   153  		log.Errorf("Json marshalling error %v", zap.Error(err))
   154  		return nil, err
   155  	}
   156  	err = json.Unmarshal(bdata, &backup)
   157  	if err != nil {
   158  		log.Errorf("Json unmarshall error %v", zap.Error(err))
   159  		return nil, err
   160  	}
   161  
   162  	return &backup, nil
   163  }
   164  
   165  // GetVeleroRestore Retrieves Velero restore object from the cluster
   166  func GetVeleroRestore(namespace, restoreName string, log *zap.SugaredLogger) (*VeleroRestoreModel, error) {
   167  
   168  	restoreFetched, err := getUnstructuredData("velero.io", "v1", "restores", restoreName, namespace, "velero", log)
   169  	if err != nil {
   170  		log.Errorf("Unable to fetch velero restore '%s' due to '%v'", restoreName, zap.Error(err))
   171  		return nil, err
   172  	}
   173  
   174  	if restoreFetched == nil {
   175  		log.Infof("No Velero restore with name '%s' in namespace '%s' was detected", restoreName, namespace)
   176  	}
   177  
   178  	var restore VeleroRestoreModel
   179  	bdata, err := json.Marshal(restoreFetched)
   180  	if err != nil {
   181  		log.Errorf("Json marshalling error %v", zap.Error(err))
   182  		return nil, err
   183  	}
   184  	err = json.Unmarshal(bdata, &restore)
   185  	if err != nil {
   186  		log.Errorf("Json unmarshall error %v", zap.Error(err))
   187  		return nil, err
   188  	}
   189  
   190  	return &restore, nil
   191  }
   192  
   193  // GetPodVolumeBackups Retrieves Velero pod volume backups object from the cluster
   194  func GetPodVolumeBackups(namespace string, log *zap.SugaredLogger) error {
   195  
   196  	podVolumeBackupsFetched, err := getUnstructuredDataList("velero.io", "v1", BackupPodVolumeResource, namespace, "velero", log)
   197  	if err != nil {
   198  		log.Errorf("Unable to fetch velero podvolumebackups due to '%v'", zap.Error(err))
   199  		return err
   200  	}
   201  
   202  	if podVolumeBackupsFetched == nil {
   203  		log.Infof("No Velero podvolumebackups in namespace '%s' was detected ", namespace)
   204  	}
   205  	writer := tabwriter.NewWriter(os.Stdout, 0, 8, 1, '\t', tabwriter.AlignRight)
   206  	fmt.Fprintln(writer, "Name\tStatus\tNamespace\tPod\tVolume\tRepo\tStorage")
   207  	for _, item := range podVolumeBackupsFetched.Items {
   208  		var podVolumeBackup VeleroPodVolumeBackups
   209  		bdata, err := json.Marshal(item.Object)
   210  		if err != nil {
   211  			log.Errorf("Json marshalling error %v", zap.Error(err))
   212  			return err
   213  		}
   214  		err = json.Unmarshal(bdata, &podVolumeBackup)
   215  		if err != nil {
   216  			log.Errorf("Json unmarshall error %v", zap.Error(err))
   217  			return err
   218  		}
   219  		fmt.Fprintf(writer, "%v\n", fmt.Sprintf("%v\t%v\t%v\t%v\t%v\t%v\t%v", podVolumeBackup.Metadata.Name, podVolumeBackup.Status.Phase,
   220  			podVolumeBackup.Metadata.Namespace, podVolumeBackup.Spec.Pod.Name,
   221  			podVolumeBackup.Spec.Volume, podVolumeBackup.Spec.RepoIdentifier, podVolumeBackup.Spec.BackupStorageLocation))
   222  	}
   223  	writer.Flush()
   224  	return nil
   225  }
   226  
   227  // GetPodVolumeRestores Retrieves Velero pod volume restores object from the cluster
   228  func GetPodVolumeRestores(namespace string, log *zap.SugaredLogger) error {
   229  
   230  	restoreFetched, err := getUnstructuredDataList("velero.io", "v1", RestorePodVolumeResource, namespace, "velero", log)
   231  	if err != nil {
   232  		log.Errorf("Unable to fetch velero podvolumebackups due to '%v'", zap.Error(err))
   233  		return err
   234  	}
   235  
   236  	if restoreFetched == nil {
   237  		log.Infof("No Velero podvolumebackups in namespace '%s' was detected ", namespace)
   238  	}
   239  
   240  	writer := tabwriter.NewWriter(os.Stdout, 0, 8, 1, '\t', tabwriter.AlignRight)
   241  	fmt.Fprintln(writer, "Name\tnamespace\tPod\tVolume\tStatus\tTotalBytes\tBytesDone")
   242  	for _, item := range restoreFetched.Items {
   243  		var podVolumeRestore VeleroPodVolumeRestores
   244  		bdata, err := json.Marshal(item.Object)
   245  		if err != nil {
   246  			log.Errorf("Json marshalling error %v", zap.Error(err))
   247  			return err
   248  		}
   249  		err = json.Unmarshal(bdata, &podVolumeRestore)
   250  		if err != nil {
   251  			log.Errorf("Json unmarshall error %v", zap.Error(err))
   252  			return err
   253  		}
   254  		fmt.Fprintf(writer, "%v\n", fmt.Sprintf("%v\t%v\t%v\t%v\t%v\t%v\t%v", podVolumeRestore.Metadata.Name,
   255  			podVolumeRestore.Metadata.Namespace, podVolumeRestore.Spec.Pod.Name, podVolumeRestore.Spec.Volume,
   256  			podVolumeRestore.Status.Phase, podVolumeRestore.Status.Progress.TotalBytes, podVolumeRestore.Status.Progress.BytesDone))
   257  	}
   258  	writer.Flush()
   259  	return nil
   260  }
   261  
   262  // GetRancherBackup Retrieves rancher backup object from the cluster
   263  func GetRancherBackup(backupName string, log *zap.SugaredLogger) (*RancherBackupModel, error) {
   264  
   265  	backupFetched, err := getUnstructuredData("resources.cattle.io", "v1", "backups", backupName, "", "rancher", log)
   266  	if err != nil {
   267  		log.Errorf("Unable to fetch Rancher backup '%s' due to '%v'", backupName, zap.Error(err))
   268  		return nil, err
   269  	}
   270  
   271  	if backupFetched == nil {
   272  		log.Infof("No Rancher backup with name '%s'' was detected", backupName)
   273  	}
   274  
   275  	var backup RancherBackupModel
   276  	bdata, err := json.Marshal(backupFetched)
   277  	if err != nil {
   278  		log.Errorf("Json marshalling error %v", zap.Error(err))
   279  		return nil, err
   280  	}
   281  	err = json.Unmarshal(bdata, &backup)
   282  	if err != nil {
   283  		log.Errorf("Json unmarshall error %v", zap.Error(err))
   284  		return nil, err
   285  	}
   286  
   287  	return &backup, nil
   288  }
   289  
   290  // GetRancherRestore Retrieves rancher restore object from the cluster
   291  func GetRancherRestore(restoreName string, log *zap.SugaredLogger) (*RancherRestoreModel, error) {
   292  
   293  	restoreFetched, err := getUnstructuredData("resources.cattle.io", "v1", "restores", restoreName, "", "rancher", log)
   294  	if err != nil {
   295  		log.Errorf("Unable to fetch Rancher restore '%s' due to '%v'", restoreName, zap.Error(err))
   296  		return nil, err
   297  	}
   298  
   299  	if restoreFetched == nil {
   300  		log.Infof("No Rancher restore with name '%s'' was detected", restoreName)
   301  	}
   302  
   303  	var restore RancherRestoreModel
   304  	bdata, err := json.Marshal(restoreFetched)
   305  	if err != nil {
   306  		log.Errorf("Json marshalling error %v", zap.Error(err))
   307  		return nil, err
   308  	}
   309  	err = json.Unmarshal(bdata, &restore)
   310  	if err != nil {
   311  		log.Errorf("Json unmarshall error %v", zap.Error(err))
   312  		return nil, err
   313  	}
   314  
   315  	return &restore, nil
   316  }
   317  
   318  func GetMySQLBackup(namespace, backupName string, log *zap.SugaredLogger) (*MySQLBackupModel, error) {
   319  	backupFetched, err := getUnstructuredData("mysql.oracle.com", "v2", "mysqlbackups", backupName, namespace, "MySQL", log)
   320  	if err != nil {
   321  		log.Errorf("Unable to fetch MySQL backup '%s' due to '%v'", backupName, zap.Error(err))
   322  		return nil, err
   323  	}
   324  
   325  	if backupFetched == nil {
   326  		log.Infof("No MySQL backup with name '%s' in namespace '%s' was detected", backupName, namespace)
   327  	}
   328  
   329  	var backup MySQLBackupModel
   330  	bdata, err := json.Marshal(backupFetched)
   331  	if err != nil {
   332  		log.Errorf("Json marshalling error %v", zap.Error(err))
   333  		return nil, err
   334  	}
   335  	err = json.Unmarshal(bdata, &backup)
   336  	if err != nil {
   337  		log.Errorf("Json unmarshall error %v", zap.Error(err))
   338  		return nil, err
   339  	}
   340  	return &backup, nil
   341  }
   342  
   343  func GetMySQLInnoDBStatus(namespace, backupName string, log *zap.SugaredLogger) (*InnoDBIcsModel, error) {
   344  	innodbFetched, err := getUnstructuredData("mysql.oracle.com", "v2", "innodbclusters", backupName, namespace, "MySQL", log)
   345  	if err != nil {
   346  		log.Errorf("Unable to fetch MySQL backup '%s' due to '%v'", backupName, zap.Error(err))
   347  		return nil, err
   348  	}
   349  
   350  	if innodbFetched == nil {
   351  		log.Infof("No MySQL backup with name '%s' in namespace '%s' was detected", backupName, namespace)
   352  	}
   353  
   354  	var ics InnoDBIcsModel
   355  	bdata, err := json.Marshal(innodbFetched)
   356  	if err != nil {
   357  		log.Errorf("Json marshalling error %v", zap.Error(err))
   358  		return nil, err
   359  	}
   360  	err = json.Unmarshal(bdata, &ics)
   361  	if err != nil {
   362  		log.Errorf("Json unmarshall error %v", zap.Error(err))
   363  		return nil, err
   364  	}
   365  	return &ics, nil
   366  }
   367  
   368  // TrackOperationProgress used to track operation status for a given gvr
   369  func TrackOperationProgress(operator, operation, objectName, namespace string, log *zap.SugaredLogger) error {
   370  	var response string
   371  	switch operator {
   372  	case "mysql":
   373  		switch operation {
   374  		case "backups":
   375  			backupInfo, err := GetMySQLBackup(namespace, objectName, log)
   376  			if err != nil {
   377  				log.Errorf("Unable to fetch backup '%s' due to '%v'", objectName, zap.Error(err))
   378  			}
   379  			if backupInfo == nil {
   380  				response = "Nil"
   381  			} else {
   382  				response = backupInfo.Status.Status
   383  			}
   384  
   385  		case "restores":
   386  			restoreInfo, err := GetMySQLInnoDBStatus(namespace, objectName, log)
   387  			if err != nil {
   388  				log.Errorf("Unable to fetch restore '%s' due to '%v'", objectName, zap.Error(err))
   389  			}
   390  			if restoreInfo == nil {
   391  				response = "Nil"
   392  			} else {
   393  				response = restoreInfo.Status.Cluster.Status
   394  			}
   395  		default:
   396  			log.Errorf("Invalid operation specified for Velero = %s'", operation)
   397  			response = "NAN"
   398  		}
   399  
   400  	case "velero":
   401  		switch operation {
   402  		case "backups":
   403  			backupInfo, err := GetVeleroBackup(namespace, objectName, log)
   404  			if err != nil {
   405  				log.Errorf("Unable to fetch backup '%s' due to '%v'", objectName, zap.Error(err))
   406  			}
   407  			if backupInfo == nil {
   408  				response = "Nil"
   409  			} else {
   410  				response = backupInfo.Status.Phase
   411  			}
   412  
   413  		case "restores":
   414  			restoreInfo, err := GetVeleroRestore(namespace, objectName, log)
   415  			if err != nil {
   416  				log.Errorf("Unable to fetch restore '%s' due to '%v'", objectName, zap.Error(err))
   417  			}
   418  			if restoreInfo == nil {
   419  				response = "Nil"
   420  			} else {
   421  				response = restoreInfo.Status.Phase
   422  			}
   423  		default:
   424  			log.Errorf("Invalid operation specified for Velero = %s'", operation)
   425  			response = "NAN"
   426  		}
   427  
   428  	case "rancher":
   429  		switch operation {
   430  		case "backups":
   431  			backupInfo, err := GetRancherBackup(objectName, log)
   432  			if err != nil {
   433  				log.Errorf("Unable to fetch backup '%s' due to '%v'", objectName, zap.Error(err))
   434  			}
   435  			if backupInfo == nil {
   436  				response = "Nil"
   437  			} else {
   438  				if backupInfo.Status.Conditions != nil {
   439  					for _, cond := range backupInfo.Status.Conditions {
   440  						if cond.Type == "Ready" {
   441  							response = cond.Message
   442  						} else {
   443  							log.Infof("Rancher backup status : Type = %v, Status = %v", cond.Type, cond.Status)
   444  						}
   445  					}
   446  				}
   447  			}
   448  
   449  		case "restores":
   450  			restoreInfo, err := GetRancherRestore(objectName, log)
   451  			if err != nil {
   452  				log.Errorf("Unable to fetch restore '%s' due to '%v'", objectName, zap.Error(err))
   453  			}
   454  			if restoreInfo == nil {
   455  				response = "Nil"
   456  			} else {
   457  				if restoreInfo.Status.Conditions != nil {
   458  					for _, cond := range restoreInfo.Status.Conditions {
   459  						if cond.Type == "Ready" {
   460  							response = cond.Message
   461  						} else {
   462  							log.Infof("Rancher restore status : Type = %v, Status = %v", cond.Type, cond.Status)
   463  						}
   464  					}
   465  				}
   466  			}
   467  
   468  		default:
   469  			log.Errorf("Invalid operation specified for Rancher = %s'", operation)
   470  			response = "NAN"
   471  		}
   472  
   473  	default:
   474  		log.Errorf("Invalid operator specified = %s'", operator)
   475  		response = "NAN"
   476  
   477  	}
   478  
   479  	switch response {
   480  	case "InProgress", "", "PENDING", "INITIALIZING":
   481  		log.Infof("%s '%s' is in progress. Status = %s", strings.ToTitle(operation), objectName, response)
   482  		return fmt.Errorf("%s '%s' is in progress", strings.ToTitle(operation), objectName)
   483  	case "Completed", "ONLINE":
   484  		log.Infof("%s '%s' completed successfully", strings.ToTitle(operation), objectName)
   485  		return nil
   486  	default:
   487  		return fmt.Errorf("%s failed. State = '%s'", strings.ToTitle(operation), response)
   488  	}
   489  }
   490  
   491  // CrdPruner is a gvr based pruner
   492  func CrdPruner(group, version, resource, resourceName, nameSpaceName string, log *zap.SugaredLogger) error {
   493  	config, err := k8sutil.GetKubeConfig()
   494  	if err != nil {
   495  		log.Errorf("Unable to fetch kubeconfig %v", zap.Error(err))
   496  		return err
   497  	}
   498  
   499  	dclient, err := dynamic.NewForConfig(config)
   500  	if err != nil {
   501  		log.Errorf("Unable to create dynamic client %v", zap.Error(err))
   502  		return err
   503  	}
   504  
   505  	gvr := schema.GroupVersionResource{
   506  		Group:    group,
   507  		Version:  version,
   508  		Resource: resource,
   509  	}
   510  
   511  	if strings.Contains(group, "velero") || strings.Contains(group, "mysql") {
   512  		err = dclient.Resource(gvr).Namespace(nameSpaceName).Delete(context.TODO(), resourceName, metav1.DeleteOptions{})
   513  		if err != nil {
   514  			if !k8serror.IsNotFound(err) {
   515  				log.Errorf("Unable to delete resource '%s' from namespace '%s' due to '%v'", resourceName, nameSpaceName, zap.Error(err))
   516  				return err
   517  			}
   518  		}
   519  	}
   520  
   521  	if strings.Contains(group, "cattle") {
   522  		err = dclient.Resource(gvr).Delete(context.TODO(), resourceName, metav1.DeleteOptions{})
   523  		if err != nil {
   524  			if !k8serror.IsNotFound(err) {
   525  				log.Errorf("Unable to delete resource '%s' due to '%v'", resourceName, zap.Error(err))
   526  				return err
   527  			}
   528  		}
   529  	}
   530  	return nil
   531  }