github.com/verrazzano/verrazzano@v1.7.1/tests/e2e/backup/helpers/crd_helpers.go (about) 1 // Copyright (c) 2022, Oracle and/or its affiliates. 2 // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. 3 4 package helpers 5 6 import ( 7 "bytes" 8 "context" 9 "encoding/json" 10 "fmt" 11 "os" 12 "strings" 13 "text/tabwriter" 14 "text/template" 15 16 "github.com/verrazzano/verrazzano/pkg/k8s/resource" 17 "github.com/verrazzano/verrazzano/pkg/k8sutil" 18 "go.uber.org/zap" 19 k8serror "k8s.io/apimachinery/pkg/api/errors" 20 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 21 "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" 22 "k8s.io/apimachinery/pkg/runtime/schema" 23 "k8s.io/client-go/dynamic" 24 ) 25 26 // getUnstructuredData common utility to fetch unstructured data 27 func getUnstructuredData(group, version, resource, resourceName, nameSpaceName, component string, log *zap.SugaredLogger) (*unstructured.Unstructured, error) { 28 var dataFetched *unstructured.Unstructured 29 var err error 30 config, err := k8sutil.GetKubeConfig() 31 if err != nil { 32 log.Errorf("Unable to fetch kubeconfig %v", zap.Error(err)) 33 return nil, err 34 } 35 dclient, err := dynamic.NewForConfig(config) 36 if err != nil { 37 log.Errorf("Unable to create dynamic client %v", zap.Error(err)) 38 return nil, err 39 } 40 41 gvr := schema.GroupVersionResource{ 42 Group: group, 43 Version: version, 44 Resource: resource, 45 } 46 47 if nameSpaceName != "" { 48 log.Infof("Fetching '%s' '%s' '%s' in namespace '%s'", component, resource, resourceName, nameSpaceName) 49 dataFetched, err = dclient.Resource(gvr).Namespace(nameSpaceName).Get(context.TODO(), resourceName, metav1.GetOptions{}) 50 } else { 51 log.Infof("Fetching '%s' '%s' '%s'", component, resource, resourceName) 52 dataFetched, err = dclient.Resource(gvr).Get(context.TODO(), resourceName, metav1.GetOptions{}) 53 } 54 if err != nil { 55 log.Errorf("Unable to fetch %s %s %s due to '%v'", component, resource, resourceName, zap.Error(err)) 56 return nil, err 57 } 58 return dataFetched, nil 59 } 60 61 // getUnstructuredData common utility to fetch list of unstructured data 62 func getUnstructuredDataList(group, version, resource, nameSpaceName, component string, log *zap.SugaredLogger) (*unstructured.UnstructuredList, error) { 63 config, err := k8sutil.GetKubeConfig() 64 if err != nil { 65 log.Errorf("Unable to fetch kubeconfig %v", zap.Error(err)) 66 return nil, err 67 } 68 dclient, err := dynamic.NewForConfig(config) 69 if err != nil { 70 log.Errorf("Unable to create dynamic client %v", zap.Error(err)) 71 return nil, err 72 } 73 74 log.Infof("Fetching %s %s", component, resource) 75 gvr := schema.GroupVersionResource{ 76 Group: group, 77 Version: version, 78 Resource: resource, 79 } 80 81 dataFetched, err := dclient.Resource(gvr).Namespace(nameSpaceName).List(context.TODO(), metav1.ListOptions{}) 82 if err != nil { 83 log.Errorf("Unable to fetch %s %s due to '%v'", component, resource, zap.Error(err)) 84 return nil, err 85 } 86 return dataFetched, nil 87 } 88 89 // CreateVeleroBackupLocationObject creates velero backup object location 90 func CreateVeleroBackupLocationObject(backupStorageName, backupSecretName string, log *zap.SugaredLogger) error { 91 var b bytes.Buffer 92 template, _ := template.New("velero-backup-location").Parse(VeleroBackupLocation) 93 94 data := VeleroBackupLocationObjectData{ 95 VeleroBackupStorageName: backupStorageName, 96 VeleroNamespaceName: VeleroNameSpace, 97 VeleroObjectStoreBucketName: OciBucketName, 98 VeleroSecretName: backupSecretName, 99 VeleroObjectStorageNamespaceName: OciNamespaceName, 100 VeleroBackupRegion: BackupRegion, 101 } 102 template.Execute(&b, data) 103 err := resource.CreateOrUpdateResourceFromBytes(b.Bytes(), log) 104 if err != nil { 105 log.Errorf("Error creating velero backup location ", zap.Error(err)) 106 return err 107 } 108 return nil 109 } 110 111 // GetRancherBackupFileName gets the filename backed up to object store 112 func GetRancherBackupFileName(backupName string, log *zap.SugaredLogger) (string, error) { 113 114 log.Infof("Fetching uploaded filename from backup '%s'", backupName) 115 backupFetched, err := getUnstructuredData("resources.cattle.io", "v1", "backups", backupName, "", "rancher", log) 116 if err != nil { 117 log.Errorf("Unable to fetch Rancher backup '%s' due to '%v'", backupName, zap.Error(err)) 118 return "", err 119 } 120 if backupFetched == nil { 121 log.Infof("No Rancher backup with name '%s'' was detected", backupName) 122 } 123 124 var backup RancherBackupModel 125 bdata, err := json.Marshal(backupFetched) 126 if err != nil { 127 log.Errorf("Json marshalling error %v", zap.Error(err)) 128 return "", err 129 } 130 err = json.Unmarshal(bdata, &backup) 131 if err != nil { 132 log.Errorf("Json unmarshall error %v", zap.Error(err)) 133 return "", err 134 } 135 return backup.Status.Filename, nil 136 } 137 138 // GetVeleroBackup Retrieves Velero backup object from the cluster 139 func GetVeleroBackup(namespace, backupName string, log *zap.SugaredLogger) (*VeleroBackupModel, error) { 140 backupFetched, err := getUnstructuredData("velero.io", "v1", "backups", backupName, namespace, "velero", log) 141 if err != nil { 142 log.Errorf("Unable to fetch Velero backup '%s' due to '%v'", backupName, zap.Error(err)) 143 return nil, err 144 } 145 146 if backupFetched == nil { 147 log.Infof("No Velero backup with name '%s' in namespace '%s' was detected", backupName, namespace) 148 } 149 150 var backup VeleroBackupModel 151 bdata, err := json.Marshal(backupFetched) 152 if err != nil { 153 log.Errorf("Json marshalling error %v", zap.Error(err)) 154 return nil, err 155 } 156 err = json.Unmarshal(bdata, &backup) 157 if err != nil { 158 log.Errorf("Json unmarshall error %v", zap.Error(err)) 159 return nil, err 160 } 161 162 return &backup, nil 163 } 164 165 // GetVeleroRestore Retrieves Velero restore object from the cluster 166 func GetVeleroRestore(namespace, restoreName string, log *zap.SugaredLogger) (*VeleroRestoreModel, error) { 167 168 restoreFetched, err := getUnstructuredData("velero.io", "v1", "restores", restoreName, namespace, "velero", log) 169 if err != nil { 170 log.Errorf("Unable to fetch velero restore '%s' due to '%v'", restoreName, zap.Error(err)) 171 return nil, err 172 } 173 174 if restoreFetched == nil { 175 log.Infof("No Velero restore with name '%s' in namespace '%s' was detected", restoreName, namespace) 176 } 177 178 var restore VeleroRestoreModel 179 bdata, err := json.Marshal(restoreFetched) 180 if err != nil { 181 log.Errorf("Json marshalling error %v", zap.Error(err)) 182 return nil, err 183 } 184 err = json.Unmarshal(bdata, &restore) 185 if err != nil { 186 log.Errorf("Json unmarshall error %v", zap.Error(err)) 187 return nil, err 188 } 189 190 return &restore, nil 191 } 192 193 // GetPodVolumeBackups Retrieves Velero pod volume backups object from the cluster 194 func GetPodVolumeBackups(namespace string, log *zap.SugaredLogger) error { 195 196 podVolumeBackupsFetched, err := getUnstructuredDataList("velero.io", "v1", BackupPodVolumeResource, namespace, "velero", log) 197 if err != nil { 198 log.Errorf("Unable to fetch velero podvolumebackups due to '%v'", zap.Error(err)) 199 return err 200 } 201 202 if podVolumeBackupsFetched == nil { 203 log.Infof("No Velero podvolumebackups in namespace '%s' was detected ", namespace) 204 } 205 writer := tabwriter.NewWriter(os.Stdout, 0, 8, 1, '\t', tabwriter.AlignRight) 206 fmt.Fprintln(writer, "Name\tStatus\tNamespace\tPod\tVolume\tRepo\tStorage") 207 for _, item := range podVolumeBackupsFetched.Items { 208 var podVolumeBackup VeleroPodVolumeBackups 209 bdata, err := json.Marshal(item.Object) 210 if err != nil { 211 log.Errorf("Json marshalling error %v", zap.Error(err)) 212 return err 213 } 214 err = json.Unmarshal(bdata, &podVolumeBackup) 215 if err != nil { 216 log.Errorf("Json unmarshall error %v", zap.Error(err)) 217 return err 218 } 219 fmt.Fprintf(writer, "%v\n", fmt.Sprintf("%v\t%v\t%v\t%v\t%v\t%v\t%v", podVolumeBackup.Metadata.Name, podVolumeBackup.Status.Phase, 220 podVolumeBackup.Metadata.Namespace, podVolumeBackup.Spec.Pod.Name, 221 podVolumeBackup.Spec.Volume, podVolumeBackup.Spec.RepoIdentifier, podVolumeBackup.Spec.BackupStorageLocation)) 222 } 223 writer.Flush() 224 return nil 225 } 226 227 // GetPodVolumeRestores Retrieves Velero pod volume restores object from the cluster 228 func GetPodVolumeRestores(namespace string, log *zap.SugaredLogger) error { 229 230 restoreFetched, err := getUnstructuredDataList("velero.io", "v1", RestorePodVolumeResource, namespace, "velero", log) 231 if err != nil { 232 log.Errorf("Unable to fetch velero podvolumebackups due to '%v'", zap.Error(err)) 233 return err 234 } 235 236 if restoreFetched == nil { 237 log.Infof("No Velero podvolumebackups in namespace '%s' was detected ", namespace) 238 } 239 240 writer := tabwriter.NewWriter(os.Stdout, 0, 8, 1, '\t', tabwriter.AlignRight) 241 fmt.Fprintln(writer, "Name\tnamespace\tPod\tVolume\tStatus\tTotalBytes\tBytesDone") 242 for _, item := range restoreFetched.Items { 243 var podVolumeRestore VeleroPodVolumeRestores 244 bdata, err := json.Marshal(item.Object) 245 if err != nil { 246 log.Errorf("Json marshalling error %v", zap.Error(err)) 247 return err 248 } 249 err = json.Unmarshal(bdata, &podVolumeRestore) 250 if err != nil { 251 log.Errorf("Json unmarshall error %v", zap.Error(err)) 252 return err 253 } 254 fmt.Fprintf(writer, "%v\n", fmt.Sprintf("%v\t%v\t%v\t%v\t%v\t%v\t%v", podVolumeRestore.Metadata.Name, 255 podVolumeRestore.Metadata.Namespace, podVolumeRestore.Spec.Pod.Name, podVolumeRestore.Spec.Volume, 256 podVolumeRestore.Status.Phase, podVolumeRestore.Status.Progress.TotalBytes, podVolumeRestore.Status.Progress.BytesDone)) 257 } 258 writer.Flush() 259 return nil 260 } 261 262 // GetRancherBackup Retrieves rancher backup object from the cluster 263 func GetRancherBackup(backupName string, log *zap.SugaredLogger) (*RancherBackupModel, error) { 264 265 backupFetched, err := getUnstructuredData("resources.cattle.io", "v1", "backups", backupName, "", "rancher", log) 266 if err != nil { 267 log.Errorf("Unable to fetch Rancher backup '%s' due to '%v'", backupName, zap.Error(err)) 268 return nil, err 269 } 270 271 if backupFetched == nil { 272 log.Infof("No Rancher backup with name '%s'' was detected", backupName) 273 } 274 275 var backup RancherBackupModel 276 bdata, err := json.Marshal(backupFetched) 277 if err != nil { 278 log.Errorf("Json marshalling error %v", zap.Error(err)) 279 return nil, err 280 } 281 err = json.Unmarshal(bdata, &backup) 282 if err != nil { 283 log.Errorf("Json unmarshall error %v", zap.Error(err)) 284 return nil, err 285 } 286 287 return &backup, nil 288 } 289 290 // GetRancherRestore Retrieves rancher restore object from the cluster 291 func GetRancherRestore(restoreName string, log *zap.SugaredLogger) (*RancherRestoreModel, error) { 292 293 restoreFetched, err := getUnstructuredData("resources.cattle.io", "v1", "restores", restoreName, "", "rancher", log) 294 if err != nil { 295 log.Errorf("Unable to fetch Rancher restore '%s' due to '%v'", restoreName, zap.Error(err)) 296 return nil, err 297 } 298 299 if restoreFetched == nil { 300 log.Infof("No Rancher restore with name '%s'' was detected", restoreName) 301 } 302 303 var restore RancherRestoreModel 304 bdata, err := json.Marshal(restoreFetched) 305 if err != nil { 306 log.Errorf("Json marshalling error %v", zap.Error(err)) 307 return nil, err 308 } 309 err = json.Unmarshal(bdata, &restore) 310 if err != nil { 311 log.Errorf("Json unmarshall error %v", zap.Error(err)) 312 return nil, err 313 } 314 315 return &restore, nil 316 } 317 318 func GetMySQLBackup(namespace, backupName string, log *zap.SugaredLogger) (*MySQLBackupModel, error) { 319 backupFetched, err := getUnstructuredData("mysql.oracle.com", "v2", "mysqlbackups", backupName, namespace, "MySQL", log) 320 if err != nil { 321 log.Errorf("Unable to fetch MySQL backup '%s' due to '%v'", backupName, zap.Error(err)) 322 return nil, err 323 } 324 325 if backupFetched == nil { 326 log.Infof("No MySQL backup with name '%s' in namespace '%s' was detected", backupName, namespace) 327 } 328 329 var backup MySQLBackupModel 330 bdata, err := json.Marshal(backupFetched) 331 if err != nil { 332 log.Errorf("Json marshalling error %v", zap.Error(err)) 333 return nil, err 334 } 335 err = json.Unmarshal(bdata, &backup) 336 if err != nil { 337 log.Errorf("Json unmarshall error %v", zap.Error(err)) 338 return nil, err 339 } 340 return &backup, nil 341 } 342 343 func GetMySQLInnoDBStatus(namespace, backupName string, log *zap.SugaredLogger) (*InnoDBIcsModel, error) { 344 innodbFetched, err := getUnstructuredData("mysql.oracle.com", "v2", "innodbclusters", backupName, namespace, "MySQL", log) 345 if err != nil { 346 log.Errorf("Unable to fetch MySQL backup '%s' due to '%v'", backupName, zap.Error(err)) 347 return nil, err 348 } 349 350 if innodbFetched == nil { 351 log.Infof("No MySQL backup with name '%s' in namespace '%s' was detected", backupName, namespace) 352 } 353 354 var ics InnoDBIcsModel 355 bdata, err := json.Marshal(innodbFetched) 356 if err != nil { 357 log.Errorf("Json marshalling error %v", zap.Error(err)) 358 return nil, err 359 } 360 err = json.Unmarshal(bdata, &ics) 361 if err != nil { 362 log.Errorf("Json unmarshall error %v", zap.Error(err)) 363 return nil, err 364 } 365 return &ics, nil 366 } 367 368 // TrackOperationProgress used to track operation status for a given gvr 369 func TrackOperationProgress(operator, operation, objectName, namespace string, log *zap.SugaredLogger) error { 370 var response string 371 switch operator { 372 case "mysql": 373 switch operation { 374 case "backups": 375 backupInfo, err := GetMySQLBackup(namespace, objectName, log) 376 if err != nil { 377 log.Errorf("Unable to fetch backup '%s' due to '%v'", objectName, zap.Error(err)) 378 } 379 if backupInfo == nil { 380 response = "Nil" 381 } else { 382 response = backupInfo.Status.Status 383 } 384 385 case "restores": 386 restoreInfo, err := GetMySQLInnoDBStatus(namespace, objectName, log) 387 if err != nil { 388 log.Errorf("Unable to fetch restore '%s' due to '%v'", objectName, zap.Error(err)) 389 } 390 if restoreInfo == nil { 391 response = "Nil" 392 } else { 393 response = restoreInfo.Status.Cluster.Status 394 } 395 default: 396 log.Errorf("Invalid operation specified for Velero = %s'", operation) 397 response = "NAN" 398 } 399 400 case "velero": 401 switch operation { 402 case "backups": 403 backupInfo, err := GetVeleroBackup(namespace, objectName, log) 404 if err != nil { 405 log.Errorf("Unable to fetch backup '%s' due to '%v'", objectName, zap.Error(err)) 406 } 407 if backupInfo == nil { 408 response = "Nil" 409 } else { 410 response = backupInfo.Status.Phase 411 } 412 413 case "restores": 414 restoreInfo, err := GetVeleroRestore(namespace, objectName, log) 415 if err != nil { 416 log.Errorf("Unable to fetch restore '%s' due to '%v'", objectName, zap.Error(err)) 417 } 418 if restoreInfo == nil { 419 response = "Nil" 420 } else { 421 response = restoreInfo.Status.Phase 422 } 423 default: 424 log.Errorf("Invalid operation specified for Velero = %s'", operation) 425 response = "NAN" 426 } 427 428 case "rancher": 429 switch operation { 430 case "backups": 431 backupInfo, err := GetRancherBackup(objectName, log) 432 if err != nil { 433 log.Errorf("Unable to fetch backup '%s' due to '%v'", objectName, zap.Error(err)) 434 } 435 if backupInfo == nil { 436 response = "Nil" 437 } else { 438 if backupInfo.Status.Conditions != nil { 439 for _, cond := range backupInfo.Status.Conditions { 440 if cond.Type == "Ready" { 441 response = cond.Message 442 } else { 443 log.Infof("Rancher backup status : Type = %v, Status = %v", cond.Type, cond.Status) 444 } 445 } 446 } 447 } 448 449 case "restores": 450 restoreInfo, err := GetRancherRestore(objectName, log) 451 if err != nil { 452 log.Errorf("Unable to fetch restore '%s' due to '%v'", objectName, zap.Error(err)) 453 } 454 if restoreInfo == nil { 455 response = "Nil" 456 } else { 457 if restoreInfo.Status.Conditions != nil { 458 for _, cond := range restoreInfo.Status.Conditions { 459 if cond.Type == "Ready" { 460 response = cond.Message 461 } else { 462 log.Infof("Rancher restore status : Type = %v, Status = %v", cond.Type, cond.Status) 463 } 464 } 465 } 466 } 467 468 default: 469 log.Errorf("Invalid operation specified for Rancher = %s'", operation) 470 response = "NAN" 471 } 472 473 default: 474 log.Errorf("Invalid operator specified = %s'", operator) 475 response = "NAN" 476 477 } 478 479 switch response { 480 case "InProgress", "", "PENDING", "INITIALIZING": 481 log.Infof("%s '%s' is in progress. Status = %s", strings.ToTitle(operation), objectName, response) 482 return fmt.Errorf("%s '%s' is in progress", strings.ToTitle(operation), objectName) 483 case "Completed", "ONLINE": 484 log.Infof("%s '%s' completed successfully", strings.ToTitle(operation), objectName) 485 return nil 486 default: 487 return fmt.Errorf("%s failed. State = '%s'", strings.ToTitle(operation), response) 488 } 489 } 490 491 // CrdPruner is a gvr based pruner 492 func CrdPruner(group, version, resource, resourceName, nameSpaceName string, log *zap.SugaredLogger) error { 493 config, err := k8sutil.GetKubeConfig() 494 if err != nil { 495 log.Errorf("Unable to fetch kubeconfig %v", zap.Error(err)) 496 return err 497 } 498 499 dclient, err := dynamic.NewForConfig(config) 500 if err != nil { 501 log.Errorf("Unable to create dynamic client %v", zap.Error(err)) 502 return err 503 } 504 505 gvr := schema.GroupVersionResource{ 506 Group: group, 507 Version: version, 508 Resource: resource, 509 } 510 511 if strings.Contains(group, "velero") || strings.Contains(group, "mysql") { 512 err = dclient.Resource(gvr).Namespace(nameSpaceName).Delete(context.TODO(), resourceName, metav1.DeleteOptions{}) 513 if err != nil { 514 if !k8serror.IsNotFound(err) { 515 log.Errorf("Unable to delete resource '%s' from namespace '%s' due to '%v'", resourceName, nameSpaceName, zap.Error(err)) 516 return err 517 } 518 } 519 } 520 521 if strings.Contains(group, "cattle") { 522 err = dclient.Resource(gvr).Delete(context.TODO(), resourceName, metav1.DeleteOptions{}) 523 if err != nil { 524 if !k8serror.IsNotFound(err) { 525 log.Errorf("Unable to delete resource '%s' due to '%v'", resourceName, zap.Error(err)) 526 return err 527 } 528 } 529 } 530 return nil 531 }