github.com/cloudberrydb/gpbackup@v1.0.3-0.20240118031043-5410fd45eed6/restore/remote.go (about) 1 package restore 2 3 import ( 4 "fmt" 5 "strconv" 6 "strings" 7 8 "github.com/cloudberrydb/gp-common-go-libs/cluster" 9 "github.com/cloudberrydb/gp-common-go-libs/gplog" 10 "github.com/cloudberrydb/gp-common-go-libs/iohelper" 11 "github.com/cloudberrydb/gpbackup/options" 12 "github.com/pkg/errors" 13 ) 14 15 /* 16 * Functions to run commands on entire cluster during restore 17 */ 18 19 func VerifyBackupDirectoriesExistOnAllHosts() { 20 _, err := globalCluster.ExecuteLocalCommand(fmt.Sprintf("test -d %s", globalFPInfo.GetDirForContent(-1))) 21 gplog.FatalOnError(err, "Backup directory %s missing or inaccessible", globalFPInfo.GetDirForContent(-1)) 22 if MustGetFlagString(options.PLUGIN_CONFIG) == "" || backupConfig.SingleDataFile { 23 origSize, destSize, isResizeRestore := GetResizeClusterInfo() 24 25 remoteOutput := globalCluster.GenerateAndExecuteCommand("Verifying backup directories exist", cluster.ON_SEGMENTS, func(contentID int) string { 26 if isResizeRestore { // Map origin content to destination content to find where the original files have been placed 27 if contentID >= origSize { // Don't check for directories for contents that aren't part of the backup set 28 return "" 29 } 30 contentID = contentID % destSize 31 } 32 return fmt.Sprintf("test -d %s", globalFPInfo.GetDirForContent(contentID)) 33 }) 34 globalCluster.CheckClusterError(remoteOutput, "Backup directories missing or inaccessible", func(contentID int) string { 35 return fmt.Sprintf("Backup directory %s missing or inaccessible", globalFPInfo.GetDirForContent(contentID)) 36 }) 37 } 38 } 39 40 func VerifyBackupFileCountOnSegments() { 41 remoteOutput := globalCluster.GenerateAndExecuteCommand("Verifying backup file count", cluster.ON_SEGMENTS, func(contentID int) string { 42 return fmt.Sprintf("find %s -type f | wc -l", globalFPInfo.GetDirForContent(contentID)) 43 }) 44 globalCluster.CheckClusterError(remoteOutput, "Could not verify backup file count", func(contentID int) string { 45 return "Could not verify backup file count" 46 }) 47 48 // these are the file counts for non-resize restores. 49 fileCount := 2 // 1 for the actual data file, 1 for the segment TOC file 50 if !backupConfig.SingleDataFile { 51 fileCount = len(globalTOC.DataEntries) 52 } 53 54 origSize, destSize, isResizeRestore := GetResizeClusterInfo() 55 batchMap := make(map[int]int, len(remoteOutput.Commands)) 56 for i := 0; i < origSize; i++ { 57 batchMap[i%destSize] += fileCount 58 } 59 60 numIncorrect := 0 61 for contentID, cmd := range remoteOutput.Commands { 62 numFound, _ := strconv.Atoi(strings.TrimSpace(cmd.Stdout)) 63 if isResizeRestore { 64 fileCount = batchMap[contentID] 65 } 66 if numFound != fileCount { 67 gplog.Verbose("Expected to find %d file(s) on segment %d on host %s, but found %d instead.", fileCount, contentID, globalCluster.GetHostForContent(contentID), numFound) 68 numIncorrect++ 69 } 70 } 71 if numIncorrect > 0 { 72 cluster.LogFatalClusterError("Found incorrect number of backup files", cluster.ON_SEGMENTS, numIncorrect) 73 } 74 } 75 76 func VerifyMetadataFilePaths(withStats bool) { 77 filetypes := []string{"config", "table of contents", "metadata"} 78 missing := false 79 for _, filetype := range filetypes { 80 filepath := globalFPInfo.GetBackupFilePath(filetype) 81 if !iohelper.FileExistsAndIsReadable(filepath) { 82 missing = true 83 gplog.Error("Cannot access %s file %s", filetype, filepath) 84 } 85 } 86 if withStats { 87 filepath := globalFPInfo.GetStatisticsFilePath() 88 if !iohelper.FileExistsAndIsReadable(filepath) { 89 missing = true 90 gplog.Error("Cannot access statistics file %s", filepath) 91 gplog.Error(`Note that the "-with-stats" flag must be passed to gpbackup to generate a statistics file.`) 92 } 93 } 94 if missing { 95 gplog.Fatal(errors.Errorf("One or more metadata files do not exist or are not readable."), "Cannot proceed with restore") 96 } 97 } 98 99 func GetResizeClusterInfo() (int, int, bool) { 100 isResizeCluster := MustGetFlagBool(options.RESIZE_CLUSTER) 101 origSize := backupConfig.SegmentCount 102 destSize := len(globalCluster.ContentIDs) - 1 103 return origSize, destSize, isResizeCluster 104 }