github.com/cloudberrydb/gpbackup@v1.0.3-0.20240118031043-5410fd45eed6/restore/remote.go (about)

     1  package restore
     2  
     3  import (
     4  	"fmt"
     5  	"strconv"
     6  	"strings"
     7  
     8  	"github.com/cloudberrydb/gp-common-go-libs/cluster"
     9  	"github.com/cloudberrydb/gp-common-go-libs/gplog"
    10  	"github.com/cloudberrydb/gp-common-go-libs/iohelper"
    11  	"github.com/cloudberrydb/gpbackup/options"
    12  	"github.com/pkg/errors"
    13  )
    14  
    15  /*
    16   * Functions to run commands on entire cluster during restore
    17   */
    18  
    19  func VerifyBackupDirectoriesExistOnAllHosts() {
    20  	_, err := globalCluster.ExecuteLocalCommand(fmt.Sprintf("test -d %s", globalFPInfo.GetDirForContent(-1)))
    21  	gplog.FatalOnError(err, "Backup directory %s missing or inaccessible", globalFPInfo.GetDirForContent(-1))
    22  	if MustGetFlagString(options.PLUGIN_CONFIG) == "" || backupConfig.SingleDataFile {
    23  		origSize, destSize, isResizeRestore := GetResizeClusterInfo()
    24  
    25  		remoteOutput := globalCluster.GenerateAndExecuteCommand("Verifying backup directories exist", cluster.ON_SEGMENTS, func(contentID int) string {
    26  			if isResizeRestore { // Map origin content to destination content to find where the original files have been placed
    27  				if contentID >= origSize { // Don't check for directories for contents that aren't part of the backup set
    28  					return ""
    29  				}
    30  				contentID = contentID % destSize
    31  			}
    32  			return fmt.Sprintf("test -d %s", globalFPInfo.GetDirForContent(contentID))
    33  		})
    34  		globalCluster.CheckClusterError(remoteOutput, "Backup directories missing or inaccessible", func(contentID int) string {
    35  			return fmt.Sprintf("Backup directory %s missing or inaccessible", globalFPInfo.GetDirForContent(contentID))
    36  		})
    37  	}
    38  }
    39  
    40  func VerifyBackupFileCountOnSegments() {
    41  	remoteOutput := globalCluster.GenerateAndExecuteCommand("Verifying backup file count", cluster.ON_SEGMENTS, func(contentID int) string {
    42  		return fmt.Sprintf("find %s -type f | wc -l", globalFPInfo.GetDirForContent(contentID))
    43  	})
    44  	globalCluster.CheckClusterError(remoteOutput, "Could not verify backup file count", func(contentID int) string {
    45  		return "Could not verify backup file count"
    46  	})
    47  
    48  	// these are the file counts for non-resize restores.
    49  	fileCount := 2 // 1 for the actual data file, 1 for the segment TOC file
    50  	if !backupConfig.SingleDataFile {
    51  		fileCount = len(globalTOC.DataEntries)
    52  	}
    53  
    54  	origSize, destSize, isResizeRestore := GetResizeClusterInfo()
    55  	batchMap := make(map[int]int, len(remoteOutput.Commands))
    56  	for i := 0; i < origSize; i++ {
    57  		batchMap[i%destSize] += fileCount
    58  	}
    59  
    60  	numIncorrect := 0
    61  	for contentID, cmd := range remoteOutput.Commands {
    62  		numFound, _ := strconv.Atoi(strings.TrimSpace(cmd.Stdout))
    63  		if isResizeRestore {
    64  			fileCount = batchMap[contentID]
    65  		}
    66  		if numFound != fileCount {
    67  			gplog.Verbose("Expected to find %d file(s) on segment %d on host %s, but found %d instead.", fileCount, contentID, globalCluster.GetHostForContent(contentID), numFound)
    68  			numIncorrect++
    69  		}
    70  	}
    71  	if numIncorrect > 0 {
    72  		cluster.LogFatalClusterError("Found incorrect number of backup files", cluster.ON_SEGMENTS, numIncorrect)
    73  	}
    74  }
    75  
    76  func VerifyMetadataFilePaths(withStats bool) {
    77  	filetypes := []string{"config", "table of contents", "metadata"}
    78  	missing := false
    79  	for _, filetype := range filetypes {
    80  		filepath := globalFPInfo.GetBackupFilePath(filetype)
    81  		if !iohelper.FileExistsAndIsReadable(filepath) {
    82  			missing = true
    83  			gplog.Error("Cannot access %s file %s", filetype, filepath)
    84  		}
    85  	}
    86  	if withStats {
    87  		filepath := globalFPInfo.GetStatisticsFilePath()
    88  		if !iohelper.FileExistsAndIsReadable(filepath) {
    89  			missing = true
    90  			gplog.Error("Cannot access statistics file %s", filepath)
    91  			gplog.Error(`Note that the "-with-stats" flag must be passed to gpbackup to generate a statistics file.`)
    92  		}
    93  	}
    94  	if missing {
    95  		gplog.Fatal(errors.Errorf("One or more metadata files do not exist or are not readable."), "Cannot proceed with restore")
    96  	}
    97  }
    98  
    99  func GetResizeClusterInfo() (int, int, bool) {
   100  	isResizeCluster := MustGetFlagBool(options.RESIZE_CLUSTER)
   101  	origSize := backupConfig.SegmentCount
   102  	destSize := len(globalCluster.ContentIDs) - 1
   103  	return origSize, destSize, isResizeCluster
   104  }