github.com/tuhaihe/gpbackup@v1.0.3/restore/wrappers.go (about) 1 package restore 2 3 import ( 4 "fmt" 5 path "path/filepath" 6 "strconv" 7 "strings" 8 9 "github.com/tuhaihe/gp-common-go-libs/dbconn" 10 "github.com/tuhaihe/gp-common-go-libs/gplog" 11 "github.com/tuhaihe/gp-common-go-libs/iohelper" 12 "github.com/tuhaihe/gpbackup/filepath" 13 "github.com/tuhaihe/gpbackup/history" 14 "github.com/tuhaihe/gpbackup/options" 15 "github.com/tuhaihe/gpbackup/report" 16 "github.com/tuhaihe/gpbackup/toc" 17 "github.com/tuhaihe/gpbackup/utils" 18 ) 19 20 /* 21 * This file contains wrapper functions that group together functions relating 22 * to querying and restoring metadata, so that the logic for each object type 23 * can all be in one place and restore.go can serve as a high-level look at the 24 * overall restore flow. 25 */ 26 27 /* 28 * Setup and validation wrapper functions 29 */ 30 31 /* 32 * Filter structure to filter schemas and relations 33 */ 34 type Filters struct { 35 includeSchemas []string 36 excludeSchemas []string 37 includeRelations []string 38 excludeRelations []string 39 } 40 41 func NewFilters(inSchema []string, exSchemas []string, inRelations []string, exRelations []string) Filters { 42 f := Filters{} 43 f.includeSchemas = inSchema 44 f.excludeSchemas = exSchemas 45 f.includeRelations = inRelations 46 f.excludeRelations = exRelations 47 return f 48 } 49 50 func filtersEmpty(filters Filters) bool { 51 return len(filters.includeSchemas) == 0 && len(filters.excludeSchemas) == 0 && len(filters.includeRelations) == 0 && len(filters.excludeRelations) == 0 52 } 53 54 func SetLoggerVerbosity() { 55 if MustGetFlagBool(options.QUIET) { 56 gplog.SetVerbosity(gplog.LOGERROR) 57 } else if MustGetFlagBool(options.DEBUG) { 58 gplog.SetVerbosity(gplog.LOGDEBUG) 59 } else if MustGetFlagBool(options.VERBOSE) { 60 gplog.SetVerbosity(gplog.LOGVERBOSE) 61 } 62 } 63 64 func CreateConnectionPool(unquotedDBName string) { 65 connectionPool = dbconn.NewDBConnFromEnvironment(unquotedDBName) 66 if FlagChanged(options.COPY_QUEUE_SIZE) { 67 connectionPool.MustConnect(MustGetFlagInt(options.COPY_QUEUE_SIZE)) 68 } else { 69 connectionPool.MustConnect(MustGetFlagInt(options.JOBS)) 70 } 71 utils.ValidateGPDBVersionCompatibility(connectionPool) 72 } 73 74 func InitializeConnectionPool(backupTimestamp string, restoreTimestamp string, unquotedDBName string) { 75 CreateConnectionPool(unquotedDBName) 76 resizeRestore := MustGetFlagBool(options.RESIZE_CLUSTER) 77 setupQuery := fmt.Sprintf("SET application_name TO 'gprestore_%s_%s';", backupTimestamp, restoreTimestamp) 78 setupQuery += ` 79 SET search_path TO pg_catalog; 80 SET gp_default_storage_options=''; 81 SET statement_timeout = 0; 82 SET check_function_bodies = false; 83 SET client_min_messages = error; 84 SET standard_conforming_strings = on; 85 SET default_with_oids = off; 86 ` 87 88 setupQuery += "SET gp_ignore_error_table = on;\n" 89 90 setupQuery += "SET allow_system_table_mods = true;\n" 91 setupQuery += "SET lock_timeout = 0;\n" 92 setupQuery += "SET default_transaction_read_only = off;\n" 93 setupQuery += "SET xmloption = content;\n" 94 95 // If the backup is from a GPDB version less than 6.0, 96 // we need to use legacy hash operators when restoring 97 // the tables, unless we're restoring to a cluster of 98 // a different size since in that case the data will be 99 // redistributed during the restore process. 100 backupConfigMajorVer, _ := strconv.Atoi(strings.Split(backupConfig.DatabaseVersion, ".")[0]) 101 if false && backupConfigMajorVer < 6 && !resizeRestore { 102 setupQuery += "SET gp_use_legacy_hashops = on;\n" 103 gplog.Warn("This backup set was taken on a version of Greenplum prior to 6.x. This restore will use the legacy hash operators when loading data.") 104 gplog.Warn("To use the new Greenplum 6.x default hash operators, these tables will need to be redistributed.") 105 gplog.Warn("For more information, refer to the migration guide located as https://docs.greenplum.org/latest/install_guide/migrate.html.") 106 } 107 108 // If we're restoring to a different-sized cluster, disable the 109 // distribution key check because the data won't necessarily 110 // match initially and will be redistributed after the restore. 111 if resizeRestore { 112 setupQuery += "SET gp_enable_segment_copy_checking TO off;\n" 113 } 114 115 setupQuery += SetMaxCsvLineLengthQuery(connectionPool) 116 117 // Always disable gp_autostats_mode to prevent automatic ANALYZE 118 // during COPY FROM SEGMENT. ANALYZE should be run separately. 119 setupQuery += "SET gp_autostats_mode = 'none';\n" 120 121 for i := 0; i < connectionPool.NumConns; i++ { 122 connectionPool.MustExec(setupQuery, i) 123 } 124 } 125 126 func SetMaxCsvLineLengthQuery(connectionPool *dbconn.DBConn) string { 127 return "" 128 } 129 130 func InitializeBackupConfig() { 131 backupConfig = history.ReadConfigFile(globalFPInfo.GetConfigFilePath()) 132 utils.InitializePipeThroughParameters(backupConfig.Compressed, backupConfig.CompressionType, 0) 133 report.EnsureBackupVersionCompatibility(backupConfig.BackupVersion, version) 134 report.EnsureDatabaseVersionCompatibility(backupConfig.DatabaseVersion, connectionPool.Version) 135 } 136 137 func BackupConfigurationValidation() { 138 if !backupConfig.MetadataOnly { 139 gplog.Verbose("Gathering information on backup directories") 140 VerifyBackupDirectoriesExistOnAllHosts() 141 } 142 143 VerifyMetadataFilePaths(MustGetFlagBool(options.WITH_STATS)) 144 145 tocFilename := globalFPInfo.GetTOCFilePath() 146 globalTOC = toc.NewTOC(tocFilename) 147 globalTOC.InitializeMetadataEntryMap() 148 149 // Legacy backups prior to the incremental feature would have no restoreplan yaml element 150 if isLegacyBackup := backupConfig.RestorePlan == nil; isLegacyBackup { 151 SetRestorePlanForLegacyBackup(globalTOC, globalFPInfo.Timestamp, backupConfig) 152 } 153 154 ValidateBackupFlagCombinations() 155 156 validateFilterListsInBackupSet() 157 } 158 159 func SetRestorePlanForLegacyBackup(toc *toc.TOC, backupTimestamp string, backupConfig *history.BackupConfig) { 160 tableFQNs := make([]string, 0, len(toc.DataEntries)) 161 for _, entry := range toc.DataEntries { 162 entryFQN := utils.MakeFQN(entry.Schema, entry.Name) 163 tableFQNs = append(tableFQNs, entryFQN) 164 } 165 backupConfig.RestorePlan = []history.RestorePlanEntry{ 166 {Timestamp: backupTimestamp, TableFQNs: tableFQNs}, 167 } 168 } 169 170 func RecoverMetadataFilesUsingPlugin() { 171 var err error 172 pluginConfig, err = utils.ReadPluginConfig(MustGetFlagString(options.PLUGIN_CONFIG)) 173 gplog.FatalOnError(err) 174 configFilename := path.Base(pluginConfig.ConfigPath) 175 configDirname := path.Dir(pluginConfig.ConfigPath) 176 pluginConfig.ConfigPath = path.Join(configDirname, history.CurrentTimestamp()+"_"+configFilename) 177 _ = cmdFlags.Set(options.PLUGIN_CONFIG, pluginConfig.ConfigPath) 178 gplog.Info("plugin config path: %s", pluginConfig.ConfigPath) 179 180 pluginConfig.CheckPluginExistsOnAllHosts(globalCluster) 181 182 timestamp := MustGetFlagString(options.TIMESTAMP) 183 historicalPluginVersion := FindHistoricalPluginVersion(timestamp) 184 pluginConfig.SetBackupPluginVersion(timestamp, historicalPluginVersion) 185 186 pluginConfig.CopyPluginConfigToAllHosts(globalCluster) 187 pluginConfig.SetupPluginForRestore(globalCluster, globalFPInfo) 188 189 metadataFiles := []string{globalFPInfo.GetConfigFilePath(), globalFPInfo.GetMetadataFilePath(), 190 globalFPInfo.GetBackupReportFilePath()} 191 if MustGetFlagBool(options.WITH_STATS) { 192 metadataFiles = append(metadataFiles, globalFPInfo.GetStatisticsFilePath()) 193 } 194 for _, filename := range metadataFiles { 195 pluginConfig.MustRestoreFile(filename) 196 } 197 198 InitializeBackupConfig() 199 200 var fpInfoList []filepath.FilePathInfo 201 if backupConfig.MetadataOnly { 202 fpInfoList = []filepath.FilePathInfo{globalFPInfo} 203 } else { 204 fpInfoList = GetBackupFPInfoListFromRestorePlan() 205 } 206 207 for _, fpInfo := range fpInfoList { 208 pluginConfig.MustRestoreFile(fpInfo.GetTOCFilePath()) 209 if backupConfig.SingleDataFile { 210 origSize, destSize, isResizeRestore := GetResizeClusterInfo() 211 pluginConfig.RestoreSegmentTOCs(globalCluster, fpInfo, isResizeRestore, origSize, destSize) 212 } 213 } 214 } 215 216 func FindHistoricalPluginVersion(timestamp string) string { 217 // in order for plugins to implement backwards compatibility, 218 // first, read history from coordinator and provide the historical version 219 // of the plugin that was used to create the original backup 220 221 // adapted from incremental GetLatestMatchingBackupTimestamp 222 var historicalPluginVersion string 223 if iohelper.FileExistsAndIsReadable(globalFPInfo.GetBackupHistoryFilePath()) { 224 hist, _, err := history.NewHistory(globalFPInfo.GetBackupHistoryFilePath()) 225 gplog.FatalOnError(err) 226 foundBackupConfig := hist.FindBackupConfig(timestamp) 227 if foundBackupConfig != nil { 228 historicalPluginVersion = foundBackupConfig.PluginVersion 229 } 230 } 231 return historicalPluginVersion 232 } 233 234 /* 235 * Metadata and/or data restore wrapper functions 236 */ 237 238 func GetRestoreMetadataStatements(section string, filename string, includeObjectTypes []string, excludeObjectTypes []string) []toc.StatementWithType { 239 var statements []toc.StatementWithType 240 statements = GetRestoreMetadataStatementsFiltered(section, filename, includeObjectTypes, excludeObjectTypes, Filters{}) 241 return statements 242 } 243 244 func GetRestoreMetadataStatementsFiltered(section string, filename string, includeObjectTypes []string, excludeObjectTypes []string, filters Filters) []toc.StatementWithType { 245 metadataFile := iohelper.MustOpenFileForReading(filename) 246 var statements []toc.StatementWithType 247 var inSchemas, exSchemas, inRelations, exRelations []string 248 if !filtersEmpty(filters) { 249 inSchemas = filters.includeSchemas 250 exSchemas = filters.excludeSchemas 251 inRelations = filters.includeRelations 252 exRelations = filters.excludeRelations 253 fpInfoList := GetBackupFPInfoListFromRestorePlan() 254 for _, fpInfo := range fpInfoList { 255 tocFilename := fpInfo.GetTOCFilePath() 256 tocfile := toc.NewTOC(tocFilename) 257 inRelations = append(inRelations, toc.GetIncludedPartitionRoots(tocfile.DataEntries, inRelations)...) 258 } 259 // Update include schemas for schema restore if include table is set 260 if utils.Exists(includeObjectTypes, "SCHEMA") { 261 for _, inRelation := range inRelations { 262 schema := inRelation[:strings.Index(inRelation, ".")] 263 if !utils.Exists(inSchemas, schema) { 264 inSchemas = append(inSchemas, schema) 265 } 266 } 267 // reset relation list as these were required only to extract schemas from inRelations 268 inRelations = nil 269 exRelations = nil 270 } 271 } 272 statements = globalTOC.GetSQLStatementForObjectTypes(section, metadataFile, includeObjectTypes, excludeObjectTypes, inSchemas, exSchemas, inRelations, exRelations) 273 return statements 274 } 275 276 func ExecuteRestoreMetadataStatements(statements []toc.StatementWithType, objectsTitle string, progressBar utils.ProgressBar, showProgressBar int, executeInParallel bool) int32 { 277 var numErrors int32 278 if progressBar == nil { 279 numErrors = ExecuteStatementsAndCreateProgressBar(statements, objectsTitle, showProgressBar, executeInParallel) 280 } else { 281 numErrors = ExecuteStatements(statements, progressBar, executeInParallel) 282 } 283 284 return numErrors 285 } 286 287 func GetBackupFPInfoListFromRestorePlan() []filepath.FilePathInfo { 288 fpInfoList := make([]filepath.FilePathInfo, 0) 289 for _, entry := range backupConfig.RestorePlan { 290 segPrefix, err := filepath.ParseSegPrefix(MustGetFlagString(options.BACKUP_DIR)) 291 gplog.FatalOnError(err) 292 293 fpInfo := filepath.NewFilePathInfo(globalCluster, MustGetFlagString(options.BACKUP_DIR), entry.Timestamp, segPrefix) 294 fpInfoList = append(fpInfoList, fpInfo) 295 } 296 297 return fpInfoList 298 } 299 300 func GetBackupFPInfoForTimestamp(timestamp string) filepath.FilePathInfo { 301 segPrefix, err := filepath.ParseSegPrefix(MustGetFlagString(options.BACKUP_DIR)) 302 gplog.FatalOnError(err) 303 fpInfo := filepath.NewFilePathInfo(globalCluster, MustGetFlagString(options.BACKUP_DIR), timestamp, segPrefix) 304 return fpInfo 305 } 306 307 /* 308 * The first time this function is called, it retrieves the session GUCs from the 309 * predata file and processes them appropriately, then it returns them so they 310 * can be used in later calls without the file access and processing overhead. 311 */ 312 func setGUCsForConnection(gucStatements []toc.StatementWithType, whichConn int) []toc.StatementWithType { 313 if gucStatements == nil { 314 objectTypes := []string{"SESSION GUCS"} 315 gucStatements = GetRestoreMetadataStatements("global", globalFPInfo.GetMetadataFilePath(), objectTypes, []string{}) 316 } 317 ExecuteStatementsAndCreateProgressBar(gucStatements, "", utils.PB_NONE, false, whichConn) 318 return gucStatements 319 } 320 321 func RestoreSchemas(schemaStatements []toc.StatementWithType, progressBar utils.ProgressBar) { 322 numErrors := 0 323 for _, schema := range schemaStatements { 324 _, err := connectionPool.Exec(schema.Statement, 0) 325 if err != nil { 326 if strings.Contains(err.Error(), "already exists") { 327 gplog.Warn("Schema %s already exists", schema.Name) 328 } else { 329 errMsg := fmt.Sprintf("Error encountered while creating schema %s", schema.Name) 330 if MustGetFlagBool(options.ON_ERROR_CONTINUE) { 331 gplog.Verbose(fmt.Sprintf("%s: %s", errMsg, err.Error())) 332 numErrors++ 333 } else { 334 gplog.Fatal(err, errMsg) 335 } 336 } 337 } 338 progressBar.Increment() 339 } 340 if numErrors > 0 { 341 gplog.Error("Encountered %d errors during schema restore; see log file %s for a list of errors.", numErrors, gplog.GetLogFilePath()) 342 } 343 } 344 345 func GetExistingTableFQNs() ([]string, error) { 346 existingTableFQNs := make([]string, 0) 347 var relkindFilter string 348 349 relkindFilter = "'r', 'S', 'f', 'p'" 350 query := fmt.Sprintf(`SELECT quote_ident(n.nspname) || '.' || quote_ident(c.relname) 351 FROM pg_catalog.pg_class c 352 LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace 353 WHERE c.relkind IN (%s) 354 AND n.nspname !~ '^pg_' 355 AND n.nspname !~ '^gp_' 356 AND n.nspname <> 'information_schema' 357 ORDER BY 1;`, relkindFilter) 358 359 err := connectionPool.Select(&existingTableFQNs, query) 360 return existingTableFQNs, err 361 } 362 363 func GetExistingSchemas() ([]string, error) { 364 existingSchemas := make([]string, 0) 365 366 query := `SELECT n.nspname AS "Name" 367 FROM pg_catalog.pg_namespace n 368 WHERE n.nspname !~ '^pg_' AND n.nspname <> 'information_schema' 369 ORDER BY 1;` 370 371 err := connectionPool.Select(&existingSchemas, query) 372 return existingSchemas, err 373 } 374 375 func TruncateTable(tableFQN string, whichConn int) error { 376 gplog.Verbose("Truncating table %s prior to restoring data", tableFQN) 377 _, err := connectionPool.Exec(`TRUNCATE `+tableFQN, whichConn) 378 return err 379 }