github.com/cloudberrydb/gpbackup@v1.0.3-0.20240118031043-5410fd45eed6/utils/plugin.go (about) 1 package utils 2 3 import ( 4 "fmt" 5 "os" 6 "os/exec" 7 path "path/filepath" 8 "strconv" 9 "strings" 10 "time" 11 12 "github.com/blang/semver" 13 "github.com/cloudberrydb/gp-common-go-libs/cluster" 14 "github.com/cloudberrydb/gp-common-go-libs/gplog" 15 "github.com/cloudberrydb/gp-common-go-libs/iohelper" 16 "github.com/cloudberrydb/gp-common-go-libs/operating" 17 "github.com/cloudberrydb/gpbackup/filepath" 18 "github.com/pkg/errors" 19 "gopkg.in/yaml.v2" 20 ) 21 22 const RequiredPluginVersion = "0.3.0" 23 const SecretKeyFile = ".encrypt" 24 25 type PluginConfig struct { 26 ExecutablePath string `yaml:"executablepath"` 27 ConfigPath string `yaml:"-"` 28 Options map[string]string `yaml:"options"` 29 backupPluginVersion string `yaml:"-"` 30 } 31 32 type PluginScope string 33 34 // The COORDINATOR and MASTER scopes are identical in function, we just support 35 // both so that creators of existing plugins as of GPDB 6 need not (re)write 36 // them to support the GPDB 7 verbiage. Plugin code should use COORDINATOR when 37 // carrying out internal functionality, but check for both COORDINATOR and MASTER 38 // when expecting external input. 39 const ( 40 COORDINATOR PluginScope = "coordinator" 41 MASTER PluginScope = "master" 42 SEGMENT_HOST PluginScope = "segment_host" 43 SEGMENT PluginScope = "segment" 44 ) 45 46 func ReadPluginConfig(configFile string) (*PluginConfig, error) { 47 gplog.Info("Reading Plugin Config %s", configFile) 48 config := &PluginConfig{} 49 contents, err := operating.System.ReadFile(configFile) 50 if err != nil { 51 return nil, err 52 } 53 err = yaml.UnmarshalStrict(contents, config) 54 if err != nil { 55 return nil, errors.New("plugin config file is formatted incorrectly") 56 } 57 if config.ExecutablePath == "" { 58 return nil, errors.New("executablepath is required in config file") 59 } 60 if config.Options == nil { 61 config.Options = make(map[string]string) 62 } 63 config.ExecutablePath = os.ExpandEnv(config.ExecutablePath) 64 err = ValidateFullPath(config.ExecutablePath) 65 if err != nil { 66 return nil, err 67 } 68 configFilename := path.Base(configFile) 69 config.ConfigPath = path.Join("/tmp", configFilename) 70 return config, nil 71 } 72 73 func (plugin *PluginConfig) BackupFile(filenamePath string) error { 74 command := fmt.Sprintf("%s backup_file %s %s", plugin.ExecutablePath, plugin.ConfigPath, filenamePath) 75 gplog.Debug("%s", command) 76 output, err := exec.Command("bash", "-c", command).CombinedOutput() 77 if err != nil { 78 return fmt.Errorf("ERROR: Plugin failed to process %s. %s", filenamePath, string(output)) 79 } 80 err = operating.System.Chmod(filenamePath, 0755) 81 return err 82 } 83 84 func (plugin *PluginConfig) MustBackupFile(filenamePath string) { 85 err := plugin.BackupFile(filenamePath) 86 gplog.FatalOnError(err) 87 } 88 89 func (plugin *PluginConfig) MustRestoreFile(filenamePath string) { 90 directory, _ := path.Split(filenamePath) 91 err := operating.System.MkdirAll(directory, 0755) 92 gplog.FatalOnError(err) 93 command := fmt.Sprintf("%s restore_file %s %s", plugin.ExecutablePath, plugin.ConfigPath, filenamePath) 94 gplog.Debug("%s", command) 95 output, err := exec.Command("bash", "-c", command).CombinedOutput() 96 gplog.FatalOnError(err, string(output)) 97 } 98 99 func (plugin *PluginConfig) CheckPluginExistsOnAllHosts(c *cluster.Cluster) string { 100 plugin.checkPluginAPIVersion(c) 101 102 return plugin.getPluginNativeVersion(c) 103 } 104 105 func (plugin *PluginConfig) checkPluginAPIVersion(c *cluster.Cluster) { 106 command := fmt.Sprintf("source %s/greenplum_path.sh && %s plugin_api_version", 107 operating.System.Getenv("GPHOME"), plugin.ExecutablePath) 108 remoteOutput := c.GenerateAndExecuteCommand( 109 "Checking plugin api version on all hosts", 110 cluster.ON_HOSTS&cluster.INCLUDE_COORDINATOR, 111 func(contentID int) string { 112 return command 113 }) 114 gplog.Debug("%s", command) 115 c.CheckClusterError( 116 remoteOutput, 117 fmt.Sprintf("Unable to execute plugin %s", plugin.ExecutablePath), 118 func(contentID int) string { 119 return fmt.Sprintf("Unable to execute plugin %s", plugin.ExecutablePath) 120 }) 121 requiredVersion, err := semver.Make(RequiredPluginVersion) 122 if err != nil { 123 gplog.Fatal(fmt.Errorf("cannot parse hardcoded internal string of required version: %s", 124 err.Error()), RequiredPluginVersion) 125 } 126 numIncorrect := 0 127 var pluginVersion string 128 var version semver.Version 129 index := 0 130 for contentID, cmd := range remoteOutput.Commands { 131 // check consistency of plugin version across all segments 132 tempPluginVersion := strings.TrimSpace(cmd.Stdout) 133 if pluginVersion != "" && tempPluginVersion != "" { 134 if pluginVersion != tempPluginVersion { 135 gplog.Verbose("Plugin %s on content ID %v with API version %s is not consistent "+ 136 "with version on another segment", plugin.ExecutablePath, contentID, version) 137 cluster.LogFatalClusterError("Plugin API version is inconsistent "+ 138 "across segments; please reinstall plugin across segments", 139 cluster.ON_HOSTS&cluster.INCLUDE_COORDINATOR, numIncorrect) 140 } 141 } 142 143 pluginVersion = tempPluginVersion 144 version, err = semver.Make(pluginVersion) 145 if err != nil { 146 gplog.Fatal(fmt.Errorf("ERROR: Unable to parse plugin API version: %s", err.Error()), "") 147 } 148 if !version.GE(requiredVersion) { 149 gplog.Verbose("Plugin %s API version %s is not compatible with supported API "+ 150 "version %s", plugin.ExecutablePath, version, requiredVersion) 151 numIncorrect++ 152 } 153 index++ 154 } 155 if numIncorrect > 0 { 156 cluster.LogFatalClusterError("Plugin API version incorrect", 157 cluster.ON_HOSTS|cluster.INCLUDE_COORDINATOR, numIncorrect) 158 } 159 } 160 161 func (plugin *PluginConfig) getPluginNativeVersion(c *cluster.Cluster) string { 162 command := fmt.Sprintf("source %s/greenplum_path.sh && %s --version", 163 operating.System.Getenv("GPHOME"), plugin.ExecutablePath) 164 remoteOutput := c.GenerateAndExecuteCommand( 165 "Checking plugin version on all hosts", 166 cluster.ON_HOSTS|cluster.INCLUDE_COORDINATOR, 167 func(contentID int) string { 168 return command 169 }) 170 gplog.Debug("%s", command) 171 c.CheckClusterError( 172 remoteOutput, 173 fmt.Sprintf("Unable to execute plugin %s", plugin.ExecutablePath), 174 func(contentID int) string { 175 return fmt.Sprintf("Unable to execute plugin %s", plugin.ExecutablePath) 176 }) 177 numIncorrect := 0 178 var pluginVersion string 179 index := 0 180 badPluginVersion := "" 181 var parts []string 182 for contentID, cmd := range remoteOutput.Commands { 183 tempPluginVersion := strings.TrimSpace(cmd.Stdout) 184 // check consistency of plugin version across all segments 185 if pluginVersion != "" && tempPluginVersion != "" { 186 if pluginVersion != tempPluginVersion { 187 gplog.Verbose("Plugin %s on content ID %v with --version %s is not consistent "+ 188 "with version on another segment", plugin.ExecutablePath, contentID, pluginVersion) 189 cluster.LogFatalClusterError("Plugin --version is inconsistent "+ 190 "across segments; please reinstall plugin across segments", 191 cluster.ON_HOSTS&cluster.INCLUDE_COORDINATOR, numIncorrect) 192 } 193 } 194 195 parts = strings.Split(tempPluginVersion, " ") 196 if len(parts) < 3 { 197 numIncorrect++ 198 badPluginVersion = tempPluginVersion 199 } else { 200 pluginVersion = tempPluginVersion 201 } 202 index++ 203 } 204 if numIncorrect > 0 || pluginVersion == "" { 205 cluster.LogFatalClusterError(fmt.Sprintf("Plugin --version response '%s' incorrect", badPluginVersion), 206 cluster.ON_HOSTS&cluster.INCLUDE_COORDINATOR, numIncorrect) 207 } 208 return parts[2] 209 } 210 211 /*-----------------------------Hooks------------------------------------------*/ 212 213 func (plugin *PluginConfig) SetupPluginForBackup(c *cluster.Cluster, fpInfo filepath.FilePathInfo) { 214 const command = "setup_plugin_for_backup" 215 const verboseCommandMsg = "Running plugin setup for backup on %s" 216 plugin.executeHook(c, verboseCommandMsg, command, fpInfo, false) 217 } 218 219 func (plugin *PluginConfig) SetupPluginForRestore(c *cluster.Cluster, fpInfo filepath.FilePathInfo) { 220 const command = "setup_plugin_for_restore" 221 const verboseCommandMsg = "Running plugin setup for restore on %s" 222 plugin.executeHook(c, verboseCommandMsg, command, fpInfo, false) 223 } 224 225 func (plugin *PluginConfig) CleanupPluginForBackup(c *cluster.Cluster, fpInfo filepath.FilePathInfo) { 226 const command = "cleanup_plugin_for_backup" 227 const verboseCommandMsg = "Running plugin cleanup for backup on %s" 228 plugin.executeHook(c, verboseCommandMsg, command, fpInfo, true) 229 } 230 231 func (plugin *PluginConfig) CleanupPluginForRestore(c *cluster.Cluster, fpInfo filepath.FilePathInfo) { 232 const command = "cleanup_plugin_for_restore" 233 const verboseCommandMsg = "Running plugin cleanup for restore on %s" 234 plugin.executeHook(c, verboseCommandMsg, command, fpInfo, true) 235 } 236 237 func (plugin *PluginConfig) executeHook(c *cluster.Cluster, verboseCommandMsg string, 238 command string, fpInfo filepath.FilePathInfo, noFatal bool) { 239 240 // Execute command once on coordinator 241 scope := MASTER 242 _, _ = plugin.buildHookErrorMsgAndFunc(command, scope) 243 coordinatorContentID := -1 244 coordinatorOutput, coordinatorErr := c.ExecuteLocalCommand( 245 plugin.buildHookString(command, fpInfo, scope, coordinatorContentID)) 246 if coordinatorErr != nil { 247 if noFatal { 248 gplog.Error(coordinatorOutput) 249 return 250 } 251 gplog.Fatal(coordinatorErr, coordinatorOutput) 252 } 253 254 // Execute command once on each segment host 255 scope = SEGMENT_HOST 256 hookFunc := plugin.buildHookFunc(command, fpInfo, scope) 257 verboseErrorMsg, errorMsgFunc := plugin.buildHookErrorMsgAndFunc(command, scope) 258 verboseCommandHostCoordinatorMsg := fmt.Sprintf(verboseCommandMsg, "segment hosts") 259 remoteOutput := c.GenerateAndExecuteCommand(verboseCommandHostCoordinatorMsg, cluster.ON_HOSTS, hookFunc) 260 gplog.Debug("Execute Hook: %s", command) 261 c.CheckClusterError(remoteOutput, verboseErrorMsg, errorMsgFunc, noFatal) 262 263 // Execute command once for each segment 264 scope = SEGMENT 265 hookFunc = plugin.buildHookFunc(command, fpInfo, scope) 266 verboseErrorMsg, errorMsgFunc = plugin.buildHookErrorMsgAndFunc(command, scope) 267 verboseCommandSegMsg := fmt.Sprintf(verboseCommandMsg, "segments") 268 remoteOutput = c.GenerateAndExecuteCommand(verboseCommandSegMsg, cluster.ON_SEGMENTS, hookFunc) 269 c.CheckClusterError(remoteOutput, verboseErrorMsg, errorMsgFunc, noFatal) 270 } 271 272 func (plugin *PluginConfig) buildHookFunc(command string, 273 fpInfo filepath.FilePathInfo, scope PluginScope) func(int) string { 274 return func(contentID int) string { 275 return plugin.buildHookString(command, fpInfo, scope, contentID) 276 } 277 } 278 279 func (plugin *PluginConfig) buildHookString(command string, 280 fpInfo filepath.FilePathInfo, scope PluginScope, contentID int) string { 281 contentIDStr := "" 282 if scope == COORDINATOR || scope == MASTER || scope == SEGMENT { 283 contentIDStr = fmt.Sprintf(`\"%d\"`, contentID) 284 } 285 286 backupDir := fpInfo.GetDirForContent(contentID) 287 return fmt.Sprintf("source %s/greenplum_path.sh && %s %s %s %s %s %s", 288 operating.System.Getenv("GPHOME"), plugin.ExecutablePath, command, 289 plugin.ConfigPath, backupDir, scope, contentIDStr) 290 } 291 292 func (plugin *PluginConfig) buildHookErrorMsgAndFunc(command string, 293 scope PluginScope) (string, func(int) string) { 294 errorMsg := fmt.Sprintf("Unable to execute command: %s at: %s, on: %s", 295 command, plugin.ExecutablePath, scope) 296 return errorMsg, func(contentID int) string { 297 return errorMsg 298 } 299 } 300 301 /*---------------------------------------------------------------------------------------------------*/ 302 303 func (plugin *PluginConfig) CopyPluginConfigToAllHosts(c *cluster.Cluster) { 304 // create a unique config file per segment in order to convey the PGPORT for the segment 305 // to the plugin. At some point in the future, the plugin MAY be able to get PGPORT as 306 // an environmental var, at which time the code to write *specific* config files per segment 307 // can be removed 308 var command string 309 rsync_exists := CommandExists("rsync") 310 if !rsync_exists { 311 gplog.Fatal(errors.New("Failed to find rsync on PATH. Please ensure rsync is installed."), "") 312 } 313 remoteOutput := c.GenerateAndExecuteCommand( 314 "Copying plugin config to all hosts", 315 cluster.ON_LOCAL|cluster.ON_HOSTS|cluster.INCLUDE_COORDINATOR, 316 func(contentIDForSegmentOnHost int) string { 317 hostConfigFile := plugin.createHostPluginConfig(contentIDForSegmentOnHost, c) 318 command = fmt.Sprintf("rsync -e ssh %[1]s %s:%s; rm %[1]s", hostConfigFile, 319 c.GetHostForContent(contentIDForSegmentOnHost), plugin.ConfigPath) 320 return command 321 }) 322 gplog.Debug("%s", command) 323 errMsg := "Unable to copy plugin config" 324 c.CheckClusterError( 325 remoteOutput, 326 errMsg, 327 func(contentID int) string { 328 return errMsg 329 }, 330 ) 331 } 332 333 func (plugin *PluginConfig) DeletePluginConfigWhenEncrypting(c *cluster.Cluster) { 334 if !plugin.UsesEncryption() { 335 return 336 } 337 338 verboseMsg := "Removing plugin config from all hosts" 339 scope := cluster.ON_HOSTS | cluster.INCLUDE_COORDINATOR 340 command := fmt.Sprintf("rm -f %s", plugin.ConfigPath) 341 f := func(contentIDForSegmentOnHost int) string { 342 return command 343 } 344 remoteOutput := c.GenerateAndExecuteCommand(verboseMsg, scope, f) 345 gplog.Debug("%s", command) 346 errMsg := "Unable to remove plugin config" 347 c.CheckClusterError( 348 remoteOutput, 349 errMsg, 350 func(contentID int) string { 351 return errMsg 352 }, 353 true, 354 ) 355 } 356 357 // Creates a valid segment-specific plugin configuration file with unique name 358 func (plugin *PluginConfig) createHostPluginConfig(contentIDForSegmentOnHost int, 359 c *cluster.Cluster) string { 360 // copy "general" config file to temp, and add segment-specific PGPORT value 361 362 segmentSpecificConfigFile := plugin.ConfigPath + "_" + strconv.FormatInt(time.Now().UnixNano(), 10) + "_" + strconv.Itoa(contentIDForSegmentOnHost) 363 file := iohelper.MustOpenFileForWriting(segmentSpecificConfigFile) 364 365 // add current pgport as attribute 366 plugin.Options["pgport"] = strconv.Itoa(c.GetPortForContent(contentIDForSegmentOnHost)) 367 plugin.Options["backup_plugin_version"] = plugin.BackupPluginVersion() 368 if plugin.UsesEncryption() { 369 pluginName, err := plugin.GetPluginName(c) 370 if err != nil { 371 _, _ = fmt.Fprintf(operating.System.Stdout, err.Error()) 372 gplog.Fatal(nil, err.Error()) 373 } 374 375 secret, err := GetSecretKey(pluginName, c.GetDirForContent(-1)) 376 if err != nil { 377 _, _ = fmt.Fprintf(operating.System.Stdout, err.Error()) 378 gplog.Fatal(nil, err.Error()) 379 } 380 plugin.Options[pluginName] = secret 381 } 382 out, err := yaml.Marshal(plugin) 383 gplog.FatalOnError(err) 384 bytes, err := file.Write(out) 385 gplog.FatalOnError(err) 386 err = file.Close() 387 gplog.FatalOnError(err) 388 gplog.Debug("Wrote %d bytes to plugin config %s", bytes, segmentSpecificConfigFile) 389 return segmentSpecificConfigFile 390 } 391 392 func GetSecretKey(pluginName string, mdd string) (string, error) { 393 secretFilePath := path.Join(mdd, SecretKeyFile) 394 contents, err := operating.System.ReadFile(secretFilePath) 395 396 errMsg := fmt.Sprintf("Cannot find encryption key for plugin %s. "+ 397 "Please re-encrypt password(s) so that key becomes available.", pluginName) 398 if err != nil { 399 return "", errors.New(errMsg) 400 } 401 keys := make(map[string]string) 402 _ = yaml.Unmarshal(contents, keys) // if error happens, we catch it because no keys exist 403 key, exists := keys[pluginName] 404 if !exists { 405 return "", errors.New(errMsg) 406 } 407 return key, nil 408 409 } 410 411 func (plugin *PluginConfig) BackupSegmentTOCs(c *cluster.Cluster, fpInfo filepath.FilePathInfo) { 412 var command string 413 remoteOutput := c.GenerateAndExecuteCommand("Waiting for remaining data to be uploaded to plugin destination", 414 cluster.ON_SEGMENTS, 415 func(contentID int) string { 416 tocFile := fpInfo.GetSegmentTOCFilePath(contentID) 417 errorFile := fmt.Sprintf("%s_error", fpInfo.GetSegmentPipeFilePath(contentID)) 418 command = fmt.Sprintf(`while [[ ! -f "%s" && ! -f "%s" ]]; do sleep 1; done; ls "%s"`, tocFile, errorFile, tocFile) 419 return command 420 }) 421 gplog.Debug("%s", command) 422 c.CheckClusterError(remoteOutput, "Error occurred in gpbackup_helper", func(contentID int) string { 423 return "See gpAdminLog for gpbackup_helper on segment host for details: Error occurred with plugin" 424 }) 425 426 remoteOutput = c.GenerateAndExecuteCommand("Processing segment TOC files with plugin", cluster.ON_SEGMENTS, 427 func(contentID int) string { 428 tocFile := fpInfo.GetSegmentTOCFilePath(contentID) 429 return fmt.Sprintf("source %s/greenplum_path.sh && %s backup_file %s %s && "+ 430 "chmod 0755 %s", operating.System.Getenv("GPHOME"), plugin.ExecutablePath, plugin.ConfigPath, tocFile, tocFile) 431 }) 432 c.CheckClusterError(remoteOutput, "Unable to process segment TOC files using plugin", func(contentID int) string { 433 return "See gpAdminLog for gpbackup_helper on segment host for details: Error occurred with plugin" 434 }) 435 } 436 437 func (plugin *PluginConfig) RestoreSegmentTOCs(c *cluster.Cluster, fpInfo filepath.FilePathInfo, isResizeRestore bool, origSize int, destSize int) { 438 var command string 439 batches := 1 440 if isResizeRestore { 441 batches = origSize / destSize 442 if origSize%destSize != 0 { 443 batches += 1 444 } 445 } 446 for b := 0; b < batches; b++ { 447 remoteOutput := c.GenerateAndExecuteCommand("Processing segment TOC files with plugin", cluster.ON_SEGMENTS, func(contentID int) string { 448 origContent := contentID + b*destSize 449 if origContent >= origSize { // Don't try to restore files for contents that aren't part of the backup set 450 return "" 451 } 452 tocFile := fpInfo.GetSegmentTOCFilePath(contentID) 453 // Restore the filename with the origin content to the directory with the destination content 454 tocFile = strings.ReplaceAll(tocFile, fmt.Sprintf("gpbackup_%d", contentID), fmt.Sprintf("gpbackup_%d", origContent)) 455 command = fmt.Sprintf("mkdir -p %s && source %s/greenplum_path.sh && %s restore_file %s %s", 456 fpInfo.GetDirForContent(contentID), operating.System.Getenv("GPHOME"), 457 plugin.ExecutablePath, plugin.ConfigPath, tocFile) 458 return command 459 }) 460 gplog.Debug("%s", command) 461 c.CheckClusterError(remoteOutput, "Unable to process segment TOC files using plugin", func(contentID int) string { 462 return fmt.Sprintf("Unable to process segment TOC files using plugin") 463 }) 464 } 465 } 466 467 func (plugin *PluginConfig) UsesEncryption() bool { 468 return plugin.Options["password_encryption"] == "on" || 469 (plugin.Options["replication"] == "on" && plugin.Options["remote_password_encryption"] == "on") 470 } 471 472 func (plugin *PluginConfig) GetPluginName(c *cluster.Cluster) (pluginName string, err error) { 473 pluginCall := fmt.Sprintf("%s --version", plugin.ExecutablePath) 474 output, err := c.ExecuteLocalCommand(pluginCall) 475 if err != nil { 476 return "", fmt.Errorf("ERROR: Failed to get plugin name. Failed with error: %s", err.Error()) 477 } 478 479 // expects the output to be in "[plugin_name] version [git_version]" 480 s := strings.Split(output, " ") 481 if len(s) != 3 { 482 return "", fmt.Errorf("Unexpected plugin version format: "+ 483 "\"%s\"\nExpected: \"[plugin_name] version [git_version]\"", strings.Join(s, " ")) 484 } 485 486 return s[0], nil 487 } 488 489 func (plugin *PluginConfig) BackupPluginVersion() string { 490 return plugin.backupPluginVersion 491 } 492 493 func (plugin *PluginConfig) SetBackupPluginVersion(timestamp string, historicalPluginVersion string) { 494 if historicalPluginVersion == "" { 495 gplog.Warn("cannot recover plugin version from history using timestamp %s, "+ 496 "so using current plugin version. This is fine unless there is a backwards "+ 497 "compatibility consideration within the plugin", timestamp) 498 plugin.backupPluginVersion = "" 499 } else { 500 plugin.backupPluginVersion = historicalPluginVersion 501 } 502 } 503 504 func (plugin *PluginConfig) CanRestoreSubset() bool { 505 return (plugin.Options["restore_subset"] == "on") || 506 (strings.HasSuffix(plugin.ExecutablePath, "ddboost_plugin") && 507 plugin.Options["restore_subset"] != "off") 508 }