vitess.io/vitess@v0.16.2/go/vt/mysqlctl/backupengine.go (about) 1 /* 2 Copyright 2019 The Vitess Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package mysqlctl 18 19 import ( 20 "context" 21 "encoding/json" 22 "fmt" 23 "os" 24 "path" 25 "path/filepath" 26 "strings" 27 "time" 28 29 "github.com/spf13/pflag" 30 31 "vitess.io/vitess/go/mysql" 32 "vitess.io/vitess/go/vt/logutil" 33 "vitess.io/vitess/go/vt/mysqlctl/backupstorage" 34 "vitess.io/vitess/go/vt/proto/vtrpc" 35 "vitess.io/vitess/go/vt/servenv" 36 "vitess.io/vitess/go/vt/topo" 37 "vitess.io/vitess/go/vt/vterrors" 38 ) 39 40 var ( 41 // BackupEngineImplementation is the implementation to use for BackupEngine 42 backupEngineImplementation = builtinBackupEngineName 43 ) 44 45 // BackupEngine is the interface to take a backup with a given engine. 46 type BackupEngine interface { 47 ExecuteBackup(ctx context.Context, params BackupParams, bh backupstorage.BackupHandle) (bool, error) 48 ShouldDrainForBackup() bool 49 } 50 51 // BackupParams is the struct that holds all params passed to ExecuteBackup 52 type BackupParams struct { 53 Cnf *Mycnf 54 Mysqld MysqlDaemon 55 Logger logutil.Logger 56 // Concurrency is the value of -concurrency flag given to Backup command 57 // It determines how many files are processed in parallel 58 Concurrency int 59 // Extra env variables used while stopping and starting mysqld 60 HookExtraEnv map[string]string 61 // TopoServer, Keyspace and Shard are used to discover primary tablet 62 TopoServer *topo.Server 63 // Keyspace and Shard are used to infer the directory where backups should be stored 64 Keyspace string 65 Shard string 66 // TabletAlias is used along with backupTime to construct the backup name 67 TabletAlias string 68 // BackupTime is the time at which the backup is being started 69 BackupTime time.Time 70 // Position of last known backup. If non empty, then this value indicates the backup should be incremental 71 // and as of this position 72 IncrementalFromPos string 73 } 74 75 // RestoreParams is the struct that holds all params passed to ExecuteRestore 76 type RestoreParams struct { 77 Cnf *Mycnf 78 Mysqld MysqlDaemon 79 Logger logutil.Logger 80 // Concurrency is the value of --restore_concurrency flag (init restore parameter) 81 // It determines how many files are processed in parallel 82 Concurrency int 83 // Extra env variables for pre-restore and post-restore transform hooks 84 HookExtraEnv map[string]string 85 // DeleteBeforeRestore tells us whether existing data should be deleted before 86 // restoring. This is always set to false when starting a tablet with -restore_from_backup, 87 // but is set to true when executing a RestoreFromBackup command on an already running vttablet 88 DeleteBeforeRestore bool 89 // DbName is the name of the managed database / schema 90 DbName string 91 // Keyspace and Shard are used to infer the directory where backups are stored 92 Keyspace string 93 Shard string 94 // StartTime: if non-zero, look for a backup that was taken at or before this time 95 // Otherwise, find the most recent backup 96 StartTime time.Time 97 // RestoreToPos hints that a point in time recovery is requested, to recover up to the specific given pos. 98 // When empty, the restore is a normal from full backup 99 RestoreToPos mysql.Position 100 // When DryRun is set, no restore actually takes place; but some of its steps are validated. 101 DryRun bool 102 } 103 104 func (p *RestoreParams) IsIncrementalRecovery() bool { 105 return !p.RestoreToPos.IsZero() 106 } 107 108 // RestoreEngine is the interface to restore a backup with a given engine. 109 // Returns the manifest of a backup if successful, otherwise returns an error 110 type RestoreEngine interface { 111 ExecuteRestore(ctx context.Context, params RestoreParams, bh backupstorage.BackupHandle) (*BackupManifest, error) 112 } 113 114 // BackupRestoreEngine is a combination of BackupEngine and RestoreEngine. 115 type BackupRestoreEngine interface { 116 BackupEngine 117 RestoreEngine 118 } 119 120 // BackupRestoreEngineMap contains the registered implementations for 121 // BackupEngine and RestoreEngine. 122 var BackupRestoreEngineMap = make(map[string]BackupRestoreEngine) 123 124 func init() { 125 for _, cmd := range []string{"vtcombo", "vttablet", "vttestserver", "vtctld", "vtbackup"} { 126 servenv.OnParseFor(cmd, registerBackupEngineFlags) 127 } 128 } 129 130 func registerBackupEngineFlags(fs *pflag.FlagSet) { 131 fs.StringVar(&backupEngineImplementation, "backup_engine_implementation", backupEngineImplementation, "Specifies which implementation to use for creating new backups (builtin or xtrabackup). Restores will always be done with whichever engine created a given backup.") 132 } 133 134 // GetBackupEngine returns the BackupEngine implementation that should be used 135 // to create new backups. 136 // 137 // To restore a backup, you should instead get the appropriate RestoreEngine for 138 // a particular backup by calling GetRestoreEngine(). 139 // 140 // This must only be called after flags have been parsed. 141 func GetBackupEngine() (BackupEngine, error) { 142 name := backupEngineImplementation 143 be, ok := BackupRestoreEngineMap[name] 144 if !ok { 145 return nil, vterrors.Errorf(vtrpc.Code_NOT_FOUND, "unknown BackupEngine implementation %q", name) 146 } 147 return be, nil 148 } 149 150 // GetRestoreEngine returns the RestoreEngine implementation to restore a given backup. 151 // It reads the MANIFEST file from the backup to check which engine was used to create it. 152 func GetRestoreEngine(ctx context.Context, backup backupstorage.BackupHandle) (RestoreEngine, error) { 153 manifest, err := GetBackupManifest(ctx, backup) 154 if err != nil { 155 return nil, vterrors.Wrap(err, "can't get backup MANIFEST") 156 } 157 engine := manifest.BackupMethod 158 if engine == "" { 159 // The builtin engine is the only one that ever left BackupMethod unset. 160 engine = builtinBackupEngineName 161 } 162 re, ok := BackupRestoreEngineMap[engine] 163 if !ok { 164 return nil, vterrors.Errorf(vtrpc.Code_NOT_FOUND, "can't restore backup created with %q engine; no such BackupEngine implementation is registered", manifest.BackupMethod) 165 } 166 return re, nil 167 } 168 169 // GetBackupManifest returns the common fields of the MANIFEST file for a given backup. 170 func GetBackupManifest(ctx context.Context, backup backupstorage.BackupHandle) (*BackupManifest, error) { 171 manifest := &BackupManifest{} 172 if err := getBackupManifestInto(ctx, backup, manifest); err != nil { 173 return nil, err 174 } 175 return manifest, nil 176 } 177 178 // getBackupManifestInto fetches and decodes a MANIFEST file into the specified object. 179 func getBackupManifestInto(ctx context.Context, backup backupstorage.BackupHandle, outManifest any) error { 180 file, err := backup.ReadFile(ctx, backupManifestFileName) 181 if err != nil { 182 return vterrors.Wrap(err, "can't read MANIFEST") 183 } 184 defer file.Close() 185 186 if err := json.NewDecoder(file).Decode(outManifest); err != nil { 187 return vterrors.Wrap(err, "can't decode MANIFEST") 188 } 189 return nil 190 } 191 192 // BackupManifest defines the common fields in the MANIFEST file. 193 // All backup engines must include at least these fields. They are free to add 194 // their own custom fields by embedding this struct anonymously into their own 195 // custom struct, as long as their custom fields don't have conflicting names. 196 type BackupManifest struct { 197 // BackupMethod is the name of the backup engine that created this backup. 198 // If this is empty, the backup engine is assumed to be "builtin" since that 199 // was the only engine that ever left this field empty. All new backup 200 // engines are required to set this field to the backup engine name. 201 BackupMethod string 202 203 // Position is the replication position at which the backup was taken. 204 Position mysql.Position 205 206 // PurgedPosition stands for purged GTIDs, information that is necessary for PITR recovery. This is specific to MySQL56 207 PurgedPosition mysql.Position 208 209 // FromPosition is only applicable to incremental backups, and stands for the position from 210 // which incremental changes are backed up. 211 FromPosition mysql.Position 212 213 // Incremental indicates whether this is an incremental backup 214 Incremental bool 215 216 // BackupTime is when the backup was taken in UTC time (RFC 3339 format) 217 BackupTime string 218 219 // FinishedTime is the time (in RFC 3339 format, UTC) at which the backup finished, if known. 220 // Some backups may not set this field if they were created before the field was added. 221 FinishedTime string 222 223 // ServerUUID identifies the server from which backup was taken 224 ServerUUID string 225 226 TabletAlias string 227 228 Keyspace string 229 230 Shard string 231 } 232 233 func (m *BackupManifest) HashKey() string { 234 return fmt.Sprintf("%v/%v/%v/%t/%v", m.BackupMethod, m.Position, m.FromPosition, m.Incremental, m.BackupTime) 235 } 236 237 // ManifestHandleMap is a utility container to map manifests to handles, making it possible to search for, and iterate, handles based on manifests. 238 type ManifestHandleMap struct { 239 mp map[string]backupstorage.BackupHandle 240 } 241 242 func NewManifestHandleMap() *ManifestHandleMap { 243 return &ManifestHandleMap{ 244 mp: map[string]backupstorage.BackupHandle{}, 245 } 246 } 247 248 // Map assigns a handle to a manifest 249 func (m *ManifestHandleMap) Map(manifest *BackupManifest, handle backupstorage.BackupHandle) { 250 if manifest == nil { 251 return 252 } 253 m.mp[manifest.HashKey()] = handle 254 } 255 256 // Handle returns the backup handles assigned to given manifest 257 func (m *ManifestHandleMap) Handle(manifest *BackupManifest) (handle backupstorage.BackupHandle) { 258 return m.mp[manifest.HashKey()] 259 } 260 261 // Handles returns an ordered list of handles, by given list of manifests 262 func (m *ManifestHandleMap) Handles(manifests []*BackupManifest) (handles []backupstorage.BackupHandle) { 263 handles = make([]backupstorage.BackupHandle, 0, len(manifests)) 264 for _, manifest := range manifests { 265 handles = append(handles, m.mp[manifest.HashKey()]) 266 } 267 return handles 268 } 269 270 // RestorePath is an ordered sequence of backup handles & manifests, that can be used to restore from backup. 271 // The path could be empty, in which case it's invalid, there's no way to restore. Otherwise, the path 272 // consists of exactly one full backup, followed by zero or more incremental backups. 273 type RestorePath struct { 274 manifests []*BackupManifest 275 manifestHandleMap *ManifestHandleMap 276 } 277 278 func (p *RestorePath) IsEmpty() bool { 279 return len(p.manifests) == 0 280 } 281 282 func (p *RestorePath) Len() int { 283 return len(p.manifests) 284 } 285 286 func (p *RestorePath) Add(m *BackupManifest) { 287 p.manifests = append(p.manifests, m) 288 } 289 290 // FullBackupHandle returns the single (if any) full backup handle, which is always the first handle in the sequence 291 func (p *RestorePath) FullBackupHandle() backupstorage.BackupHandle { 292 if p.IsEmpty() { 293 return nil 294 } 295 return p.manifestHandleMap.Handle(p.manifests[0]) 296 } 297 298 // IncrementalBackupHandles returns an ordered list of backup handles comprising of the incremental (non-full) path 299 func (p *RestorePath) IncrementalBackupHandles() []backupstorage.BackupHandle { 300 if p.IsEmpty() { 301 return nil 302 } 303 return p.manifestHandleMap.Handles(p.manifests[1:]) 304 } 305 306 func (p *RestorePath) String() string { 307 var sb strings.Builder 308 sb.WriteString("RestorePath: [") 309 for i, m := range p.manifests { 310 if i > 0 { 311 sb.WriteString(", ") 312 } 313 if m.Incremental { 314 sb.WriteString("incremental:") 315 } else { 316 sb.WriteString("full:") 317 } 318 sb.WriteString(p.manifestHandleMap.Handle(m).Name()) 319 } 320 sb.WriteString("]") 321 return sb.String() 322 } 323 324 // FindLatestSuccessfulBackup returns the handle and manifest for the last good backup, 325 // which can be either full or increment 326 func FindLatestSuccessfulBackup(ctx context.Context, logger logutil.Logger, bhs []backupstorage.BackupHandle) (backupstorage.BackupHandle, *BackupManifest, error) { 327 for index := len(bhs) - 1; index >= 0; index-- { 328 bh := bhs[index] 329 // Check that the backup MANIFEST exists and can be successfully decoded. 330 bm, err := GetBackupManifest(ctx, bh) 331 if err != nil { 332 logger.Warningf("Possibly incomplete backup %v on BackupStorage: can't read MANIFEST: %v)", bh.Name(), err) 333 continue 334 } 335 return bh, bm, nil 336 } 337 return nil, nil, ErrNoCompleteBackup 338 } 339 340 // FindBackupToRestore returns a path, a sequence of backup handles, to be restored. 341 // The returned handles stand for valid backups with complete manifests. 342 func FindBackupToRestore(ctx context.Context, params RestoreParams, bhs []backupstorage.BackupHandle) (*RestorePath, error) { 343 // if a StartTime is provided in params, then find a backup that was taken at or before that time 344 checkBackupTime := !params.StartTime.IsZero() 345 backupDir := GetBackupDir(params.Keyspace, params.Shard) 346 347 manifests := make([]*BackupManifest, len(bhs)) 348 manifestHandleMap := NewManifestHandleMap() 349 350 fullBackupIndex := func() int { 351 for index := len(bhs) - 1; index >= 0; index-- { 352 bh := bhs[index] 353 // Check that the backup MANIFEST exists and can be successfully decoded. 354 bm, err := GetBackupManifest(ctx, bh) 355 if err != nil { 356 params.Logger.Warningf("Possibly incomplete backup %v in directory %v on BackupStorage: can't read MANIFEST: %v)", bh.Name(), backupDir, err) 357 continue 358 } 359 // the manifest is valid 360 manifests[index] = bm // manifests's order is insignificant, it will be sorted later on 361 manifestHandleMap.Map(bm, bh) 362 if bm.Incremental { 363 // We're looking for a full backup 364 continue 365 } 366 367 var backupTime time.Time 368 if checkBackupTime { 369 backupTime, err = time.Parse(time.RFC3339, bm.BackupTime) 370 if err != nil { 371 params.Logger.Warningf("Restore: skipping backup %v/%v with invalid time %v: %v", backupDir, bh.Name(), bm.BackupTime, err) 372 continue 373 } 374 } 375 376 switch { 377 case checkBackupTime: 378 // restore to specific time 379 if backupTime.Equal(params.StartTime) || backupTime.Before(params.StartTime) { 380 params.Logger.Infof("Restore: found backup %v %v to restore using the specified timestamp of '%v'", bh.Directory(), bh.Name(), params.StartTime.Format(BackupTimestampFormat)) 381 return index 382 } 383 case !params.RestoreToPos.IsZero(): 384 // restore to specific pos 385 if params.RestoreToPos.GTIDSet.Contains(bm.Position.GTIDSet) { 386 // this is the most recent backup which is <= desired position 387 return index 388 } 389 default: 390 // restore latest full backup 391 params.Logger.Infof("Restore: found latest backup %v %v to restore", bh.Directory(), bh.Name()) 392 return index 393 } 394 } 395 return -1 396 }() 397 if fullBackupIndex < 0 { 398 if checkBackupTime { 399 params.Logger.Errorf("No valid backup found before time %v", params.StartTime.Format(BackupTimestampFormat)) 400 } 401 // There is at least one attempted backup, but none could be read. 402 // This implies there is data we ought to have, so it's not safe to start 403 // up empty. 404 return nil, ErrNoCompleteBackup 405 } 406 // Anything taken before the full backup that we picked, is not of interest: 407 manifests = manifests[fullBackupIndex:] 408 restorePath := &RestorePath{ 409 manifestHandleMap: manifestHandleMap, 410 } 411 if params.RestoreToPos.IsZero() { 412 // restoring from a single full backup: 413 restorePath.Add(manifests[0]) 414 return restorePath, nil 415 } 416 // restore to a position (using incremental backups): 417 // we calculate a possible restore path based on the manifests. The resulting manifests are 418 // a sorted subsequence, with the full backup first, and zero or more incremental backups to follow. 419 manifests, err := FindPITRPath(params.RestoreToPos.GTIDSet, manifests) 420 if err != nil { 421 return nil, err 422 } 423 restorePath.manifests = manifests 424 return restorePath, nil 425 } 426 427 func prepareToRestore(ctx context.Context, cnf *Mycnf, mysqld MysqlDaemon, logger logutil.Logger) error { 428 // shutdown mysqld if it is running 429 logger.Infof("Restore: shutdown mysqld") 430 if err := mysqld.Shutdown(ctx, cnf, true); err != nil { 431 return err 432 } 433 434 logger.Infof("Restore: deleting existing files") 435 if err := removeExistingFiles(cnf); err != nil { 436 return err 437 } 438 439 logger.Infof("Restore: reinit config file") 440 if err := mysqld.ReinitConfig(ctx, cnf); err != nil { 441 return err 442 } 443 return nil 444 } 445 446 // create restore state file 447 func createStateFile(cnf *Mycnf) error { 448 // if we start writing content to this file: 449 // change RD_ONLY to RDWR 450 // change Create to Open 451 // rename func to openStateFile 452 // change to return a *File 453 fname := filepath.Join(cnf.TabletDir(), RestoreState) 454 fd, err := os.Create(fname) 455 if err != nil { 456 return fmt.Errorf("unable to create file: %v", err) 457 } 458 if err = fd.Close(); err != nil { 459 return fmt.Errorf("unable to close file: %v", err) 460 } 461 return nil 462 } 463 464 // delete restore state file 465 func removeStateFile(cnf *Mycnf) error { 466 fname := filepath.Join(cnf.TabletDir(), RestoreState) 467 if err := os.Remove(fname); err != nil { 468 return fmt.Errorf("unable to delete file: %v", err) 469 } 470 return nil 471 } 472 473 // RestoreWasInterrupted tells us whether a previous restore 474 // was interrupted and we are now retrying it 475 func RestoreWasInterrupted(cnf *Mycnf) bool { 476 name := filepath.Join(cnf.TabletDir(), RestoreState) 477 _, err := os.Stat(name) 478 return err == nil 479 } 480 481 // GetBackupDir returns the directory where backups for the 482 // given keyspace/shard are (or will be) stored 483 func GetBackupDir(keyspace, shard string) string { 484 return fmt.Sprintf("%v/%v", keyspace, shard) 485 } 486 487 // isDbDir returns true if the given directory contains a DB 488 func isDbDir(p string) bool { 489 // db.opt is there 490 if _, err := os.Stat(path.Join(p, "db.opt")); err == nil { 491 return true 492 } 493 494 // Look for at least one database file 495 fis, err := os.ReadDir(p) 496 if err != nil { 497 return false 498 } 499 for _, fi := range fis { 500 if strings.HasSuffix(fi.Name(), ".frm") { 501 return true 502 } 503 504 // the MyRocks engine stores data in RocksDB .sst files 505 // https://github.com/facebook/rocksdb/wiki/Rocksdb-BlockBasedTable-Format 506 if strings.HasSuffix(fi.Name(), ".sst") { 507 return true 508 } 509 510 // .frm files were removed in MySQL 8, so we need to check for two other file types 511 // https://dev.mysql.com/doc/refman/8.0/en/data-dictionary-file-removal.html 512 if strings.HasSuffix(fi.Name(), ".ibd") { 513 return true 514 } 515 // https://dev.mysql.com/doc/refman/8.0/en/serialized-dictionary-information.html 516 if strings.HasSuffix(fi.Name(), ".sdi") { 517 return true 518 } 519 } 520 521 return false 522 } 523 524 func addDirectory(fes []FileEntry, base string, baseDir string, subDir string) ([]FileEntry, int64, error) { 525 p := path.Join(baseDir, subDir) 526 var size int64 527 528 entries, err := os.ReadDir(p) 529 if err != nil { 530 return nil, 0, err 531 } 532 for _, entry := range entries { 533 fi, err := entry.Info() 534 if err != nil { 535 return nil, 0, err 536 } 537 538 fes = append(fes, FileEntry{ 539 Base: base, 540 Name: path.Join(subDir, fi.Name()), 541 }) 542 size = size + fi.Size() 543 } 544 return fes, size, nil 545 } 546 547 // addMySQL8DataDictionary checks to see if the new data dictionary introduced in MySQL 8 exists 548 // and adds it to the backup manifest if it does 549 // https://dev.mysql.com/doc/refman/8.0/en/data-dictionary-transactional-storage.html 550 func addMySQL8DataDictionary(fes []FileEntry, base string, baseDir string) ([]FileEntry, int64, error) { 551 filePath := path.Join(baseDir, dataDictionaryFile) 552 553 // no-op if this file doesn't exist 554 fi, err := os.Stat(filePath) 555 if os.IsNotExist(err) { 556 return fes, 0, nil 557 } 558 559 fes = append(fes, FileEntry{ 560 Base: base, 561 Name: dataDictionaryFile, 562 }) 563 564 return fes, fi.Size(), nil 565 } 566 567 func hasDynamicRedoLog(cnf *Mycnf) bool { 568 dynamicRedoLogPath := path.Join(cnf.InnodbLogGroupHomeDir, mysql.DynamicRedoLogSubdir) 569 info, err := os.Stat(dynamicRedoLogPath) 570 return !os.IsNotExist(err) && info.IsDir() 571 } 572 573 func findFilesToBackup(cnf *Mycnf) ([]FileEntry, int64, error) { 574 var err error 575 var result []FileEntry 576 var size, totalSize int64 577 578 // first add innodb files 579 result, totalSize, err = addDirectory(result, backupInnodbDataHomeDir, cnf.InnodbDataHomeDir, "") 580 if err != nil { 581 return nil, 0, err 582 } 583 584 if hasDynamicRedoLog(cnf) { 585 result, size, err = addDirectory(result, backupInnodbLogGroupHomeDir, cnf.InnodbLogGroupHomeDir, mysql.DynamicRedoLogSubdir) 586 } else { 587 result, size, err = addDirectory(result, backupInnodbLogGroupHomeDir, cnf.InnodbLogGroupHomeDir, "") 588 } 589 if err != nil { 590 return nil, 0, err 591 } 592 totalSize = totalSize + size 593 // then add the transactional data dictionary if it exists 594 result, size, err = addMySQL8DataDictionary(result, backupData, cnf.DataDir) 595 if err != nil { 596 return nil, 0, err 597 } 598 totalSize = totalSize + size 599 600 // then add DB directories 601 fis, err := os.ReadDir(cnf.DataDir) 602 if err != nil { 603 return nil, 0, err 604 } 605 606 for _, fi := range fis { 607 p := path.Join(cnf.DataDir, fi.Name()) 608 if isDbDir(p) { 609 result, size, err = addDirectory(result, backupData, cnf.DataDir, fi.Name()) 610 if err != nil { 611 return nil, 0, err 612 } 613 totalSize = totalSize + size 614 } 615 } 616 617 return result, totalSize, nil 618 } 619 620 // binlogFilesToBackup returns the file entries for given binlog files (identified by file name, no path) 621 func binlogFilesToBackup(cnf *Mycnf, binlogFiles []string) (result []FileEntry, totalSize int64, err error) { 622 binlogsDirectory := filepath.Dir(cnf.BinLogPath) 623 entries, err := os.ReadDir(binlogsDirectory) 624 if err != nil { 625 return nil, 0, err 626 } 627 binlogFilesMap := map[string]bool{} 628 for _, b := range binlogFiles { 629 binlogFilesMap[b] = true 630 } 631 for _, entry := range entries { 632 if !binlogFilesMap[entry.Name()] { 633 // not a file we're looking for 634 continue 635 } 636 fi, err := entry.Info() 637 if err != nil { 638 return nil, 0, err 639 } 640 641 result = append(result, FileEntry{ 642 Base: backupBinlogDir, 643 Name: fi.Name(), 644 }) 645 totalSize = totalSize + fi.Size() 646 } 647 return result, totalSize, nil 648 }