vitess.io/vitess@v0.16.2/go/vt/mysqlctl/backupengine.go (about)

     1  /*
     2  Copyright 2019 The Vitess Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package mysqlctl
    18  
    19  import (
    20  	"context"
    21  	"encoding/json"
    22  	"fmt"
    23  	"os"
    24  	"path"
    25  	"path/filepath"
    26  	"strings"
    27  	"time"
    28  
    29  	"github.com/spf13/pflag"
    30  
    31  	"vitess.io/vitess/go/mysql"
    32  	"vitess.io/vitess/go/vt/logutil"
    33  	"vitess.io/vitess/go/vt/mysqlctl/backupstorage"
    34  	"vitess.io/vitess/go/vt/proto/vtrpc"
    35  	"vitess.io/vitess/go/vt/servenv"
    36  	"vitess.io/vitess/go/vt/topo"
    37  	"vitess.io/vitess/go/vt/vterrors"
    38  )
    39  
    40  var (
    41  	// BackupEngineImplementation is the implementation to use for BackupEngine
    42  	backupEngineImplementation = builtinBackupEngineName
    43  )
    44  
    45  // BackupEngine is the interface to take a backup with a given engine.
    46  type BackupEngine interface {
    47  	ExecuteBackup(ctx context.Context, params BackupParams, bh backupstorage.BackupHandle) (bool, error)
    48  	ShouldDrainForBackup() bool
    49  }
    50  
    51  // BackupParams is the struct that holds all params passed to ExecuteBackup
    52  type BackupParams struct {
    53  	Cnf    *Mycnf
    54  	Mysqld MysqlDaemon
    55  	Logger logutil.Logger
    56  	// Concurrency is the value of -concurrency flag given to Backup command
    57  	// It determines how many files are processed in parallel
    58  	Concurrency int
    59  	// Extra env variables used while stopping and starting mysqld
    60  	HookExtraEnv map[string]string
    61  	// TopoServer, Keyspace and Shard are used to discover primary tablet
    62  	TopoServer *topo.Server
    63  	// Keyspace and Shard are used to infer the directory where backups should be stored
    64  	Keyspace string
    65  	Shard    string
    66  	// TabletAlias is used along with backupTime to construct the backup name
    67  	TabletAlias string
    68  	// BackupTime is the time at which the backup is being started
    69  	BackupTime time.Time
    70  	// Position of last known backup. If non empty, then this value indicates the backup should be incremental
    71  	// and as of this position
    72  	IncrementalFromPos string
    73  }
    74  
    75  // RestoreParams is the struct that holds all params passed to ExecuteRestore
    76  type RestoreParams struct {
    77  	Cnf    *Mycnf
    78  	Mysqld MysqlDaemon
    79  	Logger logutil.Logger
    80  	// Concurrency is the value of --restore_concurrency flag (init restore parameter)
    81  	// It determines how many files are processed in parallel
    82  	Concurrency int
    83  	// Extra env variables for pre-restore and post-restore transform hooks
    84  	HookExtraEnv map[string]string
    85  	// DeleteBeforeRestore tells us whether existing data should be deleted before
    86  	// restoring. This is always set to false when starting a tablet with -restore_from_backup,
    87  	// but is set to true when executing a RestoreFromBackup command on an already running vttablet
    88  	DeleteBeforeRestore bool
    89  	// DbName is the name of the managed database / schema
    90  	DbName string
    91  	// Keyspace and Shard are used to infer the directory where backups are stored
    92  	Keyspace string
    93  	Shard    string
    94  	// StartTime: if non-zero, look for a backup that was taken at or before this time
    95  	// Otherwise, find the most recent backup
    96  	StartTime time.Time
    97  	// RestoreToPos hints that a point in time recovery is requested, to recover up to the specific given pos.
    98  	// When empty, the restore is a normal from full backup
    99  	RestoreToPos mysql.Position
   100  	// When DryRun is set, no restore actually takes place; but some of its steps are validated.
   101  	DryRun bool
   102  }
   103  
   104  func (p *RestoreParams) IsIncrementalRecovery() bool {
   105  	return !p.RestoreToPos.IsZero()
   106  }
   107  
   108  // RestoreEngine is the interface to restore a backup with a given engine.
   109  // Returns the manifest of a backup if successful, otherwise returns an error
   110  type RestoreEngine interface {
   111  	ExecuteRestore(ctx context.Context, params RestoreParams, bh backupstorage.BackupHandle) (*BackupManifest, error)
   112  }
   113  
   114  // BackupRestoreEngine is a combination of BackupEngine and RestoreEngine.
   115  type BackupRestoreEngine interface {
   116  	BackupEngine
   117  	RestoreEngine
   118  }
   119  
   120  // BackupRestoreEngineMap contains the registered implementations for
   121  // BackupEngine and RestoreEngine.
   122  var BackupRestoreEngineMap = make(map[string]BackupRestoreEngine)
   123  
   124  func init() {
   125  	for _, cmd := range []string{"vtcombo", "vttablet", "vttestserver", "vtctld", "vtbackup"} {
   126  		servenv.OnParseFor(cmd, registerBackupEngineFlags)
   127  	}
   128  }
   129  
   130  func registerBackupEngineFlags(fs *pflag.FlagSet) {
   131  	fs.StringVar(&backupEngineImplementation, "backup_engine_implementation", backupEngineImplementation, "Specifies which implementation to use for creating new backups (builtin or xtrabackup). Restores will always be done with whichever engine created a given backup.")
   132  }
   133  
   134  // GetBackupEngine returns the BackupEngine implementation that should be used
   135  // to create new backups.
   136  //
   137  // To restore a backup, you should instead get the appropriate RestoreEngine for
   138  // a particular backup by calling GetRestoreEngine().
   139  //
   140  // This must only be called after flags have been parsed.
   141  func GetBackupEngine() (BackupEngine, error) {
   142  	name := backupEngineImplementation
   143  	be, ok := BackupRestoreEngineMap[name]
   144  	if !ok {
   145  		return nil, vterrors.Errorf(vtrpc.Code_NOT_FOUND, "unknown BackupEngine implementation %q", name)
   146  	}
   147  	return be, nil
   148  }
   149  
   150  // GetRestoreEngine returns the RestoreEngine implementation to restore a given backup.
   151  // It reads the MANIFEST file from the backup to check which engine was used to create it.
   152  func GetRestoreEngine(ctx context.Context, backup backupstorage.BackupHandle) (RestoreEngine, error) {
   153  	manifest, err := GetBackupManifest(ctx, backup)
   154  	if err != nil {
   155  		return nil, vterrors.Wrap(err, "can't get backup MANIFEST")
   156  	}
   157  	engine := manifest.BackupMethod
   158  	if engine == "" {
   159  		// The builtin engine is the only one that ever left BackupMethod unset.
   160  		engine = builtinBackupEngineName
   161  	}
   162  	re, ok := BackupRestoreEngineMap[engine]
   163  	if !ok {
   164  		return nil, vterrors.Errorf(vtrpc.Code_NOT_FOUND, "can't restore backup created with %q engine; no such BackupEngine implementation is registered", manifest.BackupMethod)
   165  	}
   166  	return re, nil
   167  }
   168  
   169  // GetBackupManifest returns the common fields of the MANIFEST file for a given backup.
   170  func GetBackupManifest(ctx context.Context, backup backupstorage.BackupHandle) (*BackupManifest, error) {
   171  	manifest := &BackupManifest{}
   172  	if err := getBackupManifestInto(ctx, backup, manifest); err != nil {
   173  		return nil, err
   174  	}
   175  	return manifest, nil
   176  }
   177  
   178  // getBackupManifestInto fetches and decodes a MANIFEST file into the specified object.
   179  func getBackupManifestInto(ctx context.Context, backup backupstorage.BackupHandle, outManifest any) error {
   180  	file, err := backup.ReadFile(ctx, backupManifestFileName)
   181  	if err != nil {
   182  		return vterrors.Wrap(err, "can't read MANIFEST")
   183  	}
   184  	defer file.Close()
   185  
   186  	if err := json.NewDecoder(file).Decode(outManifest); err != nil {
   187  		return vterrors.Wrap(err, "can't decode MANIFEST")
   188  	}
   189  	return nil
   190  }
   191  
   192  // BackupManifest defines the common fields in the MANIFEST file.
   193  // All backup engines must include at least these fields. They are free to add
   194  // their own custom fields by embedding this struct anonymously into their own
   195  // custom struct, as long as their custom fields don't have conflicting names.
   196  type BackupManifest struct {
   197  	// BackupMethod is the name of the backup engine that created this backup.
   198  	// If this is empty, the backup engine is assumed to be "builtin" since that
   199  	// was the only engine that ever left this field empty. All new backup
   200  	// engines are required to set this field to the backup engine name.
   201  	BackupMethod string
   202  
   203  	// Position is the replication position at which the backup was taken.
   204  	Position mysql.Position
   205  
   206  	// PurgedPosition stands for purged GTIDs, information that is necessary for PITR recovery. This is specific to MySQL56
   207  	PurgedPosition mysql.Position
   208  
   209  	// FromPosition is only applicable to incremental backups, and stands for the position from
   210  	// which incremental changes are backed up.
   211  	FromPosition mysql.Position
   212  
   213  	// Incremental indicates whether this is an incremental backup
   214  	Incremental bool
   215  
   216  	// BackupTime is when the backup was taken in UTC time (RFC 3339 format)
   217  	BackupTime string
   218  
   219  	// FinishedTime is the time (in RFC 3339 format, UTC) at which the backup finished, if known.
   220  	// Some backups may not set this field if they were created before the field was added.
   221  	FinishedTime string
   222  
   223  	// ServerUUID identifies the server from which backup was taken
   224  	ServerUUID string
   225  
   226  	TabletAlias string
   227  
   228  	Keyspace string
   229  
   230  	Shard string
   231  }
   232  
   233  func (m *BackupManifest) HashKey() string {
   234  	return fmt.Sprintf("%v/%v/%v/%t/%v", m.BackupMethod, m.Position, m.FromPosition, m.Incremental, m.BackupTime)
   235  }
   236  
   237  // ManifestHandleMap is a utility container to map manifests to handles, making it possible to search for, and iterate, handles based on manifests.
   238  type ManifestHandleMap struct {
   239  	mp map[string]backupstorage.BackupHandle
   240  }
   241  
   242  func NewManifestHandleMap() *ManifestHandleMap {
   243  	return &ManifestHandleMap{
   244  		mp: map[string]backupstorage.BackupHandle{},
   245  	}
   246  }
   247  
   248  // Map assigns a handle to a manifest
   249  func (m *ManifestHandleMap) Map(manifest *BackupManifest, handle backupstorage.BackupHandle) {
   250  	if manifest == nil {
   251  		return
   252  	}
   253  	m.mp[manifest.HashKey()] = handle
   254  }
   255  
   256  // Handle returns the backup handles assigned to given manifest
   257  func (m *ManifestHandleMap) Handle(manifest *BackupManifest) (handle backupstorage.BackupHandle) {
   258  	return m.mp[manifest.HashKey()]
   259  }
   260  
   261  // Handles returns an ordered list of handles, by given list of manifests
   262  func (m *ManifestHandleMap) Handles(manifests []*BackupManifest) (handles []backupstorage.BackupHandle) {
   263  	handles = make([]backupstorage.BackupHandle, 0, len(manifests))
   264  	for _, manifest := range manifests {
   265  		handles = append(handles, m.mp[manifest.HashKey()])
   266  	}
   267  	return handles
   268  }
   269  
   270  // RestorePath is an ordered sequence of backup handles & manifests, that can be used to restore from backup.
   271  // The path could be empty, in which case it's invalid, there's no way to restore. Otherwise, the path
   272  // consists of exactly one full backup, followed by zero or more incremental backups.
   273  type RestorePath struct {
   274  	manifests         []*BackupManifest
   275  	manifestHandleMap *ManifestHandleMap
   276  }
   277  
   278  func (p *RestorePath) IsEmpty() bool {
   279  	return len(p.manifests) == 0
   280  }
   281  
   282  func (p *RestorePath) Len() int {
   283  	return len(p.manifests)
   284  }
   285  
   286  func (p *RestorePath) Add(m *BackupManifest) {
   287  	p.manifests = append(p.manifests, m)
   288  }
   289  
   290  // FullBackupHandle returns the single (if any) full backup handle, which is always the first handle in the sequence
   291  func (p *RestorePath) FullBackupHandle() backupstorage.BackupHandle {
   292  	if p.IsEmpty() {
   293  		return nil
   294  	}
   295  	return p.manifestHandleMap.Handle(p.manifests[0])
   296  }
   297  
   298  // IncrementalBackupHandles returns an ordered list of backup handles comprising of the incremental (non-full) path
   299  func (p *RestorePath) IncrementalBackupHandles() []backupstorage.BackupHandle {
   300  	if p.IsEmpty() {
   301  		return nil
   302  	}
   303  	return p.manifestHandleMap.Handles(p.manifests[1:])
   304  }
   305  
   306  func (p *RestorePath) String() string {
   307  	var sb strings.Builder
   308  	sb.WriteString("RestorePath: [")
   309  	for i, m := range p.manifests {
   310  		if i > 0 {
   311  			sb.WriteString(", ")
   312  		}
   313  		if m.Incremental {
   314  			sb.WriteString("incremental:")
   315  		} else {
   316  			sb.WriteString("full:")
   317  		}
   318  		sb.WriteString(p.manifestHandleMap.Handle(m).Name())
   319  	}
   320  	sb.WriteString("]")
   321  	return sb.String()
   322  }
   323  
   324  // FindLatestSuccessfulBackup returns the handle and manifest for the last good backup,
   325  // which can be either full or increment
   326  func FindLatestSuccessfulBackup(ctx context.Context, logger logutil.Logger, bhs []backupstorage.BackupHandle) (backupstorage.BackupHandle, *BackupManifest, error) {
   327  	for index := len(bhs) - 1; index >= 0; index-- {
   328  		bh := bhs[index]
   329  		// Check that the backup MANIFEST exists and can be successfully decoded.
   330  		bm, err := GetBackupManifest(ctx, bh)
   331  		if err != nil {
   332  			logger.Warningf("Possibly incomplete backup %v on BackupStorage: can't read MANIFEST: %v)", bh.Name(), err)
   333  			continue
   334  		}
   335  		return bh, bm, nil
   336  	}
   337  	return nil, nil, ErrNoCompleteBackup
   338  }
   339  
   340  // FindBackupToRestore returns a path, a sequence of backup handles, to be restored.
   341  // The returned handles stand for valid backups with complete manifests.
   342  func FindBackupToRestore(ctx context.Context, params RestoreParams, bhs []backupstorage.BackupHandle) (*RestorePath, error) {
   343  	// if a StartTime is provided in params, then find a backup that was taken at or before that time
   344  	checkBackupTime := !params.StartTime.IsZero()
   345  	backupDir := GetBackupDir(params.Keyspace, params.Shard)
   346  
   347  	manifests := make([]*BackupManifest, len(bhs))
   348  	manifestHandleMap := NewManifestHandleMap()
   349  
   350  	fullBackupIndex := func() int {
   351  		for index := len(bhs) - 1; index >= 0; index-- {
   352  			bh := bhs[index]
   353  			// Check that the backup MANIFEST exists and can be successfully decoded.
   354  			bm, err := GetBackupManifest(ctx, bh)
   355  			if err != nil {
   356  				params.Logger.Warningf("Possibly incomplete backup %v in directory %v on BackupStorage: can't read MANIFEST: %v)", bh.Name(), backupDir, err)
   357  				continue
   358  			}
   359  			// the manifest is valid
   360  			manifests[index] = bm // manifests's order is insignificant, it will be sorted later on
   361  			manifestHandleMap.Map(bm, bh)
   362  			if bm.Incremental {
   363  				// We're looking for a full backup
   364  				continue
   365  			}
   366  
   367  			var backupTime time.Time
   368  			if checkBackupTime {
   369  				backupTime, err = time.Parse(time.RFC3339, bm.BackupTime)
   370  				if err != nil {
   371  					params.Logger.Warningf("Restore: skipping backup %v/%v with invalid time %v: %v", backupDir, bh.Name(), bm.BackupTime, err)
   372  					continue
   373  				}
   374  			}
   375  
   376  			switch {
   377  			case checkBackupTime:
   378  				// restore to specific time
   379  				if backupTime.Equal(params.StartTime) || backupTime.Before(params.StartTime) {
   380  					params.Logger.Infof("Restore: found backup %v %v to restore using the specified timestamp of '%v'", bh.Directory(), bh.Name(), params.StartTime.Format(BackupTimestampFormat))
   381  					return index
   382  				}
   383  			case !params.RestoreToPos.IsZero():
   384  				// restore to specific pos
   385  				if params.RestoreToPos.GTIDSet.Contains(bm.Position.GTIDSet) {
   386  					// this is the most recent backup which is <= desired position
   387  					return index
   388  				}
   389  			default:
   390  				// restore latest full backup
   391  				params.Logger.Infof("Restore: found latest backup %v %v to restore", bh.Directory(), bh.Name())
   392  				return index
   393  			}
   394  		}
   395  		return -1
   396  	}()
   397  	if fullBackupIndex < 0 {
   398  		if checkBackupTime {
   399  			params.Logger.Errorf("No valid backup found before time %v", params.StartTime.Format(BackupTimestampFormat))
   400  		}
   401  		// There is at least one attempted backup, but none could be read.
   402  		// This implies there is data we ought to have, so it's not safe to start
   403  		// up empty.
   404  		return nil, ErrNoCompleteBackup
   405  	}
   406  	// Anything taken before the full backup that we picked, is not of interest:
   407  	manifests = manifests[fullBackupIndex:]
   408  	restorePath := &RestorePath{
   409  		manifestHandleMap: manifestHandleMap,
   410  	}
   411  	if params.RestoreToPos.IsZero() {
   412  		// restoring from a single full backup:
   413  		restorePath.Add(manifests[0])
   414  		return restorePath, nil
   415  	}
   416  	// restore to a position (using incremental backups):
   417  	// we calculate a possible restore path based on the manifests. The resulting manifests are
   418  	// a sorted subsequence, with the full backup first, and zero or more incremental backups to follow.
   419  	manifests, err := FindPITRPath(params.RestoreToPos.GTIDSet, manifests)
   420  	if err != nil {
   421  		return nil, err
   422  	}
   423  	restorePath.manifests = manifests
   424  	return restorePath, nil
   425  }
   426  
   427  func prepareToRestore(ctx context.Context, cnf *Mycnf, mysqld MysqlDaemon, logger logutil.Logger) error {
   428  	// shutdown mysqld if it is running
   429  	logger.Infof("Restore: shutdown mysqld")
   430  	if err := mysqld.Shutdown(ctx, cnf, true); err != nil {
   431  		return err
   432  	}
   433  
   434  	logger.Infof("Restore: deleting existing files")
   435  	if err := removeExistingFiles(cnf); err != nil {
   436  		return err
   437  	}
   438  
   439  	logger.Infof("Restore: reinit config file")
   440  	if err := mysqld.ReinitConfig(ctx, cnf); err != nil {
   441  		return err
   442  	}
   443  	return nil
   444  }
   445  
   446  // create restore state file
   447  func createStateFile(cnf *Mycnf) error {
   448  	// if we start writing content to this file:
   449  	// change RD_ONLY to RDWR
   450  	// change Create to Open
   451  	// rename func to openStateFile
   452  	// change to return a *File
   453  	fname := filepath.Join(cnf.TabletDir(), RestoreState)
   454  	fd, err := os.Create(fname)
   455  	if err != nil {
   456  		return fmt.Errorf("unable to create file: %v", err)
   457  	}
   458  	if err = fd.Close(); err != nil {
   459  		return fmt.Errorf("unable to close file: %v", err)
   460  	}
   461  	return nil
   462  }
   463  
   464  // delete restore state file
   465  func removeStateFile(cnf *Mycnf) error {
   466  	fname := filepath.Join(cnf.TabletDir(), RestoreState)
   467  	if err := os.Remove(fname); err != nil {
   468  		return fmt.Errorf("unable to delete file: %v", err)
   469  	}
   470  	return nil
   471  }
   472  
   473  // RestoreWasInterrupted tells us whether a previous restore
   474  // was interrupted and we are now retrying it
   475  func RestoreWasInterrupted(cnf *Mycnf) bool {
   476  	name := filepath.Join(cnf.TabletDir(), RestoreState)
   477  	_, err := os.Stat(name)
   478  	return err == nil
   479  }
   480  
   481  // GetBackupDir returns the directory where backups for the
   482  // given keyspace/shard are (or will be) stored
   483  func GetBackupDir(keyspace, shard string) string {
   484  	return fmt.Sprintf("%v/%v", keyspace, shard)
   485  }
   486  
   487  // isDbDir returns true if the given directory contains a DB
   488  func isDbDir(p string) bool {
   489  	// db.opt is there
   490  	if _, err := os.Stat(path.Join(p, "db.opt")); err == nil {
   491  		return true
   492  	}
   493  
   494  	// Look for at least one database file
   495  	fis, err := os.ReadDir(p)
   496  	if err != nil {
   497  		return false
   498  	}
   499  	for _, fi := range fis {
   500  		if strings.HasSuffix(fi.Name(), ".frm") {
   501  			return true
   502  		}
   503  
   504  		// the MyRocks engine stores data in RocksDB .sst files
   505  		// https://github.com/facebook/rocksdb/wiki/Rocksdb-BlockBasedTable-Format
   506  		if strings.HasSuffix(fi.Name(), ".sst") {
   507  			return true
   508  		}
   509  
   510  		// .frm files were removed in MySQL 8, so we need to check for two other file types
   511  		// https://dev.mysql.com/doc/refman/8.0/en/data-dictionary-file-removal.html
   512  		if strings.HasSuffix(fi.Name(), ".ibd") {
   513  			return true
   514  		}
   515  		// https://dev.mysql.com/doc/refman/8.0/en/serialized-dictionary-information.html
   516  		if strings.HasSuffix(fi.Name(), ".sdi") {
   517  			return true
   518  		}
   519  	}
   520  
   521  	return false
   522  }
   523  
   524  func addDirectory(fes []FileEntry, base string, baseDir string, subDir string) ([]FileEntry, int64, error) {
   525  	p := path.Join(baseDir, subDir)
   526  	var size int64
   527  
   528  	entries, err := os.ReadDir(p)
   529  	if err != nil {
   530  		return nil, 0, err
   531  	}
   532  	for _, entry := range entries {
   533  		fi, err := entry.Info()
   534  		if err != nil {
   535  			return nil, 0, err
   536  		}
   537  
   538  		fes = append(fes, FileEntry{
   539  			Base: base,
   540  			Name: path.Join(subDir, fi.Name()),
   541  		})
   542  		size = size + fi.Size()
   543  	}
   544  	return fes, size, nil
   545  }
   546  
   547  // addMySQL8DataDictionary checks to see if the new data dictionary introduced in MySQL 8 exists
   548  // and adds it to the backup manifest if it does
   549  // https://dev.mysql.com/doc/refman/8.0/en/data-dictionary-transactional-storage.html
   550  func addMySQL8DataDictionary(fes []FileEntry, base string, baseDir string) ([]FileEntry, int64, error) {
   551  	filePath := path.Join(baseDir, dataDictionaryFile)
   552  
   553  	// no-op if this file doesn't exist
   554  	fi, err := os.Stat(filePath)
   555  	if os.IsNotExist(err) {
   556  		return fes, 0, nil
   557  	}
   558  
   559  	fes = append(fes, FileEntry{
   560  		Base: base,
   561  		Name: dataDictionaryFile,
   562  	})
   563  
   564  	return fes, fi.Size(), nil
   565  }
   566  
   567  func hasDynamicRedoLog(cnf *Mycnf) bool {
   568  	dynamicRedoLogPath := path.Join(cnf.InnodbLogGroupHomeDir, mysql.DynamicRedoLogSubdir)
   569  	info, err := os.Stat(dynamicRedoLogPath)
   570  	return !os.IsNotExist(err) && info.IsDir()
   571  }
   572  
   573  func findFilesToBackup(cnf *Mycnf) ([]FileEntry, int64, error) {
   574  	var err error
   575  	var result []FileEntry
   576  	var size, totalSize int64
   577  
   578  	// first add innodb files
   579  	result, totalSize, err = addDirectory(result, backupInnodbDataHomeDir, cnf.InnodbDataHomeDir, "")
   580  	if err != nil {
   581  		return nil, 0, err
   582  	}
   583  
   584  	if hasDynamicRedoLog(cnf) {
   585  		result, size, err = addDirectory(result, backupInnodbLogGroupHomeDir, cnf.InnodbLogGroupHomeDir, mysql.DynamicRedoLogSubdir)
   586  	} else {
   587  		result, size, err = addDirectory(result, backupInnodbLogGroupHomeDir, cnf.InnodbLogGroupHomeDir, "")
   588  	}
   589  	if err != nil {
   590  		return nil, 0, err
   591  	}
   592  	totalSize = totalSize + size
   593  	// then add the transactional data dictionary if it exists
   594  	result, size, err = addMySQL8DataDictionary(result, backupData, cnf.DataDir)
   595  	if err != nil {
   596  		return nil, 0, err
   597  	}
   598  	totalSize = totalSize + size
   599  
   600  	// then add DB directories
   601  	fis, err := os.ReadDir(cnf.DataDir)
   602  	if err != nil {
   603  		return nil, 0, err
   604  	}
   605  
   606  	for _, fi := range fis {
   607  		p := path.Join(cnf.DataDir, fi.Name())
   608  		if isDbDir(p) {
   609  			result, size, err = addDirectory(result, backupData, cnf.DataDir, fi.Name())
   610  			if err != nil {
   611  				return nil, 0, err
   612  			}
   613  			totalSize = totalSize + size
   614  		}
   615  	}
   616  
   617  	return result, totalSize, nil
   618  }
   619  
   620  // binlogFilesToBackup returns the file entries for given binlog files (identified by file name, no path)
   621  func binlogFilesToBackup(cnf *Mycnf, binlogFiles []string) (result []FileEntry, totalSize int64, err error) {
   622  	binlogsDirectory := filepath.Dir(cnf.BinLogPath)
   623  	entries, err := os.ReadDir(binlogsDirectory)
   624  	if err != nil {
   625  		return nil, 0, err
   626  	}
   627  	binlogFilesMap := map[string]bool{}
   628  	for _, b := range binlogFiles {
   629  		binlogFilesMap[b] = true
   630  	}
   631  	for _, entry := range entries {
   632  		if !binlogFilesMap[entry.Name()] {
   633  			// not a file we're looking for
   634  			continue
   635  		}
   636  		fi, err := entry.Info()
   637  		if err != nil {
   638  			return nil, 0, err
   639  		}
   640  
   641  		result = append(result, FileEntry{
   642  			Base: backupBinlogDir,
   643  			Name: fi.Name(),
   644  		})
   645  		totalSize = totalSize + fi.Size()
   646  	}
   647  	return result, totalSize, nil
   648  }