github.com/pingcap/br@v5.3.0-alpha.0.20220125034240-ec59c7b6ce30+incompatible/pkg/task/restore_log.go (about)

     1  // Copyright 2020 PingCAP, Inc. Licensed under Apache-2.0.
     2  
     3  package task
     4  
     5  import (
     6  	"context"
     7  
     8  	"github.com/pingcap/errors"
     9  	"github.com/spf13/cobra"
    10  	"github.com/spf13/pflag"
    11  
    12  	"github.com/pingcap/br/pkg/glue"
    13  	"github.com/pingcap/br/pkg/restore"
    14  	"github.com/pingcap/br/pkg/storage"
    15  )
    16  
    17  const (
    18  	flagStartTS         = "start-ts"
    19  	flagEndTS           = "end-ts"
    20  	flagBatchWriteCount = "write-kvs"
    21  	flagBatchFlushCount = "flush-kvs"
    22  
    23  	// represents kv flush to storage for each table.
    24  	defaultFlushKV = 5120
    25  	// represents kv size flush to storage for each table.
    26  	defaultFlushKVSize = 5 << 20
    27  	// represents kv that write to TiKV once at at time.
    28  	defaultWriteKV = 1280
    29  )
    30  
    31  // LogRestoreConfig is the configuration specific for restore tasks.
    32  type LogRestoreConfig struct {
    33  	Config
    34  
    35  	StartTS uint64
    36  	EndTS   uint64
    37  
    38  	BatchFlushKVPairs int
    39  	BatchFlushKVSize  int64
    40  	BatchWriteKVPairs int
    41  }
    42  
    43  // DefineLogRestoreFlags defines common flags for the backup command.
    44  func DefineLogRestoreFlags(command *cobra.Command) {
    45  	command.Flags().Uint64P(flagStartTS, "", 0, "restore log start ts")
    46  	command.Flags().Uint64P(flagEndTS, "", 0, "restore log end ts")
    47  
    48  	command.Flags().Uint64P(flagBatchWriteCount, "", 0, "the kv count that write to TiKV once at a time")
    49  	command.Flags().Uint64P(flagBatchFlushCount, "", 0, "the kv count that flush from memory to TiKV")
    50  }
    51  
    52  // ParseFromFlags parses the restore-related flags from the flag set.
    53  func (cfg *LogRestoreConfig) ParseFromFlags(flags *pflag.FlagSet) error {
    54  	var err error
    55  	cfg.StartTS, err = flags.GetUint64(flagStartTS)
    56  	if err != nil {
    57  		return errors.Trace(err)
    58  	}
    59  	cfg.EndTS, err = flags.GetUint64(flagEndTS)
    60  	if err != nil {
    61  		return errors.Trace(err)
    62  	}
    63  	err = cfg.Config.ParseFromFlags(flags)
    64  	if err != nil {
    65  		return errors.Trace(err)
    66  	}
    67  	return nil
    68  }
    69  
    70  // adjustRestoreConfig is use for BR(binary) and BR in TiDB.
    71  // When new config was add and not included in parser.
    72  // we should set proper value in this function.
    73  // so that both binary and TiDB will use same default value.
    74  func (cfg *LogRestoreConfig) adjustRestoreConfig() {
    75  	cfg.adjust()
    76  
    77  	if cfg.Config.Concurrency == 0 {
    78  		cfg.Config.Concurrency = defaultRestoreConcurrency
    79  	}
    80  	if cfg.BatchFlushKVPairs == 0 {
    81  		cfg.BatchFlushKVPairs = defaultFlushKV
    82  	}
    83  	if cfg.BatchWriteKVPairs == 0 {
    84  		cfg.BatchWriteKVPairs = defaultWriteKV
    85  	}
    86  	if cfg.BatchFlushKVSize == 0 {
    87  		cfg.BatchFlushKVSize = defaultFlushKVSize
    88  	}
    89  	// write kv count doesn't have to excceed flush kv count.
    90  	if cfg.BatchWriteKVPairs > cfg.BatchFlushKVPairs {
    91  		cfg.BatchWriteKVPairs = cfg.BatchFlushKVPairs
    92  	}
    93  }
    94  
    95  // RunLogRestore starts a restore task inside the current goroutine.
    96  func RunLogRestore(c context.Context, g glue.Glue, cfg *LogRestoreConfig) error {
    97  	cfg.adjustRestoreConfig()
    98  
    99  	ctx, cancel := context.WithCancel(c)
   100  	defer cancel()
   101  
   102  	// Restore needs domain to do DDL.
   103  	needDomain := true
   104  	mgr, err := NewMgr(ctx, g, cfg.PD, cfg.TLS, GetKeepalive(&cfg.Config), cfg.CheckRequirements, needDomain)
   105  	if err != nil {
   106  		return errors.Trace(err)
   107  	}
   108  	defer mgr.Close()
   109  
   110  	u, err := storage.ParseBackend(cfg.Storage, &cfg.BackendOptions)
   111  	if err != nil {
   112  		return errors.Trace(err)
   113  	}
   114  	keepaliveCfg := GetKeepalive(&cfg.Config)
   115  	keepaliveCfg.PermitWithoutStream = true
   116  	client, err := restore.NewRestoreClient(g, mgr.GetPDClient(), mgr.GetStorage(), mgr.GetTLSConfig(), keepaliveCfg)
   117  	if err != nil {
   118  		return errors.Trace(err)
   119  	}
   120  	defer client.Close()
   121  
   122  	opts := storage.ExternalStorageOptions{
   123  		NoCredentials:   cfg.NoCreds,
   124  		SendCredentials: cfg.SendCreds,
   125  		SkipCheckPath:   cfg.SkipCheckPath,
   126  	}
   127  	if err = client.SetStorage(ctx, u, &opts); err != nil {
   128  		return errors.Trace(err)
   129  	}
   130  
   131  	err = client.LoadRestoreStores(ctx)
   132  	if err != nil {
   133  		return errors.Trace(err)
   134  	}
   135  
   136  	logClient, err := restore.NewLogRestoreClient(
   137  		ctx, client, cfg.StartTS, cfg.EndTS, cfg.TableFilter, uint(cfg.Concurrency),
   138  		cfg.BatchFlushKVPairs, cfg.BatchFlushKVSize, cfg.BatchWriteKVPairs)
   139  	if err != nil {
   140  		return errors.Trace(err)
   141  	}
   142  
   143  	return logClient.RestoreLogData(ctx, mgr.GetDomain())
   144  }