github.com/pingcap/tidb-lightning@v5.0.0-rc.0.20210428090220-84b649866577+incompatible/lightning/lightning.go (about)

     1  // Copyright 2019 PingCAP, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // See the License for the specific language governing permissions and
    12  // limitations under the License.
    13  
    14  package lightning
    15  
    16  import (
    17  	"compress/gzip"
    18  	"context"
    19  	"encoding/json"
    20  	"fmt"
    21  	"io"
    22  	"io/ioutil"
    23  	"net"
    24  	"net/http"
    25  	"net/http/pprof"
    26  	"os"
    27  	"sort"
    28  	"strconv"
    29  	"strings"
    30  	"sync"
    31  	"time"
    32  
    33  	"github.com/pingcap/br/pkg/storage"
    34  	"github.com/pingcap/errors"
    35  	"github.com/pingcap/failpoint"
    36  	"github.com/pingcap/tidb-lightning/lightning/glue"
    37  	"github.com/prometheus/client_golang/prometheus/promhttp"
    38  	"github.com/shurcooL/httpgzip"
    39  	"go.uber.org/zap"
    40  	"go.uber.org/zap/zapcore"
    41  	"golang.org/x/net/http/httpproxy"
    42  
    43  	"github.com/pingcap/tidb-lightning/lightning/backend"
    44  	"github.com/pingcap/tidb-lightning/lightning/checkpoints"
    45  	"github.com/pingcap/tidb-lightning/lightning/common"
    46  	"github.com/pingcap/tidb-lightning/lightning/config"
    47  	"github.com/pingcap/tidb-lightning/lightning/log"
    48  	"github.com/pingcap/tidb-lightning/lightning/mydump"
    49  	"github.com/pingcap/tidb-lightning/lightning/restore"
    50  	"github.com/pingcap/tidb-lightning/lightning/web"
    51  )
    52  
    53  type Lightning struct {
    54  	globalCfg *config.GlobalConfig
    55  	globalTLS *common.TLS
    56  	// taskCfgs is the list of task configurations enqueued in the server mode
    57  	taskCfgs   *config.ConfigList
    58  	ctx        context.Context
    59  	shutdown   context.CancelFunc // for whole lightning context
    60  	server     http.Server
    61  	serverAddr net.Addr
    62  	serverLock sync.Mutex
    63  
    64  	cancelLock sync.Mutex
    65  	curTask    *config.Config
    66  	cancel     context.CancelFunc // for per task context, which maybe different from lightning context
    67  }
    68  
    69  func initEnv(cfg *config.GlobalConfig) error {
    70  	return log.InitLogger(&cfg.App.Config, cfg.TiDB.LogLevel)
    71  }
    72  
    73  func New(globalCfg *config.GlobalConfig) *Lightning {
    74  	if err := initEnv(globalCfg); err != nil {
    75  		fmt.Println("Failed to initialize environment:", err)
    76  		os.Exit(1)
    77  	}
    78  
    79  	tls, err := common.NewTLS(globalCfg.Security.CAPath, globalCfg.Security.CertPath, globalCfg.Security.KeyPath, globalCfg.App.StatusAddr)
    80  	if err != nil {
    81  		log.L().Fatal("failed to load TLS certificates", zap.Error(err))
    82  	}
    83  
    84  	log.InitRedact(globalCfg.Security.RedactInfoLog)
    85  
    86  	ctx, shutdown := context.WithCancel(context.Background())
    87  	return &Lightning{
    88  		globalCfg: globalCfg,
    89  		globalTLS: tls,
    90  		ctx:       ctx,
    91  		shutdown:  shutdown,
    92  	}
    93  }
    94  
    95  func (l *Lightning) GoServe() error {
    96  	handleSigUsr1(func() {
    97  		l.serverLock.Lock()
    98  		statusAddr := l.globalCfg.App.StatusAddr
    99  		shouldStartServer := len(statusAddr) == 0
   100  		if shouldStartServer {
   101  			l.globalCfg.App.StatusAddr = ":"
   102  		}
   103  		l.serverLock.Unlock()
   104  
   105  		if shouldStartServer {
   106  			// open a random port and start the server if SIGUSR1 is received.
   107  			if err := l.goServe(":", os.Stderr); err != nil {
   108  				log.L().Warn("failed to start HTTP server", log.ShortError(err))
   109  			}
   110  		} else {
   111  			// just prints the server address if it is already started.
   112  			log.L().Info("already started HTTP server", zap.Stringer("address", l.serverAddr))
   113  		}
   114  	})
   115  
   116  	l.serverLock.Lock()
   117  	statusAddr := l.globalCfg.App.StatusAddr
   118  	l.serverLock.Unlock()
   119  
   120  	if len(statusAddr) == 0 {
   121  		return nil
   122  	}
   123  	return l.goServe(statusAddr, ioutil.Discard)
   124  }
   125  
   126  func (l *Lightning) goServe(statusAddr string, realAddrWriter io.Writer) error {
   127  	mux := http.NewServeMux()
   128  	mux.Handle("/", http.RedirectHandler("/web/", http.StatusFound))
   129  	mux.Handle("/metrics", promhttp.Handler())
   130  
   131  	mux.HandleFunc("/debug/pprof/", pprof.Index)
   132  	mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
   133  	mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
   134  	mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
   135  	mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
   136  
   137  	handleTasks := http.StripPrefix("/tasks", http.HandlerFunc(l.handleTask))
   138  	mux.Handle("/tasks", handleTasks)
   139  	mux.Handle("/tasks/", handleTasks)
   140  	mux.HandleFunc("/progress/task", handleProgressTask)
   141  	mux.HandleFunc("/progress/table", handleProgressTable)
   142  	mux.HandleFunc("/pause", handlePause)
   143  	mux.HandleFunc("/resume", handleResume)
   144  	mux.HandleFunc("/loglevel", handleLogLevel)
   145  
   146  	mux.Handle("/web/", http.StripPrefix("/web", httpgzip.FileServer(web.Res, httpgzip.FileServerOptions{
   147  		IndexHTML: true,
   148  		ServeError: func(w http.ResponseWriter, req *http.Request, err error) {
   149  			if os.IsNotExist(err) && !strings.Contains(req.URL.Path, ".") {
   150  				http.Redirect(w, req, "/web/", http.StatusFound)
   151  			} else {
   152  				httpgzip.NonSpecific(w, req, err)
   153  			}
   154  		},
   155  	})))
   156  
   157  	listener, err := net.Listen("tcp", statusAddr)
   158  	if err != nil {
   159  		return err
   160  	}
   161  	l.serverAddr = listener.Addr()
   162  	log.L().Info("starting HTTP server", zap.Stringer("address", l.serverAddr))
   163  	fmt.Fprintln(realAddrWriter, "started HTTP server on", l.serverAddr)
   164  	l.server.Handler = mux
   165  	listener = l.globalTLS.WrapListener(listener)
   166  
   167  	go func() {
   168  		err := l.server.Serve(listener)
   169  		log.L().Info("stopped HTTP server", log.ShortError(err))
   170  	}()
   171  	return nil
   172  }
   173  
   174  // RunOnce is used by binary lightning and host when using lightning as a library.
   175  // - for binary lightning, taskCtx could be context.Background which means taskCtx wouldn't be canceled directly by its
   176  //   cancel function, but only by Lightning.Stop or HTTP DELETE using l.cancel. and glue could be nil to let lightning
   177  //   use a default glue later.
   178  // - for lightning as a library, taskCtx could be a meaningful context that get canceled outside, and glue could be a
   179  //   caller implemented glue.
   180  func (l *Lightning) RunOnce(taskCtx context.Context, taskCfg *config.Config, glue glue.Glue, replaceLogger *zap.Logger) error {
   181  	if err := taskCfg.Adjust(taskCtx); err != nil {
   182  		return err
   183  	}
   184  
   185  	taskCfg.TaskID = time.Now().UnixNano()
   186  	failpoint.Inject("SetTaskID", func(val failpoint.Value) {
   187  		taskCfg.TaskID = int64(val.(int))
   188  	})
   189  
   190  	if replaceLogger != nil {
   191  		log.SetAppLogger(replaceLogger)
   192  	}
   193  	return l.run(taskCtx, taskCfg, glue)
   194  }
   195  
   196  func (l *Lightning) RunServer() error {
   197  	l.taskCfgs = config.NewConfigList()
   198  	log.L().Info(
   199  		"Lightning server is running, post to /tasks to start an import task",
   200  		zap.Stringer("address", l.serverAddr),
   201  	)
   202  
   203  	for {
   204  		task, err := l.taskCfgs.Pop(l.ctx)
   205  		if err != nil {
   206  			return err
   207  		}
   208  		err = l.run(context.Background(), task, nil)
   209  		if err != nil {
   210  			restore.DeliverPauser.Pause() // force pause the progress on error
   211  			log.L().Error("tidb lightning encountered error", zap.Error(err))
   212  		}
   213  	}
   214  }
   215  
   216  var taskCfgRecorderKey struct{}
   217  
   218  func (l *Lightning) run(taskCtx context.Context, taskCfg *config.Config, g glue.Glue) (err error) {
   219  	common.PrintInfo("lightning", func() {
   220  		log.L().Info("cfg", zap.Stringer("cfg", taskCfg))
   221  	})
   222  
   223  	logEnvVariables()
   224  
   225  	ctx, cancel := context.WithCancel(taskCtx)
   226  	l.cancelLock.Lock()
   227  	l.cancel = cancel
   228  	l.curTask = taskCfg
   229  	l.cancelLock.Unlock()
   230  	web.BroadcastStartTask()
   231  
   232  	defer func() {
   233  		cancel()
   234  		l.cancelLock.Lock()
   235  		l.cancel = nil
   236  		l.cancelLock.Unlock()
   237  		web.BroadcastEndTask(err)
   238  	}()
   239  
   240  	failpoint.Inject("SkipRunTask", func() {
   241  		if recorder, ok := l.ctx.Value(&taskCfgRecorderKey).(chan *config.Config); ok {
   242  			select {
   243  			case recorder <- taskCfg:
   244  			case <-ctx.Done():
   245  				failpoint.Return(ctx.Err())
   246  			}
   247  		}
   248  		failpoint.Return(nil)
   249  	})
   250  
   251  	if err := taskCfg.TiDB.Security.RegisterMySQL(); err != nil {
   252  		return err
   253  	}
   254  	defer func() {
   255  		// deregister TLS config with name "cluster"
   256  		if taskCfg.TiDB.Security == nil {
   257  			return
   258  		}
   259  		taskCfg.TiDB.Security.CAPath = ""
   260  		taskCfg.TiDB.Security.RegisterMySQL()
   261  	}()
   262  
   263  	// initiation of default glue should be after RegisterMySQL, which is ready to be called after taskCfg.Adjust
   264  	// and also put it here could avoid injecting another two SkipRunTask failpoint to caller
   265  	if g == nil {
   266  		db, err := restore.DBFromConfig(taskCfg.TiDB)
   267  		if err != nil {
   268  			return err
   269  		}
   270  		g = glue.NewExternalTiDBGlue(db, taskCfg.TiDB.SQLMode)
   271  	}
   272  
   273  	u, err := storage.ParseBackend(taskCfg.Mydumper.SourceDir, &storage.BackendOptions{})
   274  	if err != nil {
   275  		return errors.Annotate(err, "parse backend failed")
   276  	}
   277  	s, err := storage.Create(ctx, u, true)
   278  	if err != nil {
   279  		return errors.Annotate(err, "create storage failed")
   280  	}
   281  
   282  	loadTask := log.L().Begin(zap.InfoLevel, "load data source")
   283  	var mdl *mydump.MDLoader
   284  	mdl, err = mydump.NewMyDumpLoaderWithStore(ctx, taskCfg, s)
   285  	loadTask.End(zap.ErrorLevel, err)
   286  	if err != nil {
   287  		return errors.Trace(err)
   288  	}
   289  	err = checkSystemRequirement(taskCfg, mdl.GetDatabases())
   290  	if err != nil {
   291  		log.L().Error("check system requirements failed", zap.Error(err))
   292  		return errors.Trace(err)
   293  	}
   294  	// check table schema conflicts
   295  	err = checkSchemaConflict(taskCfg, mdl.GetDatabases())
   296  	if err != nil {
   297  		log.L().Error("checkpoint schema conflicts with data files", zap.Error(err))
   298  		return errors.Trace(err)
   299  	}
   300  
   301  	dbMetas := mdl.GetDatabases()
   302  	web.BroadcastInitProgress(dbMetas)
   303  
   304  	var procedure *restore.RestoreController
   305  	procedure, err = restore.NewRestoreController(ctx, dbMetas, taskCfg, s, g)
   306  	if err != nil {
   307  		log.L().Error("restore failed", log.ShortError(err))
   308  		return errors.Trace(err)
   309  	}
   310  	defer procedure.Close()
   311  
   312  	err = procedure.Run(ctx)
   313  	return errors.Trace(err)
   314  }
   315  
   316  func (l *Lightning) Stop() {
   317  	l.cancelLock.Lock()
   318  	if l.cancel != nil {
   319  		l.cancel()
   320  	}
   321  	l.cancelLock.Unlock()
   322  	if err := l.server.Shutdown(l.ctx); err != nil {
   323  		log.L().Warn("failed to shutdown HTTP server", log.ShortError(err))
   324  	}
   325  	l.shutdown()
   326  }
   327  
   328  // logEnvVariables add related environment variables to log
   329  func logEnvVariables() {
   330  	// log http proxy settings, it will be used in gRPC connection by default
   331  	proxyCfg := httpproxy.FromEnvironment()
   332  	if proxyCfg.HTTPProxy != "" || proxyCfg.HTTPSProxy != "" {
   333  		log.L().Info("environment variables", zap.Reflect("httpproxy", proxyCfg))
   334  	}
   335  }
   336  
   337  func writeJSONError(w http.ResponseWriter, code int, prefix string, err error) {
   338  	type errorResponse struct {
   339  		Error string `json:"error"`
   340  	}
   341  
   342  	w.WriteHeader(code)
   343  
   344  	if err != nil {
   345  		prefix += ": " + err.Error()
   346  	}
   347  	json.NewEncoder(w).Encode(errorResponse{Error: prefix})
   348  }
   349  
   350  func parseTaskID(req *http.Request) (int64, string, error) {
   351  	path := strings.TrimPrefix(req.URL.Path, "/")
   352  	taskIDString := path
   353  	verb := ""
   354  	if i := strings.IndexByte(path, '/'); i >= 0 {
   355  		taskIDString = path[:i]
   356  		verb = path[i+1:]
   357  	}
   358  
   359  	taskID, err := strconv.ParseInt(taskIDString, 10, 64)
   360  	if err != nil {
   361  		return 0, "", err
   362  	}
   363  
   364  	return taskID, verb, nil
   365  }
   366  
   367  func (l *Lightning) handleTask(w http.ResponseWriter, req *http.Request) {
   368  	w.Header().Set("Content-Type", "application/json")
   369  
   370  	switch req.Method {
   371  	case http.MethodGet:
   372  		taskID, _, err := parseTaskID(req)
   373  		if e, ok := err.(*strconv.NumError); ok && e.Num == "" {
   374  			l.handleGetTask(w)
   375  		} else if err == nil {
   376  			l.handleGetOneTask(w, req, taskID)
   377  		} else {
   378  			writeJSONError(w, http.StatusBadRequest, "invalid task ID", err)
   379  		}
   380  	case http.MethodPost:
   381  		l.handlePostTask(w, req)
   382  	case http.MethodDelete:
   383  		l.handleDeleteOneTask(w, req)
   384  	case http.MethodPatch:
   385  		l.handlePatchOneTask(w, req)
   386  	default:
   387  		w.Header().Set("Allow", http.MethodGet+", "+http.MethodPost+", "+http.MethodDelete+", "+http.MethodPatch)
   388  		writeJSONError(w, http.StatusMethodNotAllowed, "only GET, POST, DELETE and PATCH are allowed", nil)
   389  	}
   390  }
   391  
   392  func (l *Lightning) handleGetTask(w http.ResponseWriter) {
   393  	var response struct {
   394  		Current   *int64  `json:"current"`
   395  		QueuedIDs []int64 `json:"queue"`
   396  	}
   397  
   398  	if l.taskCfgs != nil {
   399  		response.QueuedIDs = l.taskCfgs.AllIDs()
   400  	} else {
   401  		response.QueuedIDs = []int64{}
   402  	}
   403  
   404  	l.cancelLock.Lock()
   405  	if l.cancel != nil && l.curTask != nil {
   406  		response.Current = new(int64)
   407  		*response.Current = l.curTask.TaskID
   408  	}
   409  	l.cancelLock.Unlock()
   410  
   411  	w.WriteHeader(http.StatusOK)
   412  	json.NewEncoder(w).Encode(response)
   413  }
   414  
   415  func (l *Lightning) handleGetOneTask(w http.ResponseWriter, req *http.Request, taskID int64) {
   416  	var task *config.Config
   417  
   418  	l.cancelLock.Lock()
   419  	if l.curTask != nil && l.curTask.TaskID == taskID {
   420  		task = l.curTask
   421  	}
   422  	l.cancelLock.Unlock()
   423  
   424  	if task == nil && l.taskCfgs != nil {
   425  		task, _ = l.taskCfgs.Get(taskID)
   426  	}
   427  
   428  	if task == nil {
   429  		writeJSONError(w, http.StatusNotFound, "task ID not found", nil)
   430  		return
   431  	}
   432  
   433  	json, err := json.Marshal(task)
   434  	if err != nil {
   435  		writeJSONError(w, http.StatusInternalServerError, "unable to serialize task", err)
   436  		return
   437  	}
   438  
   439  	writeBytesCompressed(w, req, json)
   440  }
   441  
   442  func (l *Lightning) handlePostTask(w http.ResponseWriter, req *http.Request) {
   443  	w.Header().Set("Cache-Control", "no-store")
   444  
   445  	if l.taskCfgs == nil {
   446  		// l.taskCfgs is non-nil only if Lightning is started with RunServer().
   447  		// Without the server mode this pointer is default to be nil.
   448  		writeJSONError(w, http.StatusNotImplemented, "server-mode not enabled", nil)
   449  		return
   450  	}
   451  
   452  	type taskResponse struct {
   453  		ID int64 `json:"id"`
   454  	}
   455  
   456  	data, err := ioutil.ReadAll(req.Body)
   457  	if err != nil {
   458  		writeJSONError(w, http.StatusBadRequest, "cannot read request", err)
   459  		return
   460  	}
   461  	log.L().Debug("received task config", zap.ByteString("content", data))
   462  
   463  	cfg := config.NewConfig()
   464  	if err = cfg.LoadFromGlobal(l.globalCfg); err != nil {
   465  		writeJSONError(w, http.StatusInternalServerError, "cannot restore from global config", err)
   466  		return
   467  	}
   468  	if err = cfg.LoadFromTOML(data); err != nil {
   469  		writeJSONError(w, http.StatusBadRequest, "cannot parse task (must be TOML)", err)
   470  		return
   471  	}
   472  	if err = cfg.Adjust(l.ctx); err != nil {
   473  		writeJSONError(w, http.StatusBadRequest, "invalid task configuration", err)
   474  		return
   475  	}
   476  
   477  	l.taskCfgs.Push(cfg)
   478  	w.WriteHeader(http.StatusOK)
   479  	json.NewEncoder(w).Encode(taskResponse{ID: cfg.TaskID})
   480  }
   481  
   482  func (l *Lightning) handleDeleteOneTask(w http.ResponseWriter, req *http.Request) {
   483  	w.Header().Set("Content-Type", "application/json")
   484  
   485  	taskID, _, err := parseTaskID(req)
   486  	if err != nil {
   487  		writeJSONError(w, http.StatusBadRequest, "invalid task ID", err)
   488  		return
   489  	}
   490  
   491  	var cancel context.CancelFunc
   492  	cancelSuccess := false
   493  
   494  	l.cancelLock.Lock()
   495  	if l.cancel != nil && l.curTask != nil && l.curTask.TaskID == taskID {
   496  		cancel = l.cancel
   497  		l.cancel = nil
   498  	}
   499  	l.cancelLock.Unlock()
   500  
   501  	if cancel != nil {
   502  		cancel()
   503  		cancelSuccess = true
   504  	} else if l.taskCfgs != nil {
   505  		cancelSuccess = l.taskCfgs.Remove(taskID)
   506  	}
   507  
   508  	log.L().Info("canceled task", zap.Int64("taskID", taskID), zap.Bool("success", cancelSuccess))
   509  
   510  	if cancelSuccess {
   511  		w.WriteHeader(http.StatusOK)
   512  		w.Write([]byte("{}"))
   513  	} else {
   514  		writeJSONError(w, http.StatusNotFound, "task ID not found", nil)
   515  	}
   516  }
   517  
   518  func (l *Lightning) handlePatchOneTask(w http.ResponseWriter, req *http.Request) {
   519  	if l.taskCfgs == nil {
   520  		writeJSONError(w, http.StatusNotImplemented, "server-mode not enabled", nil)
   521  		return
   522  	}
   523  
   524  	taskID, verb, err := parseTaskID(req)
   525  	if err != nil {
   526  		writeJSONError(w, http.StatusBadRequest, "invalid task ID", err)
   527  		return
   528  	}
   529  
   530  	moveSuccess := false
   531  	switch verb {
   532  	case "front":
   533  		moveSuccess = l.taskCfgs.MoveToFront(taskID)
   534  	case "back":
   535  		moveSuccess = l.taskCfgs.MoveToBack(taskID)
   536  	default:
   537  		writeJSONError(w, http.StatusBadRequest, "unknown patch action", nil)
   538  		return
   539  	}
   540  
   541  	if moveSuccess {
   542  		w.WriteHeader(http.StatusOK)
   543  		w.Write([]byte("{}"))
   544  	} else {
   545  		writeJSONError(w, http.StatusNotFound, "task ID not found", nil)
   546  	}
   547  }
   548  
   549  func writeBytesCompressed(w http.ResponseWriter, req *http.Request, b []byte) {
   550  	if !strings.Contains(req.Header.Get("Accept-Encoding"), "gzip") {
   551  		w.Write(b)
   552  		return
   553  	}
   554  
   555  	w.Header().Set("Content-Encoding", "gzip")
   556  	w.WriteHeader(http.StatusOK)
   557  	gw, _ := gzip.NewWriterLevel(w, gzip.BestSpeed)
   558  	gw.Write(b)
   559  	gw.Close()
   560  }
   561  
   562  func handleProgressTask(w http.ResponseWriter, req *http.Request) {
   563  	w.Header().Set("Content-Type", "application/json")
   564  	res, err := web.MarshalTaskProgress()
   565  	if err == nil {
   566  		writeBytesCompressed(w, req, res)
   567  	} else {
   568  		w.WriteHeader(http.StatusInternalServerError)
   569  		json.NewEncoder(w).Encode(err.Error())
   570  	}
   571  }
   572  
   573  func handleProgressTable(w http.ResponseWriter, req *http.Request) {
   574  	w.Header().Set("Content-Type", "application/json")
   575  	tableName := req.URL.Query().Get("t")
   576  	res, err := web.MarshalTableCheckpoints(tableName)
   577  	if err == nil {
   578  		writeBytesCompressed(w, req, res)
   579  	} else {
   580  		if errors.IsNotFound(err) {
   581  			w.WriteHeader(http.StatusNotFound)
   582  		} else {
   583  			w.WriteHeader(http.StatusInternalServerError)
   584  		}
   585  		json.NewEncoder(w).Encode(err.Error())
   586  	}
   587  }
   588  
   589  func handlePause(w http.ResponseWriter, req *http.Request) {
   590  	w.Header().Set("Content-Type", "application/json")
   591  
   592  	switch req.Method {
   593  	case http.MethodGet:
   594  		w.WriteHeader(http.StatusOK)
   595  		fmt.Fprintf(w, `{"paused":%v}`, restore.DeliverPauser.IsPaused())
   596  
   597  	case http.MethodPut:
   598  		w.WriteHeader(http.StatusOK)
   599  		restore.DeliverPauser.Pause()
   600  		log.L().Info("progress paused")
   601  		w.Write([]byte("{}"))
   602  
   603  	default:
   604  		w.Header().Set("Allow", http.MethodGet+", "+http.MethodPut)
   605  		writeJSONError(w, http.StatusMethodNotAllowed, "only GET and PUT are allowed", nil)
   606  	}
   607  }
   608  
   609  func handleResume(w http.ResponseWriter, req *http.Request) {
   610  	w.Header().Set("Content-Type", "application/json")
   611  
   612  	switch req.Method {
   613  	case http.MethodPut:
   614  		w.WriteHeader(http.StatusOK)
   615  		restore.DeliverPauser.Resume()
   616  		log.L().Info("progress resumed")
   617  		w.Write([]byte("{}"))
   618  
   619  	default:
   620  		w.Header().Set("Allow", http.MethodPut)
   621  		writeJSONError(w, http.StatusMethodNotAllowed, "only PUT is allowed", nil)
   622  	}
   623  }
   624  
   625  func handleLogLevel(w http.ResponseWriter, req *http.Request) {
   626  	w.Header().Set("Content-Type", "application/json")
   627  
   628  	var logLevel struct {
   629  		Level zapcore.Level `json:"level"`
   630  	}
   631  
   632  	switch req.Method {
   633  	case http.MethodGet:
   634  		logLevel.Level = log.Level()
   635  		w.WriteHeader(http.StatusOK)
   636  		json.NewEncoder(w).Encode(logLevel)
   637  
   638  	case http.MethodPut, http.MethodPost:
   639  		if err := json.NewDecoder(req.Body).Decode(&logLevel); err != nil {
   640  			writeJSONError(w, http.StatusBadRequest, "invalid log level", err)
   641  			return
   642  		}
   643  		oldLevel := log.SetLevel(zapcore.InfoLevel)
   644  		log.L().Info("changed log level", zap.Stringer("old", oldLevel), zap.Stringer("new", logLevel.Level))
   645  		log.SetLevel(logLevel.Level)
   646  		w.WriteHeader(http.StatusOK)
   647  		w.Write([]byte("{}"))
   648  
   649  	default:
   650  		w.Header().Set("Allow", http.MethodGet+", "+http.MethodPut+", "+http.MethodPost)
   651  		writeJSONError(w, http.StatusMethodNotAllowed, "only GET, PUT and POST are allowed", nil)
   652  	}
   653  }
   654  
   655  func checkSystemRequirement(cfg *config.Config, dbsMeta []*mydump.MDDatabaseMeta) error {
   656  	if !cfg.App.CheckRequirements {
   657  		log.L().Info("check-requirement is disabled, skip check system rlimit")
   658  		return nil
   659  	}
   660  
   661  	// in local mode, we need to read&write a lot of L0 sst files, so we need to check system max open files limit
   662  	if cfg.TikvImporter.Backend == config.BackendLocal {
   663  		// estimate max open files = {top N(TableConcurrency) table sizes} / {MemoryTableSize}
   664  		tableTotalSizes := make([]int64, 0)
   665  		for _, dbs := range dbsMeta {
   666  			for _, tb := range dbs.Tables {
   667  				tableTotalSizes = append(tableTotalSizes, tb.TotalSize)
   668  			}
   669  		}
   670  		sort.Slice(tableTotalSizes, func(i, j int) bool {
   671  			return tableTotalSizes[i] > tableTotalSizes[j]
   672  		})
   673  		topNTotalSize := int64(0)
   674  		for i := 0; i < len(tableTotalSizes) && i < cfg.App.TableConcurrency; i++ {
   675  			topNTotalSize += tableTotalSizes[i]
   676  		}
   677  
   678  		estimateMaxFiles := uint64(topNTotalSize/backend.LocalMemoryTableSize) * 2
   679  		if err := backend.VerifyRLimit(estimateMaxFiles); err != nil {
   680  			return err
   681  		}
   682  	}
   683  
   684  	return nil
   685  }
   686  
   687  /// checkSchemaConflict return error if checkpoint table scheme is conflict with data files
   688  func checkSchemaConflict(cfg *config.Config, dbsMeta []*mydump.MDDatabaseMeta) error {
   689  	if cfg.Checkpoint.Enable && cfg.Checkpoint.Driver == config.CheckpointDriverMySQL {
   690  		for _, db := range dbsMeta {
   691  			if db.Name == cfg.Checkpoint.Schema {
   692  				for _, tb := range db.Tables {
   693  					if checkpoints.IsCheckpointTable(tb.Name) {
   694  						return errors.Errorf("checkpoint table `%s`.`%s` conflict with data files. Please change the `checkpoint.schema` config or set `checkpoint.driver` to \"file\" instead", db.Name, tb.Name)
   695  					}
   696  				}
   697  			}
   698  		}
   699  	}
   700  	return nil
   701  }