github.com/pingcap/tidb-lightning@v5.0.0-rc.0.20210428090220-84b649866577+incompatible/cmd/tidb-lightning/main.go (about)

     1  // Copyright 2019 PingCAP, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // See the License for the specific language governing permissions and
    12  // limitations under the License.
    13  
    14  package main
    15  
    16  import (
    17  	"context"
    18  	"fmt"
    19  	"os"
    20  	"os/signal"
    21  	"runtime/debug"
    22  	"syscall"
    23  
    24  	"go.uber.org/zap"
    25  
    26  	"github.com/pingcap/tidb-lightning/lightning"
    27  	"github.com/pingcap/tidb-lightning/lightning/config"
    28  	"github.com/pingcap/tidb-lightning/lightning/log"
    29  )
    30  
    31  func main() {
    32  	globalCfg := config.Must(config.LoadGlobalConfig(os.Args[1:], nil))
    33  	fmt.Fprintf(os.Stdout, "Verbose debug logs will be written to %s\n\n", globalCfg.App.Config.File)
    34  
    35  	app := lightning.New(globalCfg)
    36  
    37  	sc := make(chan os.Signal, 1)
    38  	signal.Notify(sc,
    39  		syscall.SIGHUP,
    40  		syscall.SIGINT,
    41  		syscall.SIGTERM,
    42  		syscall.SIGQUIT)
    43  
    44  	go func() {
    45  		sig := <-sc
    46  		log.L().Info("got signal to exit", zap.Stringer("signal", sig))
    47  		app.Stop()
    48  	}()
    49  
    50  	logger := log.L()
    51  
    52  	// Lightning allocates too many transient objects and heap size is small,
    53  	// so garbage collections happen too frequently and lots of time is spent in GC component.
    54  	//
    55  	// In a test of loading the table `order_line.csv` of 14k TPCC.
    56  	// The time need of `encode kv data and write` step reduce from 52m4s to 37m30s when change
    57  	// GOGC from 100 to 500, the total time needed reduce near 15m too.
    58  	// The cost of this is the memory of lightnin at runtime grow from about 200M to 700M, but it's acceptable.
    59  	// So we set the gc percentage as 500 default to reduce the GC frequency instead of 100.
    60  	//
    61  	// Local mode need much more memory than importer/tidb mode, if the gc percentage is too high,
    62  	// lightning memory usage will also be high.
    63  	if globalCfg.TikvImporter.Backend != config.BackendLocal {
    64  		gogc := os.Getenv("GOGC")
    65  		if gogc == "" {
    66  			old := debug.SetGCPercent(500)
    67  			log.L().Debug("set gc percentage", zap.Int("old", old), zap.Int("new", 500))
    68  		}
    69  	}
    70  
    71  	err := app.GoServe()
    72  	if err != nil {
    73  		logger.Error("failed to start HTTP server", zap.Error(err))
    74  		fmt.Fprintln(os.Stderr, "failed to start HTTP server:", err)
    75  		return
    76  	}
    77  
    78  	err = func() error {
    79  		if globalCfg.App.ServerMode {
    80  			return app.RunServer()
    81  		} else {
    82  			cfg := config.NewConfig()
    83  			if err := cfg.LoadFromGlobal(globalCfg); err != nil {
    84  				return err
    85  			}
    86  			return app.RunOnce(context.Background(), cfg, nil, nil)
    87  		}
    88  	}()
    89  
    90  	if err != nil {
    91  		logger.Error("tidb lightning encountered error stack info", zap.Error(err))
    92  		logger.Error("tidb lightning encountered error", log.ShortError(err))
    93  		fmt.Fprintln(os.Stderr, "tidb lightning encountered error: ", err)
    94  	} else {
    95  		logger.Info("tidb lightning exit")
    96  		fmt.Fprintln(os.Stdout, "tidb lightning exit")
    97  	}
    98  
    99  	// call Sync() with log to stdout may return error in some case, so just skip it
   100  	if globalCfg.App.File != "" {
   101  		syncErr := logger.Sync()
   102  		if syncErr != nil {
   103  			fmt.Fprintln(os.Stderr, "sync log failed", syncErr)
   104  		}
   105  	}
   106  
   107  	if err != nil {
   108  		os.Exit(1)
   109  	}
   110  }