github.com/nsqio/nsq@v1.3.0/apps/nsq_to_file/nsq_to_file.go (about)

     1  // This is a client that writes out to a file, and optionally rolls the file
     2  
     3  package main
     4  
     5  import (
     6  	"flag"
     7  	"fmt"
     8  	"log"
     9  	"os"
    10  	"os/signal"
    11  	"syscall"
    12  	"time"
    13  
    14  	"github.com/mreiferson/go-options"
    15  	"github.com/nsqio/go-nsq"
    16  	"github.com/nsqio/nsq/internal/app"
    17  	"github.com/nsqio/nsq/internal/lg"
    18  	"github.com/nsqio/nsq/internal/version"
    19  )
    20  
    21  func flagSet() *flag.FlagSet {
    22  	fs := flag.NewFlagSet("nsqd", flag.ExitOnError)
    23  
    24  	fs.Bool("version", false, "print version string")
    25  	fs.String("log-level", "info", "set log verbosity: debug, info, warn, error, or fatal")
    26  	fs.String("log-prefix", "[nsq_to_file] ", "log message prefix")
    27  
    28  	fs.String("channel", "nsq_to_file", "nsq channel")
    29  	fs.Int("max-in-flight", 200, "max number of messages to allow in flight")
    30  
    31  	fs.String("output-dir", "/tmp", "directory to write output files to")
    32  	fs.String("work-dir", "", "directory for in-progress files before moving to output-dir")
    33  	fs.String("datetime-format", "%Y-%m-%d_%H", "strftime compatible format for <DATETIME> in filename format")
    34  	fs.String("filename-format", "<TOPIC>.<HOST><REV>.<DATETIME>.log", "output filename format (<TOPIC>, <HOST>, <PID>, <DATETIME>, <REV> are replaced. <REV> is increased when file already exists)")
    35  	fs.String("host-identifier", "", "value to output in log filename in place of hostname. <SHORT_HOST> and <HOSTNAME> are valid replacement tokens")
    36  	fs.Int("gzip-level", 6, "gzip compression level (1-9, 1=BestSpeed, 9=BestCompression)")
    37  	fs.Bool("gzip", false, "gzip output files.")
    38  	fs.Bool("skip-empty-files", false, "skip writing empty files")
    39  	fs.Duration("topic-refresh", time.Minute, "how frequently the topic list should be refreshed")
    40  	fs.String("topic-pattern", "", "only log topics matching the following pattern")
    41  
    42  	fs.Int64("rotate-size", 0, "rotate the file when it grows bigger than `rotate-size` bytes")
    43  	fs.Duration("rotate-interval", 0, "rotate the file every duration")
    44  	fs.Duration("sync-interval", 30*time.Second, "sync file to disk every duration")
    45  
    46  	fs.Duration("http-client-connect-timeout", 2*time.Second, "timeout for HTTP connect")
    47  	fs.Duration("http-client-request-timeout", 5*time.Second, "timeout for HTTP request")
    48  
    49  	nsqdTCPAddrs := app.StringArray{}
    50  	lookupdHTTPAddrs := app.StringArray{}
    51  	topics := app.StringArray{}
    52  	consumerOpts := app.StringArray{}
    53  	fs.Var(&nsqdTCPAddrs, "nsqd-tcp-address", "nsqd TCP address (may be given multiple times)")
    54  	fs.Var(&lookupdHTTPAddrs, "lookupd-http-address", "lookupd HTTP address (may be given multiple times)")
    55  	fs.Var(&topics, "topic", "nsq topic (may be given multiple times)")
    56  	fs.Var(&consumerOpts, "consumer-opt", "option to passthrough to nsq.Consumer (may be given multiple times, http://godoc.org/github.com/nsqio/go-nsq#Config)")
    57  
    58  	return fs
    59  }
    60  
    61  func main() {
    62  	fs := flagSet()
    63  	fs.Parse(os.Args[1:])
    64  
    65  	if args := fs.Args(); len(args) > 0 {
    66  		log.Fatalf("unknown arguments: %s", args)
    67  	}
    68  
    69  	opts := NewOptions()
    70  	options.Resolve(opts, fs, nil)
    71  
    72  	logger := log.New(os.Stderr, opts.LogPrefix, log.Ldate|log.Ltime|log.Lmicroseconds)
    73  	logLevel, err := lg.ParseLogLevel(opts.LogLevel)
    74  	if err != nil {
    75  		log.Fatal("--log-level is invalid")
    76  	}
    77  	logf := func(lvl lg.LogLevel, f string, args ...interface{}) {
    78  		lg.Logf(logger, logLevel, lvl, f, args...)
    79  	}
    80  
    81  	if fs.Lookup("version").Value.(flag.Getter).Get().(bool) {
    82  		fmt.Printf("nsq_to_file v%s\n", version.Binary)
    83  		return
    84  	}
    85  
    86  	if opts.Channel == "" {
    87  		log.Fatal("--channel is required")
    88  	}
    89  
    90  	if opts.HTTPClientConnectTimeout <= 0 {
    91  		log.Fatal("--http-client-connect-timeout should be positive")
    92  	}
    93  
    94  	if opts.HTTPClientRequestTimeout <= 0 {
    95  		log.Fatal("--http-client-request-timeout should be positive")
    96  	}
    97  
    98  	if len(opts.NSQDTCPAddrs) == 0 && len(opts.NSQLookupdHTTPAddrs) == 0 {
    99  		log.Fatal("--nsqd-tcp-address or --lookupd-http-address required.")
   100  	}
   101  	if len(opts.NSQDTCPAddrs) != 0 && len(opts.NSQLookupdHTTPAddrs) != 0 {
   102  		log.Fatal("use --nsqd-tcp-address or --lookupd-http-address not both")
   103  	}
   104  
   105  	if opts.GZIPLevel < 1 || opts.GZIPLevel > 9 {
   106  		log.Fatalf("invalid --gzip-level value (%d), should be 1-9", opts.GZIPLevel)
   107  	}
   108  
   109  	if len(opts.Topics) == 0 && len(opts.TopicPattern) == 0 {
   110  		log.Fatal("--topic or --topic-pattern required")
   111  	}
   112  
   113  	if len(opts.Topics) == 0 && len(opts.NSQLookupdHTTPAddrs) == 0 {
   114  		log.Fatal("--lookupd-http-address must be specified when no --topic specified")
   115  	}
   116  
   117  	if opts.WorkDir == "" {
   118  		opts.WorkDir = opts.OutputDir
   119  	}
   120  
   121  	cfg := nsq.NewConfig()
   122  	cfgFlag := nsq.ConfigFlag{cfg}
   123  	for _, opt := range opts.ConsumerOpts {
   124  		cfgFlag.Set(opt)
   125  	}
   126  	cfg.UserAgent = fmt.Sprintf("nsq_to_file/%s go-nsq/%s", version.Binary, nsq.VERSION)
   127  	cfg.MaxInFlight = opts.MaxInFlight
   128  
   129  	hupChan := make(chan os.Signal, 1)
   130  	termChan := make(chan os.Signal, 1)
   131  	signal.Notify(hupChan, syscall.SIGHUP)
   132  	signal.Notify(termChan, syscall.SIGINT, syscall.SIGTERM)
   133  
   134  	discoverer := newTopicDiscoverer(logf, opts, cfg, hupChan, termChan)
   135  	discoverer.run()
   136  }