github.com/prestonp/nomad@v0.10.4/command/agent/command.go (about)

     1  package agent
     2  
     3  import (
     4  	"flag"
     5  	"fmt"
     6  	"io"
     7  	"log"
     8  	"os"
     9  	"os/signal"
    10  	"path/filepath"
    11  	"reflect"
    12  	"sort"
    13  	"strconv"
    14  	"strings"
    15  	"syscall"
    16  	"time"
    17  
    18  	metrics "github.com/armon/go-metrics"
    19  	"github.com/armon/go-metrics/circonus"
    20  	"github.com/armon/go-metrics/datadog"
    21  	"github.com/armon/go-metrics/prometheus"
    22  	"github.com/hashicorp/consul/lib"
    23  	checkpoint "github.com/hashicorp/go-checkpoint"
    24  	discover "github.com/hashicorp/go-discover"
    25  	hclog "github.com/hashicorp/go-hclog"
    26  	gsyslog "github.com/hashicorp/go-syslog"
    27  	"github.com/hashicorp/logutils"
    28  	"github.com/hashicorp/nomad/helper"
    29  	flaghelper "github.com/hashicorp/nomad/helper/flag-helpers"
    30  	gatedwriter "github.com/hashicorp/nomad/helper/gated-writer"
    31  	"github.com/hashicorp/nomad/helper/logging"
    32  	"github.com/hashicorp/nomad/helper/winsvc"
    33  	"github.com/hashicorp/nomad/nomad/structs/config"
    34  	"github.com/hashicorp/nomad/version"
    35  	"github.com/mitchellh/cli"
    36  	"github.com/posener/complete"
    37  )
    38  
    39  // gracefulTimeout controls how long we wait before forcefully terminating
    40  const gracefulTimeout = 5 * time.Second
    41  
    42  // Command is a Command implementation that runs a Nomad agent.
    43  // The command will not end unless a shutdown message is sent on the
    44  // ShutdownCh. If two messages are sent on the ShutdownCh it will forcibly
    45  // exit.
    46  type Command struct {
    47  	Version    *version.VersionInfo
    48  	Ui         cli.Ui
    49  	ShutdownCh <-chan struct{}
    50  
    51  	args           []string
    52  	agent          *Agent
    53  	httpServer     *HTTPServer
    54  	logFilter      *logutils.LevelFilter
    55  	logOutput      io.Writer
    56  	retryJoinErrCh chan struct{}
    57  }
    58  
    59  func (c *Command) readConfig() *Config {
    60  	var dev *devModeConfig
    61  	var configPath []string
    62  	var servers string
    63  	var meta []string
    64  
    65  	// Make a new, empty config.
    66  	cmdConfig := &Config{
    67  		Client: &ClientConfig{},
    68  		Consul: &config.ConsulConfig{},
    69  		Ports:  &Ports{},
    70  		Server: &ServerConfig{
    71  			ServerJoin: &ServerJoin{},
    72  		},
    73  		Vault: &config.VaultConfig{},
    74  		ACL:   &ACLConfig{},
    75  	}
    76  
    77  	flags := flag.NewFlagSet("agent", flag.ContinueOnError)
    78  	flags.Usage = func() { c.Ui.Error(c.Help()) }
    79  
    80  	// Role options
    81  	var devMode bool
    82  	var devConnectMode bool
    83  	flags.BoolVar(&devMode, "dev", false, "")
    84  	flags.BoolVar(&devConnectMode, "dev-connect", false, "")
    85  	flags.BoolVar(&cmdConfig.Server.Enabled, "server", false, "")
    86  	flags.BoolVar(&cmdConfig.Client.Enabled, "client", false, "")
    87  
    88  	// Server-only options
    89  	flags.IntVar(&cmdConfig.Server.BootstrapExpect, "bootstrap-expect", 0, "")
    90  	flags.StringVar(&cmdConfig.Server.EncryptKey, "encrypt", "", "gossip encryption key")
    91  	flags.IntVar(&cmdConfig.Server.RaftProtocol, "raft-protocol", 0, "")
    92  	flags.BoolVar(&cmdConfig.Server.RejoinAfterLeave, "rejoin", false, "")
    93  	flags.Var((*flaghelper.StringFlag)(&cmdConfig.Server.ServerJoin.StartJoin), "join", "")
    94  	flags.Var((*flaghelper.StringFlag)(&cmdConfig.Server.ServerJoin.RetryJoin), "retry-join", "")
    95  	flags.IntVar(&cmdConfig.Server.ServerJoin.RetryMaxAttempts, "retry-max", 0, "")
    96  	flags.Var((flaghelper.FuncDurationVar)(func(d time.Duration) error {
    97  		cmdConfig.Server.ServerJoin.RetryInterval = d
    98  		return nil
    99  	}), "retry-interval", "")
   100  
   101  	// Client-only options
   102  	flags.StringVar(&cmdConfig.Client.StateDir, "state-dir", "", "")
   103  	flags.StringVar(&cmdConfig.Client.AllocDir, "alloc-dir", "", "")
   104  	flags.StringVar(&cmdConfig.Client.NodeClass, "node-class", "", "")
   105  	flags.StringVar(&servers, "servers", "", "")
   106  	flags.Var((*flaghelper.StringFlag)(&meta), "meta", "")
   107  	flags.StringVar(&cmdConfig.Client.NetworkInterface, "network-interface", "", "")
   108  	flags.IntVar(&cmdConfig.Client.NetworkSpeed, "network-speed", 0, "")
   109  
   110  	// General options
   111  	flags.Var((*flaghelper.StringFlag)(&configPath), "config", "config")
   112  	flags.StringVar(&cmdConfig.BindAddr, "bind", "", "")
   113  	flags.StringVar(&cmdConfig.Region, "region", "", "")
   114  	flags.StringVar(&cmdConfig.DataDir, "data-dir", "", "")
   115  	flags.StringVar(&cmdConfig.PluginDir, "plugin-dir", "", "")
   116  	flags.StringVar(&cmdConfig.Datacenter, "dc", "", "")
   117  	flags.StringVar(&cmdConfig.LogLevel, "log-level", "", "")
   118  	flags.BoolVar(&cmdConfig.LogJson, "log-json", false, "")
   119  	flags.StringVar(&cmdConfig.NodeName, "node", "", "")
   120  
   121  	// Consul options
   122  	flags.StringVar(&cmdConfig.Consul.Auth, "consul-auth", "", "")
   123  	flags.Var((flaghelper.FuncBoolVar)(func(b bool) error {
   124  		cmdConfig.Consul.AutoAdvertise = &b
   125  		return nil
   126  	}), "consul-auto-advertise", "")
   127  	flags.StringVar(&cmdConfig.Consul.CAFile, "consul-ca-file", "", "")
   128  	flags.StringVar(&cmdConfig.Consul.CertFile, "consul-cert-file", "", "")
   129  	flags.StringVar(&cmdConfig.Consul.KeyFile, "consul-key-file", "", "")
   130  	flags.Var((flaghelper.FuncBoolVar)(func(b bool) error {
   131  		cmdConfig.Consul.ChecksUseAdvertise = &b
   132  		return nil
   133  	}), "consul-checks-use-advertise", "")
   134  	flags.Var((flaghelper.FuncBoolVar)(func(b bool) error {
   135  		cmdConfig.Consul.ClientAutoJoin = &b
   136  		return nil
   137  	}), "consul-client-auto-join", "")
   138  	flags.StringVar(&cmdConfig.Consul.ClientServiceName, "consul-client-service-name", "", "")
   139  	flags.StringVar(&cmdConfig.Consul.ClientHTTPCheckName, "consul-client-http-check-name", "", "")
   140  	flags.StringVar(&cmdConfig.Consul.ServerServiceName, "consul-server-service-name", "", "")
   141  	flags.StringVar(&cmdConfig.Consul.ServerHTTPCheckName, "consul-server-http-check-name", "", "")
   142  	flags.StringVar(&cmdConfig.Consul.ServerSerfCheckName, "consul-server-serf-check-name", "", "")
   143  	flags.StringVar(&cmdConfig.Consul.ServerRPCCheckName, "consul-server-rpc-check-name", "", "")
   144  	flags.Var((flaghelper.FuncBoolVar)(func(b bool) error {
   145  		cmdConfig.Consul.ServerAutoJoin = &b
   146  		return nil
   147  	}), "consul-server-auto-join", "")
   148  	flags.Var((flaghelper.FuncBoolVar)(func(b bool) error {
   149  		cmdConfig.Consul.EnableSSL = &b
   150  		return nil
   151  	}), "consul-ssl", "")
   152  	flags.StringVar(&cmdConfig.Consul.Token, "consul-token", "", "")
   153  	flags.Var((flaghelper.FuncBoolVar)(func(b bool) error {
   154  		cmdConfig.Consul.VerifySSL = &b
   155  		return nil
   156  	}), "consul-verify-ssl", "")
   157  	flags.StringVar(&cmdConfig.Consul.Addr, "consul-address", "", "")
   158  	flags.Var((flaghelper.FuncBoolVar)(func(b bool) error {
   159  		cmdConfig.Consul.AllowUnauthenticated = &b
   160  		return nil
   161  	}), "consul-allow-unauthenticated", "")
   162  
   163  	// Vault options
   164  	flags.Var((flaghelper.FuncBoolVar)(func(b bool) error {
   165  		cmdConfig.Vault.Enabled = &b
   166  		return nil
   167  	}), "vault-enabled", "")
   168  	flags.Var((flaghelper.FuncBoolVar)(func(b bool) error {
   169  		cmdConfig.Vault.AllowUnauthenticated = &b
   170  		return nil
   171  	}), "vault-allow-unauthenticated", "")
   172  	flags.StringVar(&cmdConfig.Vault.Token, "vault-token", "", "")
   173  	flags.StringVar(&cmdConfig.Vault.Addr, "vault-address", "", "")
   174  	flags.StringVar(&cmdConfig.Vault.Namespace, "vault-namespace", "", "")
   175  	flags.StringVar(&cmdConfig.Vault.Role, "vault-create-from-role", "", "")
   176  	flags.StringVar(&cmdConfig.Vault.TLSCaFile, "vault-ca-file", "", "")
   177  	flags.StringVar(&cmdConfig.Vault.TLSCaPath, "vault-ca-path", "", "")
   178  	flags.StringVar(&cmdConfig.Vault.TLSCertFile, "vault-cert-file", "", "")
   179  	flags.StringVar(&cmdConfig.Vault.TLSKeyFile, "vault-key-file", "", "")
   180  	flags.Var((flaghelper.FuncBoolVar)(func(b bool) error {
   181  		cmdConfig.Vault.TLSSkipVerify = &b
   182  		return nil
   183  	}), "vault-tls-skip-verify", "")
   184  	flags.StringVar(&cmdConfig.Vault.TLSServerName, "vault-tls-server-name", "", "")
   185  
   186  	// ACL options
   187  	flags.BoolVar(&cmdConfig.ACL.Enabled, "acl-enabled", false, "")
   188  	flags.StringVar(&cmdConfig.ACL.ReplicationToken, "acl-replication-token", "", "")
   189  
   190  	if err := flags.Parse(c.args); err != nil {
   191  		return nil
   192  	}
   193  
   194  	// Split the servers.
   195  	if servers != "" {
   196  		cmdConfig.Client.Servers = strings.Split(servers, ",")
   197  	}
   198  
   199  	// Parse the meta flags.
   200  	metaLength := len(meta)
   201  	if metaLength != 0 {
   202  		cmdConfig.Client.Meta = make(map[string]string, metaLength)
   203  		for _, kv := range meta {
   204  			parts := strings.SplitN(kv, "=", 2)
   205  			if len(parts) != 2 {
   206  				c.Ui.Error(fmt.Sprintf("Error parsing Client.Meta value: %v", kv))
   207  				return nil
   208  			}
   209  			cmdConfig.Client.Meta[parts[0]] = parts[1]
   210  		}
   211  	}
   212  
   213  	// Load the configuration
   214  	dev, err := newDevModeConfig(devMode, devConnectMode)
   215  	if err != nil {
   216  		c.Ui.Error(err.Error())
   217  		return nil
   218  	}
   219  	var config *Config
   220  	if dev != nil {
   221  		config = DevConfig(dev)
   222  	} else {
   223  		config = DefaultConfig()
   224  	}
   225  
   226  	// Merge in the enterprise overlay
   227  	config.Merge(DefaultEntConfig())
   228  
   229  	for _, path := range configPath {
   230  		current, err := LoadConfig(path)
   231  		if err != nil {
   232  			c.Ui.Error(fmt.Sprintf(
   233  				"Error loading configuration from %s: %s", path, err))
   234  			return nil
   235  		}
   236  
   237  		// The user asked us to load some config here but we didn't find any,
   238  		// so we'll complain but continue.
   239  		if current == nil || reflect.DeepEqual(current, &Config{}) {
   240  			c.Ui.Warn(fmt.Sprintf("No configuration loaded from %s", path))
   241  		}
   242  
   243  		if config == nil {
   244  			config = current
   245  		} else {
   246  			config = config.Merge(current)
   247  		}
   248  	}
   249  
   250  	// Ensure the sub-structs at least exist
   251  	if config.Client == nil {
   252  		config.Client = &ClientConfig{}
   253  	}
   254  
   255  	if config.Server == nil {
   256  		config.Server = &ServerConfig{}
   257  	}
   258  
   259  	// Merge any CLI options over config file options
   260  	config = config.Merge(cmdConfig)
   261  
   262  	// Set the version info
   263  	config.Version = c.Version
   264  
   265  	// Normalize binds, ports, addresses, and advertise
   266  	if err := config.normalizeAddrs(); err != nil {
   267  		c.Ui.Error(err.Error())
   268  		return nil
   269  	}
   270  
   271  	// Check to see if we should read the Vault token from the environment
   272  	if config.Vault.Token == "" {
   273  		config.Vault.Token = os.Getenv("VAULT_TOKEN")
   274  	}
   275  
   276  	// Check to see if we should read the Vault namespace from the environment
   277  	if config.Vault.Namespace == "" {
   278  		config.Vault.Namespace = os.Getenv("VAULT_NAMESPACE")
   279  	}
   280  
   281  	// Default the plugin directory to be under that of the data directory if it
   282  	// isn't explicitly specified.
   283  	if config.PluginDir == "" && config.DataDir != "" {
   284  		config.PluginDir = filepath.Join(config.DataDir, "plugins")
   285  	}
   286  
   287  	if !c.isValidConfig(config, cmdConfig) {
   288  		return nil
   289  	}
   290  
   291  	return config
   292  }
   293  
   294  func (c *Command) isValidConfig(config, cmdConfig *Config) bool {
   295  
   296  	// Check that the server is running in at least one mode.
   297  	if !(config.Server.Enabled || config.Client.Enabled) {
   298  		c.Ui.Error("Must specify either server, client or dev mode for the agent.")
   299  		return false
   300  	}
   301  
   302  	// Set up the TLS configuration properly if we have one.
   303  	// XXX chelseakomlo: set up a TLSConfig New method which would wrap
   304  	// constructor-type actions like this.
   305  	if config.TLSConfig != nil && !config.TLSConfig.IsEmpty() {
   306  		if err := config.TLSConfig.SetChecksum(); err != nil {
   307  			c.Ui.Error(fmt.Sprintf("WARNING: Error when parsing TLS configuration: %v", err))
   308  		}
   309  	}
   310  
   311  	if config.Server.EncryptKey != "" {
   312  		if _, err := config.Server.EncryptBytes(); err != nil {
   313  			c.Ui.Error(fmt.Sprintf("Invalid encryption key: %s", err))
   314  			return false
   315  		}
   316  		keyfile := filepath.Join(config.DataDir, serfKeyring)
   317  		if _, err := os.Stat(keyfile); err == nil {
   318  			c.Ui.Warn("WARNING: keyring exists but -encrypt given, using keyring")
   319  		}
   320  	}
   321  
   322  	// Verify the paths are absolute.
   323  	dirs := map[string]string{
   324  		"data-dir":   config.DataDir,
   325  		"plugin-dir": config.PluginDir,
   326  		"alloc-dir":  config.Client.AllocDir,
   327  		"state-dir":  config.Client.StateDir,
   328  	}
   329  	for k, dir := range dirs {
   330  		if dir == "" {
   331  			continue
   332  		}
   333  
   334  		if !filepath.IsAbs(dir) {
   335  			c.Ui.Error(fmt.Sprintf("%s must be given as an absolute path: got %v", k, dir))
   336  			return false
   337  		}
   338  	}
   339  
   340  	if config.Client.Enabled {
   341  		for k := range config.Client.Meta {
   342  			if !helper.IsValidInterpVariable(k) {
   343  				c.Ui.Error(fmt.Sprintf("Invalid Client.Meta key: %v", k))
   344  				return false
   345  			}
   346  		}
   347  	}
   348  
   349  	if config.DevMode {
   350  		// Skip the rest of the validation for dev mode
   351  		return true
   352  	}
   353  
   354  	// Ensure that we have the directories we need to run.
   355  	if config.Server.Enabled && config.DataDir == "" {
   356  		c.Ui.Error("Must specify data directory")
   357  		return false
   358  	}
   359  
   360  	// The config is valid if the top-level data-dir is set or if both
   361  	// alloc-dir and state-dir are set.
   362  	if config.Client.Enabled && config.DataDir == "" {
   363  		if config.Client.AllocDir == "" || config.Client.StateDir == "" || config.PluginDir == "" {
   364  			c.Ui.Error("Must specify the state, alloc dir, and plugin dir if data-dir is omitted.")
   365  			return false
   366  		}
   367  	}
   368  
   369  	// Check the bootstrap flags
   370  	if !config.Server.Enabled && cmdConfig.Server.BootstrapExpect > 0 {
   371  		// report an error if BootstrapExpect is set in CLI but server is disabled
   372  		c.Ui.Error("Bootstrap requires server mode to be enabled")
   373  		return false
   374  	}
   375  	if config.Server.Enabled && config.Server.BootstrapExpect == 1 {
   376  		c.Ui.Error("WARNING: Bootstrap mode enabled! Potentially unsafe operation.")
   377  	}
   378  
   379  	return true
   380  }
   381  
   382  // setupLoggers is used to setup the logGate, and our logOutput
   383  func (c *Command) setupLoggers(config *Config) (*gatedwriter.Writer, io.Writer) {
   384  	// Setup logging. First create the gated log writer, which will
   385  	// store logs until we're ready to show them. Then create the level
   386  	// filter, filtering logs of the specified level.
   387  	logGate := &gatedwriter.Writer{
   388  		Writer: &cli.UiWriter{Ui: c.Ui},
   389  	}
   390  
   391  	c.logFilter = LevelFilter()
   392  	c.logFilter.MinLevel = logutils.LogLevel(strings.ToUpper(config.LogLevel))
   393  	c.logFilter.Writer = logGate
   394  	if !ValidateLevelFilter(c.logFilter.MinLevel, c.logFilter) {
   395  		c.Ui.Error(fmt.Sprintf(
   396  			"Invalid log level: %s. Valid log levels are: %v",
   397  			c.logFilter.MinLevel, c.logFilter.Levels))
   398  		return nil, nil
   399  	}
   400  
   401  	// Create a log writer, and wrap a logOutput around it
   402  	writers := []io.Writer{c.logFilter}
   403  
   404  	// Check if syslog is enabled
   405  	if config.EnableSyslog {
   406  		l, err := gsyslog.NewLogger(gsyslog.LOG_NOTICE, config.SyslogFacility, "nomad")
   407  		if err != nil {
   408  			c.Ui.Error(fmt.Sprintf("Syslog setup failed: %v", err))
   409  			return nil, nil
   410  		}
   411  		writers = append(writers, &SyslogWrapper{l, c.logFilter})
   412  	}
   413  
   414  	// Check if file logging is enabled
   415  	if config.LogFile != "" {
   416  		dir, fileName := filepath.Split(config.LogFile)
   417  
   418  		// if a path is provided, but has no filename, then a default is used.
   419  		if fileName == "" {
   420  			fileName = "nomad.log"
   421  		}
   422  
   423  		// Try to enter the user specified log rotation duration first
   424  		var logRotateDuration time.Duration
   425  		if config.LogRotateDuration != "" {
   426  			duration, err := time.ParseDuration(config.LogRotateDuration)
   427  			if err != nil {
   428  				c.Ui.Error(fmt.Sprintf("Failed to parse log rotation duration: %v", err))
   429  				return nil, nil
   430  			}
   431  			logRotateDuration = duration
   432  		} else {
   433  			// Default to 24 hrs if no rotation period is specified
   434  			logRotateDuration = 24 * time.Hour
   435  		}
   436  
   437  		logFile := &logFile{
   438  			logFilter: c.logFilter,
   439  			fileName:  fileName,
   440  			logPath:   dir,
   441  			duration:  logRotateDuration,
   442  			MaxBytes:  config.LogRotateBytes,
   443  			MaxFiles:  config.LogRotateMaxFiles,
   444  		}
   445  
   446  		writers = append(writers, logFile)
   447  	}
   448  
   449  	c.logOutput = io.MultiWriter(writers...)
   450  	log.SetOutput(c.logOutput)
   451  	return logGate, c.logOutput
   452  }
   453  
   454  // setupAgent is used to start the agent and various interfaces
   455  func (c *Command) setupAgent(config *Config, logger hclog.InterceptLogger, logOutput io.Writer, inmem *metrics.InmemSink) error {
   456  	c.Ui.Output("Starting Nomad agent...")
   457  	agent, err := NewAgent(config, logger, logOutput, inmem)
   458  	if err != nil {
   459  		c.Ui.Error(fmt.Sprintf("Error starting agent: %s", err))
   460  		return err
   461  	}
   462  	c.agent = agent
   463  
   464  	// Setup the HTTP server
   465  	http, err := NewHTTPServer(agent, config)
   466  	if err != nil {
   467  		agent.Shutdown()
   468  		c.Ui.Error(fmt.Sprintf("Error starting http server: %s", err))
   469  		return err
   470  	}
   471  	c.httpServer = http
   472  
   473  	// If DisableUpdateCheck is not enabled, set up update checking
   474  	// (DisableUpdateCheck is false by default)
   475  	if config.DisableUpdateCheck != nil && !*config.DisableUpdateCheck {
   476  		version := config.Version.Version
   477  		if config.Version.VersionPrerelease != "" {
   478  			version += fmt.Sprintf("-%s", config.Version.VersionPrerelease)
   479  		}
   480  		updateParams := &checkpoint.CheckParams{
   481  			Product: "nomad",
   482  			Version: version,
   483  		}
   484  		if !config.DisableAnonymousSignature {
   485  			updateParams.SignatureFile = filepath.Join(config.DataDir, "checkpoint-signature")
   486  		}
   487  
   488  		// Schedule a periodic check with expected interval of 24 hours
   489  		checkpoint.CheckInterval(updateParams, 24*time.Hour, c.checkpointResults)
   490  
   491  		// Do an immediate check within the next 30 seconds
   492  		go func() {
   493  			time.Sleep(lib.RandomStagger(30 * time.Second))
   494  			c.checkpointResults(checkpoint.Check(updateParams))
   495  		}()
   496  	}
   497  
   498  	return nil
   499  }
   500  
   501  // checkpointResults is used to handler periodic results from our update checker
   502  func (c *Command) checkpointResults(results *checkpoint.CheckResponse, err error) {
   503  	if err != nil {
   504  		c.Ui.Error(fmt.Sprintf("Failed to check for updates: %v", err))
   505  		return
   506  	}
   507  	if results.Outdated {
   508  		c.Ui.Error(fmt.Sprintf("Newer Nomad version available: %s (currently running: %s)", results.CurrentVersion, c.Version.VersionNumber()))
   509  	}
   510  	for _, alert := range results.Alerts {
   511  		switch alert.Level {
   512  		case "info":
   513  			c.Ui.Info(fmt.Sprintf("Bulletin [%s]: %s (%s)", alert.Level, alert.Message, alert.URL))
   514  		default:
   515  			c.Ui.Error(fmt.Sprintf("Bulletin [%s]: %s (%s)", alert.Level, alert.Message, alert.URL))
   516  		}
   517  	}
   518  }
   519  
   520  func (c *Command) AutocompleteFlags() complete.Flags {
   521  	configFilePredictor := complete.PredictOr(
   522  		complete.PredictFiles("*.json"),
   523  		complete.PredictFiles("*.hcl"))
   524  
   525  	return map[string]complete.Predictor{
   526  		"-dev":                           complete.PredictNothing,
   527  		"-dev-connect":                   complete.PredictNothing,
   528  		"-server":                        complete.PredictNothing,
   529  		"-client":                        complete.PredictNothing,
   530  		"-bootstrap-expect":              complete.PredictAnything,
   531  		"-encrypt":                       complete.PredictAnything,
   532  		"-raft-protocol":                 complete.PredictAnything,
   533  		"-rejoin":                        complete.PredictNothing,
   534  		"-join":                          complete.PredictAnything,
   535  		"-retry-join":                    complete.PredictAnything,
   536  		"-retry-max":                     complete.PredictAnything,
   537  		"-state-dir":                     complete.PredictDirs("*"),
   538  		"-alloc-dir":                     complete.PredictDirs("*"),
   539  		"-node-class":                    complete.PredictAnything,
   540  		"-servers":                       complete.PredictAnything,
   541  		"-meta":                          complete.PredictAnything,
   542  		"-config":                        configFilePredictor,
   543  		"-bind":                          complete.PredictAnything,
   544  		"-region":                        complete.PredictAnything,
   545  		"-data-dir":                      complete.PredictDirs("*"),
   546  		"-plugin-dir":                    complete.PredictDirs("*"),
   547  		"-dc":                            complete.PredictAnything,
   548  		"-log-level":                     complete.PredictAnything,
   549  		"-json-logs":                     complete.PredictNothing,
   550  		"-node":                          complete.PredictAnything,
   551  		"-consul-auth":                   complete.PredictAnything,
   552  		"-consul-auto-advertise":         complete.PredictNothing,
   553  		"-consul-ca-file":                complete.PredictAnything,
   554  		"-consul-cert-file":              complete.PredictAnything,
   555  		"-consul-key-file":               complete.PredictAnything,
   556  		"-consul-checks-use-advertise":   complete.PredictNothing,
   557  		"-consul-client-auto-join":       complete.PredictNothing,
   558  		"-consul-client-service-name":    complete.PredictAnything,
   559  		"-consul-client-http-check-name": complete.PredictAnything,
   560  		"-consul-server-service-name":    complete.PredictAnything,
   561  		"-consul-server-http-check-name": complete.PredictAnything,
   562  		"-consul-server-serf-check-name": complete.PredictAnything,
   563  		"-consul-server-rpc-check-name":  complete.PredictAnything,
   564  		"-consul-server-auto-join":       complete.PredictNothing,
   565  		"-consul-ssl":                    complete.PredictNothing,
   566  		"-consul-verify-ssl":             complete.PredictNothing,
   567  		"-consul-address":                complete.PredictAnything,
   568  		"-consul-token":                  complete.PredictAnything,
   569  		"-vault-enabled":                 complete.PredictNothing,
   570  		"-vault-allow-unauthenticated":   complete.PredictNothing,
   571  		"-vault-token":                   complete.PredictAnything,
   572  		"-vault-address":                 complete.PredictAnything,
   573  		"-vault-create-from-role":        complete.PredictAnything,
   574  		"-vault-ca-file":                 complete.PredictAnything,
   575  		"-vault-ca-path":                 complete.PredictAnything,
   576  		"-vault-cert-file":               complete.PredictAnything,
   577  		"-vault-key-file":                complete.PredictAnything,
   578  		"-vault-tls-skip-verify":         complete.PredictNothing,
   579  		"-vault-tls-server-name":         complete.PredictAnything,
   580  		"-acl-enabled":                   complete.PredictNothing,
   581  		"-acl-replication-token":         complete.PredictAnything,
   582  	}
   583  }
   584  
   585  func (c *Command) AutocompleteArgs() complete.Predictor {
   586  	return nil
   587  }
   588  
   589  func (c *Command) Run(args []string) int {
   590  	c.Ui = &cli.PrefixedUi{
   591  		OutputPrefix: "==> ",
   592  		InfoPrefix:   "    ",
   593  		ErrorPrefix:  "==> ",
   594  		Ui:           c.Ui,
   595  	}
   596  
   597  	// Parse our configs
   598  	c.args = args
   599  	config := c.readConfig()
   600  	if config == nil {
   601  		return 1
   602  	}
   603  
   604  	// Setup the log outputs
   605  	logGate, logOutput := c.setupLoggers(config)
   606  	if logGate == nil {
   607  		return 1
   608  	}
   609  
   610  	// Create logger
   611  	logger := hclog.NewInterceptLogger(&hclog.LoggerOptions{
   612  		Name:       "agent",
   613  		Level:      hclog.LevelFromString(config.LogLevel),
   614  		Output:     logOutput,
   615  		JSONFormat: config.LogJson,
   616  	})
   617  
   618  	// Swap out UI implementation if json logging is enabled
   619  	if config.LogJson {
   620  		c.Ui = &logging.HcLogUI{Log: logger}
   621  	}
   622  
   623  	// Log config files
   624  	if len(config.Files) > 0 {
   625  		c.Ui.Output(fmt.Sprintf("Loaded configuration from %s", strings.Join(config.Files, ", ")))
   626  	} else {
   627  		c.Ui.Output("No configuration files loaded")
   628  	}
   629  
   630  	// Initialize the telemetry
   631  	inmem, err := c.setupTelemetry(config)
   632  	if err != nil {
   633  		c.Ui.Error(fmt.Sprintf("Error initializing telemetry: %s", err))
   634  		return 1
   635  	}
   636  
   637  	// Create the agent
   638  	if err := c.setupAgent(config, logger, logOutput, inmem); err != nil {
   639  		logGate.Flush()
   640  		return 1
   641  	}
   642  	defer c.agent.Shutdown()
   643  
   644  	// Shutdown the HTTP server at the end
   645  	defer func() {
   646  		if c.httpServer != nil {
   647  			c.httpServer.Shutdown()
   648  		}
   649  	}()
   650  
   651  	// Join startup nodes if specified
   652  	if err := c.startupJoin(config); err != nil {
   653  		c.Ui.Error(err.Error())
   654  		return 1
   655  	}
   656  
   657  	// Compile agent information for output later
   658  	info := make(map[string]string)
   659  	info["version"] = config.Version.VersionNumber()
   660  	info["client"] = strconv.FormatBool(config.Client.Enabled)
   661  	info["log level"] = config.LogLevel
   662  	info["server"] = strconv.FormatBool(config.Server.Enabled)
   663  	info["region"] = fmt.Sprintf("%s (DC: %s)", config.Region, config.Datacenter)
   664  	info["bind addrs"] = c.getBindAddrSynopsis()
   665  	info["advertise addrs"] = c.getAdvertiseAddrSynopsis()
   666  
   667  	// Sort the keys for output
   668  	infoKeys := make([]string, 0, len(info))
   669  	for key := range info {
   670  		infoKeys = append(infoKeys, key)
   671  	}
   672  	sort.Strings(infoKeys)
   673  
   674  	// Agent configuration output
   675  	padding := 18
   676  	c.Ui.Output("Nomad agent configuration:\n")
   677  	for _, k := range infoKeys {
   678  		c.Ui.Info(fmt.Sprintf(
   679  			"%s%s: %s",
   680  			strings.Repeat(" ", padding-len(k)),
   681  			strings.Title(k),
   682  			info[k]))
   683  	}
   684  	c.Ui.Output("")
   685  
   686  	// Output the header that the server has started
   687  	c.Ui.Output("Nomad agent started! Log data will stream in below:\n")
   688  
   689  	// Enable log streaming
   690  	logGate.Flush()
   691  
   692  	// Start retry join process
   693  	if err := c.handleRetryJoin(config); err != nil {
   694  		c.Ui.Error(err.Error())
   695  		return 1
   696  	}
   697  
   698  	// Wait for exit
   699  	return c.handleSignals()
   700  }
   701  
   702  // handleRetryJoin is used to start retry joining if it is configured.
   703  func (c *Command) handleRetryJoin(config *Config) error {
   704  	c.retryJoinErrCh = make(chan struct{})
   705  
   706  	if config.Server.Enabled && len(config.Server.RetryJoin) != 0 {
   707  		joiner := retryJoiner{
   708  			discover:      &discover.Discover{},
   709  			errCh:         c.retryJoinErrCh,
   710  			logger:        c.agent.logger.Named("joiner"),
   711  			serverJoin:    c.agent.server.Join,
   712  			serverEnabled: true,
   713  		}
   714  
   715  		if err := joiner.Validate(config); err != nil {
   716  			return err
   717  		}
   718  
   719  		// Remove the duplicate fields
   720  		if len(config.Server.RetryJoin) != 0 {
   721  			config.Server.ServerJoin.RetryJoin = config.Server.RetryJoin
   722  			config.Server.RetryJoin = nil
   723  		}
   724  		if config.Server.RetryMaxAttempts != 0 {
   725  			config.Server.ServerJoin.RetryMaxAttempts = config.Server.RetryMaxAttempts
   726  			config.Server.RetryMaxAttempts = 0
   727  		}
   728  		if config.Server.RetryInterval != 0 {
   729  			config.Server.ServerJoin.RetryInterval = config.Server.RetryInterval
   730  			config.Server.RetryInterval = 0
   731  		}
   732  
   733  		c.agent.logger.Warn("using deprecated retry_join fields. Upgrade configuration to use server_join")
   734  	}
   735  
   736  	if config.Server.Enabled &&
   737  		config.Server.ServerJoin != nil &&
   738  		len(config.Server.ServerJoin.RetryJoin) != 0 {
   739  
   740  		joiner := retryJoiner{
   741  			discover:      &discover.Discover{},
   742  			errCh:         c.retryJoinErrCh,
   743  			logger:        c.agent.logger.Named("joiner"),
   744  			serverJoin:    c.agent.server.Join,
   745  			serverEnabled: true,
   746  		}
   747  
   748  		if err := joiner.Validate(config); err != nil {
   749  			return err
   750  		}
   751  
   752  		go joiner.RetryJoin(config.Server.ServerJoin)
   753  	}
   754  
   755  	if config.Client.Enabled &&
   756  		config.Client.ServerJoin != nil &&
   757  		len(config.Client.ServerJoin.RetryJoin) != 0 {
   758  		joiner := retryJoiner{
   759  			discover:      &discover.Discover{},
   760  			errCh:         c.retryJoinErrCh,
   761  			logger:        c.agent.logger.Named("joiner"),
   762  			clientJoin:    c.agent.client.SetServers,
   763  			clientEnabled: true,
   764  		}
   765  
   766  		if err := joiner.Validate(config); err != nil {
   767  			return err
   768  		}
   769  
   770  		go joiner.RetryJoin(config.Client.ServerJoin)
   771  	}
   772  
   773  	return nil
   774  }
   775  
   776  // handleSignals blocks until we get an exit-causing signal
   777  func (c *Command) handleSignals() int {
   778  	signalCh := make(chan os.Signal, 4)
   779  	signal.Notify(signalCh, os.Interrupt, syscall.SIGTERM, syscall.SIGHUP, syscall.SIGPIPE)
   780  
   781  	// Wait for a signal
   782  WAIT:
   783  	var sig os.Signal
   784  	select {
   785  	case s := <-signalCh:
   786  		sig = s
   787  	case <-winsvc.ShutdownChannel():
   788  		sig = os.Interrupt
   789  	case <-c.ShutdownCh:
   790  		sig = os.Interrupt
   791  	case <-c.retryJoinErrCh:
   792  		return 1
   793  	}
   794  
   795  	// Skip any SIGPIPE signal and don't try to log it (See issues #1798, #3554)
   796  	if sig == syscall.SIGPIPE {
   797  		goto WAIT
   798  	}
   799  
   800  	c.Ui.Output(fmt.Sprintf("Caught signal: %v", sig))
   801  
   802  	// Check if this is a SIGHUP
   803  	if sig == syscall.SIGHUP {
   804  		c.handleReload()
   805  		goto WAIT
   806  	}
   807  
   808  	// Check if we should do a graceful leave
   809  	graceful := false
   810  	if sig == os.Interrupt && c.agent.GetConfig().LeaveOnInt {
   811  		graceful = true
   812  	} else if sig == syscall.SIGTERM && c.agent.GetConfig().LeaveOnTerm {
   813  		graceful = true
   814  	}
   815  
   816  	// Bail fast if not doing a graceful leave
   817  	if !graceful {
   818  		return 1
   819  	}
   820  
   821  	// Attempt a graceful leave
   822  	gracefulCh := make(chan struct{})
   823  	c.Ui.Output("Gracefully shutting down agent...")
   824  	go func() {
   825  		if err := c.agent.Leave(); err != nil {
   826  			c.Ui.Error(fmt.Sprintf("Error: %s", err))
   827  			return
   828  		}
   829  		close(gracefulCh)
   830  	}()
   831  
   832  	// Wait for leave or another signal
   833  	select {
   834  	case <-signalCh:
   835  		return 1
   836  	case <-time.After(gracefulTimeout):
   837  		return 1
   838  	case <-gracefulCh:
   839  		return 0
   840  	}
   841  }
   842  
   843  // reloadHTTPServer shuts down the existing HTTP server and restarts it. This
   844  // is helpful when reloading the agent configuration.
   845  func (c *Command) reloadHTTPServer() error {
   846  	c.agent.logger.Info("reloading HTTP server with new TLS configuration")
   847  
   848  	c.httpServer.Shutdown()
   849  
   850  	http, err := NewHTTPServer(c.agent, c.agent.config)
   851  	if err != nil {
   852  		return err
   853  	}
   854  	c.httpServer = http
   855  
   856  	return nil
   857  }
   858  
   859  // handleReload is invoked when we should reload our configs, e.g. SIGHUP
   860  func (c *Command) handleReload() {
   861  	c.Ui.Output("Reloading configuration...")
   862  	newConf := c.readConfig()
   863  	if newConf == nil {
   864  		c.Ui.Error(fmt.Sprintf("Failed to reload configs"))
   865  		return
   866  	}
   867  
   868  	// Change the log level
   869  	minLevel := logutils.LogLevel(strings.ToUpper(newConf.LogLevel))
   870  	if ValidateLevelFilter(minLevel, c.logFilter) {
   871  		c.logFilter.SetMinLevel(minLevel)
   872  	} else {
   873  		c.Ui.Error(fmt.Sprintf(
   874  			"Invalid log level: %s. Valid log levels are: %v",
   875  			minLevel, c.logFilter.Levels))
   876  
   877  		// Keep the current log level
   878  		newConf.LogLevel = c.agent.GetConfig().LogLevel
   879  	}
   880  
   881  	shouldReloadAgent, shouldReloadHTTP := c.agent.ShouldReload(newConf)
   882  	if shouldReloadAgent {
   883  		c.agent.logger.Debug("starting reload of agent config")
   884  		err := c.agent.Reload(newConf)
   885  		if err != nil {
   886  			c.agent.logger.Error("failed to reload the config", "error", err)
   887  			return
   888  		}
   889  	}
   890  
   891  	if s := c.agent.Server(); s != nil {
   892  		c.agent.logger.Debug("starting reload of server config")
   893  		sconf, err := convertServerConfig(newConf)
   894  		if err != nil {
   895  			c.agent.logger.Error("failed to convert server config", "error", err)
   896  			return
   897  		}
   898  
   899  		// Finalize the config to get the agent objects injected in
   900  		c.agent.finalizeServerConfig(sconf)
   901  
   902  		// Reload the config
   903  		if err := s.Reload(sconf); err != nil {
   904  			c.agent.logger.Error("reloading server config failed", "error", err)
   905  			return
   906  		}
   907  	}
   908  
   909  	if s := c.agent.Client(); s != nil {
   910  		c.agent.logger.Debug("starting reload of client config")
   911  		clientConfig, err := convertClientConfig(newConf)
   912  		if err != nil {
   913  			c.agent.logger.Error("failed to convert client config", "error", err)
   914  			return
   915  		}
   916  
   917  		// Finalize the config to get the agent objects injected in
   918  		if err := c.agent.finalizeClientConfig(clientConfig); err != nil {
   919  			c.agent.logger.Error("failed to finalize client config", "error", err)
   920  			return
   921  		}
   922  
   923  		if err := c.agent.Client().Reload(clientConfig); err != nil {
   924  			c.agent.logger.Error("reloading client config failed", "error", err)
   925  			return
   926  		}
   927  	}
   928  
   929  	// reload HTTP server after we have reloaded both client and server, in case
   930  	// we error in either of the above cases. For example, reloading the http
   931  	// server to a TLS connection could succeed, while reloading the server's rpc
   932  	// connections could fail.
   933  	if shouldReloadHTTP {
   934  		err := c.reloadHTTPServer()
   935  		if err != nil {
   936  			c.agent.httpLogger.Error("reloading config failed", "error", err)
   937  			return
   938  		}
   939  	}
   940  }
   941  
   942  // setupTelemetry is used ot setup the telemetry sub-systems
   943  func (c *Command) setupTelemetry(config *Config) (*metrics.InmemSink, error) {
   944  	/* Setup telemetry
   945  	Aggregate on 10 second intervals for 1 minute. Expose the
   946  	metrics over stderr when there is a SIGUSR1 received.
   947  	*/
   948  	inm := metrics.NewInmemSink(10*time.Second, time.Minute)
   949  	metrics.DefaultInmemSignal(inm)
   950  
   951  	var telConfig *Telemetry
   952  	if config.Telemetry == nil {
   953  		telConfig = &Telemetry{}
   954  	} else {
   955  		telConfig = config.Telemetry
   956  	}
   957  
   958  	metricsConf := metrics.DefaultConfig("nomad")
   959  	metricsConf.EnableHostname = !telConfig.DisableHostname
   960  
   961  	// Prefer the hostname as a label.
   962  	metricsConf.EnableHostnameLabel = !telConfig.DisableHostname &&
   963  		!telConfig.DisableTaggedMetrics && !telConfig.BackwardsCompatibleMetrics
   964  
   965  	if telConfig.UseNodeName {
   966  		metricsConf.HostName = config.NodeName
   967  		metricsConf.EnableHostname = true
   968  	}
   969  
   970  	allowedPrefixes, blockedPrefixes, err := telConfig.PrefixFilters()
   971  	if err != nil {
   972  		return inm, err
   973  	}
   974  
   975  	metricsConf.AllowedPrefixes = allowedPrefixes
   976  	metricsConf.BlockedPrefixes = blockedPrefixes
   977  
   978  	if telConfig.FilterDefault != nil {
   979  		metricsConf.FilterDefault = *telConfig.FilterDefault
   980  	}
   981  
   982  	// Configure the statsite sink
   983  	var fanout metrics.FanoutSink
   984  	if telConfig.StatsiteAddr != "" {
   985  		sink, err := metrics.NewStatsiteSink(telConfig.StatsiteAddr)
   986  		if err != nil {
   987  			return inm, err
   988  		}
   989  		fanout = append(fanout, sink)
   990  	}
   991  
   992  	// Configure the statsd sink
   993  	if telConfig.StatsdAddr != "" {
   994  		sink, err := metrics.NewStatsdSink(telConfig.StatsdAddr)
   995  		if err != nil {
   996  			return inm, err
   997  		}
   998  		fanout = append(fanout, sink)
   999  	}
  1000  
  1001  	// Configure the prometheus sink
  1002  	if telConfig.PrometheusMetrics {
  1003  		promSink, err := prometheus.NewPrometheusSink()
  1004  		if err != nil {
  1005  			return inm, err
  1006  		}
  1007  		fanout = append(fanout, promSink)
  1008  	}
  1009  
  1010  	// Configure the datadog sink
  1011  	if telConfig.DataDogAddr != "" {
  1012  		sink, err := datadog.NewDogStatsdSink(telConfig.DataDogAddr, config.NodeName)
  1013  		if err != nil {
  1014  			return inm, err
  1015  		}
  1016  		sink.SetTags(telConfig.DataDogTags)
  1017  		fanout = append(fanout, sink)
  1018  	}
  1019  
  1020  	// Configure the Circonus sink
  1021  	if telConfig.CirconusAPIToken != "" || telConfig.CirconusCheckSubmissionURL != "" {
  1022  		cfg := &circonus.Config{}
  1023  		cfg.Interval = telConfig.CirconusSubmissionInterval
  1024  		cfg.CheckManager.API.TokenKey = telConfig.CirconusAPIToken
  1025  		cfg.CheckManager.API.TokenApp = telConfig.CirconusAPIApp
  1026  		cfg.CheckManager.API.URL = telConfig.CirconusAPIURL
  1027  		cfg.CheckManager.Check.SubmissionURL = telConfig.CirconusCheckSubmissionURL
  1028  		cfg.CheckManager.Check.ID = telConfig.CirconusCheckID
  1029  		cfg.CheckManager.Check.ForceMetricActivation = telConfig.CirconusCheckForceMetricActivation
  1030  		cfg.CheckManager.Check.InstanceID = telConfig.CirconusCheckInstanceID
  1031  		cfg.CheckManager.Check.SearchTag = telConfig.CirconusCheckSearchTag
  1032  		cfg.CheckManager.Check.Tags = telConfig.CirconusCheckTags
  1033  		cfg.CheckManager.Check.DisplayName = telConfig.CirconusCheckDisplayName
  1034  		cfg.CheckManager.Broker.ID = telConfig.CirconusBrokerID
  1035  		cfg.CheckManager.Broker.SelectTag = telConfig.CirconusBrokerSelectTag
  1036  
  1037  		if cfg.CheckManager.Check.DisplayName == "" {
  1038  			cfg.CheckManager.Check.DisplayName = "Nomad"
  1039  		}
  1040  
  1041  		if cfg.CheckManager.API.TokenApp == "" {
  1042  			cfg.CheckManager.API.TokenApp = "nomad"
  1043  		}
  1044  
  1045  		if cfg.CheckManager.Check.SearchTag == "" {
  1046  			cfg.CheckManager.Check.SearchTag = "service:nomad"
  1047  		}
  1048  
  1049  		sink, err := circonus.NewCirconusSink(cfg)
  1050  		if err != nil {
  1051  			return inm, err
  1052  		}
  1053  		sink.Start()
  1054  		fanout = append(fanout, sink)
  1055  	}
  1056  
  1057  	// Initialize the global sink
  1058  	if len(fanout) > 0 {
  1059  		fanout = append(fanout, inm)
  1060  		metrics.NewGlobal(metricsConf, fanout)
  1061  	} else {
  1062  		metricsConf.EnableHostname = false
  1063  		metrics.NewGlobal(metricsConf, inm)
  1064  	}
  1065  
  1066  	return inm, nil
  1067  }
  1068  
  1069  func (c *Command) startupJoin(config *Config) error {
  1070  	// Nothing to do
  1071  	if !config.Server.Enabled {
  1072  		return nil
  1073  	}
  1074  
  1075  	// Validate both old and new aren't being set
  1076  	old := len(config.Server.StartJoin)
  1077  	var new int
  1078  	if config.Server.ServerJoin != nil {
  1079  		new = len(config.Server.ServerJoin.StartJoin)
  1080  	}
  1081  	if old != 0 && new != 0 {
  1082  		return fmt.Errorf("server_join and start_join cannot both be defined; prefer setting the server_join stanza")
  1083  	}
  1084  
  1085  	// Nothing to do
  1086  	if old+new == 0 {
  1087  		return nil
  1088  	}
  1089  
  1090  	// Combine the lists and join
  1091  	joining := config.Server.StartJoin
  1092  	if new != 0 {
  1093  		joining = append(joining, config.Server.ServerJoin.StartJoin...)
  1094  	}
  1095  
  1096  	c.Ui.Output("Joining cluster...")
  1097  	n, err := c.agent.server.Join(joining)
  1098  	if err != nil {
  1099  		return err
  1100  	}
  1101  
  1102  	c.Ui.Output(fmt.Sprintf("Join completed. Synced with %d initial agents", n))
  1103  	return nil
  1104  }
  1105  
  1106  // getBindAddrSynopsis returns a string that describes the addresses the agent
  1107  // is bound to.
  1108  func (c *Command) getBindAddrSynopsis() string {
  1109  	if c == nil || c.agent == nil || c.agent.config == nil || c.agent.config.normalizedAddrs == nil {
  1110  		return ""
  1111  	}
  1112  
  1113  	b := new(strings.Builder)
  1114  	fmt.Fprintf(b, "HTTP: %s", c.agent.config.normalizedAddrs.HTTP)
  1115  
  1116  	if c.agent.server != nil {
  1117  		if c.agent.config.normalizedAddrs.RPC != "" {
  1118  			fmt.Fprintf(b, "; RPC: %s", c.agent.config.normalizedAddrs.RPC)
  1119  		}
  1120  		if c.agent.config.normalizedAddrs.Serf != "" {
  1121  			fmt.Fprintf(b, "; Serf: %s", c.agent.config.normalizedAddrs.Serf)
  1122  		}
  1123  	}
  1124  
  1125  	return b.String()
  1126  }
  1127  
  1128  // getAdvertiseAddrSynopsis returns a string that describes the addresses the agent
  1129  // is advertising.
  1130  func (c *Command) getAdvertiseAddrSynopsis() string {
  1131  	if c == nil || c.agent == nil || c.agent.config == nil || c.agent.config.AdvertiseAddrs == nil {
  1132  		return ""
  1133  	}
  1134  
  1135  	b := new(strings.Builder)
  1136  	fmt.Fprintf(b, "HTTP: %s", c.agent.config.AdvertiseAddrs.HTTP)
  1137  
  1138  	if c.agent.server != nil {
  1139  		if c.agent.config.AdvertiseAddrs.RPC != "" {
  1140  			fmt.Fprintf(b, "; RPC: %s", c.agent.config.AdvertiseAddrs.RPC)
  1141  		}
  1142  		if c.agent.config.AdvertiseAddrs.Serf != "" {
  1143  			fmt.Fprintf(b, "; Serf: %s", c.agent.config.AdvertiseAddrs.Serf)
  1144  		}
  1145  	}
  1146  
  1147  	return b.String()
  1148  }
  1149  
  1150  func (c *Command) Synopsis() string {
  1151  	return "Runs a Nomad agent"
  1152  }
  1153  
  1154  func (c *Command) Help() string {
  1155  	helpText := `
  1156  Usage: nomad agent [options]
  1157  
  1158    Starts the Nomad agent and runs until an interrupt is received.
  1159    The agent may be a client and/or server.
  1160  
  1161    The Nomad agent's configuration primarily comes from the config
  1162    files used, but a subset of the options may also be passed directly
  1163    as CLI arguments, listed below.
  1164  
  1165  General Options (clients and servers):
  1166  
  1167    -bind=<addr>
  1168      The address the agent will bind to for all of its various network
  1169      services. The individual services that run bind to individual
  1170      ports on this address. Defaults to the loopback 127.0.0.1.
  1171  
  1172    -config=<path>
  1173      The path to either a single config file or a directory of config
  1174      files to use for configuring the Nomad agent. This option may be
  1175      specified multiple times. If multiple config files are used, the
  1176      values from each will be merged together. During merging, values
  1177      from files found later in the list are merged over values from
  1178      previously parsed files.
  1179  
  1180    -data-dir=<path>
  1181      The data directory used to store state and other persistent data.
  1182      On client machines this is used to house allocation data such as
  1183      downloaded artifacts used by drivers. On server nodes, the data
  1184      dir is also used to store the replicated log.
  1185  
  1186    -plugin-dir=<path>
  1187      The plugin directory is used to discover Nomad plugins. If not specified,
  1188      the plugin directory defaults to be that of <data-dir>/plugins/.
  1189  
  1190    -dc=<datacenter>
  1191      The name of the datacenter this Nomad agent is a member of. By
  1192      default this is set to "dc1".
  1193  
  1194    -log-level=<level>
  1195      Specify the verbosity level of Nomad's logs. Valid values include
  1196      DEBUG, INFO, and WARN, in decreasing order of verbosity. The
  1197      default is INFO.
  1198  
  1199    -log-json
  1200      Output logs in a JSON format. The default is false.
  1201  
  1202    -node=<name>
  1203      The name of the local agent. This name is used to identify the node
  1204      in the cluster. The name must be unique per region. The default is
  1205      the current hostname of the machine.
  1206  
  1207    -region=<region>
  1208      Name of the region the Nomad agent will be a member of. By default
  1209      this value is set to "global".
  1210  
  1211    -dev
  1212      Start the agent in development mode. This enables a pre-configured
  1213      dual-role agent (client + server) which is useful for developing
  1214      or testing Nomad. No other configuration is required to start the
  1215      agent in this mode, but you may pass an optional comma-separated
  1216      list of mode configurations:
  1217  
  1218    -dev-connect
  1219  	Start the agent in development mode, but bind to a public network
  1220  	interface rather than localhost for using Consul Connect. This
  1221  	mode is supported only on Linux as root.
  1222  
  1223  Server Options:
  1224  
  1225    -server
  1226      Enable server mode for the agent. Agents in server mode are
  1227      clustered together and handle the additional responsibility of
  1228      leader election, data replication, and scheduling work onto
  1229      eligible client nodes.
  1230  
  1231    -bootstrap-expect=<num>
  1232      Configures the expected number of servers nodes to wait for before
  1233      bootstrapping the cluster. Once <num> servers have joined each other,
  1234      Nomad initiates the bootstrap process.
  1235  
  1236    -encrypt=<key>
  1237      Provides the gossip encryption key
  1238  
  1239    -join=<address>
  1240      Address of an agent to join at start time. Can be specified
  1241      multiple times.
  1242  
  1243    -raft-protocol=<num>
  1244      The Raft protocol version to use. Used for enabling certain Autopilot
  1245      features. Defaults to 2.
  1246  
  1247    -retry-join=<address>
  1248      Address of an agent to join at start time with retries enabled.
  1249      Can be specified multiple times.
  1250  
  1251    -retry-max=<num>
  1252      Maximum number of join attempts. Defaults to 0, which will retry
  1253      indefinitely.
  1254  
  1255    -retry-interval=<dur>
  1256      Time to wait between join attempts.
  1257  
  1258    -rejoin
  1259      Ignore a previous leave and attempts to rejoin the cluster.
  1260  
  1261  Client Options:
  1262  
  1263    -client
  1264      Enable client mode for the agent. Client mode enables a given node to be
  1265      evaluated for allocations. If client mode is not enabled, no work will be
  1266      scheduled to the agent.
  1267  
  1268    -state-dir
  1269      The directory used to store state and other persistent data. If not
  1270      specified a subdirectory under the "-data-dir" will be used.
  1271  
  1272    -alloc-dir
  1273      The directory used to store allocation data such as downloaded artifacts as
  1274      well as data produced by tasks. If not specified, a subdirectory under the
  1275      "-data-dir" will be used.
  1276  
  1277    -servers
  1278      A list of known server addresses to connect to given as "host:port" and
  1279      delimited by commas.
  1280  
  1281    -node-class
  1282      Mark this node as a member of a node-class. This can be used to label
  1283      similar node types.
  1284  
  1285    -meta
  1286      User specified metadata to associated with the node. Each instance of -meta
  1287      parses a single KEY=VALUE pair. Repeat the meta flag for each key/value pair
  1288      to be added.
  1289  
  1290    -network-interface
  1291      Forces the network fingerprinter to use the specified network interface.
  1292  
  1293    -network-speed
  1294      The default speed for network interfaces in MBits if the link speed can not
  1295      be determined dynamically.
  1296  
  1297  ACL Options:
  1298  
  1299    -acl-enabled
  1300      Specifies whether the agent should enable ACLs.
  1301  
  1302    -acl-replication-token
  1303      The replication token for servers to use when replicating from the
  1304      authoritative region. The token must be a valid management token from the
  1305      authoritative region.
  1306  
  1307  Consul Options:
  1308  
  1309    -consul-address=<addr>
  1310      Specifies the address to the local Consul agent, given in the format host:port.
  1311      Supports Unix sockets with the format: unix:///tmp/consul/consul.sock
  1312  
  1313    -consul-auth=<auth>
  1314      Specifies the HTTP Basic Authentication information to use for access to the
  1315      Consul Agent, given in the format username:password.
  1316  
  1317    -consul-auto-advertise
  1318      Specifies if Nomad should advertise its services in Consul. The services
  1319      are named according to server_service_name and client_service_name. Nomad
  1320      servers and clients advertise their respective services, each tagged
  1321      appropriately with either http or rpc tag. Nomad servers also advertise a
  1322      serf tagged service.
  1323  
  1324    -consul-ca-file=<path>
  1325      Specifies an optional path to the CA certificate used for Consul communication.
  1326      This defaults to the system bundle if unspecified.
  1327  
  1328    -consul-cert-file=<path>
  1329      Specifies the path to the certificate used for Consul communication. If this
  1330      is set then you need to also set key_file.
  1331  
  1332    -consul-checks-use-advertise
  1333      Specifies if Consul heath checks should bind to the advertise address. By
  1334      default, this is the bind address.
  1335  
  1336    -consul-client-auto-join
  1337      Specifies if the Nomad clients should automatically discover servers in the
  1338      same region by searching for the Consul service name defined in the
  1339      server_service_name option.
  1340  
  1341    -consul-client-service-name=<name>
  1342      Specifies the name of the service in Consul for the Nomad clients.
  1343  
  1344    -consul-client-http-check-name=<name>
  1345      Specifies the HTTP health check name in Consul for the Nomad clients.
  1346  
  1347    -consul-key-file=<path>
  1348      Specifies the path to the private key used for Consul communication. If this
  1349      is set then you need to also set cert_file.
  1350  
  1351    -consul-server-service-name=<name>
  1352      Specifies the name of the service in Consul for the Nomad servers.
  1353  
  1354    -consul-server-http-check-name=<name>
  1355      Specifies the HTTP health check name in Consul for the Nomad servers.
  1356  
  1357    -consul-server-serf-check-name=<name>
  1358      Specifies the Serf health check name in Consul for the Nomad servers.
  1359  
  1360    -consul-server-rpc-check-name=<name>
  1361      Specifies the RPC health check name in Consul for the Nomad servers.
  1362  
  1363    -consul-server-auto-join
  1364      Specifies if the Nomad servers should automatically discover and join other
  1365      Nomad servers by searching for the Consul service name defined in the
  1366      server_service_name option. This search only happens if the server does not
  1367      have a leader.
  1368  
  1369    -consul-ssl
  1370      Specifies if the transport scheme should use HTTPS to communicate with the
  1371      Consul agent.
  1372  
  1373    -consul-token=<token>
  1374      Specifies the token used to provide a per-request ACL token.
  1375  
  1376    -consul-verify-ssl
  1377      Specifies if SSL peer verification should be used when communicating to the
  1378      Consul API client over HTTPS.
  1379  
  1380  Vault Options:
  1381  
  1382    -vault-enabled
  1383      Whether to enable or disable Vault integration.
  1384  
  1385    -vault-address=<addr>
  1386      The address to communicate with Vault. This should be provided with the http://
  1387      or https:// prefix.
  1388  
  1389    -vault-token=<token>
  1390      The Vault token used to derive tokens from Vault on behalf of clients.
  1391      This only needs to be set on Servers. Overrides the Vault token read from
  1392      the VAULT_TOKEN environment variable.
  1393  
  1394    -vault-create-from-role=<role>
  1395      The role name to create tokens for tasks from.
  1396  
  1397    -vault-allow-unauthenticated
  1398      Whether to allow jobs to be submitted that request Vault Tokens but do not
  1399      authentication. The flag only applies to Servers.
  1400  
  1401    -vault-ca-file=<path>
  1402      The path to a PEM-encoded CA cert file to use to verify the Vault server SSL
  1403      certificate.
  1404  
  1405    -vault-ca-path=<path>
  1406      The path to a directory of PEM-encoded CA cert files to verify the Vault server
  1407      certificate.
  1408  
  1409    -vault-cert-file=<token>
  1410      The path to the certificate for Vault communication.
  1411  
  1412    -vault-key-file=<addr>
  1413      The path to the private key for Vault communication.
  1414  
  1415    -vault-tls-skip-verify=<token>
  1416      Enables or disables SSL certificate verification.
  1417  
  1418    -vault-tls-server-name=<token>
  1419      Used to set the SNI host when connecting over TLS.
  1420   `
  1421  	return strings.TrimSpace(helpText)
  1422  }