github.com/mongey/nomad@v0.5.2/command/agent/agent.go (about)

     1  package agent
     2  
     3  import (
     4  	"fmt"
     5  	"io"
     6  	"log"
     7  	"net"
     8  	"os"
     9  	"path/filepath"
    10  	"runtime"
    11  	"strconv"
    12  	"strings"
    13  	"sync"
    14  	"sync/atomic"
    15  	"time"
    16  
    17  	"github.com/hashicorp/nomad/client"
    18  	clientconfig "github.com/hashicorp/nomad/client/config"
    19  	"github.com/hashicorp/nomad/command/agent/consul"
    20  	"github.com/hashicorp/nomad/nomad"
    21  	"github.com/hashicorp/nomad/nomad/structs"
    22  )
    23  
    24  const (
    25  	clientHttpCheckInterval = 10 * time.Second
    26  	clientHttpCheckTimeout  = 3 * time.Second
    27  	serverHttpCheckInterval = 10 * time.Second
    28  	serverHttpCheckTimeout  = 6 * time.Second
    29  	serverRpcCheckInterval  = 10 * time.Second
    30  	serverRpcCheckTimeout   = 3 * time.Second
    31  	serverSerfCheckInterval = 10 * time.Second
    32  	serverSerfCheckTimeout  = 3 * time.Second
    33  )
    34  
    35  // Agent is a long running daemon that is used to run both
    36  // clients and servers. Servers are responsible for managing
    37  // state and making scheduling decisions. Clients can be
    38  // scheduled to, and are responsible for interfacing with
    39  // servers to run allocations.
    40  type Agent struct {
    41  	config    *Config
    42  	logger    *log.Logger
    43  	logOutput io.Writer
    44  
    45  	// consulSyncer registers the Nomad agent with the Consul Agent
    46  	consulSyncer *consul.Syncer
    47  
    48  	client *client.Client
    49  
    50  	server *nomad.Server
    51  
    52  	shutdown     bool
    53  	shutdownCh   chan struct{}
    54  	shutdownLock sync.Mutex
    55  }
    56  
    57  // NewAgent is used to create a new agent with the given configuration
    58  func NewAgent(config *Config, logOutput io.Writer) (*Agent, error) {
    59  	a := &Agent{
    60  		config:     config,
    61  		logger:     log.New(logOutput, "", log.LstdFlags|log.Lmicroseconds),
    62  		logOutput:  logOutput,
    63  		shutdownCh: make(chan struct{}),
    64  	}
    65  
    66  	if err := a.setupConsulSyncer(); err != nil {
    67  		return nil, fmt.Errorf("Failed to initialize Consul syncer task: %v", err)
    68  	}
    69  	if err := a.setupServer(); err != nil {
    70  		return nil, err
    71  	}
    72  	if err := a.setupClient(); err != nil {
    73  		return nil, err
    74  	}
    75  	if a.client == nil && a.server == nil {
    76  		return nil, fmt.Errorf("must have at least client or server mode enabled")
    77  	}
    78  
    79  	// The Nomad Agent runs the consul.Syncer regardless of whether or not the
    80  	// Agent is running in Client or Server mode (or both), and regardless of
    81  	// the consul.auto_advertise parameter. The Client and Server both reuse the
    82  	// same consul.Syncer instance. This Syncer task periodically executes
    83  	// callbacks that update Consul. The reason the Syncer is always running is
    84  	// because one of the callbacks is attempts to self-bootstrap Nomad using
    85  	// information found in Consul.
    86  	go a.consulSyncer.Run()
    87  
    88  	return a, nil
    89  }
    90  
    91  // serverConfig is used to generate a new server configuration struct
    92  // for initializing a nomad server.
    93  func (a *Agent) serverConfig() (*nomad.Config, error) {
    94  	conf := a.config.NomadConfig
    95  	if conf == nil {
    96  		conf = nomad.DefaultConfig()
    97  	}
    98  	conf.LogOutput = a.logOutput
    99  	conf.DevMode = a.config.DevMode
   100  	conf.Build = fmt.Sprintf("%s%s", a.config.Version, a.config.VersionPrerelease)
   101  	if a.config.Region != "" {
   102  		conf.Region = a.config.Region
   103  	}
   104  	if a.config.Datacenter != "" {
   105  		conf.Datacenter = a.config.Datacenter
   106  	}
   107  	if a.config.NodeName != "" {
   108  		conf.NodeName = a.config.NodeName
   109  	}
   110  	if a.config.Server.BootstrapExpect > 0 {
   111  		if a.config.Server.BootstrapExpect == 1 {
   112  			conf.Bootstrap = true
   113  		} else {
   114  			atomic.StoreInt32(&conf.BootstrapExpect, int32(a.config.Server.BootstrapExpect))
   115  		}
   116  	}
   117  	if a.config.DataDir != "" {
   118  		conf.DataDir = filepath.Join(a.config.DataDir, "server")
   119  	}
   120  	if a.config.Server.DataDir != "" {
   121  		conf.DataDir = a.config.Server.DataDir
   122  	}
   123  	if a.config.Server.ProtocolVersion != 0 {
   124  		conf.ProtocolVersion = uint8(a.config.Server.ProtocolVersion)
   125  	}
   126  	if a.config.Server.NumSchedulers != 0 {
   127  		conf.NumSchedulers = a.config.Server.NumSchedulers
   128  	}
   129  	if len(a.config.Server.EnabledSchedulers) != 0 {
   130  		conf.EnabledSchedulers = a.config.Server.EnabledSchedulers
   131  	}
   132  
   133  	// Set up the bind addresses
   134  	rpcAddr, err := net.ResolveTCPAddr("tcp", a.config.normalizedAddrs.RPC)
   135  	if err != nil {
   136  		return nil, fmt.Errorf("Failed to parse RPC address %q: %v", a.config.normalizedAddrs.RPC, err)
   137  	}
   138  	serfAddr, err := net.ResolveTCPAddr("tcp", a.config.normalizedAddrs.Serf)
   139  	if err != nil {
   140  		return nil, fmt.Errorf("Failed to parse Serf address %q: %v", a.config.normalizedAddrs.Serf, err)
   141  	}
   142  	conf.RPCAddr.Port = rpcAddr.Port
   143  	conf.RPCAddr.IP = rpcAddr.IP
   144  	conf.SerfConfig.MemberlistConfig.BindPort = serfAddr.Port
   145  	conf.SerfConfig.MemberlistConfig.BindAddr = serfAddr.IP.String()
   146  
   147  	// Set up the advertise addresses
   148  	rpcAddr, err = net.ResolveTCPAddr("tcp", a.config.AdvertiseAddrs.RPC)
   149  	if err != nil {
   150  		return nil, fmt.Errorf("Failed to parse RPC advertise address %q: %v", a.config.AdvertiseAddrs.RPC, err)
   151  	}
   152  	serfAddr, err = net.ResolveTCPAddr("tcp", a.config.AdvertiseAddrs.Serf)
   153  	if err != nil {
   154  		return nil, fmt.Errorf("Failed to parse Serf advertise address %q: %v", a.config.AdvertiseAddrs.Serf, err)
   155  	}
   156  	conf.RPCAdvertise = rpcAddr
   157  	conf.SerfConfig.MemberlistConfig.AdvertiseAddr = serfAddr.IP.String()
   158  	conf.SerfConfig.MemberlistConfig.AdvertisePort = serfAddr.Port
   159  
   160  	// Set up gc threshold and heartbeat grace period
   161  	if gcThreshold := a.config.Server.NodeGCThreshold; gcThreshold != "" {
   162  		dur, err := time.ParseDuration(gcThreshold)
   163  		if err != nil {
   164  			return nil, err
   165  		}
   166  		conf.NodeGCThreshold = dur
   167  	}
   168  
   169  	if heartbeatGrace := a.config.Server.HeartbeatGrace; heartbeatGrace != "" {
   170  		dur, err := time.ParseDuration(heartbeatGrace)
   171  		if err != nil {
   172  			return nil, err
   173  		}
   174  		conf.HeartbeatGrace = dur
   175  	}
   176  
   177  	if a.config.Consul.AutoAdvertise && a.config.Consul.ServerServiceName == "" {
   178  		return nil, fmt.Errorf("server_service_name must be set when auto_advertise is enabled")
   179  	}
   180  
   181  	// Add the Consul and Vault configs
   182  	conf.ConsulConfig = a.config.Consul
   183  	conf.VaultConfig = a.config.Vault
   184  
   185  	// Set the TLS config
   186  	conf.TLSConfig = a.config.TLSConfig
   187  
   188  	return conf, nil
   189  }
   190  
   191  // clientConfig is used to generate a new client configuration struct
   192  // for initializing a Nomad client.
   193  func (a *Agent) clientConfig() (*clientconfig.Config, error) {
   194  	// Setup the configuration
   195  	conf := a.config.ClientConfig
   196  	if conf == nil {
   197  		conf = clientconfig.DefaultConfig()
   198  	}
   199  	if a.server != nil {
   200  		conf.RPCHandler = a.server
   201  	}
   202  	conf.LogOutput = a.logOutput
   203  	conf.DevMode = a.config.DevMode
   204  	if a.config.Region != "" {
   205  		conf.Region = a.config.Region
   206  	}
   207  	if a.config.DataDir != "" {
   208  		conf.StateDir = filepath.Join(a.config.DataDir, "client")
   209  		conf.AllocDir = filepath.Join(a.config.DataDir, "alloc")
   210  	}
   211  	if a.config.Client.StateDir != "" {
   212  		conf.StateDir = a.config.Client.StateDir
   213  	}
   214  	if a.config.Client.AllocDir != "" {
   215  		conf.AllocDir = a.config.Client.AllocDir
   216  	}
   217  	conf.Servers = a.config.Client.Servers
   218  	if a.config.Client.NetworkInterface != "" {
   219  		conf.NetworkInterface = a.config.Client.NetworkInterface
   220  	}
   221  	conf.ChrootEnv = a.config.Client.ChrootEnv
   222  	conf.Options = a.config.Client.Options
   223  	// Logging deprecation messages about consul related configuration in client
   224  	// options
   225  	var invalidConsulKeys []string
   226  	for key := range conf.Options {
   227  		if strings.HasPrefix(key, "consul") {
   228  			invalidConsulKeys = append(invalidConsulKeys, fmt.Sprintf("options.%s", key))
   229  		}
   230  	}
   231  	if len(invalidConsulKeys) > 0 {
   232  		a.logger.Printf("[WARN] agent: Invalid keys: %v", strings.Join(invalidConsulKeys, ","))
   233  		a.logger.Printf(`Nomad client ignores consul related configuration in client options.
   234  		Please refer to the guide https://www.nomadproject.io/docs/agent/configuration/consul.html
   235  		to configure Nomad to work with Consul.`)
   236  	}
   237  
   238  	if a.config.Client.NetworkSpeed != 0 {
   239  		conf.NetworkSpeed = a.config.Client.NetworkSpeed
   240  	}
   241  	if a.config.Client.MaxKillTimeout != "" {
   242  		dur, err := time.ParseDuration(a.config.Client.MaxKillTimeout)
   243  		if err != nil {
   244  			return nil, fmt.Errorf("Error parsing max kill timeout: %s", err)
   245  		}
   246  		conf.MaxKillTimeout = dur
   247  	}
   248  	conf.ClientMaxPort = uint(a.config.Client.ClientMaxPort)
   249  	conf.ClientMinPort = uint(a.config.Client.ClientMinPort)
   250  
   251  	// Setup the node
   252  	conf.Node = new(structs.Node)
   253  	conf.Node.Datacenter = a.config.Datacenter
   254  	conf.Node.Name = a.config.NodeName
   255  	conf.Node.Meta = a.config.Client.Meta
   256  	conf.Node.NodeClass = a.config.Client.NodeClass
   257  
   258  	// Set up the HTTP advertise address
   259  	conf.Node.HTTPAddr = a.config.AdvertiseAddrs.HTTP
   260  
   261  	// Reserve resources on the node.
   262  	r := conf.Node.Reserved
   263  	if r == nil {
   264  		r = new(structs.Resources)
   265  		conf.Node.Reserved = r
   266  	}
   267  	r.CPU = a.config.Client.Reserved.CPU
   268  	r.MemoryMB = a.config.Client.Reserved.MemoryMB
   269  	r.DiskMB = a.config.Client.Reserved.DiskMB
   270  	r.IOPS = a.config.Client.Reserved.IOPS
   271  	conf.GloballyReservedPorts = a.config.Client.Reserved.ParsedReservedPorts
   272  
   273  	conf.Version = fmt.Sprintf("%s%s", a.config.Version, a.config.VersionPrerelease)
   274  	conf.Revision = a.config.Revision
   275  
   276  	if a.config.Consul.AutoAdvertise && a.config.Consul.ClientServiceName == "" {
   277  		return nil, fmt.Errorf("client_service_name must be set when auto_advertise is enabled")
   278  	}
   279  
   280  	conf.ConsulConfig = a.config.Consul
   281  	conf.VaultConfig = a.config.Vault
   282  	conf.StatsCollectionInterval = a.config.Telemetry.collectionInterval
   283  	conf.PublishNodeMetrics = a.config.Telemetry.PublishNodeMetrics
   284  	conf.PublishAllocationMetrics = a.config.Telemetry.PublishAllocationMetrics
   285  
   286  	// Set the TLS related configs
   287  	conf.TLSConfig = a.config.TLSConfig
   288  	conf.Node.TLSEnabled = conf.TLSConfig.EnableHTTP
   289  
   290  	return conf, nil
   291  }
   292  
   293  // setupServer is used to setup the server if enabled
   294  func (a *Agent) setupServer() error {
   295  	if !a.config.Server.Enabled {
   296  		return nil
   297  	}
   298  
   299  	// Setup the configuration
   300  	conf, err := a.serverConfig()
   301  	if err != nil {
   302  		return fmt.Errorf("server config setup failed: %s", err)
   303  	}
   304  
   305  	// Sets up the keyring for gossip encryption
   306  	if err := a.setupKeyrings(conf); err != nil {
   307  		return fmt.Errorf("failed to configure keyring: %v", err)
   308  	}
   309  
   310  	// Create the server
   311  	server, err := nomad.NewServer(conf, a.consulSyncer, a.logger)
   312  	if err != nil {
   313  		return fmt.Errorf("server setup failed: %v", err)
   314  	}
   315  	a.server = server
   316  
   317  	// Consul check addresses default to bind but can be toggled to use advertise
   318  	httpCheckAddr := a.config.normalizedAddrs.HTTP
   319  	rpcCheckAddr := a.config.normalizedAddrs.RPC
   320  	serfCheckAddr := a.config.normalizedAddrs.Serf
   321  	if a.config.Consul.ChecksUseAdvertise {
   322  		httpCheckAddr = a.config.AdvertiseAddrs.HTTP
   323  		rpcCheckAddr = a.config.AdvertiseAddrs.RPC
   324  		serfCheckAddr = a.config.AdvertiseAddrs.Serf
   325  	}
   326  
   327  	// Create the Nomad Server services for Consul
   328  	// TODO re-introduce HTTP/S checks when Consul 0.7.1 comes out
   329  	if a.config.Consul.AutoAdvertise {
   330  		httpServ := &structs.Service{
   331  			Name:      a.config.Consul.ServerServiceName,
   332  			PortLabel: a.config.AdvertiseAddrs.HTTP,
   333  			Tags:      []string{consul.ServiceTagHTTP},
   334  			Checks: []*structs.ServiceCheck{
   335  				&structs.ServiceCheck{
   336  					Name:      "Nomad Server HTTP Check",
   337  					Type:      "http",
   338  					Path:      "/v1/status/peers",
   339  					Protocol:  "http",
   340  					Interval:  serverHttpCheckInterval,
   341  					Timeout:   serverHttpCheckTimeout,
   342  					PortLabel: httpCheckAddr,
   343  				},
   344  			},
   345  		}
   346  		rpcServ := &structs.Service{
   347  			Name:      a.config.Consul.ServerServiceName,
   348  			PortLabel: a.config.AdvertiseAddrs.RPC,
   349  			Tags:      []string{consul.ServiceTagRPC},
   350  			Checks: []*structs.ServiceCheck{
   351  				&structs.ServiceCheck{
   352  					Name:      "Nomad Server RPC Check",
   353  					Type:      "tcp",
   354  					Interval:  serverRpcCheckInterval,
   355  					Timeout:   serverRpcCheckTimeout,
   356  					PortLabel: rpcCheckAddr,
   357  				},
   358  			},
   359  		}
   360  		serfServ := &structs.Service{
   361  			Name:      a.config.Consul.ServerServiceName,
   362  			PortLabel: a.config.AdvertiseAddrs.Serf,
   363  			Tags:      []string{consul.ServiceTagSerf},
   364  			Checks: []*structs.ServiceCheck{
   365  				&structs.ServiceCheck{
   366  					Name:      "Nomad Server Serf Check",
   367  					Type:      "tcp",
   368  					Interval:  serverSerfCheckInterval,
   369  					Timeout:   serverSerfCheckTimeout,
   370  					PortLabel: serfCheckAddr,
   371  				},
   372  			},
   373  		}
   374  
   375  		// Add the http port check if TLS isn't enabled
   376  		// TODO Add TLS check when Consul 0.7.1 comes out.
   377  		consulServices := map[consul.ServiceKey]*structs.Service{
   378  			consul.GenerateServiceKey(rpcServ):  rpcServ,
   379  			consul.GenerateServiceKey(serfServ): serfServ,
   380  		}
   381  		if !conf.TLSConfig.EnableHTTP {
   382  			consulServices[consul.GenerateServiceKey(httpServ)] = httpServ
   383  		}
   384  		a.consulSyncer.SetServices(consul.ServerDomain, consulServices)
   385  	}
   386  
   387  	return nil
   388  }
   389  
   390  // setupKeyrings is used to initialize and load keyrings during agent startup
   391  func (a *Agent) setupKeyrings(config *nomad.Config) error {
   392  	file := filepath.Join(a.config.DataDir, serfKeyring)
   393  
   394  	if a.config.Server.EncryptKey == "" {
   395  		goto LOAD
   396  	}
   397  	if _, err := os.Stat(file); err != nil {
   398  		if err := initKeyring(file, a.config.Server.EncryptKey); err != nil {
   399  			return err
   400  		}
   401  	}
   402  
   403  LOAD:
   404  	if _, err := os.Stat(file); err == nil {
   405  		config.SerfConfig.KeyringFile = file
   406  	}
   407  	if err := loadKeyringFile(config.SerfConfig); err != nil {
   408  		return err
   409  	}
   410  	// Success!
   411  	return nil
   412  }
   413  
   414  // setupClient is used to setup the client if enabled
   415  func (a *Agent) setupClient() error {
   416  	if !a.config.Client.Enabled {
   417  		return nil
   418  	}
   419  
   420  	// Setup the configuration
   421  	conf, err := a.clientConfig()
   422  	if err != nil {
   423  		return fmt.Errorf("client setup failed: %v", err)
   424  	}
   425  
   426  	// Reserve some ports for the plugins if we are on Windows
   427  	if runtime.GOOS == "windows" {
   428  		if err := a.reservePortsForClient(conf); err != nil {
   429  			return err
   430  		}
   431  	}
   432  
   433  	// Create the client
   434  	client, err := client.NewClient(conf, a.consulSyncer, a.logger)
   435  	if err != nil {
   436  		return fmt.Errorf("client setup failed: %v", err)
   437  	}
   438  	a.client = client
   439  
   440  	// Resolve the http check address
   441  	httpCheckAddr := a.config.normalizedAddrs.HTTP
   442  	if a.config.Consul.ChecksUseAdvertise {
   443  		httpCheckAddr = a.config.AdvertiseAddrs.HTTP
   444  	}
   445  
   446  	// Create the Nomad Client  services for Consul
   447  	// TODO think how we can re-introduce HTTP/S checks when Consul 0.7.1 comes
   448  	// out
   449  	if a.config.Consul.AutoAdvertise {
   450  		httpServ := &structs.Service{
   451  			Name:      a.config.Consul.ClientServiceName,
   452  			PortLabel: a.config.AdvertiseAddrs.HTTP,
   453  			Tags:      []string{consul.ServiceTagHTTP},
   454  			Checks: []*structs.ServiceCheck{
   455  				&structs.ServiceCheck{
   456  					Name:      "Nomad Client HTTP Check",
   457  					Type:      "http",
   458  					Path:      "/v1/agent/servers",
   459  					Protocol:  "http",
   460  					Interval:  clientHttpCheckInterval,
   461  					Timeout:   clientHttpCheckTimeout,
   462  					PortLabel: httpCheckAddr,
   463  				},
   464  			},
   465  		}
   466  		if !conf.TLSConfig.EnableHTTP {
   467  			a.consulSyncer.SetServices(consul.ClientDomain, map[consul.ServiceKey]*structs.Service{
   468  				consul.GenerateServiceKey(httpServ): httpServ,
   469  			})
   470  		}
   471  	}
   472  
   473  	return nil
   474  }
   475  
   476  // reservePortsForClient reserves a range of ports for the client to use when
   477  // it creates various plugins for log collection, executors, drivers, etc
   478  func (a *Agent) reservePortsForClient(conf *clientconfig.Config) error {
   479  	// finding the device name for loopback
   480  	deviceName, addr, mask, err := a.findLoopbackDevice()
   481  	if err != nil {
   482  		return fmt.Errorf("error finding the device name for loopback: %v", err)
   483  	}
   484  
   485  	// seeing if the user has already reserved some resources on this device
   486  	var nr *structs.NetworkResource
   487  	if conf.Node.Reserved == nil {
   488  		conf.Node.Reserved = &structs.Resources{}
   489  	}
   490  	for _, n := range conf.Node.Reserved.Networks {
   491  		if n.Device == deviceName {
   492  			nr = n
   493  		}
   494  	}
   495  	// If the user hasn't already created the device, we create it
   496  	if nr == nil {
   497  		nr = &structs.NetworkResource{
   498  			Device:        deviceName,
   499  			IP:            addr,
   500  			CIDR:          mask,
   501  			ReservedPorts: make([]structs.Port, 0),
   502  		}
   503  	}
   504  	// appending the port ranges we want to use for the client to the list of
   505  	// reserved ports for this device
   506  	for i := conf.ClientMinPort; i <= conf.ClientMaxPort; i++ {
   507  		nr.ReservedPorts = append(nr.ReservedPorts, structs.Port{Label: fmt.Sprintf("plugin-%d", i), Value: int(i)})
   508  	}
   509  	conf.Node.Reserved.Networks = append(conf.Node.Reserved.Networks, nr)
   510  	return nil
   511  }
   512  
   513  // findLoopbackDevice iterates through all the interfaces on a machine and
   514  // returns the ip addr, mask of the loopback device
   515  func (a *Agent) findLoopbackDevice() (string, string, string, error) {
   516  	var ifcs []net.Interface
   517  	var err error
   518  	ifcs, err = net.Interfaces()
   519  	if err != nil {
   520  		return "", "", "", err
   521  	}
   522  	for _, ifc := range ifcs {
   523  		addrs, err := ifc.Addrs()
   524  		if err != nil {
   525  			return "", "", "", err
   526  		}
   527  		for _, addr := range addrs {
   528  			var ip net.IP
   529  			switch v := addr.(type) {
   530  			case *net.IPNet:
   531  				ip = v.IP
   532  			case *net.IPAddr:
   533  				ip = v.IP
   534  			}
   535  			if ip.IsLoopback() {
   536  				if ip.To4() == nil {
   537  					continue
   538  				}
   539  				return ifc.Name, ip.String(), addr.String(), nil
   540  			}
   541  		}
   542  	}
   543  
   544  	return "", "", "", fmt.Errorf("no loopback devices with IPV4 addr found")
   545  }
   546  
   547  // Leave is used gracefully exit. Clients will inform servers
   548  // of their departure so that allocations can be rescheduled.
   549  func (a *Agent) Leave() error {
   550  	if a.client != nil {
   551  		if err := a.client.Leave(); err != nil {
   552  			a.logger.Printf("[ERR] agent: client leave failed: %v", err)
   553  		}
   554  	}
   555  	if a.server != nil {
   556  		if err := a.server.Leave(); err != nil {
   557  			a.logger.Printf("[ERR] agent: server leave failed: %v", err)
   558  		}
   559  	}
   560  	return nil
   561  }
   562  
   563  // Shutdown is used to terminate the agent.
   564  func (a *Agent) Shutdown() error {
   565  	a.shutdownLock.Lock()
   566  	defer a.shutdownLock.Unlock()
   567  
   568  	if a.shutdown {
   569  		return nil
   570  	}
   571  
   572  	a.logger.Println("[INFO] agent: requesting shutdown")
   573  	if a.client != nil {
   574  		if err := a.client.Shutdown(); err != nil {
   575  			a.logger.Printf("[ERR] agent: client shutdown failed: %v", err)
   576  		}
   577  	}
   578  	if a.server != nil {
   579  		if err := a.server.Shutdown(); err != nil {
   580  			a.logger.Printf("[ERR] agent: server shutdown failed: %v", err)
   581  		}
   582  	}
   583  
   584  	if err := a.consulSyncer.Shutdown(); err != nil {
   585  		a.logger.Printf("[ERR] agent: shutting down consul service failed: %v", err)
   586  	}
   587  
   588  	a.logger.Println("[INFO] agent: shutdown complete")
   589  	a.shutdown = true
   590  	close(a.shutdownCh)
   591  	return nil
   592  }
   593  
   594  // RPC is used to make an RPC call to the Nomad servers
   595  func (a *Agent) RPC(method string, args interface{}, reply interface{}) error {
   596  	if a.server != nil {
   597  		return a.server.RPC(method, args, reply)
   598  	}
   599  	return a.client.RPC(method, args, reply)
   600  }
   601  
   602  // Client returns the configured client or nil
   603  func (a *Agent) Client() *client.Client {
   604  	return a.client
   605  }
   606  
   607  // Server returns the configured server or nil
   608  func (a *Agent) Server() *nomad.Server {
   609  	return a.server
   610  }
   611  
   612  // Stats is used to return statistics for debugging and insight
   613  // for various sub-systems
   614  func (a *Agent) Stats() map[string]map[string]string {
   615  	stats := make(map[string]map[string]string)
   616  	if a.server != nil {
   617  		subStat := a.server.Stats()
   618  		for k, v := range subStat {
   619  			stats[k] = v
   620  		}
   621  	}
   622  	if a.client != nil {
   623  		subStat := a.client.Stats()
   624  		for k, v := range subStat {
   625  			stats[k] = v
   626  		}
   627  	}
   628  	return stats
   629  }
   630  
   631  // setupConsulSyncer creates the Consul tasks used by this Nomad Agent
   632  // (either Client or Server mode).
   633  func (a *Agent) setupConsulSyncer() error {
   634  	var err error
   635  	a.consulSyncer, err = consul.NewSyncer(a.config.Consul, a.shutdownCh, a.logger)
   636  	if err != nil {
   637  		return err
   638  	}
   639  
   640  	a.consulSyncer.SetAddrFinder(func(portLabel string) (string, int) {
   641  		host, port, err := net.SplitHostPort(portLabel)
   642  		if err != nil {
   643  			p, err := strconv.Atoi(port)
   644  			if err != nil {
   645  				return "", 0
   646  			}
   647  			return "", p
   648  		}
   649  
   650  		// If the addr for the service is ":port", then we fall back
   651  		// to Nomad's default address resolution protocol.
   652  		//
   653  		// TODO(sean@): This should poll Consul to figure out what
   654  		// its advertise address is and use that in order to handle
   655  		// the case where there is something funky like NAT on this
   656  		// host.  For now we just use the BindAddr if set, otherwise
   657  		// we fall back to a loopback addr.
   658  		if host == "" {
   659  			if a.config.BindAddr != "" {
   660  				host = a.config.BindAddr
   661  			} else {
   662  				host = "127.0.0.1"
   663  			}
   664  		}
   665  		p, err := strconv.Atoi(port)
   666  		if err != nil {
   667  			return host, 0
   668  		}
   669  		return host, p
   670  	})
   671  
   672  	return nil
   673  }