github.com/hhrutter/nomad@v0.6.0-rc2.0.20170723054333-80c4b03f0705/command/agent/agent.go (about)

     1  package agent
     2  
     3  import (
     4  	"fmt"
     5  	"io"
     6  	"log"
     7  	"net"
     8  	"os"
     9  	"path/filepath"
    10  	"runtime"
    11  	"strings"
    12  	"sync"
    13  	"sync/atomic"
    14  	"time"
    15  
    16  	"github.com/hashicorp/consul/api"
    17  	version "github.com/hashicorp/go-version"
    18  	"github.com/hashicorp/nomad/client"
    19  	clientconfig "github.com/hashicorp/nomad/client/config"
    20  	"github.com/hashicorp/nomad/command/agent/consul"
    21  	"github.com/hashicorp/nomad/nomad"
    22  	"github.com/hashicorp/nomad/nomad/structs"
    23  	"github.com/hashicorp/nomad/nomad/structs/config"
    24  )
    25  
    26  const (
    27  	clientHttpCheckInterval = 10 * time.Second
    28  	clientHttpCheckTimeout  = 3 * time.Second
    29  	serverHttpCheckInterval = 10 * time.Second
    30  	serverHttpCheckTimeout  = 6 * time.Second
    31  	serverRpcCheckInterval  = 10 * time.Second
    32  	serverRpcCheckTimeout   = 3 * time.Second
    33  	serverSerfCheckInterval = 10 * time.Second
    34  	serverSerfCheckTimeout  = 3 * time.Second
    35  
    36  	// roles used in identifying Consul entries for Nomad agents
    37  	consulRoleServer = "server"
    38  	consulRoleClient = "client"
    39  )
    40  
    41  // Agent is a long running daemon that is used to run both
    42  // clients and servers. Servers are responsible for managing
    43  // state and making scheduling decisions. Clients can be
    44  // scheduled to, and are responsible for interfacing with
    45  // servers to run allocations.
    46  type Agent struct {
    47  	config    *Config
    48  	logger    *log.Logger
    49  	logOutput io.Writer
    50  
    51  	// consulService is Nomad's custom Consul client for managing services
    52  	// and checks.
    53  	consulService *consul.ServiceClient
    54  
    55  	// consulCatalog is the subset of Consul's Catalog API Nomad uses.
    56  	consulCatalog consul.CatalogAPI
    57  
    58  	// consulSupportsTLSSkipVerify flags whether or not Nomad can register
    59  	// checks with TLSSkipVerify
    60  	consulSupportsTLSSkipVerify bool
    61  
    62  	client *client.Client
    63  
    64  	server *nomad.Server
    65  
    66  	shutdown     bool
    67  	shutdownCh   chan struct{}
    68  	shutdownLock sync.Mutex
    69  }
    70  
    71  // NewAgent is used to create a new agent with the given configuration
    72  func NewAgent(config *Config, logOutput io.Writer) (*Agent, error) {
    73  	a := &Agent{
    74  		config:     config,
    75  		logger:     log.New(logOutput, "", log.LstdFlags|log.Lmicroseconds),
    76  		logOutput:  logOutput,
    77  		shutdownCh: make(chan struct{}),
    78  	}
    79  
    80  	if err := a.setupConsul(config.Consul); err != nil {
    81  		return nil, fmt.Errorf("Failed to initialize Consul client: %v", err)
    82  	}
    83  	if err := a.setupServer(); err != nil {
    84  		return nil, err
    85  	}
    86  	if err := a.setupClient(); err != nil {
    87  		return nil, err
    88  	}
    89  	if a.client == nil && a.server == nil {
    90  		return nil, fmt.Errorf("must have at least client or server mode enabled")
    91  	}
    92  
    93  	return a, nil
    94  }
    95  
    96  // convertServerConfig takes an agent config and log output and returns a Nomad
    97  // Config.
    98  func convertServerConfig(agentConfig *Config, logOutput io.Writer) (*nomad.Config, error) {
    99  	conf := agentConfig.NomadConfig
   100  	if conf == nil {
   101  		conf = nomad.DefaultConfig()
   102  	}
   103  	conf.LogOutput = logOutput
   104  	conf.DevMode = agentConfig.DevMode
   105  	conf.Build = fmt.Sprintf("%s%s", agentConfig.Version, agentConfig.VersionPrerelease)
   106  	if agentConfig.Region != "" {
   107  		conf.Region = agentConfig.Region
   108  	}
   109  	if agentConfig.Datacenter != "" {
   110  		conf.Datacenter = agentConfig.Datacenter
   111  	}
   112  	if agentConfig.NodeName != "" {
   113  		conf.NodeName = agentConfig.NodeName
   114  	}
   115  	if agentConfig.Server.BootstrapExpect > 0 {
   116  		if agentConfig.Server.BootstrapExpect == 1 {
   117  			conf.Bootstrap = true
   118  		} else {
   119  			atomic.StoreInt32(&conf.BootstrapExpect, int32(agentConfig.Server.BootstrapExpect))
   120  		}
   121  	}
   122  	if agentConfig.DataDir != "" {
   123  		conf.DataDir = filepath.Join(agentConfig.DataDir, "server")
   124  	}
   125  	if agentConfig.Server.DataDir != "" {
   126  		conf.DataDir = agentConfig.Server.DataDir
   127  	}
   128  	if agentConfig.Server.ProtocolVersion != 0 {
   129  		conf.ProtocolVersion = uint8(agentConfig.Server.ProtocolVersion)
   130  	}
   131  	if agentConfig.Server.NumSchedulers != 0 {
   132  		conf.NumSchedulers = agentConfig.Server.NumSchedulers
   133  	}
   134  	if len(agentConfig.Server.EnabledSchedulers) != 0 {
   135  		conf.EnabledSchedulers = agentConfig.Server.EnabledSchedulers
   136  	}
   137  
   138  	// Set up the bind addresses
   139  	rpcAddr, err := net.ResolveTCPAddr("tcp", agentConfig.normalizedAddrs.RPC)
   140  	if err != nil {
   141  		return nil, fmt.Errorf("Failed to parse RPC address %q: %v", agentConfig.normalizedAddrs.RPC, err)
   142  	}
   143  	serfAddr, err := net.ResolveTCPAddr("tcp", agentConfig.normalizedAddrs.Serf)
   144  	if err != nil {
   145  		return nil, fmt.Errorf("Failed to parse Serf address %q: %v", agentConfig.normalizedAddrs.Serf, err)
   146  	}
   147  	conf.RPCAddr.Port = rpcAddr.Port
   148  	conf.RPCAddr.IP = rpcAddr.IP
   149  	conf.SerfConfig.MemberlistConfig.BindPort = serfAddr.Port
   150  	conf.SerfConfig.MemberlistConfig.BindAddr = serfAddr.IP.String()
   151  
   152  	// Set up the advertise addresses
   153  	rpcAddr, err = net.ResolveTCPAddr("tcp", agentConfig.AdvertiseAddrs.RPC)
   154  	if err != nil {
   155  		return nil, fmt.Errorf("Failed to parse RPC advertise address %q: %v", agentConfig.AdvertiseAddrs.RPC, err)
   156  	}
   157  	serfAddr, err = net.ResolveTCPAddr("tcp", agentConfig.AdvertiseAddrs.Serf)
   158  	if err != nil {
   159  		return nil, fmt.Errorf("Failed to parse Serf advertise address %q: %v", agentConfig.AdvertiseAddrs.Serf, err)
   160  	}
   161  	conf.RPCAdvertise = rpcAddr
   162  	conf.SerfConfig.MemberlistConfig.AdvertiseAddr = serfAddr.IP.String()
   163  	conf.SerfConfig.MemberlistConfig.AdvertisePort = serfAddr.Port
   164  
   165  	// Set up gc threshold and heartbeat grace period
   166  	if gcThreshold := agentConfig.Server.NodeGCThreshold; gcThreshold != "" {
   167  		dur, err := time.ParseDuration(gcThreshold)
   168  		if err != nil {
   169  			return nil, err
   170  		}
   171  		conf.NodeGCThreshold = dur
   172  	}
   173  	if gcThreshold := agentConfig.Server.JobGCThreshold; gcThreshold != "" {
   174  		dur, err := time.ParseDuration(gcThreshold)
   175  		if err != nil {
   176  			return nil, err
   177  		}
   178  		conf.JobGCThreshold = dur
   179  	}
   180  	if gcThreshold := agentConfig.Server.EvalGCThreshold; gcThreshold != "" {
   181  		dur, err := time.ParseDuration(gcThreshold)
   182  		if err != nil {
   183  			return nil, err
   184  		}
   185  		conf.EvalGCThreshold = dur
   186  	}
   187  	if gcThreshold := agentConfig.Server.DeploymentGCThreshold; gcThreshold != "" {
   188  		dur, err := time.ParseDuration(gcThreshold)
   189  		if err != nil {
   190  			return nil, err
   191  		}
   192  		conf.DeploymentGCThreshold = dur
   193  	}
   194  
   195  	if heartbeatGrace := agentConfig.Server.HeartbeatGrace; heartbeatGrace != 0 {
   196  		conf.HeartbeatGrace = heartbeatGrace
   197  	}
   198  	if min := agentConfig.Server.MinHeartbeatTTL; min != 0 {
   199  		conf.MinHeartbeatTTL = min
   200  	}
   201  	if maxHPS := agentConfig.Server.MaxHeartbeatsPerSecond; maxHPS != 0 {
   202  		conf.MaxHeartbeatsPerSecond = maxHPS
   203  	}
   204  
   205  	if *agentConfig.Consul.AutoAdvertise && agentConfig.Consul.ServerServiceName == "" {
   206  		return nil, fmt.Errorf("server_service_name must be set when auto_advertise is enabled")
   207  	}
   208  
   209  	// Add the Consul and Vault configs
   210  	conf.ConsulConfig = agentConfig.Consul
   211  	conf.VaultConfig = agentConfig.Vault
   212  
   213  	// Set the TLS config
   214  	conf.TLSConfig = agentConfig.TLSConfig
   215  
   216  	return conf, nil
   217  }
   218  
   219  // serverConfig is used to generate a new server configuration struct
   220  // for initializing a nomad server.
   221  func (a *Agent) serverConfig() (*nomad.Config, error) {
   222  	return convertServerConfig(a.config, a.logOutput)
   223  }
   224  
   225  // clientConfig is used to generate a new client configuration struct
   226  // for initializing a Nomad client.
   227  func (a *Agent) clientConfig() (*clientconfig.Config, error) {
   228  	// Setup the configuration
   229  	conf := a.config.ClientConfig
   230  	if conf == nil {
   231  		conf = clientconfig.DefaultConfig()
   232  	}
   233  	if a.server != nil {
   234  		conf.RPCHandler = a.server
   235  	}
   236  	conf.LogOutput = a.logOutput
   237  	conf.LogLevel = a.config.LogLevel
   238  	conf.DevMode = a.config.DevMode
   239  	if a.config.Region != "" {
   240  		conf.Region = a.config.Region
   241  	}
   242  	if a.config.DataDir != "" {
   243  		conf.StateDir = filepath.Join(a.config.DataDir, "client")
   244  		conf.AllocDir = filepath.Join(a.config.DataDir, "alloc")
   245  	}
   246  	if a.config.Client.StateDir != "" {
   247  		conf.StateDir = a.config.Client.StateDir
   248  	}
   249  	if a.config.Client.AllocDir != "" {
   250  		conf.AllocDir = a.config.Client.AllocDir
   251  	}
   252  	conf.Servers = a.config.Client.Servers
   253  	if a.config.Client.NetworkInterface != "" {
   254  		conf.NetworkInterface = a.config.Client.NetworkInterface
   255  	}
   256  	conf.ChrootEnv = a.config.Client.ChrootEnv
   257  	conf.Options = a.config.Client.Options
   258  	// Logging deprecation messages about consul related configuration in client
   259  	// options
   260  	var invalidConsulKeys []string
   261  	for key := range conf.Options {
   262  		if strings.HasPrefix(key, "consul") {
   263  			invalidConsulKeys = append(invalidConsulKeys, fmt.Sprintf("options.%s", key))
   264  		}
   265  	}
   266  	if len(invalidConsulKeys) > 0 {
   267  		a.logger.Printf("[WARN] agent: Invalid keys: %v", strings.Join(invalidConsulKeys, ","))
   268  		a.logger.Printf(`Nomad client ignores consul related configuration in client options.
   269  		Please refer to the guide https://www.nomadproject.io/docs/agent/configuration/consul.html
   270  		to configure Nomad to work with Consul.`)
   271  	}
   272  
   273  	if a.config.Client.NetworkSpeed != 0 {
   274  		conf.NetworkSpeed = a.config.Client.NetworkSpeed
   275  	}
   276  	if a.config.Client.CpuCompute != 0 {
   277  		conf.CpuCompute = a.config.Client.CpuCompute
   278  	}
   279  	if a.config.Client.MaxKillTimeout != "" {
   280  		dur, err := time.ParseDuration(a.config.Client.MaxKillTimeout)
   281  		if err != nil {
   282  			return nil, fmt.Errorf("Error parsing max kill timeout: %s", err)
   283  		}
   284  		conf.MaxKillTimeout = dur
   285  	}
   286  	conf.ClientMaxPort = uint(a.config.Client.ClientMaxPort)
   287  	conf.ClientMinPort = uint(a.config.Client.ClientMinPort)
   288  
   289  	// Setup the node
   290  	conf.Node = new(structs.Node)
   291  	conf.Node.Datacenter = a.config.Datacenter
   292  	conf.Node.Name = a.config.NodeName
   293  	conf.Node.Meta = a.config.Client.Meta
   294  	conf.Node.NodeClass = a.config.Client.NodeClass
   295  
   296  	// Set up the HTTP advertise address
   297  	conf.Node.HTTPAddr = a.config.AdvertiseAddrs.HTTP
   298  
   299  	// Reserve resources on the node.
   300  	r := conf.Node.Reserved
   301  	if r == nil {
   302  		r = new(structs.Resources)
   303  		conf.Node.Reserved = r
   304  	}
   305  	r.CPU = a.config.Client.Reserved.CPU
   306  	r.MemoryMB = a.config.Client.Reserved.MemoryMB
   307  	r.DiskMB = a.config.Client.Reserved.DiskMB
   308  	r.IOPS = a.config.Client.Reserved.IOPS
   309  	conf.GloballyReservedPorts = a.config.Client.Reserved.ParsedReservedPorts
   310  
   311  	conf.Version = fmt.Sprintf("%s%s", a.config.Version, a.config.VersionPrerelease)
   312  	conf.Revision = a.config.Revision
   313  
   314  	if *a.config.Consul.AutoAdvertise && a.config.Consul.ClientServiceName == "" {
   315  		return nil, fmt.Errorf("client_service_name must be set when auto_advertise is enabled")
   316  	}
   317  
   318  	conf.ConsulConfig = a.config.Consul
   319  	conf.VaultConfig = a.config.Vault
   320  	conf.StatsCollectionInterval = a.config.Telemetry.collectionInterval
   321  	conf.PublishNodeMetrics = a.config.Telemetry.PublishNodeMetrics
   322  	conf.PublishAllocationMetrics = a.config.Telemetry.PublishAllocationMetrics
   323  
   324  	// Set the TLS related configs
   325  	conf.TLSConfig = a.config.TLSConfig
   326  	conf.Node.TLSEnabled = conf.TLSConfig.EnableHTTP
   327  
   328  	// Set the GC related configs
   329  	conf.GCInterval = a.config.Client.GCInterval
   330  	conf.GCParallelDestroys = a.config.Client.GCParallelDestroys
   331  	conf.GCDiskUsageThreshold = a.config.Client.GCDiskUsageThreshold
   332  	conf.GCInodeUsageThreshold = a.config.Client.GCInodeUsageThreshold
   333  	conf.GCMaxAllocs = a.config.Client.GCMaxAllocs
   334  	if a.config.Client.NoHostUUID != nil {
   335  		conf.NoHostUUID = *a.config.Client.NoHostUUID
   336  	} else {
   337  		// Default no_host_uuid to true
   338  		conf.NoHostUUID = true
   339  	}
   340  
   341  	return conf, nil
   342  }
   343  
   344  // setupServer is used to setup the server if enabled
   345  func (a *Agent) setupServer() error {
   346  	if !a.config.Server.Enabled {
   347  		return nil
   348  	}
   349  
   350  	// Setup the configuration
   351  	conf, err := a.serverConfig()
   352  	if err != nil {
   353  		return fmt.Errorf("server config setup failed: %s", err)
   354  	}
   355  
   356  	// Sets up the keyring for gossip encryption
   357  	if err := a.setupKeyrings(conf); err != nil {
   358  		return fmt.Errorf("failed to configure keyring: %v", err)
   359  	}
   360  
   361  	// Create the server
   362  	server, err := nomad.NewServer(conf, a.consulCatalog, a.logger)
   363  	if err != nil {
   364  		return fmt.Errorf("server setup failed: %v", err)
   365  	}
   366  	a.server = server
   367  
   368  	// Consul check addresses default to bind but can be toggled to use advertise
   369  	rpcCheckAddr := a.config.normalizedAddrs.RPC
   370  	serfCheckAddr := a.config.normalizedAddrs.Serf
   371  	if *a.config.Consul.ChecksUseAdvertise {
   372  		rpcCheckAddr = a.config.AdvertiseAddrs.RPC
   373  		serfCheckAddr = a.config.AdvertiseAddrs.Serf
   374  	}
   375  
   376  	// Create the Nomad Server services for Consul
   377  	if *a.config.Consul.AutoAdvertise {
   378  		httpServ := &structs.Service{
   379  			Name:      a.config.Consul.ServerServiceName,
   380  			PortLabel: a.config.AdvertiseAddrs.HTTP,
   381  			Tags:      []string{consul.ServiceTagHTTP},
   382  		}
   383  		const isServer = true
   384  		if check := a.agentHTTPCheck(isServer); check != nil {
   385  			httpServ.Checks = []*structs.ServiceCheck{check}
   386  		}
   387  		rpcServ := &structs.Service{
   388  			Name:      a.config.Consul.ServerServiceName,
   389  			PortLabel: a.config.AdvertiseAddrs.RPC,
   390  			Tags:      []string{consul.ServiceTagRPC},
   391  			Checks: []*structs.ServiceCheck{
   392  				&structs.ServiceCheck{
   393  					Name:      "Nomad Server RPC Check",
   394  					Type:      "tcp",
   395  					Interval:  serverRpcCheckInterval,
   396  					Timeout:   serverRpcCheckTimeout,
   397  					PortLabel: rpcCheckAddr,
   398  				},
   399  			},
   400  		}
   401  		serfServ := &structs.Service{
   402  			Name:      a.config.Consul.ServerServiceName,
   403  			PortLabel: a.config.AdvertiseAddrs.Serf,
   404  			Tags:      []string{consul.ServiceTagSerf},
   405  			Checks: []*structs.ServiceCheck{
   406  				&structs.ServiceCheck{
   407  					Name:      "Nomad Server Serf Check",
   408  					Type:      "tcp",
   409  					Interval:  serverSerfCheckInterval,
   410  					Timeout:   serverSerfCheckTimeout,
   411  					PortLabel: serfCheckAddr,
   412  				},
   413  			},
   414  		}
   415  
   416  		// Add the http port check if TLS isn't enabled
   417  		consulServices := []*structs.Service{
   418  			rpcServ,
   419  			serfServ,
   420  			httpServ,
   421  		}
   422  		if err := a.consulService.RegisterAgent(consulRoleServer, consulServices); err != nil {
   423  			return err
   424  		}
   425  	}
   426  
   427  	return nil
   428  }
   429  
   430  // setupKeyrings is used to initialize and load keyrings during agent startup
   431  func (a *Agent) setupKeyrings(config *nomad.Config) error {
   432  	file := filepath.Join(a.config.DataDir, serfKeyring)
   433  
   434  	if a.config.Server.EncryptKey == "" {
   435  		goto LOAD
   436  	}
   437  	if _, err := os.Stat(file); err != nil {
   438  		if err := initKeyring(file, a.config.Server.EncryptKey); err != nil {
   439  			return err
   440  		}
   441  	}
   442  
   443  LOAD:
   444  	if _, err := os.Stat(file); err == nil {
   445  		config.SerfConfig.KeyringFile = file
   446  	}
   447  	if err := loadKeyringFile(config.SerfConfig); err != nil {
   448  		return err
   449  	}
   450  	// Success!
   451  	return nil
   452  }
   453  
   454  // setupClient is used to setup the client if enabled
   455  func (a *Agent) setupClient() error {
   456  	if !a.config.Client.Enabled {
   457  		return nil
   458  	}
   459  
   460  	// Setup the configuration
   461  	conf, err := a.clientConfig()
   462  	if err != nil {
   463  		return fmt.Errorf("client setup failed: %v", err)
   464  	}
   465  
   466  	// Reserve some ports for the plugins if we are on Windows
   467  	if runtime.GOOS == "windows" {
   468  		if err := a.reservePortsForClient(conf); err != nil {
   469  			return err
   470  		}
   471  	}
   472  
   473  	// Create the client
   474  	client, err := client.NewClient(conf, a.consulCatalog, a.consulService, a.logger)
   475  	if err != nil {
   476  		return fmt.Errorf("client setup failed: %v", err)
   477  	}
   478  	a.client = client
   479  
   480  	// Create the Nomad Client  services for Consul
   481  	if *a.config.Consul.AutoAdvertise {
   482  		httpServ := &structs.Service{
   483  			Name:      a.config.Consul.ClientServiceName,
   484  			PortLabel: a.config.AdvertiseAddrs.HTTP,
   485  			Tags:      []string{consul.ServiceTagHTTP},
   486  		}
   487  		const isServer = false
   488  		if check := a.agentHTTPCheck(isServer); check != nil {
   489  			httpServ.Checks = []*structs.ServiceCheck{check}
   490  		}
   491  		if err := a.consulService.RegisterAgent(consulRoleClient, []*structs.Service{httpServ}); err != nil {
   492  			return err
   493  		}
   494  	}
   495  
   496  	return nil
   497  }
   498  
   499  // agentHTTPCheck returns a health check for the agent's HTTP API if possible.
   500  // If no HTTP health check can be supported nil is returned.
   501  func (a *Agent) agentHTTPCheck(server bool) *structs.ServiceCheck {
   502  	// Resolve the http check address
   503  	httpCheckAddr := a.config.normalizedAddrs.HTTP
   504  	if *a.config.Consul.ChecksUseAdvertise {
   505  		httpCheckAddr = a.config.AdvertiseAddrs.HTTP
   506  	}
   507  	check := structs.ServiceCheck{
   508  		Name:      "Nomad Client HTTP Check",
   509  		Type:      "http",
   510  		Path:      "/v1/agent/servers",
   511  		Protocol:  "http",
   512  		Interval:  clientHttpCheckInterval,
   513  		Timeout:   clientHttpCheckTimeout,
   514  		PortLabel: httpCheckAddr,
   515  	}
   516  	// Switch to endpoint that doesn't require a leader for servers
   517  	if server {
   518  		check.Name = "Nomad Server HTTP Check"
   519  		check.Path = "/v1/status/peers"
   520  	}
   521  	if !a.config.TLSConfig.EnableHTTP {
   522  		// No HTTPS, return a plain http check
   523  		return &check
   524  	}
   525  	if !a.consulSupportsTLSSkipVerify {
   526  		a.logger.Printf("[WARN] agent: not registering Nomad HTTPS Health Check because it requires Consul>=0.7.2")
   527  		return nil
   528  	}
   529  	if a.config.TLSConfig.VerifyHTTPSClient {
   530  		a.logger.Printf("[WARN] agent: not registering Nomad HTTPS Health Check because verify_https_client enabled")
   531  		return nil
   532  	}
   533  
   534  	// HTTPS enabled; skip verification
   535  	check.Protocol = "https"
   536  	check.TLSSkipVerify = true
   537  	return &check
   538  }
   539  
   540  // reservePortsForClient reserves a range of ports for the client to use when
   541  // it creates various plugins for log collection, executors, drivers, etc
   542  func (a *Agent) reservePortsForClient(conf *clientconfig.Config) error {
   543  	// finding the device name for loopback
   544  	deviceName, addr, mask, err := a.findLoopbackDevice()
   545  	if err != nil {
   546  		return fmt.Errorf("error finding the device name for loopback: %v", err)
   547  	}
   548  
   549  	// seeing if the user has already reserved some resources on this device
   550  	var nr *structs.NetworkResource
   551  	if conf.Node.Reserved == nil {
   552  		conf.Node.Reserved = &structs.Resources{}
   553  	}
   554  	for _, n := range conf.Node.Reserved.Networks {
   555  		if n.Device == deviceName {
   556  			nr = n
   557  		}
   558  	}
   559  	// If the user hasn't already created the device, we create it
   560  	if nr == nil {
   561  		nr = &structs.NetworkResource{
   562  			Device:        deviceName,
   563  			IP:            addr,
   564  			CIDR:          mask,
   565  			ReservedPorts: make([]structs.Port, 0),
   566  		}
   567  	}
   568  	// appending the port ranges we want to use for the client to the list of
   569  	// reserved ports for this device
   570  	for i := conf.ClientMinPort; i <= conf.ClientMaxPort; i++ {
   571  		nr.ReservedPorts = append(nr.ReservedPorts, structs.Port{Label: fmt.Sprintf("plugin-%d", i), Value: int(i)})
   572  	}
   573  	conf.Node.Reserved.Networks = append(conf.Node.Reserved.Networks, nr)
   574  	return nil
   575  }
   576  
   577  // findLoopbackDevice iterates through all the interfaces on a machine and
   578  // returns the ip addr, mask of the loopback device
   579  func (a *Agent) findLoopbackDevice() (string, string, string, error) {
   580  	var ifcs []net.Interface
   581  	var err error
   582  	ifcs, err = net.Interfaces()
   583  	if err != nil {
   584  		return "", "", "", err
   585  	}
   586  	for _, ifc := range ifcs {
   587  		addrs, err := ifc.Addrs()
   588  		if err != nil {
   589  			return "", "", "", err
   590  		}
   591  		for _, addr := range addrs {
   592  			var ip net.IP
   593  			switch v := addr.(type) {
   594  			case *net.IPNet:
   595  				ip = v.IP
   596  			case *net.IPAddr:
   597  				ip = v.IP
   598  			}
   599  			if ip.IsLoopback() {
   600  				if ip.To4() == nil {
   601  					continue
   602  				}
   603  				return ifc.Name, ip.String(), addr.String(), nil
   604  			}
   605  		}
   606  	}
   607  
   608  	return "", "", "", fmt.Errorf("no loopback devices with IPV4 addr found")
   609  }
   610  
   611  // Leave is used gracefully exit. Clients will inform servers
   612  // of their departure so that allocations can be rescheduled.
   613  func (a *Agent) Leave() error {
   614  	if a.client != nil {
   615  		if err := a.client.Leave(); err != nil {
   616  			a.logger.Printf("[ERR] agent: client leave failed: %v", err)
   617  		}
   618  	}
   619  	if a.server != nil {
   620  		if err := a.server.Leave(); err != nil {
   621  			a.logger.Printf("[ERR] agent: server leave failed: %v", err)
   622  		}
   623  	}
   624  	return nil
   625  }
   626  
   627  // Shutdown is used to terminate the agent.
   628  func (a *Agent) Shutdown() error {
   629  	a.shutdownLock.Lock()
   630  	defer a.shutdownLock.Unlock()
   631  
   632  	if a.shutdown {
   633  		return nil
   634  	}
   635  
   636  	a.logger.Println("[INFO] agent: requesting shutdown")
   637  	if a.client != nil {
   638  		if err := a.client.Shutdown(); err != nil {
   639  			a.logger.Printf("[ERR] agent: client shutdown failed: %v", err)
   640  		}
   641  	}
   642  	if a.server != nil {
   643  		if err := a.server.Shutdown(); err != nil {
   644  			a.logger.Printf("[ERR] agent: server shutdown failed: %v", err)
   645  		}
   646  	}
   647  
   648  	if err := a.consulService.Shutdown(); err != nil {
   649  		a.logger.Printf("[ERR] agent: shutting down Consul client failed: %v", err)
   650  	}
   651  
   652  	a.logger.Println("[INFO] agent: shutdown complete")
   653  	a.shutdown = true
   654  	close(a.shutdownCh)
   655  	return nil
   656  }
   657  
   658  // RPC is used to make an RPC call to the Nomad servers
   659  func (a *Agent) RPC(method string, args interface{}, reply interface{}) error {
   660  	if a.server != nil {
   661  		return a.server.RPC(method, args, reply)
   662  	}
   663  	return a.client.RPC(method, args, reply)
   664  }
   665  
   666  // Client returns the configured client or nil
   667  func (a *Agent) Client() *client.Client {
   668  	return a.client
   669  }
   670  
   671  // Server returns the configured server or nil
   672  func (a *Agent) Server() *nomad.Server {
   673  	return a.server
   674  }
   675  
   676  // Stats is used to return statistics for debugging and insight
   677  // for various sub-systems
   678  func (a *Agent) Stats() map[string]map[string]string {
   679  	stats := make(map[string]map[string]string)
   680  	if a.server != nil {
   681  		subStat := a.server.Stats()
   682  		for k, v := range subStat {
   683  			stats[k] = v
   684  		}
   685  	}
   686  	if a.client != nil {
   687  		subStat := a.client.Stats()
   688  		for k, v := range subStat {
   689  			stats[k] = v
   690  		}
   691  	}
   692  	return stats
   693  }
   694  
   695  // setupConsul creates the Consul client and starts its main Run loop.
   696  func (a *Agent) setupConsul(consulConfig *config.ConsulConfig) error {
   697  	apiConf, err := consulConfig.ApiConfig()
   698  	if err != nil {
   699  		return err
   700  	}
   701  	client, err := api.NewClient(apiConf)
   702  	if err != nil {
   703  		return err
   704  	}
   705  
   706  	// Determine version for TLSSkipVerify
   707  	if self, err := client.Agent().Self(); err == nil {
   708  		a.consulSupportsTLSSkipVerify = consulSupportsTLSSkipVerify(self)
   709  	}
   710  
   711  	// Create Consul Catalog client for service discovery.
   712  	a.consulCatalog = client.Catalog()
   713  
   714  	// Create Consul Service client for service advertisement and checks.
   715  	a.consulService = consul.NewServiceClient(client.Agent(), a.consulSupportsTLSSkipVerify, a.logger)
   716  	go a.consulService.Run()
   717  	return nil
   718  }
   719  
   720  var consulTLSSkipVerifyMinVersion = version.Must(version.NewVersion("0.7.2"))
   721  
   722  // consulSupportsTLSSkipVerify returns true if Consul supports TLSSkipVerify.
   723  func consulSupportsTLSSkipVerify(self map[string]map[string]interface{}) bool {
   724  	member, ok := self["Member"]
   725  	if !ok {
   726  		return false
   727  	}
   728  	tagsI, ok := member["Tags"]
   729  	if !ok {
   730  		return false
   731  	}
   732  	tags, ok := tagsI.(map[string]interface{})
   733  	if !ok {
   734  		return false
   735  	}
   736  	buildI, ok := tags["build"]
   737  	if !ok {
   738  		return false
   739  	}
   740  	build, ok := buildI.(string)
   741  	if !ok {
   742  		return false
   743  	}
   744  	parts := strings.SplitN(build, ":", 2)
   745  	if len(parts) != 2 {
   746  		return false
   747  	}
   748  	v, err := version.NewVersion(parts[0])
   749  	if err != nil {
   750  		return false
   751  	}
   752  	if v.LessThan(consulTLSSkipVerifyMinVersion) {
   753  		return false
   754  	}
   755  	return true
   756  }