github.com/remilapeyre/nomad@v0.8.5/command/agent/agent_test.go (about)

     1  package agent
     2  
     3  import (
     4  	"io/ioutil"
     5  	"log"
     6  	"os"
     7  	"path/filepath"
     8  	"strings"
     9  	"testing"
    10  	"time"
    11  
    12  	cstructs "github.com/hashicorp/nomad/client/structs"
    13  	"github.com/hashicorp/nomad/helper"
    14  	"github.com/hashicorp/nomad/nomad/structs"
    15  	sconfig "github.com/hashicorp/nomad/nomad/structs/config"
    16  	"github.com/stretchr/testify/assert"
    17  	"github.com/stretchr/testify/require"
    18  )
    19  
    20  func tmpDir(t testing.TB) string {
    21  	dir, err := ioutil.TempDir("", "nomad")
    22  	if err != nil {
    23  		t.Fatalf("err: %v", err)
    24  	}
    25  	return dir
    26  }
    27  
    28  func TestAgent_RPC_Ping(t *testing.T) {
    29  	t.Parallel()
    30  	agent := NewTestAgent(t, t.Name(), nil)
    31  	defer agent.Shutdown()
    32  
    33  	var out struct{}
    34  	if err := agent.RPC("Status.Ping", struct{}{}, &out); err != nil {
    35  		t.Fatalf("err: %v", err)
    36  	}
    37  }
    38  
    39  func TestAgent_ServerConfig(t *testing.T) {
    40  	t.Parallel()
    41  	conf := DefaultConfig()
    42  	conf.DevMode = true // allow localhost for advertise addrs
    43  	conf.Server.Enabled = true
    44  	a := &Agent{config: conf}
    45  
    46  	conf.AdvertiseAddrs.Serf = "127.0.0.1:4000"
    47  	conf.AdvertiseAddrs.RPC = "127.0.0.1:4001"
    48  	conf.AdvertiseAddrs.HTTP = "10.10.11.1:4005"
    49  	conf.ACL.Enabled = true
    50  
    51  	// Parses the advertise addrs correctly
    52  	if err := conf.normalizeAddrs(); err != nil {
    53  		t.Fatalf("error normalizing config: %v", err)
    54  	}
    55  	out, err := a.serverConfig()
    56  	if err != nil {
    57  		t.Fatalf("err: %s", err)
    58  	}
    59  	serfAddr := out.SerfConfig.MemberlistConfig.AdvertiseAddr
    60  	if serfAddr != "127.0.0.1" {
    61  		t.Fatalf("expect 127.0.0.1, got: %s", serfAddr)
    62  	}
    63  	serfPort := out.SerfConfig.MemberlistConfig.AdvertisePort
    64  	if serfPort != 4000 {
    65  		t.Fatalf("expected 4000, got: %d", serfPort)
    66  	}
    67  	if out.AuthoritativeRegion != "global" {
    68  		t.Fatalf("bad: %#v", out.AuthoritativeRegion)
    69  	}
    70  	if !out.ACLEnabled {
    71  		t.Fatalf("ACL not enabled")
    72  	}
    73  
    74  	// Assert addresses weren't changed
    75  	if addr := conf.AdvertiseAddrs.RPC; addr != "127.0.0.1:4001" {
    76  		t.Fatalf("bad rpc advertise addr: %#v", addr)
    77  	}
    78  	if addr := conf.AdvertiseAddrs.HTTP; addr != "10.10.11.1:4005" {
    79  		t.Fatalf("expect 10.11.11.1:4005, got: %v", addr)
    80  	}
    81  	if addr := conf.Addresses.RPC; addr != "0.0.0.0" {
    82  		t.Fatalf("expect 0.0.0.0, got: %v", addr)
    83  	}
    84  
    85  	// Sets up the ports properly
    86  	conf.Addresses.RPC = ""
    87  	conf.Addresses.Serf = ""
    88  	conf.Ports.RPC = 4003
    89  	conf.Ports.Serf = 4004
    90  
    91  	if err := conf.normalizeAddrs(); err != nil {
    92  		t.Fatalf("error normalizing config: %v", err)
    93  	}
    94  	out, err = a.serverConfig()
    95  	if err != nil {
    96  		t.Fatalf("err: %s", err)
    97  	}
    98  	if addr := out.RPCAddr.Port; addr != 4003 {
    99  		t.Fatalf("expect 4003, got: %d", out.RPCAddr.Port)
   100  	}
   101  	if port := out.SerfConfig.MemberlistConfig.BindPort; port != 4004 {
   102  		t.Fatalf("expect 4004, got: %d", port)
   103  	}
   104  
   105  	// Prefers advertise over bind addr
   106  	conf.BindAddr = "127.0.0.3"
   107  	conf.Addresses.HTTP = "127.0.0.2"
   108  	conf.Addresses.RPC = "127.0.0.2"
   109  	conf.Addresses.Serf = "127.0.0.2"
   110  	conf.AdvertiseAddrs.HTTP = "10.0.0.10"
   111  	conf.AdvertiseAddrs.RPC = ""
   112  	conf.AdvertiseAddrs.Serf = "10.0.0.12:4004"
   113  
   114  	if err := conf.normalizeAddrs(); err != nil {
   115  		t.Fatalf("error normalizing config: %v", err)
   116  	}
   117  	out, err = a.serverConfig()
   118  	if err != nil {
   119  		t.Fatalf("err: %s", err)
   120  	}
   121  	if addr := out.RPCAddr.IP.String(); addr != "127.0.0.2" {
   122  		t.Fatalf("expect 127.0.0.2, got: %s", addr)
   123  	}
   124  	if port := out.RPCAddr.Port; port != 4003 {
   125  		t.Fatalf("expect 4647, got: %d", port)
   126  	}
   127  	if addr := out.SerfConfig.MemberlistConfig.BindAddr; addr != "127.0.0.2" {
   128  		t.Fatalf("expect 127.0.0.2, got: %s", addr)
   129  	}
   130  	if port := out.SerfConfig.MemberlistConfig.BindPort; port != 4004 {
   131  		t.Fatalf("expect 4648, got: %d", port)
   132  	}
   133  	if addr := conf.Addresses.HTTP; addr != "127.0.0.2" {
   134  		t.Fatalf("expect 127.0.0.2, got: %s", addr)
   135  	}
   136  	if addr := conf.Addresses.RPC; addr != "127.0.0.2" {
   137  		t.Fatalf("expect 127.0.0.2, got: %s", addr)
   138  	}
   139  	if addr := conf.Addresses.Serf; addr != "127.0.0.2" {
   140  		t.Fatalf("expect 10.0.0.12, got: %s", addr)
   141  	}
   142  	if addr := conf.normalizedAddrs.HTTP; addr != "127.0.0.2:4646" {
   143  		t.Fatalf("expect 127.0.0.2:4646, got: %s", addr)
   144  	}
   145  	if addr := conf.normalizedAddrs.RPC; addr != "127.0.0.2:4003" {
   146  		t.Fatalf("expect 127.0.0.2:4003, got: %s", addr)
   147  	}
   148  	if addr := conf.normalizedAddrs.Serf; addr != "127.0.0.2:4004" {
   149  		t.Fatalf("expect 10.0.0.12:4004, got: %s", addr)
   150  	}
   151  	if addr := conf.AdvertiseAddrs.HTTP; addr != "10.0.0.10:4646" {
   152  		t.Fatalf("expect 10.0.0.10:4646, got: %s", addr)
   153  	}
   154  	if addr := conf.AdvertiseAddrs.RPC; addr != "127.0.0.2:4003" {
   155  		t.Fatalf("expect 127.0.0.2:4003, got: %s", addr)
   156  	}
   157  	if addr := conf.AdvertiseAddrs.Serf; addr != "10.0.0.12:4004" {
   158  		t.Fatalf("expect 10.0.0.12:4004, got: %s", addr)
   159  	}
   160  
   161  	conf.Server.NodeGCThreshold = "42g"
   162  	if err := conf.normalizeAddrs(); err != nil {
   163  		t.Fatalf("error normalizing config: %v", err)
   164  	}
   165  	out, err = a.serverConfig()
   166  	if err == nil || !strings.Contains(err.Error(), "unknown unit") {
   167  		t.Fatalf("expected unknown unit error, got: %#v", err)
   168  	}
   169  
   170  	conf.Server.NodeGCThreshold = "10s"
   171  	if err := conf.normalizeAddrs(); err != nil {
   172  		t.Fatalf("error normalizing config: %v", err)
   173  	}
   174  	out, err = a.serverConfig()
   175  	if threshold := out.NodeGCThreshold; threshold != time.Second*10 {
   176  		t.Fatalf("expect 10s, got: %s", threshold)
   177  	}
   178  
   179  	conf.Server.HeartbeatGrace = 37 * time.Second
   180  	out, err = a.serverConfig()
   181  	if threshold := out.HeartbeatGrace; threshold != time.Second*37 {
   182  		t.Fatalf("expect 37s, got: %s", threshold)
   183  	}
   184  
   185  	conf.Server.MinHeartbeatTTL = 37 * time.Second
   186  	out, err = a.serverConfig()
   187  	if min := out.MinHeartbeatTTL; min != time.Second*37 {
   188  		t.Fatalf("expect 37s, got: %s", min)
   189  	}
   190  
   191  	conf.Server.MaxHeartbeatsPerSecond = 11.0
   192  	out, err = a.serverConfig()
   193  	if max := out.MaxHeartbeatsPerSecond; max != 11.0 {
   194  		t.Fatalf("expect 11, got: %v", max)
   195  	}
   196  
   197  	// Defaults to the global bind addr
   198  	conf.Addresses.RPC = ""
   199  	conf.Addresses.Serf = ""
   200  	conf.Addresses.HTTP = ""
   201  	conf.AdvertiseAddrs.RPC = ""
   202  	conf.AdvertiseAddrs.HTTP = ""
   203  	conf.AdvertiseAddrs.Serf = ""
   204  	conf.Ports.HTTP = 4646
   205  	conf.Ports.RPC = 4647
   206  	conf.Ports.Serf = 4648
   207  	if err := conf.normalizeAddrs(); err != nil {
   208  		t.Fatalf("error normalizing config: %v", err)
   209  	}
   210  	out, err = a.serverConfig()
   211  	if err != nil {
   212  		t.Fatalf("err: %s", err)
   213  	}
   214  	if addr := out.RPCAddr.IP.String(); addr != "127.0.0.3" {
   215  		t.Fatalf("expect 127.0.0.3, got: %s", addr)
   216  	}
   217  	if addr := out.SerfConfig.MemberlistConfig.BindAddr; addr != "127.0.0.3" {
   218  		t.Fatalf("expect 127.0.0.3, got: %s", addr)
   219  	}
   220  	if addr := conf.Addresses.HTTP; addr != "127.0.0.3" {
   221  		t.Fatalf("expect 127.0.0.3, got: %s", addr)
   222  	}
   223  	if addr := conf.Addresses.RPC; addr != "127.0.0.3" {
   224  		t.Fatalf("expect 127.0.0.3, got: %s", addr)
   225  	}
   226  	if addr := conf.Addresses.Serf; addr != "127.0.0.3" {
   227  		t.Fatalf("expect 127.0.0.3, got: %s", addr)
   228  	}
   229  	if addr := conf.normalizedAddrs.HTTP; addr != "127.0.0.3:4646" {
   230  		t.Fatalf("expect 127.0.0.3:4646, got: %s", addr)
   231  	}
   232  	if addr := conf.normalizedAddrs.RPC; addr != "127.0.0.3:4647" {
   233  		t.Fatalf("expect 127.0.0.3:4647, got: %s", addr)
   234  	}
   235  	if addr := conf.normalizedAddrs.Serf; addr != "127.0.0.3:4648" {
   236  		t.Fatalf("expect 127.0.0.3:4648, got: %s", addr)
   237  	}
   238  
   239  	// Properly handles the bootstrap flags
   240  	conf.Server.BootstrapExpect = 1
   241  	out, err = a.serverConfig()
   242  	if err != nil {
   243  		t.Fatalf("err: %s", err)
   244  	}
   245  	if !out.Bootstrap {
   246  		t.Fatalf("should have set bootstrap mode")
   247  	}
   248  	if out.BootstrapExpect != 0 {
   249  		t.Fatalf("bootstrap expect should be 0")
   250  	}
   251  
   252  	conf.Server.BootstrapExpect = 3
   253  	out, err = a.serverConfig()
   254  	if err != nil {
   255  		t.Fatalf("err: %s", err)
   256  	}
   257  	if out.Bootstrap {
   258  		t.Fatalf("bootstrap mode should be disabled")
   259  	}
   260  	if out.BootstrapExpect != 3 {
   261  		t.Fatalf("should have bootstrap-expect = 3")
   262  	}
   263  }
   264  
   265  func TestAgent_ClientConfig(t *testing.T) {
   266  	t.Parallel()
   267  	conf := DefaultConfig()
   268  	conf.Client.Enabled = true
   269  
   270  	// For Clients HTTP and RPC must be set (Serf can be skipped)
   271  	conf.Addresses.HTTP = "169.254.0.1"
   272  	conf.Addresses.RPC = "169.254.0.1"
   273  	conf.Ports.HTTP = 5678
   274  	a := &Agent{config: conf}
   275  
   276  	if err := conf.normalizeAddrs(); err != nil {
   277  		t.Fatalf("error normalizing config: %v", err)
   278  	}
   279  	c, err := a.clientConfig()
   280  	if err != nil {
   281  		t.Fatalf("got err: %v", err)
   282  	}
   283  
   284  	expectedHttpAddr := "169.254.0.1:5678"
   285  	if c.Node.HTTPAddr != expectedHttpAddr {
   286  		t.Fatalf("Expected http addr: %v, got: %v", expectedHttpAddr, c.Node.HTTPAddr)
   287  	}
   288  
   289  	conf = DefaultConfig()
   290  	conf.DevMode = true
   291  	a = &Agent{config: conf}
   292  	conf.Client.Enabled = true
   293  	conf.Addresses.HTTP = "169.254.0.1"
   294  
   295  	if err := conf.normalizeAddrs(); err != nil {
   296  		t.Fatalf("error normalizing config: %v", err)
   297  	}
   298  	c, err = a.clientConfig()
   299  	if err != nil {
   300  		t.Fatalf("got err: %v", err)
   301  	}
   302  
   303  	expectedHttpAddr = "169.254.0.1:4646"
   304  	if c.Node.HTTPAddr != expectedHttpAddr {
   305  		t.Fatalf("Expected http addr: %v, got: %v", expectedHttpAddr, c.Node.HTTPAddr)
   306  	}
   307  }
   308  
   309  // Clients should inherit telemetry configuration
   310  func TestAget_Client_TelemetryConfiguration(t *testing.T) {
   311  	assert := assert.New(t)
   312  
   313  	conf := DefaultConfig()
   314  	conf.DevMode = true
   315  	conf.Telemetry.DisableTaggedMetrics = true
   316  	conf.Telemetry.BackwardsCompatibleMetrics = true
   317  
   318  	a := &Agent{config: conf}
   319  
   320  	c, err := a.clientConfig()
   321  	assert.Nil(err)
   322  
   323  	telemetry := conf.Telemetry
   324  
   325  	assert.Equal(c.StatsCollectionInterval, telemetry.collectionInterval)
   326  	assert.Equal(c.PublishNodeMetrics, telemetry.PublishNodeMetrics)
   327  	assert.Equal(c.PublishAllocationMetrics, telemetry.PublishAllocationMetrics)
   328  	assert.Equal(c.DisableTaggedMetrics, telemetry.DisableTaggedMetrics)
   329  	assert.Equal(c.BackwardsCompatibleMetrics, telemetry.BackwardsCompatibleMetrics)
   330  }
   331  
   332  // TestAgent_HTTPCheck asserts Agent.agentHTTPCheck properly alters the HTTP
   333  // API health check depending on configuration.
   334  func TestAgent_HTTPCheck(t *testing.T) {
   335  	t.Parallel()
   336  	logger := log.New(ioutil.Discard, "", 0)
   337  	if testing.Verbose() {
   338  		logger = log.New(os.Stdout, "[TestAgent_HTTPCheck] ", log.Lshortfile)
   339  	}
   340  	agent := func() *Agent {
   341  		return &Agent{
   342  			logger: logger,
   343  			config: &Config{
   344  				AdvertiseAddrs:  &AdvertiseAddrs{HTTP: "advertise:4646"},
   345  				normalizedAddrs: &Addresses{HTTP: "normalized:4646"},
   346  				Consul: &sconfig.ConsulConfig{
   347  					ChecksUseAdvertise: helper.BoolToPtr(false),
   348  				},
   349  				TLSConfig: &sconfig.TLSConfig{EnableHTTP: false},
   350  			},
   351  		}
   352  	}
   353  
   354  	t.Run("Plain HTTP Check", func(t *testing.T) {
   355  		a := agent()
   356  		check := a.agentHTTPCheck(false)
   357  		if check == nil {
   358  			t.Fatalf("expected non-nil check")
   359  		}
   360  		if check.Type != "http" {
   361  			t.Errorf("expected http check not: %q", check.Type)
   362  		}
   363  		if expected := "/v1/agent/health?type=client"; check.Path != expected {
   364  			t.Errorf("expected %q path not: %q", expected, check.Path)
   365  		}
   366  		if check.Protocol != "http" {
   367  			t.Errorf("expected http proto not: %q", check.Protocol)
   368  		}
   369  		if expected := a.config.normalizedAddrs.HTTP; check.PortLabel != expected {
   370  			t.Errorf("expected normalized addr not %q", check.PortLabel)
   371  		}
   372  	})
   373  
   374  	t.Run("Plain HTTP + ChecksUseAdvertise", func(t *testing.T) {
   375  		a := agent()
   376  		a.config.Consul.ChecksUseAdvertise = helper.BoolToPtr(true)
   377  		check := a.agentHTTPCheck(false)
   378  		if check == nil {
   379  			t.Fatalf("expected non-nil check")
   380  		}
   381  		if expected := a.config.AdvertiseAddrs.HTTP; check.PortLabel != expected {
   382  			t.Errorf("expected advertise addr not %q", check.PortLabel)
   383  		}
   384  	})
   385  
   386  	t.Run("HTTPS", func(t *testing.T) {
   387  		a := agent()
   388  		a.config.TLSConfig.EnableHTTP = true
   389  
   390  		check := a.agentHTTPCheck(false)
   391  		if check == nil {
   392  			t.Fatalf("expected non-nil check")
   393  		}
   394  		if !check.TLSSkipVerify {
   395  			t.Errorf("expected tls skip verify")
   396  		}
   397  		if check.Protocol != "https" {
   398  			t.Errorf("expected https not: %q", check.Protocol)
   399  		}
   400  	})
   401  
   402  	t.Run("HTTPS + VerifyHTTPSClient", func(t *testing.T) {
   403  		a := agent()
   404  		a.config.TLSConfig.EnableHTTP = true
   405  		a.config.TLSConfig.VerifyHTTPSClient = true
   406  
   407  		if check := a.agentHTTPCheck(false); check != nil {
   408  			t.Fatalf("expected nil check not: %#v", check)
   409  		}
   410  	})
   411  }
   412  
   413  // TestAgent_HTTPCheckPath asserts clients and servers use different endpoints
   414  // for healthchecks.
   415  func TestAgent_HTTPCheckPath(t *testing.T) {
   416  	t.Parallel()
   417  	// Agent.agentHTTPCheck only needs a config and logger
   418  	a := &Agent{
   419  		config: DevConfig(),
   420  		logger: log.New(ioutil.Discard, "", 0),
   421  	}
   422  	if err := a.config.normalizeAddrs(); err != nil {
   423  		t.Fatalf("error normalizing config: %v", err)
   424  	}
   425  	if testing.Verbose() {
   426  		a.logger = log.New(os.Stderr, "", log.LstdFlags)
   427  	}
   428  
   429  	// Assert server check uses /v1/agent/health?type=server
   430  	isServer := true
   431  	check := a.agentHTTPCheck(isServer)
   432  	if expected := "Nomad Server HTTP Check"; check.Name != expected {
   433  		t.Errorf("expected server check name to be %q but found %q", expected, check.Name)
   434  	}
   435  	if expected := "/v1/agent/health?type=server"; check.Path != expected {
   436  		t.Errorf("expected server check path to be %q but found %q", expected, check.Path)
   437  	}
   438  
   439  	// Assert client check uses /v1/agent/health?type=client
   440  	isServer = false
   441  	check = a.agentHTTPCheck(isServer)
   442  	if expected := "Nomad Client HTTP Check"; check.Name != expected {
   443  		t.Errorf("expected client check name to be %q but found %q", expected, check.Name)
   444  	}
   445  	if expected := "/v1/agent/health?type=client"; check.Path != expected {
   446  		t.Errorf("expected client check path to be %q but found %q", expected, check.Path)
   447  	}
   448  }
   449  
   450  // This test asserts that the keyloader embedded in the TLS config is shared
   451  // across the Agent, Server, and Client. This is essential for certificate
   452  // reloading to work.
   453  func TestServer_Reload_TLS_Shared_Keyloader(t *testing.T) {
   454  	t.Parallel()
   455  	assert := assert.New(t)
   456  
   457  	// We will start out with a bad cert and then reload with a good one.
   458  	const (
   459  		cafile   = "../../helper/tlsutil/testdata/ca.pem"
   460  		foocert  = "../../helper/tlsutil/testdata/nomad-bad.pem"
   461  		fookey   = "../../helper/tlsutil/testdata/nomad-bad-key.pem"
   462  		foocert2 = "../../helper/tlsutil/testdata/nomad-foo.pem"
   463  		fookey2  = "../../helper/tlsutil/testdata/nomad-foo-key.pem"
   464  	)
   465  
   466  	agent := NewTestAgent(t, t.Name(), func(c *Config) {
   467  		c.TLSConfig = &sconfig.TLSConfig{
   468  			EnableHTTP:           true,
   469  			EnableRPC:            true,
   470  			VerifyServerHostname: true,
   471  			CAFile:               cafile,
   472  			CertFile:             foocert,
   473  			KeyFile:              fookey,
   474  		}
   475  	})
   476  	defer agent.Shutdown()
   477  
   478  	originalKeyloader := agent.Config.TLSConfig.GetKeyLoader()
   479  	originalCert, err := originalKeyloader.GetOutgoingCertificate(nil)
   480  	assert.NotNil(originalKeyloader)
   481  	if assert.Nil(err) {
   482  		assert.NotNil(originalCert)
   483  	}
   484  
   485  	// Switch to the correct certificates and reload
   486  	newConfig := &Config{
   487  		TLSConfig: &sconfig.TLSConfig{
   488  			EnableHTTP:           true,
   489  			EnableRPC:            true,
   490  			VerifyServerHostname: true,
   491  			CAFile:               cafile,
   492  			CertFile:             foocert2,
   493  			KeyFile:              fookey2,
   494  		},
   495  	}
   496  
   497  	assert.Nil(agent.Reload(newConfig))
   498  	assert.Equal(agent.Config.TLSConfig.CertFile, newConfig.TLSConfig.CertFile)
   499  	assert.Equal(agent.Config.TLSConfig.KeyFile, newConfig.TLSConfig.KeyFile)
   500  	assert.Equal(agent.Config.TLSConfig.GetKeyLoader(), originalKeyloader)
   501  
   502  	// Assert is passed through on the server correctly
   503  	if assert.NotNil(agent.server.GetConfig().TLSConfig) {
   504  		serverKeyloader := agent.server.GetConfig().TLSConfig.GetKeyLoader()
   505  		assert.Equal(serverKeyloader, originalKeyloader)
   506  		newCert, err := serverKeyloader.GetOutgoingCertificate(nil)
   507  		assert.Nil(err)
   508  		assert.NotEqual(originalCert, newCert)
   509  	}
   510  
   511  	// Assert is passed through on the client correctly
   512  	if assert.NotNil(agent.client.GetConfig().TLSConfig) {
   513  		clientKeyloader := agent.client.GetConfig().TLSConfig.GetKeyLoader()
   514  		assert.Equal(clientKeyloader, originalKeyloader)
   515  		newCert, err := clientKeyloader.GetOutgoingCertificate(nil)
   516  		assert.Nil(err)
   517  		assert.NotEqual(originalCert, newCert)
   518  	}
   519  }
   520  
   521  func TestServer_Reload_TLS_Certificate(t *testing.T) {
   522  	t.Parallel()
   523  	assert := assert.New(t)
   524  
   525  	const (
   526  		cafile   = "../../helper/tlsutil/testdata/ca.pem"
   527  		foocert  = "../../helper/tlsutil/testdata/nomad-bad.pem"
   528  		fookey   = "../../helper/tlsutil/testdata/nomad-bad-key.pem"
   529  		foocert2 = "../../helper/tlsutil/testdata/nomad-foo.pem"
   530  		fookey2  = "../../helper/tlsutil/testdata/nomad-foo-key.pem"
   531  	)
   532  
   533  	agentConfig := &Config{
   534  		TLSConfig: &sconfig.TLSConfig{
   535  			EnableHTTP:           true,
   536  			EnableRPC:            true,
   537  			VerifyServerHostname: true,
   538  			CAFile:               cafile,
   539  			CertFile:             foocert,
   540  			KeyFile:              fookey,
   541  		},
   542  	}
   543  
   544  	agent := &Agent{
   545  		config: agentConfig,
   546  	}
   547  
   548  	newConfig := &Config{
   549  		TLSConfig: &sconfig.TLSConfig{
   550  			EnableHTTP:           true,
   551  			EnableRPC:            true,
   552  			VerifyServerHostname: true,
   553  			CAFile:               cafile,
   554  			CertFile:             foocert2,
   555  			KeyFile:              fookey2,
   556  		},
   557  	}
   558  
   559  	originalKeyloader := agentConfig.TLSConfig.GetKeyLoader()
   560  	assert.NotNil(originalKeyloader)
   561  
   562  	err := agent.Reload(newConfig)
   563  	assert.Nil(err)
   564  	assert.Equal(agent.config.TLSConfig.CertFile, newConfig.TLSConfig.CertFile)
   565  	assert.Equal(agent.config.TLSConfig.KeyFile, newConfig.TLSConfig.KeyFile)
   566  	assert.Equal(agent.config.TLSConfig.GetKeyLoader(), originalKeyloader)
   567  }
   568  
   569  func TestServer_Reload_TLS_Certificate_Invalid(t *testing.T) {
   570  	t.Parallel()
   571  	assert := assert.New(t)
   572  
   573  	const (
   574  		cafile   = "../../helper/tlsutil/testdata/ca.pem"
   575  		foocert  = "../../helper/tlsutil/testdata/nomad-bad.pem"
   576  		fookey   = "../../helper/tlsutil/testdata/nomad-bad-key.pem"
   577  		foocert2 = "invalid_cert_path"
   578  		fookey2  = "invalid_key_path"
   579  	)
   580  
   581  	agentConfig := &Config{
   582  		TLSConfig: &sconfig.TLSConfig{
   583  			EnableHTTP:           true,
   584  			EnableRPC:            true,
   585  			VerifyServerHostname: true,
   586  			CAFile:               cafile,
   587  			CertFile:             foocert,
   588  			KeyFile:              fookey,
   589  		},
   590  	}
   591  
   592  	agent := &Agent{
   593  		config: agentConfig,
   594  	}
   595  
   596  	newConfig := &Config{
   597  		TLSConfig: &sconfig.TLSConfig{
   598  			EnableHTTP:           true,
   599  			EnableRPC:            true,
   600  			VerifyServerHostname: true,
   601  			CAFile:               cafile,
   602  			CertFile:             foocert2,
   603  			KeyFile:              fookey2,
   604  		},
   605  	}
   606  
   607  	err := agent.Reload(newConfig)
   608  	assert.NotNil(err)
   609  	assert.NotEqual(agent.config.TLSConfig.CertFile, newConfig.TLSConfig.CertFile)
   610  	assert.NotEqual(agent.config.TLSConfig.KeyFile, newConfig.TLSConfig.KeyFile)
   611  }
   612  
   613  func Test_GetConfig(t *testing.T) {
   614  	assert := assert.New(t)
   615  
   616  	agentConfig := &Config{
   617  		Telemetry:      &Telemetry{},
   618  		Client:         &ClientConfig{},
   619  		Server:         &ServerConfig{},
   620  		ACL:            &ACLConfig{},
   621  		Ports:          &Ports{},
   622  		Addresses:      &Addresses{},
   623  		AdvertiseAddrs: &AdvertiseAddrs{},
   624  		Vault:          &sconfig.VaultConfig{},
   625  		Consul:         &sconfig.ConsulConfig{},
   626  		Sentinel:       &sconfig.SentinelConfig{},
   627  	}
   628  
   629  	agent := &Agent{
   630  		config: agentConfig,
   631  	}
   632  
   633  	actualAgentConfig := agent.GetConfig()
   634  	assert.Equal(actualAgentConfig, agentConfig)
   635  }
   636  
   637  func TestServer_Reload_TLS_WithNilConfiguration(t *testing.T) {
   638  	t.Parallel()
   639  	assert := assert.New(t)
   640  
   641  	logger := log.New(ioutil.Discard, "", 0)
   642  
   643  	agent := &Agent{
   644  		logger: logger,
   645  		config: &Config{},
   646  	}
   647  
   648  	err := agent.Reload(nil)
   649  	assert.NotNil(err)
   650  	assert.Equal(err.Error(), "cannot reload agent with nil configuration")
   651  }
   652  
   653  func TestServer_Reload_TLS_UpgradeToTLS(t *testing.T) {
   654  	t.Parallel()
   655  	assert := assert.New(t)
   656  
   657  	const (
   658  		cafile  = "../../helper/tlsutil/testdata/ca.pem"
   659  		foocert = "../../helper/tlsutil/testdata/nomad-foo.pem"
   660  		fookey  = "../../helper/tlsutil/testdata/nomad-foo-key.pem"
   661  	)
   662  	dir := tmpDir(t)
   663  	defer os.RemoveAll(dir)
   664  
   665  	logger := log.New(ioutil.Discard, "", 0)
   666  
   667  	agentConfig := &Config{
   668  		TLSConfig: &sconfig.TLSConfig{},
   669  	}
   670  
   671  	agent := &Agent{
   672  		logger: logger,
   673  		config: agentConfig,
   674  	}
   675  
   676  	newConfig := &Config{
   677  		TLSConfig: &sconfig.TLSConfig{
   678  			EnableHTTP:           true,
   679  			EnableRPC:            true,
   680  			VerifyServerHostname: true,
   681  			CAFile:               cafile,
   682  			CertFile:             foocert,
   683  			KeyFile:              fookey,
   684  		},
   685  	}
   686  
   687  	err := agent.Reload(newConfig)
   688  	assert.Nil(err)
   689  
   690  	assert.Equal(agent.config.TLSConfig.CAFile, newConfig.TLSConfig.CAFile)
   691  	assert.Equal(agent.config.TLSConfig.CertFile, newConfig.TLSConfig.CertFile)
   692  	assert.Equal(agent.config.TLSConfig.KeyFile, newConfig.TLSConfig.KeyFile)
   693  }
   694  
   695  func TestServer_Reload_TLS_DowngradeFromTLS(t *testing.T) {
   696  	t.Parallel()
   697  	assert := assert.New(t)
   698  
   699  	const (
   700  		cafile  = "../../helper/tlsutil/testdata/ca.pem"
   701  		foocert = "../../helper/tlsutil/testdata/nomad-foo.pem"
   702  		fookey  = "../../helper/tlsutil/testdata/nomad-foo-key.pem"
   703  	)
   704  	dir := tmpDir(t)
   705  	defer os.RemoveAll(dir)
   706  
   707  	logger := log.New(ioutil.Discard, "", 0)
   708  
   709  	agentConfig := &Config{
   710  		TLSConfig: &sconfig.TLSConfig{
   711  			EnableHTTP:           true,
   712  			EnableRPC:            true,
   713  			VerifyServerHostname: true,
   714  			CAFile:               cafile,
   715  			CertFile:             foocert,
   716  			KeyFile:              fookey,
   717  		},
   718  	}
   719  
   720  	agent := &Agent{
   721  		logger: logger,
   722  		config: agentConfig,
   723  	}
   724  
   725  	newConfig := &Config{
   726  		TLSConfig: &sconfig.TLSConfig{},
   727  	}
   728  
   729  	assert.False(agentConfig.TLSConfig.IsEmpty())
   730  
   731  	err := agent.Reload(newConfig)
   732  	assert.Nil(err)
   733  
   734  	assert.True(agentConfig.TLSConfig.IsEmpty())
   735  }
   736  
   737  func TestServer_ShouldReload_ReturnFalseForNoChanges(t *testing.T) {
   738  	t.Parallel()
   739  	assert := assert.New(t)
   740  
   741  	const (
   742  		cafile  = "../../helper/tlsutil/testdata/ca.pem"
   743  		foocert = "../../helper/tlsutil/testdata/nomad-foo.pem"
   744  		fookey  = "../../helper/tlsutil/testdata/nomad-foo-key.pem"
   745  	)
   746  	dir := tmpDir(t)
   747  	defer os.RemoveAll(dir)
   748  
   749  	sameAgentConfig := &Config{
   750  		TLSConfig: &sconfig.TLSConfig{
   751  			EnableHTTP:           true,
   752  			EnableRPC:            true,
   753  			VerifyServerHostname: true,
   754  			CAFile:               cafile,
   755  			CertFile:             foocert,
   756  			KeyFile:              fookey,
   757  		},
   758  	}
   759  
   760  	agent := NewTestAgent(t, t.Name(), func(c *Config) {
   761  		c.TLSConfig = &sconfig.TLSConfig{
   762  			EnableHTTP:           true,
   763  			EnableRPC:            true,
   764  			VerifyServerHostname: true,
   765  			CAFile:               cafile,
   766  			CertFile:             foocert,
   767  			KeyFile:              fookey,
   768  		}
   769  	})
   770  	defer agent.Shutdown()
   771  
   772  	shouldReloadAgent, shouldReloadHTTP, shouldReloadRPC := agent.ShouldReload(sameAgentConfig)
   773  	assert.False(shouldReloadAgent)
   774  	assert.False(shouldReloadHTTP)
   775  	assert.False(shouldReloadRPC)
   776  }
   777  
   778  func TestServer_ShouldReload_ReturnTrueForOnlyHTTPChanges(t *testing.T) {
   779  	t.Parallel()
   780  	require := require.New(t)
   781  
   782  	const (
   783  		cafile  = "../../helper/tlsutil/testdata/ca.pem"
   784  		foocert = "../../helper/tlsutil/testdata/nomad-foo.pem"
   785  		fookey  = "../../helper/tlsutil/testdata/nomad-foo-key.pem"
   786  	)
   787  	dir := tmpDir(t)
   788  	defer os.RemoveAll(dir)
   789  
   790  	sameAgentConfig := &Config{
   791  		TLSConfig: &sconfig.TLSConfig{
   792  			EnableHTTP:           false,
   793  			EnableRPC:            true,
   794  			VerifyServerHostname: true,
   795  			CAFile:               cafile,
   796  			CertFile:             foocert,
   797  			KeyFile:              fookey,
   798  		},
   799  	}
   800  
   801  	agent := NewTestAgent(t, t.Name(), func(c *Config) {
   802  		c.TLSConfig = &sconfig.TLSConfig{
   803  			EnableHTTP:           true,
   804  			EnableRPC:            true,
   805  			VerifyServerHostname: true,
   806  			CAFile:               cafile,
   807  			CertFile:             foocert,
   808  			KeyFile:              fookey,
   809  		}
   810  	})
   811  	defer agent.Shutdown()
   812  
   813  	shouldReloadAgent, shouldReloadHTTP, shouldReloadRPC := agent.ShouldReload(sameAgentConfig)
   814  	require.True(shouldReloadAgent)
   815  	require.True(shouldReloadHTTP)
   816  	require.False(shouldReloadRPC)
   817  }
   818  
   819  func TestServer_ShouldReload_ReturnTrueForOnlyRPCChanges(t *testing.T) {
   820  	t.Parallel()
   821  	assert := assert.New(t)
   822  
   823  	const (
   824  		cafile  = "../../helper/tlsutil/testdata/ca.pem"
   825  		foocert = "../../helper/tlsutil/testdata/nomad-foo.pem"
   826  		fookey  = "../../helper/tlsutil/testdata/nomad-foo-key.pem"
   827  	)
   828  	dir := tmpDir(t)
   829  	defer os.RemoveAll(dir)
   830  
   831  	sameAgentConfig := &Config{
   832  		TLSConfig: &sconfig.TLSConfig{
   833  			EnableHTTP:           true,
   834  			EnableRPC:            true,
   835  			VerifyServerHostname: true,
   836  			CAFile:               cafile,
   837  			CertFile:             foocert,
   838  			KeyFile:              fookey,
   839  		},
   840  	}
   841  
   842  	agent := NewTestAgent(t, t.Name(), func(c *Config) {
   843  		c.TLSConfig = &sconfig.TLSConfig{
   844  			EnableHTTP:           true,
   845  			EnableRPC:            false,
   846  			VerifyServerHostname: true,
   847  			CAFile:               cafile,
   848  			CertFile:             foocert,
   849  			KeyFile:              fookey,
   850  		}
   851  	})
   852  	defer agent.Shutdown()
   853  
   854  	shouldReloadAgent, shouldReloadHTTP, shouldReloadRPC := agent.ShouldReload(sameAgentConfig)
   855  	assert.True(shouldReloadAgent)
   856  	assert.False(shouldReloadHTTP)
   857  	assert.True(shouldReloadRPC)
   858  }
   859  
   860  func TestServer_ShouldReload_ReturnTrueForConfigChanges(t *testing.T) {
   861  	t.Parallel()
   862  	assert := assert.New(t)
   863  
   864  	const (
   865  		cafile   = "../../helper/tlsutil/testdata/ca.pem"
   866  		foocert  = "../../helper/tlsutil/testdata/nomad-foo.pem"
   867  		fookey   = "../../helper/tlsutil/testdata/nomad-foo-key.pem"
   868  		foocert2 = "../../helper/tlsutil/testdata/nomad-bad.pem"
   869  		fookey2  = "../../helper/tlsutil/testdata/nomad-bad-key.pem"
   870  	)
   871  	dir := tmpDir(t)
   872  	defer os.RemoveAll(dir)
   873  
   874  	agent := NewTestAgent(t, t.Name(), func(c *Config) {
   875  		c.TLSConfig = &sconfig.TLSConfig{
   876  			EnableHTTP:           true,
   877  			EnableRPC:            true,
   878  			VerifyServerHostname: true,
   879  			CAFile:               cafile,
   880  			CertFile:             foocert,
   881  			KeyFile:              fookey,
   882  		}
   883  	})
   884  	defer agent.Shutdown()
   885  
   886  	newConfig := &Config{
   887  		TLSConfig: &sconfig.TLSConfig{
   888  			EnableHTTP:           true,
   889  			EnableRPC:            true,
   890  			VerifyServerHostname: true,
   891  			CAFile:               cafile,
   892  			CertFile:             foocert2,
   893  			KeyFile:              fookey2,
   894  		},
   895  	}
   896  
   897  	shouldReloadAgent, shouldReloadHTTP, shouldReloadRPC := agent.ShouldReload(newConfig)
   898  	assert.True(shouldReloadAgent)
   899  	assert.True(shouldReloadHTTP)
   900  	assert.True(shouldReloadRPC)
   901  }
   902  
   903  func TestServer_ShouldReload_ReturnTrueForFileChanges(t *testing.T) {
   904  	t.Parallel()
   905  	require := require.New(t)
   906  
   907  	oldCertificate := `
   908  	-----BEGIN CERTIFICATE-----
   909  	MIICrzCCAlagAwIBAgIUN+4rYZ6wqQCIBzYYd0sfX2e8hDowCgYIKoZIzj0EAwIw
   910  	eDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCkNhbGlmb3JuaWExFjAUBgNVBAcTDVNh
   911  	biBGcmFuY2lzY28xEjAQBgNVBAoTCUhhc2hpQ29ycDEOMAwGA1UECxMFTm9tYWQx
   912  	GDAWBgNVBAMTD25vbWFkLmhhc2hpY29ycDAgFw0xNjExMTAxOTU2MDBaGA8yMTE2
   913  	MTAxNzE5NTYwMFoweDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCkNhbGlmb3JuaWEx
   914  	FjAUBgNVBAcTDVNhbiBGcmFuY2lzY28xEjAQBgNVBAoTCUhhc2hpQ29ycDEOMAwG
   915  	A1UECxMFTm9tYWQxGDAWBgNVBAMTD3JlZ2lvbkZvby5ub21hZDBZMBMGByqGSM49
   916  	AgEGCCqGSM49AwEHA0IABOqGSFNjm+EBlLYlxmIP6SQTdX8U/6hbPWObB0ffkEO/
   917  	CFweeYIVWb3FKNPqYAlhMqg1K0ileD0FbhEzarP0sL6jgbswgbgwDgYDVR0PAQH/
   918  	BAQDAgWgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAMBgNVHRMBAf8E
   919  	AjAAMB0GA1UdDgQWBBQnMcjU4yI3k0AoMtapACpO+w9QMTAfBgNVHSMEGDAWgBQ6
   920  	NWr8F5y2eFwqfoQdQPg0kWb9QDA5BgNVHREEMjAwghZzZXJ2ZXIucmVnaW9uRm9v
   921  	Lm5vbWFkghZjbGllbnQucmVnaW9uRm9vLm5vbWFkMAoGCCqGSM49BAMCA0cAMEQC
   922  	ICrvzc5NzqhdT/HkazAx5OOUU8hqoptnmhRmwn6X+0y9AiA8bNvMUxHz3ZLjGBiw
   923  	PLBDC2UaSDqJqiiYpYegLhbQtw==
   924  	-----END CERTIFICATE-----
   925  	`
   926  
   927  	content := []byte(oldCertificate)
   928  	dir, err := ioutil.TempDir("", "certificate")
   929  	if err != nil {
   930  		log.Fatal(err)
   931  	}
   932  	defer os.RemoveAll(dir) // clean up
   933  
   934  	tmpfn := filepath.Join(dir, "testcert")
   935  	err = ioutil.WriteFile(tmpfn, content, 0666)
   936  	require.Nil(err)
   937  
   938  	const (
   939  		cafile = "../../helper/tlsutil/testdata/ca.pem"
   940  		key    = "../../helper/tlsutil/testdata/nomad-foo-key.pem"
   941  	)
   942  
   943  	logger := log.New(ioutil.Discard, "", 0)
   944  
   945  	agentConfig := &Config{
   946  		TLSConfig: &sconfig.TLSConfig{
   947  			EnableHTTP:           true,
   948  			EnableRPC:            true,
   949  			VerifyServerHostname: true,
   950  			CAFile:               cafile,
   951  			CertFile:             tmpfn,
   952  			KeyFile:              key,
   953  		},
   954  	}
   955  
   956  	agent := &Agent{
   957  		logger: logger,
   958  		config: agentConfig,
   959  	}
   960  	agent.config.TLSConfig.SetChecksum()
   961  
   962  	shouldReloadAgent, shouldReloadHTTP, shouldReloadRPC := agent.ShouldReload(agentConfig)
   963  	require.False(shouldReloadAgent)
   964  	require.False(shouldReloadHTTP)
   965  	require.False(shouldReloadRPC)
   966  
   967  	newCertificate := `
   968  	-----BEGIN CERTIFICATE-----
   969  	MIICtTCCAlqgAwIBAgIUQp/L2szbgE4b1ASlPOZMReFE27owCgYIKoZIzj0EAwIw
   970  	fDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCkNhbGlmb3JuaWExFjAUBgNVBAcTDVNh
   971  	biBGcmFuY2lzY28xEjAQBgNVBAoTCUhhc2hpQ29ycDEOMAwGA1UECxMFTm9tYWQx
   972  	HDAaBgNVBAMTE2JhZC5ub21hZC5oYXNoaWNvcnAwIBcNMTYxMTEwMjAxMDAwWhgP
   973  	MjExNjEwMTcyMDEwMDBaMHgxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpDYWxpZm9y
   974  	bmlhMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRIwEAYDVQQKEwlIYXNoaUNvcnAx
   975  	DjAMBgNVBAsTBU5vbWFkMRgwFgYDVQQDEw9yZWdpb25CYWQubm9tYWQwWTATBgcq
   976  	hkjOPQIBBggqhkjOPQMBBwNCAAQk6oXJwlxNrKvl6kpeeR4NJc5EYFI2b3y7odjY
   977  	u55Jp4sI91JVDqnpyatkyGmavdAWa4t0U6HkeaWqKk16/ZcYo4G7MIG4MA4GA1Ud
   978  	DwEB/wQEAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDAYDVR0T
   979  	AQH/BAIwADAdBgNVHQ4EFgQUxhzOftFR2L0QAPx8LOuP99WPbpgwHwYDVR0jBBgw
   980  	FoAUHPDLSgzlHqBEh+c4A7HeT0GWygIwOQYDVR0RBDIwMIIWc2VydmVyLnJlZ2lv
   981  	bkJhZC5ub21hZIIWY2xpZW50LnJlZ2lvbkJhZC5ub21hZDAKBggqhkjOPQQDAgNJ
   982  	ADBGAiEAq2rnBeX/St/8i9Cab7Yw0C7pjcaE+mrFYeQByng1Uc0CIQD/o4BrZdkX
   983  	Nm7QGTRZbUFZTHYZr0ULz08Iaz2aHQ6Mcw==
   984  	-----END CERTIFICATE-----
   985  	`
   986  
   987  	os.Remove(tmpfn)
   988  	err = ioutil.WriteFile(tmpfn, []byte(newCertificate), 0666)
   989  	require.Nil(err)
   990  
   991  	newAgentConfig := &Config{
   992  		TLSConfig: &sconfig.TLSConfig{
   993  			EnableHTTP:           true,
   994  			EnableRPC:            true,
   995  			VerifyServerHostname: true,
   996  			CAFile:               cafile,
   997  			CertFile:             tmpfn,
   998  			KeyFile:              key,
   999  		},
  1000  	}
  1001  
  1002  	shouldReloadAgent, shouldReloadHTTP, shouldReloadRPC = agent.ShouldReload(newAgentConfig)
  1003  	require.True(shouldReloadAgent)
  1004  	require.True(shouldReloadHTTP)
  1005  	require.True(shouldReloadRPC)
  1006  }
  1007  
  1008  func TestServer_ShouldReload_ShouldHandleMultipleChanges(t *testing.T) {
  1009  	t.Parallel()
  1010  	require := require.New(t)
  1011  
  1012  	const (
  1013  		cafile   = "../../helper/tlsutil/testdata/ca.pem"
  1014  		foocert  = "../../helper/tlsutil/testdata/nomad-foo.pem"
  1015  		fookey   = "../../helper/tlsutil/testdata/nomad-foo-key.pem"
  1016  		foocert2 = "../../helper/tlsutil/testdata/nomad-bad.pem"
  1017  		fookey2  = "../../helper/tlsutil/testdata/nomad-bad-key.pem"
  1018  	)
  1019  	dir := tmpDir(t)
  1020  	defer os.RemoveAll(dir)
  1021  
  1022  	sameAgentConfig := &Config{
  1023  		TLSConfig: &sconfig.TLSConfig{
  1024  			EnableHTTP:           true,
  1025  			EnableRPC:            true,
  1026  			VerifyServerHostname: true,
  1027  			CAFile:               cafile,
  1028  			CertFile:             foocert,
  1029  			KeyFile:              fookey,
  1030  		},
  1031  	}
  1032  
  1033  	agent := NewTestAgent(t, t.Name(), func(c *Config) {
  1034  		c.TLSConfig = &sconfig.TLSConfig{
  1035  			EnableHTTP:           true,
  1036  			EnableRPC:            true,
  1037  			VerifyServerHostname: true,
  1038  			CAFile:               cafile,
  1039  			CertFile:             foocert2,
  1040  			KeyFile:              fookey2,
  1041  		}
  1042  	})
  1043  	defer agent.Shutdown()
  1044  
  1045  	{
  1046  		shouldReloadAgent, shouldReloadHTTP, shouldReloadRPC := agent.ShouldReload(sameAgentConfig)
  1047  		require.True(shouldReloadAgent)
  1048  		require.True(shouldReloadHTTP)
  1049  		require.True(shouldReloadRPC)
  1050  	}
  1051  
  1052  	err := agent.Reload(sameAgentConfig)
  1053  	require.Nil(err)
  1054  
  1055  	{
  1056  		shouldReloadAgent, shouldReloadHTTP, shouldReloadRPC := agent.ShouldReload(sameAgentConfig)
  1057  		require.False(shouldReloadAgent)
  1058  		require.False(shouldReloadHTTP)
  1059  		require.False(shouldReloadRPC)
  1060  	}
  1061  }
  1062  
  1063  func TestAgent_ProxyRPC_Dev(t *testing.T) {
  1064  	t.Parallel()
  1065  	agent := NewTestAgent(t, t.Name(), nil)
  1066  	defer agent.Shutdown()
  1067  
  1068  	id := agent.client.NodeID()
  1069  	req := &structs.NodeSpecificRequest{
  1070  		NodeID: id,
  1071  		QueryOptions: structs.QueryOptions{
  1072  			Region: agent.server.Region(),
  1073  		},
  1074  	}
  1075  
  1076  	time.Sleep(100 * time.Millisecond)
  1077  
  1078  	var resp cstructs.ClientStatsResponse
  1079  	if err := agent.RPC("ClientStats.Stats", req, &resp); err != nil {
  1080  		t.Fatalf("err: %v", err)
  1081  	}
  1082  }