github.com/prestonp/nomad@v0.10.4/command/agent/agent_endpoint_test.go (about)

     1  package agent
     2  
     3  import (
     4  	"bytes"
     5  	"encoding/json"
     6  	"fmt"
     7  	"io/ioutil"
     8  	"net"
     9  	"net/http"
    10  	"net/http/httptest"
    11  	"net/url"
    12  	"strings"
    13  	"testing"
    14  	"time"
    15  
    16  	msgpackrpc "github.com/hashicorp/net-rpc-msgpackrpc"
    17  	"github.com/hashicorp/nomad/acl"
    18  	"github.com/hashicorp/nomad/helper"
    19  	"github.com/hashicorp/nomad/helper/pool"
    20  	"github.com/hashicorp/nomad/nomad/mock"
    21  	"github.com/hashicorp/nomad/nomad/structs"
    22  	"github.com/hashicorp/nomad/testutil"
    23  	"github.com/stretchr/testify/assert"
    24  	"github.com/stretchr/testify/require"
    25  )
    26  
    27  func TestHTTP_AgentSelf(t *testing.T) {
    28  	t.Parallel()
    29  	require := require.New(t)
    30  
    31  	httpTest(t, nil, func(s *TestAgent) {
    32  		// Make the HTTP request
    33  		req, err := http.NewRequest("GET", "/v1/agent/self", nil)
    34  		require.NoError(err)
    35  		respW := httptest.NewRecorder()
    36  
    37  		// Make the request
    38  		obj, err := s.Server.AgentSelfRequest(respW, req)
    39  		require.NoError(err)
    40  
    41  		// Check the job
    42  		self := obj.(agentSelf)
    43  		require.NotNil(self.Config)
    44  		require.NotNil(self.Config.ACL)
    45  		require.NotEmpty(self.Stats)
    46  
    47  		// Check the Vault config
    48  		require.Empty(self.Config.Vault.Token)
    49  
    50  		// Assign a Vault token and require it is redacted.
    51  		s.Config.Vault.Token = "badc0deb-adc0-deba-dc0d-ebadc0debadc"
    52  		respW = httptest.NewRecorder()
    53  		obj, err = s.Server.AgentSelfRequest(respW, req)
    54  		require.NoError(err)
    55  		self = obj.(agentSelf)
    56  		require.Equal("<redacted>", self.Config.Vault.Token)
    57  
    58  		// Assign a ReplicationToken token and require it is redacted.
    59  		s.Config.ACL.ReplicationToken = "badc0deb-adc0-deba-dc0d-ebadc0debadc"
    60  		respW = httptest.NewRecorder()
    61  		obj, err = s.Server.AgentSelfRequest(respW, req)
    62  		require.NoError(err)
    63  		self = obj.(agentSelf)
    64  		require.Equal("<redacted>", self.Config.ACL.ReplicationToken)
    65  
    66  		// Check the Consul config
    67  		require.Empty(self.Config.Consul.Token)
    68  
    69  		// Assign a Consul token and require it is redacted.
    70  		s.Config.Consul.Token = "badc0deb-adc0-deba-dc0d-ebadc0debadc"
    71  		respW = httptest.NewRecorder()
    72  		obj, err = s.Server.AgentSelfRequest(respW, req)
    73  		require.NoError(err)
    74  		self = obj.(agentSelf)
    75  		require.Equal("<redacted>", self.Config.Consul.Token)
    76  
    77  		// Check the Circonus config
    78  		require.Empty(self.Config.Telemetry.CirconusAPIToken)
    79  
    80  		// Assign a Consul token and require it is redacted.
    81  		s.Config.Telemetry.CirconusAPIToken = "badc0deb-adc0-deba-dc0d-ebadc0debadc"
    82  		respW = httptest.NewRecorder()
    83  		obj, err = s.Server.AgentSelfRequest(respW, req)
    84  		require.NoError(err)
    85  		self = obj.(agentSelf)
    86  		require.Equal("<redacted>", self.Config.Telemetry.CirconusAPIToken)
    87  	})
    88  }
    89  
    90  func TestHTTP_AgentSelf_ACL(t *testing.T) {
    91  	t.Parallel()
    92  	require := require.New(t)
    93  
    94  	httpACLTest(t, nil, func(s *TestAgent) {
    95  		state := s.Agent.server.State()
    96  
    97  		// Make the HTTP request
    98  		req, err := http.NewRequest("GET", "/v1/agent/self", nil)
    99  		require.Nil(err)
   100  
   101  		// Try request without a token and expect failure
   102  		{
   103  			respW := httptest.NewRecorder()
   104  			_, err := s.Server.AgentSelfRequest(respW, req)
   105  			require.NotNil(err)
   106  			require.Equal(err.Error(), structs.ErrPermissionDenied.Error())
   107  		}
   108  
   109  		// Try request with an invalid token and expect failure
   110  		{
   111  			respW := httptest.NewRecorder()
   112  			token := mock.CreatePolicyAndToken(t, state, 1005, "invalid", mock.NodePolicy(acl.PolicyWrite))
   113  			setToken(req, token)
   114  			_, err := s.Server.AgentSelfRequest(respW, req)
   115  			require.NotNil(err)
   116  			require.Equal(err.Error(), structs.ErrPermissionDenied.Error())
   117  		}
   118  
   119  		// Try request with a valid token
   120  		{
   121  			respW := httptest.NewRecorder()
   122  			token := mock.CreatePolicyAndToken(t, state, 1007, "valid", mock.AgentPolicy(acl.PolicyWrite))
   123  			setToken(req, token)
   124  			obj, err := s.Server.AgentSelfRequest(respW, req)
   125  			require.Nil(err)
   126  
   127  			self := obj.(agentSelf)
   128  			require.NotNil(self.Config)
   129  			require.NotNil(self.Stats)
   130  		}
   131  
   132  		// Try request with a root token
   133  		{
   134  			respW := httptest.NewRecorder()
   135  			setToken(req, s.RootToken)
   136  			obj, err := s.Server.AgentSelfRequest(respW, req)
   137  			require.Nil(err)
   138  
   139  			self := obj.(agentSelf)
   140  			require.NotNil(self.Config)
   141  			require.NotNil(self.Stats)
   142  		}
   143  	})
   144  }
   145  
   146  func TestHTTP_AgentJoin(t *testing.T) {
   147  	t.Parallel()
   148  	httpTest(t, nil, func(s *TestAgent) {
   149  		// Determine the join address
   150  		member := s.Agent.Server().LocalMember()
   151  		addr := fmt.Sprintf("%s:%d", member.Addr, member.Port)
   152  
   153  		// Make the HTTP request
   154  		req, err := http.NewRequest("PUT",
   155  			fmt.Sprintf("/v1/agent/join?address=%s&address=%s", addr, addr), nil)
   156  		if err != nil {
   157  			t.Fatalf("err: %v", err)
   158  		}
   159  		respW := httptest.NewRecorder()
   160  
   161  		// Make the request
   162  		obj, err := s.Server.AgentJoinRequest(respW, req)
   163  		if err != nil {
   164  			t.Fatalf("err: %v", err)
   165  		}
   166  
   167  		// Check the job
   168  		join := obj.(joinResult)
   169  		if join.NumJoined != 2 {
   170  			t.Fatalf("bad: %#v", join)
   171  		}
   172  		if join.Error != "" {
   173  			t.Fatalf("bad: %#v", join)
   174  		}
   175  	})
   176  }
   177  
   178  func TestHTTP_AgentMembers(t *testing.T) {
   179  	t.Parallel()
   180  	httpTest(t, nil, func(s *TestAgent) {
   181  		// Make the HTTP request
   182  		req, err := http.NewRequest("GET", "/v1/agent/members", nil)
   183  		if err != nil {
   184  			t.Fatalf("err: %v", err)
   185  		}
   186  		respW := httptest.NewRecorder()
   187  
   188  		// Make the request
   189  		obj, err := s.Server.AgentMembersRequest(respW, req)
   190  		if err != nil {
   191  			t.Fatalf("err: %v", err)
   192  		}
   193  
   194  		// Check the job
   195  		members := obj.(structs.ServerMembersResponse)
   196  		if len(members.Members) != 1 {
   197  			t.Fatalf("bad: %#v", members.Members)
   198  		}
   199  	})
   200  }
   201  
   202  func TestHTTP_AgentMembers_ACL(t *testing.T) {
   203  	t.Parallel()
   204  	require := require.New(t)
   205  
   206  	httpACLTest(t, nil, func(s *TestAgent) {
   207  		state := s.Agent.server.State()
   208  
   209  		// Make the HTTP request
   210  		req, err := http.NewRequest("GET", "/v1/agent/members", nil)
   211  		require.Nil(err)
   212  
   213  		// Try request without a token and expect failure
   214  		{
   215  			respW := httptest.NewRecorder()
   216  			_, err := s.Server.AgentMembersRequest(respW, req)
   217  			require.NotNil(err)
   218  			require.Equal(err.Error(), structs.ErrPermissionDenied.Error())
   219  		}
   220  
   221  		// Try request with an invalid token and expect failure
   222  		{
   223  			respW := httptest.NewRecorder()
   224  			token := mock.CreatePolicyAndToken(t, state, 1005, "invalid", mock.AgentPolicy(acl.PolicyWrite))
   225  			setToken(req, token)
   226  			_, err := s.Server.AgentMembersRequest(respW, req)
   227  			require.NotNil(err)
   228  			require.Equal(err.Error(), structs.ErrPermissionDenied.Error())
   229  		}
   230  
   231  		// Try request with a valid token
   232  		{
   233  			respW := httptest.NewRecorder()
   234  			token := mock.CreatePolicyAndToken(t, state, 1007, "valid", mock.NodePolicy(acl.PolicyRead))
   235  			setToken(req, token)
   236  			obj, err := s.Server.AgentMembersRequest(respW, req)
   237  			require.Nil(err)
   238  
   239  			members := obj.(structs.ServerMembersResponse)
   240  			require.Len(members.Members, 1)
   241  		}
   242  
   243  		// Try request with a root token
   244  		{
   245  			respW := httptest.NewRecorder()
   246  			setToken(req, s.RootToken)
   247  			obj, err := s.Server.AgentMembersRequest(respW, req)
   248  			require.Nil(err)
   249  
   250  			members := obj.(structs.ServerMembersResponse)
   251  			require.Len(members.Members, 1)
   252  		}
   253  	})
   254  }
   255  
   256  func TestHTTP_AgentMonitor(t *testing.T) {
   257  	t.Parallel()
   258  
   259  	t.Run("invalid log_json parameter", func(t *testing.T) {
   260  		httpTest(t, nil, func(s *TestAgent) {
   261  			req, err := http.NewRequest("GET", "/v1/agent/monitor?log_json=no", nil)
   262  			require.Nil(t, err)
   263  			resp := newClosableRecorder()
   264  
   265  			// Make the request
   266  			_, err = s.Server.AgentMonitor(resp, req)
   267  			httpErr := err.(HTTPCodedError).Code()
   268  			require.Equal(t, 400, httpErr)
   269  		})
   270  	})
   271  
   272  	t.Run("unknown log_level", func(t *testing.T) {
   273  		httpTest(t, nil, func(s *TestAgent) {
   274  			req, err := http.NewRequest("GET", "/v1/agent/monitor?log_level=unknown", nil)
   275  			require.Nil(t, err)
   276  			resp := newClosableRecorder()
   277  
   278  			// Make the request
   279  			_, err = s.Server.AgentMonitor(resp, req)
   280  			httpErr := err.(HTTPCodedError).Code()
   281  			require.Equal(t, 400, httpErr)
   282  		})
   283  	})
   284  
   285  	t.Run("check for specific log level", func(t *testing.T) {
   286  		httpTest(t, nil, func(s *TestAgent) {
   287  			req, err := http.NewRequest("GET", "/v1/agent/monitor?log_level=warn", nil)
   288  			require.Nil(t, err)
   289  			resp := newClosableRecorder()
   290  			defer resp.Close()
   291  
   292  			go func() {
   293  				_, err = s.Server.AgentMonitor(resp, req)
   294  				assert.NoError(t, err)
   295  			}()
   296  
   297  			// send the same log until monitor sink is set up
   298  			maxLogAttempts := 10
   299  			tried := 0
   300  			testutil.WaitForResult(func() (bool, error) {
   301  				if tried < maxLogAttempts {
   302  					s.Server.logger.Warn("log that should be sent")
   303  					tried++
   304  				}
   305  
   306  				got := resp.Body.String()
   307  				want := `{"Data":"`
   308  				if strings.Contains(got, want) {
   309  					return true, nil
   310  				}
   311  
   312  				return false, fmt.Errorf("missing expected log, got: %v, want: %v", got, want)
   313  			}, func(err error) {
   314  				require.Fail(t, err.Error())
   315  			})
   316  		})
   317  	})
   318  
   319  	t.Run("plain output", func(t *testing.T) {
   320  		httpTest(t, nil, func(s *TestAgent) {
   321  			req, err := http.NewRequest("GET", "/v1/agent/monitor?log_level=debug&plain=true", nil)
   322  			require.Nil(t, err)
   323  			resp := newClosableRecorder()
   324  			defer resp.Close()
   325  
   326  			go func() {
   327  				_, err = s.Server.AgentMonitor(resp, req)
   328  				assert.NoError(t, err)
   329  			}()
   330  
   331  			// send the same log until monitor sink is set up
   332  			maxLogAttempts := 10
   333  			tried := 0
   334  			testutil.WaitForResult(func() (bool, error) {
   335  				if tried < maxLogAttempts {
   336  					s.Server.logger.Debug("log that should be sent")
   337  					tried++
   338  				}
   339  
   340  				got := resp.Body.String()
   341  				want := `[DEBUG] http: log that should be sent`
   342  				if strings.Contains(got, want) {
   343  					return true, nil
   344  				}
   345  
   346  				return false, fmt.Errorf("missing expected log, got: %v, want: %v", got, want)
   347  			}, func(err error) {
   348  				require.Fail(t, err.Error())
   349  			})
   350  		})
   351  	})
   352  
   353  	t.Run("logs for a specific node", func(t *testing.T) {
   354  		httpTest(t, nil, func(s *TestAgent) {
   355  			req, err := http.NewRequest("GET", "/v1/agent/monitor?log_level=warn&node_id="+s.client.NodeID(), nil)
   356  			require.Nil(t, err)
   357  			resp := newClosableRecorder()
   358  			defer resp.Close()
   359  
   360  			go func() {
   361  				_, err = s.Server.AgentMonitor(resp, req)
   362  				assert.NoError(t, err)
   363  			}()
   364  
   365  			// send the same log until monitor sink is set up
   366  			maxLogAttempts := 10
   367  			tried := 0
   368  			out := ""
   369  			testutil.WaitForResult(func() (bool, error) {
   370  				if tried < maxLogAttempts {
   371  					s.Server.logger.Debug("log that should not be sent")
   372  					s.Server.logger.Warn("log that should be sent")
   373  					tried++
   374  				}
   375  				output, err := ioutil.ReadAll(resp.Body)
   376  				if err != nil {
   377  					return false, err
   378  				}
   379  
   380  				out += string(output)
   381  				want := `{"Data":"`
   382  				if strings.Contains(out, want) {
   383  					return true, nil
   384  				}
   385  
   386  				return false, fmt.Errorf("missing expected log, got: %v, want: %v", out, want)
   387  			}, func(err error) {
   388  				require.Fail(t, err.Error())
   389  			})
   390  		})
   391  	})
   392  
   393  	t.Run("logs for a local client with no server running on agent", func(t *testing.T) {
   394  		httpTest(t, nil, func(s *TestAgent) {
   395  			req, err := http.NewRequest("GET", "/v1/agent/monitor?log_level=warn", nil)
   396  			require.Nil(t, err)
   397  			resp := newClosableRecorder()
   398  			defer resp.Close()
   399  
   400  			go func() {
   401  				// set server to nil to monitor as client
   402  				s.Agent.server = nil
   403  				_, err = s.Server.AgentMonitor(resp, req)
   404  				assert.NoError(t, err)
   405  			}()
   406  
   407  			// send the same log until monitor sink is set up
   408  			maxLogAttempts := 10
   409  			tried := 0
   410  			out := ""
   411  			testutil.WaitForResult(func() (bool, error) {
   412  				if tried < maxLogAttempts {
   413  					s.Agent.logger.Warn("log that should be sent")
   414  					tried++
   415  				}
   416  				output, err := ioutil.ReadAll(resp.Body)
   417  				if err != nil {
   418  					return false, err
   419  				}
   420  
   421  				out += string(output)
   422  				want := `{"Data":"`
   423  				if strings.Contains(out, want) {
   424  					return true, nil
   425  				}
   426  
   427  				return false, fmt.Errorf("missing expected log, got: %v, want: %v", out, want)
   428  			}, func(err error) {
   429  				require.Fail(t, err.Error())
   430  			})
   431  		})
   432  	})
   433  }
   434  
   435  // Scenarios when Pprof requests should be available
   436  // see https://github.com/hashicorp/nomad/issues/6496
   437  // +---------------+------------------+--------+------------------+
   438  // |   Endpoint    |  `enable_debug`  |  ACLs  |  **Available?**  |
   439  // +---------------+------------------+--------+------------------+
   440  // | /debug/pprof  |  unset           |  n/a   |  no              |
   441  // | /debug/pprof  |  `true`          |  n/a   |  yes             |
   442  // | /debug/pprof  |  `false`         |  n/a   |  no              |
   443  // | /agent/pprof  |  unset           |  off   |  no              |
   444  // | /agent/pprof  |  unset           |  on    |  **yes**         |
   445  // | /agent/pprof  |  `true`          |  off   |  yes             |
   446  // | /agent/pprof  |  `false`         |  on    |  **yes**         |
   447  // +---------------+------------------+--------+------------------+
   448  func TestAgent_PprofRequest_Permissions(t *testing.T) {
   449  	trueP, falseP := helper.BoolToPtr(true), helper.BoolToPtr(false)
   450  	cases := []struct {
   451  		acl   *bool
   452  		debug *bool
   453  		ok    bool
   454  	}{
   455  		// manually set to false because test helpers
   456  		// enable to true by default
   457  		// enableDebug:       helper.BoolToPtr(false),
   458  		{debug: nil, ok: false},
   459  		{debug: trueP, ok: true},
   460  		{debug: falseP, ok: false},
   461  		{debug: falseP, acl: falseP, ok: false},
   462  		{acl: trueP, ok: true},
   463  		{acl: falseP, debug: trueP, ok: true},
   464  		{debug: falseP, acl: trueP, ok: true},
   465  	}
   466  
   467  	for _, tc := range cases {
   468  		ptrToStr := func(val *bool) string {
   469  			if val == nil {
   470  				return "unset"
   471  			} else if *val == true {
   472  				return "true"
   473  			} else {
   474  				return "false"
   475  			}
   476  		}
   477  
   478  		t.Run(
   479  			fmt.Sprintf("debug %s, acl %s",
   480  				ptrToStr(tc.debug),
   481  				ptrToStr(tc.acl)),
   482  			func(t *testing.T) {
   483  				cb := func(c *Config) {
   484  					if tc.acl != nil {
   485  						c.ACL.Enabled = *tc.acl
   486  					}
   487  					if tc.debug == nil {
   488  						var nodebug bool
   489  						c.EnableDebug = nodebug
   490  					} else {
   491  						c.EnableDebug = *tc.debug
   492  					}
   493  				}
   494  
   495  				httpTest(t, cb, func(s *TestAgent) {
   496  					state := s.Agent.server.State()
   497  					url := "/v1/agent/pprof/cmdline"
   498  					req, err := http.NewRequest("GET", url, nil)
   499  					require.NoError(t, err)
   500  					respW := httptest.NewRecorder()
   501  
   502  					if tc.acl != nil && *tc.acl {
   503  						token := mock.CreatePolicyAndToken(t, state, 1007, "valid", mock.AgentPolicy(acl.PolicyWrite))
   504  						setToken(req, token)
   505  					}
   506  
   507  					resp, err := s.Server.AgentPprofRequest(respW, req)
   508  					if tc.ok {
   509  						require.NoError(t, err)
   510  						require.NotNil(t, resp)
   511  					} else {
   512  						require.Error(t, err)
   513  						require.Equal(t, structs.ErrPermissionDenied.Error(), err.Error())
   514  					}
   515  				})
   516  			})
   517  	}
   518  }
   519  
   520  func TestAgent_PprofRequest(t *testing.T) {
   521  	cases := []struct {
   522  		desc        string
   523  		url         string
   524  		addNodeID   bool
   525  		addServerID bool
   526  		expectedErr string
   527  		clientOnly  bool
   528  	}{
   529  		{
   530  			desc: "cmdline local server request",
   531  			url:  "/v1/agent/pprof/cmdline",
   532  		},
   533  		{
   534  			desc:       "cmdline local node request",
   535  			url:        "/v1/agent/pprof/cmdline",
   536  			clientOnly: true,
   537  		},
   538  		{
   539  			desc:      "cmdline node request",
   540  			url:       "/v1/agent/pprof/cmdline",
   541  			addNodeID: true,
   542  		},
   543  		{
   544  			desc:        "cmdline server request",
   545  			url:         "/v1/agent/pprof/cmdline",
   546  			addServerID: true,
   547  		},
   548  		{
   549  			desc:        "invalid server request",
   550  			url:         "/v1/agent/pprof/unknown",
   551  			addServerID: true,
   552  			expectedErr: "RPC Error:: 404,Pprof profile not found profile: unknown",
   553  		},
   554  		{
   555  			desc:      "cpu profile request",
   556  			url:       "/v1/agent/pprof/profile",
   557  			addNodeID: true,
   558  		},
   559  		{
   560  			desc:      "trace request",
   561  			url:       "/v1/agent/pprof/trace",
   562  			addNodeID: true,
   563  		},
   564  		{
   565  			desc:      "pprof lookup request",
   566  			url:       "/v1/agent/pprof/goroutine",
   567  			addNodeID: true,
   568  		},
   569  		{
   570  			desc:        "unknown pprof lookup request",
   571  			url:         "/v1/agent/pprof/latency",
   572  			addNodeID:   true,
   573  			expectedErr: "RPC Error:: 404,Pprof profile not found profile: latency",
   574  		},
   575  	}
   576  
   577  	for _, tc := range cases {
   578  		t.Run(tc.desc, func(t *testing.T) {
   579  			httpTest(t, nil, func(s *TestAgent) {
   580  
   581  				// add node or server id query param
   582  				url := tc.url
   583  				if tc.addNodeID {
   584  					url = url + "?node_id=" + s.client.NodeID()
   585  				} else if tc.addServerID {
   586  					url = url + "?server_id=" + s.server.LocalMember().Name
   587  				}
   588  
   589  				if tc.clientOnly {
   590  					s.Agent.server = nil
   591  				}
   592  
   593  				req, err := http.NewRequest("GET", url, nil)
   594  				require.Nil(t, err)
   595  				respW := httptest.NewRecorder()
   596  
   597  				resp, err := s.Server.AgentPprofRequest(respW, req)
   598  
   599  				if tc.expectedErr != "" {
   600  					require.Error(t, err)
   601  					require.EqualError(t, err, tc.expectedErr)
   602  				} else {
   603  					require.NoError(t, err)
   604  					require.NotNil(t, resp)
   605  				}
   606  			})
   607  		})
   608  	}
   609  }
   610  
   611  type closableRecorder struct {
   612  	*httptest.ResponseRecorder
   613  	closer chan bool
   614  }
   615  
   616  func newClosableRecorder() *closableRecorder {
   617  	r := httptest.NewRecorder()
   618  	closer := make(chan bool)
   619  	return &closableRecorder{r, closer}
   620  }
   621  
   622  func (r *closableRecorder) Close() {
   623  	close(r.closer)
   624  }
   625  
   626  func (r *closableRecorder) CloseNotify() <-chan bool {
   627  	return r.closer
   628  }
   629  
   630  func TestHTTP_AgentForceLeave(t *testing.T) {
   631  	t.Parallel()
   632  	httpTest(t, nil, func(s *TestAgent) {
   633  		// Make the HTTP request
   634  		req, err := http.NewRequest("PUT", "/v1/agent/force-leave?node=foo", nil)
   635  		if err != nil {
   636  			t.Fatalf("err: %v", err)
   637  		}
   638  		respW := httptest.NewRecorder()
   639  
   640  		// Make the request
   641  		_, err = s.Server.AgentForceLeaveRequest(respW, req)
   642  		if err != nil {
   643  			t.Fatalf("err: %v", err)
   644  		}
   645  	})
   646  }
   647  
   648  func TestHTTP_AgentForceLeave_ACL(t *testing.T) {
   649  	t.Parallel()
   650  	require := require.New(t)
   651  
   652  	httpACLTest(t, nil, func(s *TestAgent) {
   653  		state := s.Agent.server.State()
   654  
   655  		// Make the HTTP request
   656  		req, err := http.NewRequest("PUT", "/v1/agent/force-leave?node=foo", nil)
   657  		require.Nil(err)
   658  
   659  		// Try request without a token and expect failure
   660  		{
   661  			respW := httptest.NewRecorder()
   662  			_, err := s.Server.AgentForceLeaveRequest(respW, req)
   663  			require.NotNil(err)
   664  			require.Equal(err.Error(), structs.ErrPermissionDenied.Error())
   665  		}
   666  
   667  		// Try request with an invalid token and expect failure
   668  		{
   669  			respW := httptest.NewRecorder()
   670  			token := mock.CreatePolicyAndToken(t, state, 1005, "invalid", mock.NodePolicy(acl.PolicyRead))
   671  			setToken(req, token)
   672  			_, err := s.Server.AgentForceLeaveRequest(respW, req)
   673  			require.NotNil(err)
   674  			require.Equal(err.Error(), structs.ErrPermissionDenied.Error())
   675  		}
   676  
   677  		// Try request with a valid token
   678  		{
   679  			respW := httptest.NewRecorder()
   680  			token := mock.CreatePolicyAndToken(t, state, 1007, "valid", mock.AgentPolicy(acl.PolicyWrite))
   681  			setToken(req, token)
   682  			_, err := s.Server.AgentForceLeaveRequest(respW, req)
   683  			require.Nil(err)
   684  			require.Equal(http.StatusOK, respW.Code)
   685  		}
   686  
   687  		// Try request with a root token
   688  		{
   689  			respW := httptest.NewRecorder()
   690  			setToken(req, s.RootToken)
   691  			_, err := s.Server.AgentForceLeaveRequest(respW, req)
   692  			require.Nil(err)
   693  			require.Equal(http.StatusOK, respW.Code)
   694  		}
   695  	})
   696  }
   697  
   698  func TestHTTP_AgentSetServers(t *testing.T) {
   699  	t.Parallel()
   700  	require := require.New(t)
   701  	httpTest(t, nil, func(s *TestAgent) {
   702  		addr := s.Config.AdvertiseAddrs.RPC
   703  		testutil.WaitForResult(func() (bool, error) {
   704  			conn, err := net.DialTimeout("tcp", addr, 100*time.Millisecond)
   705  			if err != nil {
   706  				return false, err
   707  			}
   708  			defer conn.Close()
   709  
   710  			// Write the Consul RPC byte to set the mode
   711  			if _, err := conn.Write([]byte{byte(pool.RpcNomad)}); err != nil {
   712  				return false, err
   713  			}
   714  
   715  			codec := pool.NewClientCodec(conn)
   716  			args := &structs.GenericRequest{}
   717  			var leader string
   718  			err = msgpackrpc.CallWithCodec(codec, "Status.Leader", args, &leader)
   719  			return leader != "", err
   720  		}, func(err error) {
   721  			t.Fatalf("failed to find leader: %v", err)
   722  		})
   723  
   724  		// Create the request
   725  		req, err := http.NewRequest("PUT", "/v1/agent/servers", nil)
   726  		require.Nil(err)
   727  
   728  		// Send the request
   729  		respW := httptest.NewRecorder()
   730  		_, err = s.Server.AgentServersRequest(respW, req)
   731  		require.NotNil(err)
   732  		require.Contains(err.Error(), "missing server address")
   733  
   734  		// Create a valid request
   735  		req, err = http.NewRequest("PUT", "/v1/agent/servers?address=127.0.0.1%3A4647&address=127.0.0.2%3A4647&address=127.0.0.3%3A4647", nil)
   736  		require.Nil(err)
   737  
   738  		// Send the request which should fail
   739  		respW = httptest.NewRecorder()
   740  		_, err = s.Server.AgentServersRequest(respW, req)
   741  		require.NotNil(err)
   742  
   743  		// Retrieve the servers again
   744  		req, err = http.NewRequest("GET", "/v1/agent/servers", nil)
   745  		require.Nil(err)
   746  		respW = httptest.NewRecorder()
   747  
   748  		// Make the request and check the result
   749  		expected := []string{
   750  			s.GetConfig().AdvertiseAddrs.RPC,
   751  		}
   752  		out, err := s.Server.AgentServersRequest(respW, req)
   753  		require.Nil(err)
   754  		servers := out.([]string)
   755  		require.Len(servers, len(expected))
   756  		require.Equal(expected, servers)
   757  	})
   758  }
   759  
   760  func TestHTTP_AgentSetServers_ACL(t *testing.T) {
   761  	t.Parallel()
   762  	require := require.New(t)
   763  
   764  	httpACLTest(t, nil, func(s *TestAgent) {
   765  		state := s.Agent.server.State()
   766  		addr := s.Config.AdvertiseAddrs.RPC
   767  		testutil.WaitForResult(func() (bool, error) {
   768  			conn, err := net.DialTimeout("tcp", addr, 100*time.Millisecond)
   769  			if err != nil {
   770  				return false, err
   771  			}
   772  			defer conn.Close()
   773  
   774  			// Write the Consul RPC byte to set the mode
   775  			if _, err := conn.Write([]byte{byte(pool.RpcNomad)}); err != nil {
   776  				return false, err
   777  			}
   778  
   779  			codec := pool.NewClientCodec(conn)
   780  			args := &structs.GenericRequest{}
   781  			var leader string
   782  			err = msgpackrpc.CallWithCodec(codec, "Status.Leader", args, &leader)
   783  			return leader != "", err
   784  		}, func(err error) {
   785  			t.Fatalf("failed to find leader: %v", err)
   786  		})
   787  
   788  		// Make the HTTP request
   789  		path := fmt.Sprintf("/v1/agent/servers?address=%s", url.QueryEscape(s.GetConfig().AdvertiseAddrs.RPC))
   790  		req, err := http.NewRequest("PUT", path, nil)
   791  		require.Nil(err)
   792  
   793  		// Try request without a token and expect failure
   794  		{
   795  			respW := httptest.NewRecorder()
   796  			_, err := s.Server.AgentServersRequest(respW, req)
   797  			require.NotNil(err)
   798  			require.Equal(err.Error(), structs.ErrPermissionDenied.Error())
   799  		}
   800  
   801  		// Try request with an invalid token and expect failure
   802  		{
   803  			respW := httptest.NewRecorder()
   804  			token := mock.CreatePolicyAndToken(t, state, 1005, "invalid", mock.NodePolicy(acl.PolicyRead))
   805  			setToken(req, token)
   806  			_, err := s.Server.AgentServersRequest(respW, req)
   807  			require.NotNil(err)
   808  			require.Equal(err.Error(), structs.ErrPermissionDenied.Error())
   809  		}
   810  
   811  		// Try request with a valid token
   812  		{
   813  			respW := httptest.NewRecorder()
   814  			token := mock.CreatePolicyAndToken(t, state, 1007, "valid", mock.AgentPolicy(acl.PolicyWrite))
   815  			setToken(req, token)
   816  			_, err := s.Server.AgentServersRequest(respW, req)
   817  			require.Nil(err)
   818  			require.Equal(http.StatusOK, respW.Code)
   819  		}
   820  
   821  		// Try request with a root token
   822  		{
   823  			respW := httptest.NewRecorder()
   824  			setToken(req, s.RootToken)
   825  			_, err := s.Server.AgentServersRequest(respW, req)
   826  			require.Nil(err)
   827  			require.Equal(http.StatusOK, respW.Code)
   828  		}
   829  	})
   830  }
   831  
   832  func TestHTTP_AgentListServers_ACL(t *testing.T) {
   833  	t.Parallel()
   834  	require := require.New(t)
   835  
   836  	httpACLTest(t, nil, func(s *TestAgent) {
   837  		state := s.Agent.server.State()
   838  
   839  		// Create list request
   840  		req, err := http.NewRequest("GET", "/v1/agent/servers", nil)
   841  		require.Nil(err)
   842  
   843  		expected := []string{
   844  			s.GetConfig().AdvertiseAddrs.RPC,
   845  		}
   846  
   847  		// Try request without a token and expect failure
   848  		{
   849  			respW := httptest.NewRecorder()
   850  			_, err := s.Server.AgentServersRequest(respW, req)
   851  			require.NotNil(err)
   852  			require.Equal(err.Error(), structs.ErrPermissionDenied.Error())
   853  		}
   854  
   855  		// Try request with an invalid token and expect failure
   856  		{
   857  			respW := httptest.NewRecorder()
   858  			token := mock.CreatePolicyAndToken(t, state, 1005, "invalid", mock.NodePolicy(acl.PolicyRead))
   859  			setToken(req, token)
   860  			_, err := s.Server.AgentServersRequest(respW, req)
   861  			require.NotNil(err)
   862  			require.Equal(err.Error(), structs.ErrPermissionDenied.Error())
   863  		}
   864  
   865  		// Wait for client to have a server
   866  		testutil.WaitForResult(func() (bool, error) {
   867  			return len(s.client.GetServers()) != 0, fmt.Errorf("no servers")
   868  		}, func(err error) {
   869  			t.Fatal(err)
   870  		})
   871  
   872  		// Try request with a valid token
   873  		{
   874  			respW := httptest.NewRecorder()
   875  			token := mock.CreatePolicyAndToken(t, state, 1007, "valid", mock.AgentPolicy(acl.PolicyRead))
   876  			setToken(req, token)
   877  			out, err := s.Server.AgentServersRequest(respW, req)
   878  			require.Nil(err)
   879  			servers := out.([]string)
   880  			require.Len(servers, len(expected))
   881  			require.Equal(expected, servers)
   882  		}
   883  
   884  		// Try request with a root token
   885  		{
   886  			respW := httptest.NewRecorder()
   887  			setToken(req, s.RootToken)
   888  			out, err := s.Server.AgentServersRequest(respW, req)
   889  			require.Nil(err)
   890  			servers := out.([]string)
   891  			require.Len(servers, len(expected))
   892  			require.Equal(expected, servers)
   893  		}
   894  	})
   895  }
   896  
   897  func TestHTTP_AgentListKeys(t *testing.T) {
   898  	t.Parallel()
   899  
   900  	key1 := "HS5lJ+XuTlYKWaeGYyG+/A=="
   901  
   902  	httpTest(t, func(c *Config) {
   903  		c.Server.EncryptKey = key1
   904  	}, func(s *TestAgent) {
   905  		req, err := http.NewRequest("GET", "/v1/agent/keyring/list", nil)
   906  		if err != nil {
   907  			t.Fatalf("err: %s", err)
   908  		}
   909  		respW := httptest.NewRecorder()
   910  
   911  		out, err := s.Server.KeyringOperationRequest(respW, req)
   912  		require.Nil(t, err)
   913  		kresp := out.(structs.KeyringResponse)
   914  		require.Len(t, kresp.Keys, 1)
   915  	})
   916  }
   917  
   918  func TestHTTP_AgentListKeys_ACL(t *testing.T) {
   919  	t.Parallel()
   920  	require := require.New(t)
   921  
   922  	key1 := "HS5lJ+XuTlYKWaeGYyG+/A=="
   923  
   924  	cb := func(c *Config) {
   925  		c.Server.EncryptKey = key1
   926  	}
   927  
   928  	httpACLTest(t, cb, func(s *TestAgent) {
   929  		state := s.Agent.server.State()
   930  
   931  		// Make the HTTP request
   932  		req, err := http.NewRequest("GET", "/v1/agent/keyring/list", nil)
   933  		require.Nil(err)
   934  
   935  		// Try request without a token and expect failure
   936  		{
   937  			respW := httptest.NewRecorder()
   938  			_, err := s.Server.KeyringOperationRequest(respW, req)
   939  			require.NotNil(err)
   940  			require.Equal(err.Error(), structs.ErrPermissionDenied.Error())
   941  		}
   942  
   943  		// Try request with an invalid token and expect failure
   944  		{
   945  			respW := httptest.NewRecorder()
   946  			token := mock.CreatePolicyAndToken(t, state, 1005, "invalid", mock.AgentPolicy(acl.PolicyRead))
   947  			setToken(req, token)
   948  			_, err := s.Server.KeyringOperationRequest(respW, req)
   949  			require.NotNil(err)
   950  			require.Equal(err.Error(), structs.ErrPermissionDenied.Error())
   951  		}
   952  
   953  		// Try request with a valid token
   954  		{
   955  			respW := httptest.NewRecorder()
   956  			token := mock.CreatePolicyAndToken(t, state, 1007, "valid", mock.AgentPolicy(acl.PolicyWrite))
   957  			setToken(req, token)
   958  			out, err := s.Server.KeyringOperationRequest(respW, req)
   959  			require.Nil(err)
   960  			kresp := out.(structs.KeyringResponse)
   961  			require.Len(kresp.Keys, 1)
   962  			require.Contains(kresp.Keys, key1)
   963  		}
   964  
   965  		// Try request with a root token
   966  		{
   967  			respW := httptest.NewRecorder()
   968  			setToken(req, s.RootToken)
   969  			out, err := s.Server.KeyringOperationRequest(respW, req)
   970  			require.Nil(err)
   971  			kresp := out.(structs.KeyringResponse)
   972  			require.Len(kresp.Keys, 1)
   973  			require.Contains(kresp.Keys, key1)
   974  		}
   975  	})
   976  }
   977  
   978  func TestHTTP_AgentInstallKey(t *testing.T) {
   979  	t.Parallel()
   980  
   981  	key1 := "HS5lJ+XuTlYKWaeGYyG+/A=="
   982  	key2 := "wH1Bn9hlJ0emgWB1JttVRA=="
   983  
   984  	httpTest(t, func(c *Config) {
   985  		c.Server.EncryptKey = key1
   986  	}, func(s *TestAgent) {
   987  		b, err := json.Marshal(&structs.KeyringRequest{Key: key2})
   988  		if err != nil {
   989  			t.Fatalf("err: %v", err)
   990  		}
   991  		req, err := http.NewRequest("GET", "/v1/agent/keyring/install", bytes.NewReader(b))
   992  		if err != nil {
   993  			t.Fatalf("err: %s", err)
   994  		}
   995  		respW := httptest.NewRecorder()
   996  
   997  		_, err = s.Server.KeyringOperationRequest(respW, req)
   998  		if err != nil {
   999  			t.Fatalf("err: %s", err)
  1000  		}
  1001  		req, err = http.NewRequest("GET", "/v1/agent/keyring/list", bytes.NewReader(b))
  1002  		if err != nil {
  1003  			t.Fatalf("err: %s", err)
  1004  		}
  1005  		respW = httptest.NewRecorder()
  1006  
  1007  		out, err := s.Server.KeyringOperationRequest(respW, req)
  1008  		if err != nil {
  1009  			t.Fatalf("err: %s", err)
  1010  		}
  1011  		kresp := out.(structs.KeyringResponse)
  1012  		if len(kresp.Keys) != 2 {
  1013  			t.Fatalf("bad: %v", kresp)
  1014  		}
  1015  	})
  1016  }
  1017  
  1018  func TestHTTP_AgentRemoveKey(t *testing.T) {
  1019  	t.Parallel()
  1020  
  1021  	key1 := "HS5lJ+XuTlYKWaeGYyG+/A=="
  1022  	key2 := "wH1Bn9hlJ0emgWB1JttVRA=="
  1023  
  1024  	httpTest(t, func(c *Config) {
  1025  		c.Server.EncryptKey = key1
  1026  	}, func(s *TestAgent) {
  1027  		b, err := json.Marshal(&structs.KeyringRequest{Key: key2})
  1028  		if err != nil {
  1029  			t.Fatalf("err: %v", err)
  1030  		}
  1031  
  1032  		req, err := http.NewRequest("GET", "/v1/agent/keyring/install", bytes.NewReader(b))
  1033  		if err != nil {
  1034  			t.Fatalf("err: %s", err)
  1035  		}
  1036  		respW := httptest.NewRecorder()
  1037  		_, err = s.Server.KeyringOperationRequest(respW, req)
  1038  		if err != nil {
  1039  			t.Fatalf("err: %s", err)
  1040  		}
  1041  
  1042  		req, err = http.NewRequest("GET", "/v1/agent/keyring/remove", bytes.NewReader(b))
  1043  		if err != nil {
  1044  			t.Fatalf("err: %s", err)
  1045  		}
  1046  		respW = httptest.NewRecorder()
  1047  		if _, err = s.Server.KeyringOperationRequest(respW, req); err != nil {
  1048  			t.Fatalf("err: %s", err)
  1049  		}
  1050  
  1051  		req, err = http.NewRequest("GET", "/v1/agent/keyring/list", nil)
  1052  		if err != nil {
  1053  			t.Fatalf("err: %s", err)
  1054  		}
  1055  		respW = httptest.NewRecorder()
  1056  		out, err := s.Server.KeyringOperationRequest(respW, req)
  1057  		if err != nil {
  1058  			t.Fatalf("err: %s", err)
  1059  		}
  1060  		kresp := out.(structs.KeyringResponse)
  1061  		if len(kresp.Keys) != 1 {
  1062  			t.Fatalf("bad: %v", kresp)
  1063  		}
  1064  	})
  1065  }
  1066  
  1067  func TestHTTP_AgentHealth_Ok(t *testing.T) {
  1068  	t.Parallel()
  1069  	require := require.New(t)
  1070  
  1071  	// Enable ACLs to ensure they're not enforced
  1072  	httpACLTest(t, nil, func(s *TestAgent) {
  1073  		// No ?type=
  1074  		{
  1075  			req, err := http.NewRequest("GET", "/v1/agent/health", nil)
  1076  			require.Nil(err)
  1077  
  1078  			respW := httptest.NewRecorder()
  1079  			healthI, err := s.Server.HealthRequest(respW, req)
  1080  			require.Nil(err)
  1081  			require.Equal(http.StatusOK, respW.Code)
  1082  			require.NotNil(healthI)
  1083  			health := healthI.(*healthResponse)
  1084  			require.NotNil(health.Client)
  1085  			require.True(health.Client.Ok)
  1086  			require.Equal("ok", health.Client.Message)
  1087  			require.NotNil(health.Server)
  1088  			require.True(health.Server.Ok)
  1089  			require.Equal("ok", health.Server.Message)
  1090  		}
  1091  
  1092  		// type=client
  1093  		{
  1094  			req, err := http.NewRequest("GET", "/v1/agent/health?type=client", nil)
  1095  			require.Nil(err)
  1096  
  1097  			respW := httptest.NewRecorder()
  1098  			healthI, err := s.Server.HealthRequest(respW, req)
  1099  			require.Nil(err)
  1100  			require.Equal(http.StatusOK, respW.Code)
  1101  			require.NotNil(healthI)
  1102  			health := healthI.(*healthResponse)
  1103  			require.NotNil(health.Client)
  1104  			require.True(health.Client.Ok)
  1105  			require.Equal("ok", health.Client.Message)
  1106  			require.Nil(health.Server)
  1107  		}
  1108  
  1109  		// type=server
  1110  		{
  1111  			req, err := http.NewRequest("GET", "/v1/agent/health?type=server", nil)
  1112  			require.Nil(err)
  1113  
  1114  			respW := httptest.NewRecorder()
  1115  			healthI, err := s.Server.HealthRequest(respW, req)
  1116  			require.Nil(err)
  1117  			require.Equal(http.StatusOK, respW.Code)
  1118  			require.NotNil(healthI)
  1119  			health := healthI.(*healthResponse)
  1120  			require.NotNil(health.Server)
  1121  			require.True(health.Server.Ok)
  1122  			require.Equal("ok", health.Server.Message)
  1123  			require.Nil(health.Client)
  1124  		}
  1125  
  1126  		// type=client&type=server
  1127  		{
  1128  			req, err := http.NewRequest("GET", "/v1/agent/health?type=client&type=server", nil)
  1129  			require.Nil(err)
  1130  
  1131  			respW := httptest.NewRecorder()
  1132  			healthI, err := s.Server.HealthRequest(respW, req)
  1133  			require.Nil(err)
  1134  			require.Equal(http.StatusOK, respW.Code)
  1135  			require.NotNil(healthI)
  1136  			health := healthI.(*healthResponse)
  1137  			require.NotNil(health.Client)
  1138  			require.True(health.Client.Ok)
  1139  			require.Equal("ok", health.Client.Message)
  1140  			require.NotNil(health.Server)
  1141  			require.True(health.Server.Ok)
  1142  			require.Equal("ok", health.Server.Message)
  1143  		}
  1144  	})
  1145  }
  1146  
  1147  func TestHTTP_AgentHealth_BadServer(t *testing.T) {
  1148  	t.Parallel()
  1149  	require := require.New(t)
  1150  
  1151  	// Enable ACLs to ensure they're not enforced
  1152  	httpACLTest(t, nil, func(s *TestAgent) {
  1153  
  1154  		// Set s.Agent.server=nil to make server unhealthy if requested
  1155  		s.Agent.server = nil
  1156  
  1157  		// No ?type= means server is just skipped
  1158  		{
  1159  			req, err := http.NewRequest("GET", "/v1/agent/health", nil)
  1160  			require.Nil(err)
  1161  
  1162  			respW := httptest.NewRecorder()
  1163  			healthI, err := s.Server.HealthRequest(respW, req)
  1164  			require.Nil(err)
  1165  			require.Equal(http.StatusOK, respW.Code)
  1166  			require.NotNil(healthI)
  1167  			health := healthI.(*healthResponse)
  1168  			require.NotNil(health.Client)
  1169  			require.True(health.Client.Ok)
  1170  			require.Equal("ok", health.Client.Message)
  1171  			require.Nil(health.Server)
  1172  		}
  1173  
  1174  		// type=server means server is considered unhealthy
  1175  		{
  1176  			req, err := http.NewRequest("GET", "/v1/agent/health?type=server", nil)
  1177  			require.Nil(err)
  1178  
  1179  			respW := httptest.NewRecorder()
  1180  			_, err = s.Server.HealthRequest(respW, req)
  1181  			require.NotNil(err)
  1182  			httpErr, ok := err.(HTTPCodedError)
  1183  			require.True(ok)
  1184  			require.Equal(500, httpErr.Code())
  1185  			require.Equal(`{"server":{"ok":false,"message":"server not enabled"}}`, err.Error())
  1186  		}
  1187  	})
  1188  }
  1189  
  1190  func TestHTTP_AgentHealth_BadClient(t *testing.T) {
  1191  	t.Parallel()
  1192  	require := require.New(t)
  1193  
  1194  	// Enable ACLs to ensure they're not enforced
  1195  	httpACLTest(t, nil, func(s *TestAgent) {
  1196  
  1197  		// Set s.Agent.client=nil to make server unhealthy if requested
  1198  		s.Agent.client = nil
  1199  
  1200  		// No ?type= means client is just skipped
  1201  		{
  1202  			req, err := http.NewRequest("GET", "/v1/agent/health", nil)
  1203  			require.Nil(err)
  1204  
  1205  			respW := httptest.NewRecorder()
  1206  			healthI, err := s.Server.HealthRequest(respW, req)
  1207  			require.Nil(err)
  1208  			require.Equal(http.StatusOK, respW.Code)
  1209  			require.NotNil(healthI)
  1210  			health := healthI.(*healthResponse)
  1211  			require.NotNil(health.Server)
  1212  			require.True(health.Server.Ok)
  1213  			require.Equal("ok", health.Server.Message)
  1214  			require.Nil(health.Client)
  1215  		}
  1216  
  1217  		// type=client means client is considered unhealthy
  1218  		{
  1219  			req, err := http.NewRequest("GET", "/v1/agent/health?type=client", nil)
  1220  			require.Nil(err)
  1221  
  1222  			respW := httptest.NewRecorder()
  1223  			_, err = s.Server.HealthRequest(respW, req)
  1224  			require.NotNil(err)
  1225  			httpErr, ok := err.(HTTPCodedError)
  1226  			require.True(ok)
  1227  			require.Equal(500, httpErr.Code())
  1228  			require.Equal(`{"client":{"ok":false,"message":"client not enabled"}}`, err.Error())
  1229  		}
  1230  	})
  1231  }