github.com/prestonp/nomad@v0.10.4/command/agent/agent_endpoint_test.go (about) 1 package agent 2 3 import ( 4 "bytes" 5 "encoding/json" 6 "fmt" 7 "io/ioutil" 8 "net" 9 "net/http" 10 "net/http/httptest" 11 "net/url" 12 "strings" 13 "testing" 14 "time" 15 16 msgpackrpc "github.com/hashicorp/net-rpc-msgpackrpc" 17 "github.com/hashicorp/nomad/acl" 18 "github.com/hashicorp/nomad/helper" 19 "github.com/hashicorp/nomad/helper/pool" 20 "github.com/hashicorp/nomad/nomad/mock" 21 "github.com/hashicorp/nomad/nomad/structs" 22 "github.com/hashicorp/nomad/testutil" 23 "github.com/stretchr/testify/assert" 24 "github.com/stretchr/testify/require" 25 ) 26 27 func TestHTTP_AgentSelf(t *testing.T) { 28 t.Parallel() 29 require := require.New(t) 30 31 httpTest(t, nil, func(s *TestAgent) { 32 // Make the HTTP request 33 req, err := http.NewRequest("GET", "/v1/agent/self", nil) 34 require.NoError(err) 35 respW := httptest.NewRecorder() 36 37 // Make the request 38 obj, err := s.Server.AgentSelfRequest(respW, req) 39 require.NoError(err) 40 41 // Check the job 42 self := obj.(agentSelf) 43 require.NotNil(self.Config) 44 require.NotNil(self.Config.ACL) 45 require.NotEmpty(self.Stats) 46 47 // Check the Vault config 48 require.Empty(self.Config.Vault.Token) 49 50 // Assign a Vault token and require it is redacted. 51 s.Config.Vault.Token = "badc0deb-adc0-deba-dc0d-ebadc0debadc" 52 respW = httptest.NewRecorder() 53 obj, err = s.Server.AgentSelfRequest(respW, req) 54 require.NoError(err) 55 self = obj.(agentSelf) 56 require.Equal("<redacted>", self.Config.Vault.Token) 57 58 // Assign a ReplicationToken token and require it is redacted. 59 s.Config.ACL.ReplicationToken = "badc0deb-adc0-deba-dc0d-ebadc0debadc" 60 respW = httptest.NewRecorder() 61 obj, err = s.Server.AgentSelfRequest(respW, req) 62 require.NoError(err) 63 self = obj.(agentSelf) 64 require.Equal("<redacted>", self.Config.ACL.ReplicationToken) 65 66 // Check the Consul config 67 require.Empty(self.Config.Consul.Token) 68 69 // Assign a Consul token and require it is redacted. 70 s.Config.Consul.Token = "badc0deb-adc0-deba-dc0d-ebadc0debadc" 71 respW = httptest.NewRecorder() 72 obj, err = s.Server.AgentSelfRequest(respW, req) 73 require.NoError(err) 74 self = obj.(agentSelf) 75 require.Equal("<redacted>", self.Config.Consul.Token) 76 77 // Check the Circonus config 78 require.Empty(self.Config.Telemetry.CirconusAPIToken) 79 80 // Assign a Consul token and require it is redacted. 81 s.Config.Telemetry.CirconusAPIToken = "badc0deb-adc0-deba-dc0d-ebadc0debadc" 82 respW = httptest.NewRecorder() 83 obj, err = s.Server.AgentSelfRequest(respW, req) 84 require.NoError(err) 85 self = obj.(agentSelf) 86 require.Equal("<redacted>", self.Config.Telemetry.CirconusAPIToken) 87 }) 88 } 89 90 func TestHTTP_AgentSelf_ACL(t *testing.T) { 91 t.Parallel() 92 require := require.New(t) 93 94 httpACLTest(t, nil, func(s *TestAgent) { 95 state := s.Agent.server.State() 96 97 // Make the HTTP request 98 req, err := http.NewRequest("GET", "/v1/agent/self", nil) 99 require.Nil(err) 100 101 // Try request without a token and expect failure 102 { 103 respW := httptest.NewRecorder() 104 _, err := s.Server.AgentSelfRequest(respW, req) 105 require.NotNil(err) 106 require.Equal(err.Error(), structs.ErrPermissionDenied.Error()) 107 } 108 109 // Try request with an invalid token and expect failure 110 { 111 respW := httptest.NewRecorder() 112 token := mock.CreatePolicyAndToken(t, state, 1005, "invalid", mock.NodePolicy(acl.PolicyWrite)) 113 setToken(req, token) 114 _, err := s.Server.AgentSelfRequest(respW, req) 115 require.NotNil(err) 116 require.Equal(err.Error(), structs.ErrPermissionDenied.Error()) 117 } 118 119 // Try request with a valid token 120 { 121 respW := httptest.NewRecorder() 122 token := mock.CreatePolicyAndToken(t, state, 1007, "valid", mock.AgentPolicy(acl.PolicyWrite)) 123 setToken(req, token) 124 obj, err := s.Server.AgentSelfRequest(respW, req) 125 require.Nil(err) 126 127 self := obj.(agentSelf) 128 require.NotNil(self.Config) 129 require.NotNil(self.Stats) 130 } 131 132 // Try request with a root token 133 { 134 respW := httptest.NewRecorder() 135 setToken(req, s.RootToken) 136 obj, err := s.Server.AgentSelfRequest(respW, req) 137 require.Nil(err) 138 139 self := obj.(agentSelf) 140 require.NotNil(self.Config) 141 require.NotNil(self.Stats) 142 } 143 }) 144 } 145 146 func TestHTTP_AgentJoin(t *testing.T) { 147 t.Parallel() 148 httpTest(t, nil, func(s *TestAgent) { 149 // Determine the join address 150 member := s.Agent.Server().LocalMember() 151 addr := fmt.Sprintf("%s:%d", member.Addr, member.Port) 152 153 // Make the HTTP request 154 req, err := http.NewRequest("PUT", 155 fmt.Sprintf("/v1/agent/join?address=%s&address=%s", addr, addr), nil) 156 if err != nil { 157 t.Fatalf("err: %v", err) 158 } 159 respW := httptest.NewRecorder() 160 161 // Make the request 162 obj, err := s.Server.AgentJoinRequest(respW, req) 163 if err != nil { 164 t.Fatalf("err: %v", err) 165 } 166 167 // Check the job 168 join := obj.(joinResult) 169 if join.NumJoined != 2 { 170 t.Fatalf("bad: %#v", join) 171 } 172 if join.Error != "" { 173 t.Fatalf("bad: %#v", join) 174 } 175 }) 176 } 177 178 func TestHTTP_AgentMembers(t *testing.T) { 179 t.Parallel() 180 httpTest(t, nil, func(s *TestAgent) { 181 // Make the HTTP request 182 req, err := http.NewRequest("GET", "/v1/agent/members", nil) 183 if err != nil { 184 t.Fatalf("err: %v", err) 185 } 186 respW := httptest.NewRecorder() 187 188 // Make the request 189 obj, err := s.Server.AgentMembersRequest(respW, req) 190 if err != nil { 191 t.Fatalf("err: %v", err) 192 } 193 194 // Check the job 195 members := obj.(structs.ServerMembersResponse) 196 if len(members.Members) != 1 { 197 t.Fatalf("bad: %#v", members.Members) 198 } 199 }) 200 } 201 202 func TestHTTP_AgentMembers_ACL(t *testing.T) { 203 t.Parallel() 204 require := require.New(t) 205 206 httpACLTest(t, nil, func(s *TestAgent) { 207 state := s.Agent.server.State() 208 209 // Make the HTTP request 210 req, err := http.NewRequest("GET", "/v1/agent/members", nil) 211 require.Nil(err) 212 213 // Try request without a token and expect failure 214 { 215 respW := httptest.NewRecorder() 216 _, err := s.Server.AgentMembersRequest(respW, req) 217 require.NotNil(err) 218 require.Equal(err.Error(), structs.ErrPermissionDenied.Error()) 219 } 220 221 // Try request with an invalid token and expect failure 222 { 223 respW := httptest.NewRecorder() 224 token := mock.CreatePolicyAndToken(t, state, 1005, "invalid", mock.AgentPolicy(acl.PolicyWrite)) 225 setToken(req, token) 226 _, err := s.Server.AgentMembersRequest(respW, req) 227 require.NotNil(err) 228 require.Equal(err.Error(), structs.ErrPermissionDenied.Error()) 229 } 230 231 // Try request with a valid token 232 { 233 respW := httptest.NewRecorder() 234 token := mock.CreatePolicyAndToken(t, state, 1007, "valid", mock.NodePolicy(acl.PolicyRead)) 235 setToken(req, token) 236 obj, err := s.Server.AgentMembersRequest(respW, req) 237 require.Nil(err) 238 239 members := obj.(structs.ServerMembersResponse) 240 require.Len(members.Members, 1) 241 } 242 243 // Try request with a root token 244 { 245 respW := httptest.NewRecorder() 246 setToken(req, s.RootToken) 247 obj, err := s.Server.AgentMembersRequest(respW, req) 248 require.Nil(err) 249 250 members := obj.(structs.ServerMembersResponse) 251 require.Len(members.Members, 1) 252 } 253 }) 254 } 255 256 func TestHTTP_AgentMonitor(t *testing.T) { 257 t.Parallel() 258 259 t.Run("invalid log_json parameter", func(t *testing.T) { 260 httpTest(t, nil, func(s *TestAgent) { 261 req, err := http.NewRequest("GET", "/v1/agent/monitor?log_json=no", nil) 262 require.Nil(t, err) 263 resp := newClosableRecorder() 264 265 // Make the request 266 _, err = s.Server.AgentMonitor(resp, req) 267 httpErr := err.(HTTPCodedError).Code() 268 require.Equal(t, 400, httpErr) 269 }) 270 }) 271 272 t.Run("unknown log_level", func(t *testing.T) { 273 httpTest(t, nil, func(s *TestAgent) { 274 req, err := http.NewRequest("GET", "/v1/agent/monitor?log_level=unknown", nil) 275 require.Nil(t, err) 276 resp := newClosableRecorder() 277 278 // Make the request 279 _, err = s.Server.AgentMonitor(resp, req) 280 httpErr := err.(HTTPCodedError).Code() 281 require.Equal(t, 400, httpErr) 282 }) 283 }) 284 285 t.Run("check for specific log level", func(t *testing.T) { 286 httpTest(t, nil, func(s *TestAgent) { 287 req, err := http.NewRequest("GET", "/v1/agent/monitor?log_level=warn", nil) 288 require.Nil(t, err) 289 resp := newClosableRecorder() 290 defer resp.Close() 291 292 go func() { 293 _, err = s.Server.AgentMonitor(resp, req) 294 assert.NoError(t, err) 295 }() 296 297 // send the same log until monitor sink is set up 298 maxLogAttempts := 10 299 tried := 0 300 testutil.WaitForResult(func() (bool, error) { 301 if tried < maxLogAttempts { 302 s.Server.logger.Warn("log that should be sent") 303 tried++ 304 } 305 306 got := resp.Body.String() 307 want := `{"Data":"` 308 if strings.Contains(got, want) { 309 return true, nil 310 } 311 312 return false, fmt.Errorf("missing expected log, got: %v, want: %v", got, want) 313 }, func(err error) { 314 require.Fail(t, err.Error()) 315 }) 316 }) 317 }) 318 319 t.Run("plain output", func(t *testing.T) { 320 httpTest(t, nil, func(s *TestAgent) { 321 req, err := http.NewRequest("GET", "/v1/agent/monitor?log_level=debug&plain=true", nil) 322 require.Nil(t, err) 323 resp := newClosableRecorder() 324 defer resp.Close() 325 326 go func() { 327 _, err = s.Server.AgentMonitor(resp, req) 328 assert.NoError(t, err) 329 }() 330 331 // send the same log until monitor sink is set up 332 maxLogAttempts := 10 333 tried := 0 334 testutil.WaitForResult(func() (bool, error) { 335 if tried < maxLogAttempts { 336 s.Server.logger.Debug("log that should be sent") 337 tried++ 338 } 339 340 got := resp.Body.String() 341 want := `[DEBUG] http: log that should be sent` 342 if strings.Contains(got, want) { 343 return true, nil 344 } 345 346 return false, fmt.Errorf("missing expected log, got: %v, want: %v", got, want) 347 }, func(err error) { 348 require.Fail(t, err.Error()) 349 }) 350 }) 351 }) 352 353 t.Run("logs for a specific node", func(t *testing.T) { 354 httpTest(t, nil, func(s *TestAgent) { 355 req, err := http.NewRequest("GET", "/v1/agent/monitor?log_level=warn&node_id="+s.client.NodeID(), nil) 356 require.Nil(t, err) 357 resp := newClosableRecorder() 358 defer resp.Close() 359 360 go func() { 361 _, err = s.Server.AgentMonitor(resp, req) 362 assert.NoError(t, err) 363 }() 364 365 // send the same log until monitor sink is set up 366 maxLogAttempts := 10 367 tried := 0 368 out := "" 369 testutil.WaitForResult(func() (bool, error) { 370 if tried < maxLogAttempts { 371 s.Server.logger.Debug("log that should not be sent") 372 s.Server.logger.Warn("log that should be sent") 373 tried++ 374 } 375 output, err := ioutil.ReadAll(resp.Body) 376 if err != nil { 377 return false, err 378 } 379 380 out += string(output) 381 want := `{"Data":"` 382 if strings.Contains(out, want) { 383 return true, nil 384 } 385 386 return false, fmt.Errorf("missing expected log, got: %v, want: %v", out, want) 387 }, func(err error) { 388 require.Fail(t, err.Error()) 389 }) 390 }) 391 }) 392 393 t.Run("logs for a local client with no server running on agent", func(t *testing.T) { 394 httpTest(t, nil, func(s *TestAgent) { 395 req, err := http.NewRequest("GET", "/v1/agent/monitor?log_level=warn", nil) 396 require.Nil(t, err) 397 resp := newClosableRecorder() 398 defer resp.Close() 399 400 go func() { 401 // set server to nil to monitor as client 402 s.Agent.server = nil 403 _, err = s.Server.AgentMonitor(resp, req) 404 assert.NoError(t, err) 405 }() 406 407 // send the same log until monitor sink is set up 408 maxLogAttempts := 10 409 tried := 0 410 out := "" 411 testutil.WaitForResult(func() (bool, error) { 412 if tried < maxLogAttempts { 413 s.Agent.logger.Warn("log that should be sent") 414 tried++ 415 } 416 output, err := ioutil.ReadAll(resp.Body) 417 if err != nil { 418 return false, err 419 } 420 421 out += string(output) 422 want := `{"Data":"` 423 if strings.Contains(out, want) { 424 return true, nil 425 } 426 427 return false, fmt.Errorf("missing expected log, got: %v, want: %v", out, want) 428 }, func(err error) { 429 require.Fail(t, err.Error()) 430 }) 431 }) 432 }) 433 } 434 435 // Scenarios when Pprof requests should be available 436 // see https://github.com/hashicorp/nomad/issues/6496 437 // +---------------+------------------+--------+------------------+ 438 // | Endpoint | `enable_debug` | ACLs | **Available?** | 439 // +---------------+------------------+--------+------------------+ 440 // | /debug/pprof | unset | n/a | no | 441 // | /debug/pprof | `true` | n/a | yes | 442 // | /debug/pprof | `false` | n/a | no | 443 // | /agent/pprof | unset | off | no | 444 // | /agent/pprof | unset | on | **yes** | 445 // | /agent/pprof | `true` | off | yes | 446 // | /agent/pprof | `false` | on | **yes** | 447 // +---------------+------------------+--------+------------------+ 448 func TestAgent_PprofRequest_Permissions(t *testing.T) { 449 trueP, falseP := helper.BoolToPtr(true), helper.BoolToPtr(false) 450 cases := []struct { 451 acl *bool 452 debug *bool 453 ok bool 454 }{ 455 // manually set to false because test helpers 456 // enable to true by default 457 // enableDebug: helper.BoolToPtr(false), 458 {debug: nil, ok: false}, 459 {debug: trueP, ok: true}, 460 {debug: falseP, ok: false}, 461 {debug: falseP, acl: falseP, ok: false}, 462 {acl: trueP, ok: true}, 463 {acl: falseP, debug: trueP, ok: true}, 464 {debug: falseP, acl: trueP, ok: true}, 465 } 466 467 for _, tc := range cases { 468 ptrToStr := func(val *bool) string { 469 if val == nil { 470 return "unset" 471 } else if *val == true { 472 return "true" 473 } else { 474 return "false" 475 } 476 } 477 478 t.Run( 479 fmt.Sprintf("debug %s, acl %s", 480 ptrToStr(tc.debug), 481 ptrToStr(tc.acl)), 482 func(t *testing.T) { 483 cb := func(c *Config) { 484 if tc.acl != nil { 485 c.ACL.Enabled = *tc.acl 486 } 487 if tc.debug == nil { 488 var nodebug bool 489 c.EnableDebug = nodebug 490 } else { 491 c.EnableDebug = *tc.debug 492 } 493 } 494 495 httpTest(t, cb, func(s *TestAgent) { 496 state := s.Agent.server.State() 497 url := "/v1/agent/pprof/cmdline" 498 req, err := http.NewRequest("GET", url, nil) 499 require.NoError(t, err) 500 respW := httptest.NewRecorder() 501 502 if tc.acl != nil && *tc.acl { 503 token := mock.CreatePolicyAndToken(t, state, 1007, "valid", mock.AgentPolicy(acl.PolicyWrite)) 504 setToken(req, token) 505 } 506 507 resp, err := s.Server.AgentPprofRequest(respW, req) 508 if tc.ok { 509 require.NoError(t, err) 510 require.NotNil(t, resp) 511 } else { 512 require.Error(t, err) 513 require.Equal(t, structs.ErrPermissionDenied.Error(), err.Error()) 514 } 515 }) 516 }) 517 } 518 } 519 520 func TestAgent_PprofRequest(t *testing.T) { 521 cases := []struct { 522 desc string 523 url string 524 addNodeID bool 525 addServerID bool 526 expectedErr string 527 clientOnly bool 528 }{ 529 { 530 desc: "cmdline local server request", 531 url: "/v1/agent/pprof/cmdline", 532 }, 533 { 534 desc: "cmdline local node request", 535 url: "/v1/agent/pprof/cmdline", 536 clientOnly: true, 537 }, 538 { 539 desc: "cmdline node request", 540 url: "/v1/agent/pprof/cmdline", 541 addNodeID: true, 542 }, 543 { 544 desc: "cmdline server request", 545 url: "/v1/agent/pprof/cmdline", 546 addServerID: true, 547 }, 548 { 549 desc: "invalid server request", 550 url: "/v1/agent/pprof/unknown", 551 addServerID: true, 552 expectedErr: "RPC Error:: 404,Pprof profile not found profile: unknown", 553 }, 554 { 555 desc: "cpu profile request", 556 url: "/v1/agent/pprof/profile", 557 addNodeID: true, 558 }, 559 { 560 desc: "trace request", 561 url: "/v1/agent/pprof/trace", 562 addNodeID: true, 563 }, 564 { 565 desc: "pprof lookup request", 566 url: "/v1/agent/pprof/goroutine", 567 addNodeID: true, 568 }, 569 { 570 desc: "unknown pprof lookup request", 571 url: "/v1/agent/pprof/latency", 572 addNodeID: true, 573 expectedErr: "RPC Error:: 404,Pprof profile not found profile: latency", 574 }, 575 } 576 577 for _, tc := range cases { 578 t.Run(tc.desc, func(t *testing.T) { 579 httpTest(t, nil, func(s *TestAgent) { 580 581 // add node or server id query param 582 url := tc.url 583 if tc.addNodeID { 584 url = url + "?node_id=" + s.client.NodeID() 585 } else if tc.addServerID { 586 url = url + "?server_id=" + s.server.LocalMember().Name 587 } 588 589 if tc.clientOnly { 590 s.Agent.server = nil 591 } 592 593 req, err := http.NewRequest("GET", url, nil) 594 require.Nil(t, err) 595 respW := httptest.NewRecorder() 596 597 resp, err := s.Server.AgentPprofRequest(respW, req) 598 599 if tc.expectedErr != "" { 600 require.Error(t, err) 601 require.EqualError(t, err, tc.expectedErr) 602 } else { 603 require.NoError(t, err) 604 require.NotNil(t, resp) 605 } 606 }) 607 }) 608 } 609 } 610 611 type closableRecorder struct { 612 *httptest.ResponseRecorder 613 closer chan bool 614 } 615 616 func newClosableRecorder() *closableRecorder { 617 r := httptest.NewRecorder() 618 closer := make(chan bool) 619 return &closableRecorder{r, closer} 620 } 621 622 func (r *closableRecorder) Close() { 623 close(r.closer) 624 } 625 626 func (r *closableRecorder) CloseNotify() <-chan bool { 627 return r.closer 628 } 629 630 func TestHTTP_AgentForceLeave(t *testing.T) { 631 t.Parallel() 632 httpTest(t, nil, func(s *TestAgent) { 633 // Make the HTTP request 634 req, err := http.NewRequest("PUT", "/v1/agent/force-leave?node=foo", nil) 635 if err != nil { 636 t.Fatalf("err: %v", err) 637 } 638 respW := httptest.NewRecorder() 639 640 // Make the request 641 _, err = s.Server.AgentForceLeaveRequest(respW, req) 642 if err != nil { 643 t.Fatalf("err: %v", err) 644 } 645 }) 646 } 647 648 func TestHTTP_AgentForceLeave_ACL(t *testing.T) { 649 t.Parallel() 650 require := require.New(t) 651 652 httpACLTest(t, nil, func(s *TestAgent) { 653 state := s.Agent.server.State() 654 655 // Make the HTTP request 656 req, err := http.NewRequest("PUT", "/v1/agent/force-leave?node=foo", nil) 657 require.Nil(err) 658 659 // Try request without a token and expect failure 660 { 661 respW := httptest.NewRecorder() 662 _, err := s.Server.AgentForceLeaveRequest(respW, req) 663 require.NotNil(err) 664 require.Equal(err.Error(), structs.ErrPermissionDenied.Error()) 665 } 666 667 // Try request with an invalid token and expect failure 668 { 669 respW := httptest.NewRecorder() 670 token := mock.CreatePolicyAndToken(t, state, 1005, "invalid", mock.NodePolicy(acl.PolicyRead)) 671 setToken(req, token) 672 _, err := s.Server.AgentForceLeaveRequest(respW, req) 673 require.NotNil(err) 674 require.Equal(err.Error(), structs.ErrPermissionDenied.Error()) 675 } 676 677 // Try request with a valid token 678 { 679 respW := httptest.NewRecorder() 680 token := mock.CreatePolicyAndToken(t, state, 1007, "valid", mock.AgentPolicy(acl.PolicyWrite)) 681 setToken(req, token) 682 _, err := s.Server.AgentForceLeaveRequest(respW, req) 683 require.Nil(err) 684 require.Equal(http.StatusOK, respW.Code) 685 } 686 687 // Try request with a root token 688 { 689 respW := httptest.NewRecorder() 690 setToken(req, s.RootToken) 691 _, err := s.Server.AgentForceLeaveRequest(respW, req) 692 require.Nil(err) 693 require.Equal(http.StatusOK, respW.Code) 694 } 695 }) 696 } 697 698 func TestHTTP_AgentSetServers(t *testing.T) { 699 t.Parallel() 700 require := require.New(t) 701 httpTest(t, nil, func(s *TestAgent) { 702 addr := s.Config.AdvertiseAddrs.RPC 703 testutil.WaitForResult(func() (bool, error) { 704 conn, err := net.DialTimeout("tcp", addr, 100*time.Millisecond) 705 if err != nil { 706 return false, err 707 } 708 defer conn.Close() 709 710 // Write the Consul RPC byte to set the mode 711 if _, err := conn.Write([]byte{byte(pool.RpcNomad)}); err != nil { 712 return false, err 713 } 714 715 codec := pool.NewClientCodec(conn) 716 args := &structs.GenericRequest{} 717 var leader string 718 err = msgpackrpc.CallWithCodec(codec, "Status.Leader", args, &leader) 719 return leader != "", err 720 }, func(err error) { 721 t.Fatalf("failed to find leader: %v", err) 722 }) 723 724 // Create the request 725 req, err := http.NewRequest("PUT", "/v1/agent/servers", nil) 726 require.Nil(err) 727 728 // Send the request 729 respW := httptest.NewRecorder() 730 _, err = s.Server.AgentServersRequest(respW, req) 731 require.NotNil(err) 732 require.Contains(err.Error(), "missing server address") 733 734 // Create a valid request 735 req, err = http.NewRequest("PUT", "/v1/agent/servers?address=127.0.0.1%3A4647&address=127.0.0.2%3A4647&address=127.0.0.3%3A4647", nil) 736 require.Nil(err) 737 738 // Send the request which should fail 739 respW = httptest.NewRecorder() 740 _, err = s.Server.AgentServersRequest(respW, req) 741 require.NotNil(err) 742 743 // Retrieve the servers again 744 req, err = http.NewRequest("GET", "/v1/agent/servers", nil) 745 require.Nil(err) 746 respW = httptest.NewRecorder() 747 748 // Make the request and check the result 749 expected := []string{ 750 s.GetConfig().AdvertiseAddrs.RPC, 751 } 752 out, err := s.Server.AgentServersRequest(respW, req) 753 require.Nil(err) 754 servers := out.([]string) 755 require.Len(servers, len(expected)) 756 require.Equal(expected, servers) 757 }) 758 } 759 760 func TestHTTP_AgentSetServers_ACL(t *testing.T) { 761 t.Parallel() 762 require := require.New(t) 763 764 httpACLTest(t, nil, func(s *TestAgent) { 765 state := s.Agent.server.State() 766 addr := s.Config.AdvertiseAddrs.RPC 767 testutil.WaitForResult(func() (bool, error) { 768 conn, err := net.DialTimeout("tcp", addr, 100*time.Millisecond) 769 if err != nil { 770 return false, err 771 } 772 defer conn.Close() 773 774 // Write the Consul RPC byte to set the mode 775 if _, err := conn.Write([]byte{byte(pool.RpcNomad)}); err != nil { 776 return false, err 777 } 778 779 codec := pool.NewClientCodec(conn) 780 args := &structs.GenericRequest{} 781 var leader string 782 err = msgpackrpc.CallWithCodec(codec, "Status.Leader", args, &leader) 783 return leader != "", err 784 }, func(err error) { 785 t.Fatalf("failed to find leader: %v", err) 786 }) 787 788 // Make the HTTP request 789 path := fmt.Sprintf("/v1/agent/servers?address=%s", url.QueryEscape(s.GetConfig().AdvertiseAddrs.RPC)) 790 req, err := http.NewRequest("PUT", path, nil) 791 require.Nil(err) 792 793 // Try request without a token and expect failure 794 { 795 respW := httptest.NewRecorder() 796 _, err := s.Server.AgentServersRequest(respW, req) 797 require.NotNil(err) 798 require.Equal(err.Error(), structs.ErrPermissionDenied.Error()) 799 } 800 801 // Try request with an invalid token and expect failure 802 { 803 respW := httptest.NewRecorder() 804 token := mock.CreatePolicyAndToken(t, state, 1005, "invalid", mock.NodePolicy(acl.PolicyRead)) 805 setToken(req, token) 806 _, err := s.Server.AgentServersRequest(respW, req) 807 require.NotNil(err) 808 require.Equal(err.Error(), structs.ErrPermissionDenied.Error()) 809 } 810 811 // Try request with a valid token 812 { 813 respW := httptest.NewRecorder() 814 token := mock.CreatePolicyAndToken(t, state, 1007, "valid", mock.AgentPolicy(acl.PolicyWrite)) 815 setToken(req, token) 816 _, err := s.Server.AgentServersRequest(respW, req) 817 require.Nil(err) 818 require.Equal(http.StatusOK, respW.Code) 819 } 820 821 // Try request with a root token 822 { 823 respW := httptest.NewRecorder() 824 setToken(req, s.RootToken) 825 _, err := s.Server.AgentServersRequest(respW, req) 826 require.Nil(err) 827 require.Equal(http.StatusOK, respW.Code) 828 } 829 }) 830 } 831 832 func TestHTTP_AgentListServers_ACL(t *testing.T) { 833 t.Parallel() 834 require := require.New(t) 835 836 httpACLTest(t, nil, func(s *TestAgent) { 837 state := s.Agent.server.State() 838 839 // Create list request 840 req, err := http.NewRequest("GET", "/v1/agent/servers", nil) 841 require.Nil(err) 842 843 expected := []string{ 844 s.GetConfig().AdvertiseAddrs.RPC, 845 } 846 847 // Try request without a token and expect failure 848 { 849 respW := httptest.NewRecorder() 850 _, err := s.Server.AgentServersRequest(respW, req) 851 require.NotNil(err) 852 require.Equal(err.Error(), structs.ErrPermissionDenied.Error()) 853 } 854 855 // Try request with an invalid token and expect failure 856 { 857 respW := httptest.NewRecorder() 858 token := mock.CreatePolicyAndToken(t, state, 1005, "invalid", mock.NodePolicy(acl.PolicyRead)) 859 setToken(req, token) 860 _, err := s.Server.AgentServersRequest(respW, req) 861 require.NotNil(err) 862 require.Equal(err.Error(), structs.ErrPermissionDenied.Error()) 863 } 864 865 // Wait for client to have a server 866 testutil.WaitForResult(func() (bool, error) { 867 return len(s.client.GetServers()) != 0, fmt.Errorf("no servers") 868 }, func(err error) { 869 t.Fatal(err) 870 }) 871 872 // Try request with a valid token 873 { 874 respW := httptest.NewRecorder() 875 token := mock.CreatePolicyAndToken(t, state, 1007, "valid", mock.AgentPolicy(acl.PolicyRead)) 876 setToken(req, token) 877 out, err := s.Server.AgentServersRequest(respW, req) 878 require.Nil(err) 879 servers := out.([]string) 880 require.Len(servers, len(expected)) 881 require.Equal(expected, servers) 882 } 883 884 // Try request with a root token 885 { 886 respW := httptest.NewRecorder() 887 setToken(req, s.RootToken) 888 out, err := s.Server.AgentServersRequest(respW, req) 889 require.Nil(err) 890 servers := out.([]string) 891 require.Len(servers, len(expected)) 892 require.Equal(expected, servers) 893 } 894 }) 895 } 896 897 func TestHTTP_AgentListKeys(t *testing.T) { 898 t.Parallel() 899 900 key1 := "HS5lJ+XuTlYKWaeGYyG+/A==" 901 902 httpTest(t, func(c *Config) { 903 c.Server.EncryptKey = key1 904 }, func(s *TestAgent) { 905 req, err := http.NewRequest("GET", "/v1/agent/keyring/list", nil) 906 if err != nil { 907 t.Fatalf("err: %s", err) 908 } 909 respW := httptest.NewRecorder() 910 911 out, err := s.Server.KeyringOperationRequest(respW, req) 912 require.Nil(t, err) 913 kresp := out.(structs.KeyringResponse) 914 require.Len(t, kresp.Keys, 1) 915 }) 916 } 917 918 func TestHTTP_AgentListKeys_ACL(t *testing.T) { 919 t.Parallel() 920 require := require.New(t) 921 922 key1 := "HS5lJ+XuTlYKWaeGYyG+/A==" 923 924 cb := func(c *Config) { 925 c.Server.EncryptKey = key1 926 } 927 928 httpACLTest(t, cb, func(s *TestAgent) { 929 state := s.Agent.server.State() 930 931 // Make the HTTP request 932 req, err := http.NewRequest("GET", "/v1/agent/keyring/list", nil) 933 require.Nil(err) 934 935 // Try request without a token and expect failure 936 { 937 respW := httptest.NewRecorder() 938 _, err := s.Server.KeyringOperationRequest(respW, req) 939 require.NotNil(err) 940 require.Equal(err.Error(), structs.ErrPermissionDenied.Error()) 941 } 942 943 // Try request with an invalid token and expect failure 944 { 945 respW := httptest.NewRecorder() 946 token := mock.CreatePolicyAndToken(t, state, 1005, "invalid", mock.AgentPolicy(acl.PolicyRead)) 947 setToken(req, token) 948 _, err := s.Server.KeyringOperationRequest(respW, req) 949 require.NotNil(err) 950 require.Equal(err.Error(), structs.ErrPermissionDenied.Error()) 951 } 952 953 // Try request with a valid token 954 { 955 respW := httptest.NewRecorder() 956 token := mock.CreatePolicyAndToken(t, state, 1007, "valid", mock.AgentPolicy(acl.PolicyWrite)) 957 setToken(req, token) 958 out, err := s.Server.KeyringOperationRequest(respW, req) 959 require.Nil(err) 960 kresp := out.(structs.KeyringResponse) 961 require.Len(kresp.Keys, 1) 962 require.Contains(kresp.Keys, key1) 963 } 964 965 // Try request with a root token 966 { 967 respW := httptest.NewRecorder() 968 setToken(req, s.RootToken) 969 out, err := s.Server.KeyringOperationRequest(respW, req) 970 require.Nil(err) 971 kresp := out.(structs.KeyringResponse) 972 require.Len(kresp.Keys, 1) 973 require.Contains(kresp.Keys, key1) 974 } 975 }) 976 } 977 978 func TestHTTP_AgentInstallKey(t *testing.T) { 979 t.Parallel() 980 981 key1 := "HS5lJ+XuTlYKWaeGYyG+/A==" 982 key2 := "wH1Bn9hlJ0emgWB1JttVRA==" 983 984 httpTest(t, func(c *Config) { 985 c.Server.EncryptKey = key1 986 }, func(s *TestAgent) { 987 b, err := json.Marshal(&structs.KeyringRequest{Key: key2}) 988 if err != nil { 989 t.Fatalf("err: %v", err) 990 } 991 req, err := http.NewRequest("GET", "/v1/agent/keyring/install", bytes.NewReader(b)) 992 if err != nil { 993 t.Fatalf("err: %s", err) 994 } 995 respW := httptest.NewRecorder() 996 997 _, err = s.Server.KeyringOperationRequest(respW, req) 998 if err != nil { 999 t.Fatalf("err: %s", err) 1000 } 1001 req, err = http.NewRequest("GET", "/v1/agent/keyring/list", bytes.NewReader(b)) 1002 if err != nil { 1003 t.Fatalf("err: %s", err) 1004 } 1005 respW = httptest.NewRecorder() 1006 1007 out, err := s.Server.KeyringOperationRequest(respW, req) 1008 if err != nil { 1009 t.Fatalf("err: %s", err) 1010 } 1011 kresp := out.(structs.KeyringResponse) 1012 if len(kresp.Keys) != 2 { 1013 t.Fatalf("bad: %v", kresp) 1014 } 1015 }) 1016 } 1017 1018 func TestHTTP_AgentRemoveKey(t *testing.T) { 1019 t.Parallel() 1020 1021 key1 := "HS5lJ+XuTlYKWaeGYyG+/A==" 1022 key2 := "wH1Bn9hlJ0emgWB1JttVRA==" 1023 1024 httpTest(t, func(c *Config) { 1025 c.Server.EncryptKey = key1 1026 }, func(s *TestAgent) { 1027 b, err := json.Marshal(&structs.KeyringRequest{Key: key2}) 1028 if err != nil { 1029 t.Fatalf("err: %v", err) 1030 } 1031 1032 req, err := http.NewRequest("GET", "/v1/agent/keyring/install", bytes.NewReader(b)) 1033 if err != nil { 1034 t.Fatalf("err: %s", err) 1035 } 1036 respW := httptest.NewRecorder() 1037 _, err = s.Server.KeyringOperationRequest(respW, req) 1038 if err != nil { 1039 t.Fatalf("err: %s", err) 1040 } 1041 1042 req, err = http.NewRequest("GET", "/v1/agent/keyring/remove", bytes.NewReader(b)) 1043 if err != nil { 1044 t.Fatalf("err: %s", err) 1045 } 1046 respW = httptest.NewRecorder() 1047 if _, err = s.Server.KeyringOperationRequest(respW, req); err != nil { 1048 t.Fatalf("err: %s", err) 1049 } 1050 1051 req, err = http.NewRequest("GET", "/v1/agent/keyring/list", nil) 1052 if err != nil { 1053 t.Fatalf("err: %s", err) 1054 } 1055 respW = httptest.NewRecorder() 1056 out, err := s.Server.KeyringOperationRequest(respW, req) 1057 if err != nil { 1058 t.Fatalf("err: %s", err) 1059 } 1060 kresp := out.(structs.KeyringResponse) 1061 if len(kresp.Keys) != 1 { 1062 t.Fatalf("bad: %v", kresp) 1063 } 1064 }) 1065 } 1066 1067 func TestHTTP_AgentHealth_Ok(t *testing.T) { 1068 t.Parallel() 1069 require := require.New(t) 1070 1071 // Enable ACLs to ensure they're not enforced 1072 httpACLTest(t, nil, func(s *TestAgent) { 1073 // No ?type= 1074 { 1075 req, err := http.NewRequest("GET", "/v1/agent/health", nil) 1076 require.Nil(err) 1077 1078 respW := httptest.NewRecorder() 1079 healthI, err := s.Server.HealthRequest(respW, req) 1080 require.Nil(err) 1081 require.Equal(http.StatusOK, respW.Code) 1082 require.NotNil(healthI) 1083 health := healthI.(*healthResponse) 1084 require.NotNil(health.Client) 1085 require.True(health.Client.Ok) 1086 require.Equal("ok", health.Client.Message) 1087 require.NotNil(health.Server) 1088 require.True(health.Server.Ok) 1089 require.Equal("ok", health.Server.Message) 1090 } 1091 1092 // type=client 1093 { 1094 req, err := http.NewRequest("GET", "/v1/agent/health?type=client", nil) 1095 require.Nil(err) 1096 1097 respW := httptest.NewRecorder() 1098 healthI, err := s.Server.HealthRequest(respW, req) 1099 require.Nil(err) 1100 require.Equal(http.StatusOK, respW.Code) 1101 require.NotNil(healthI) 1102 health := healthI.(*healthResponse) 1103 require.NotNil(health.Client) 1104 require.True(health.Client.Ok) 1105 require.Equal("ok", health.Client.Message) 1106 require.Nil(health.Server) 1107 } 1108 1109 // type=server 1110 { 1111 req, err := http.NewRequest("GET", "/v1/agent/health?type=server", nil) 1112 require.Nil(err) 1113 1114 respW := httptest.NewRecorder() 1115 healthI, err := s.Server.HealthRequest(respW, req) 1116 require.Nil(err) 1117 require.Equal(http.StatusOK, respW.Code) 1118 require.NotNil(healthI) 1119 health := healthI.(*healthResponse) 1120 require.NotNil(health.Server) 1121 require.True(health.Server.Ok) 1122 require.Equal("ok", health.Server.Message) 1123 require.Nil(health.Client) 1124 } 1125 1126 // type=client&type=server 1127 { 1128 req, err := http.NewRequest("GET", "/v1/agent/health?type=client&type=server", nil) 1129 require.Nil(err) 1130 1131 respW := httptest.NewRecorder() 1132 healthI, err := s.Server.HealthRequest(respW, req) 1133 require.Nil(err) 1134 require.Equal(http.StatusOK, respW.Code) 1135 require.NotNil(healthI) 1136 health := healthI.(*healthResponse) 1137 require.NotNil(health.Client) 1138 require.True(health.Client.Ok) 1139 require.Equal("ok", health.Client.Message) 1140 require.NotNil(health.Server) 1141 require.True(health.Server.Ok) 1142 require.Equal("ok", health.Server.Message) 1143 } 1144 }) 1145 } 1146 1147 func TestHTTP_AgentHealth_BadServer(t *testing.T) { 1148 t.Parallel() 1149 require := require.New(t) 1150 1151 // Enable ACLs to ensure they're not enforced 1152 httpACLTest(t, nil, func(s *TestAgent) { 1153 1154 // Set s.Agent.server=nil to make server unhealthy if requested 1155 s.Agent.server = nil 1156 1157 // No ?type= means server is just skipped 1158 { 1159 req, err := http.NewRequest("GET", "/v1/agent/health", nil) 1160 require.Nil(err) 1161 1162 respW := httptest.NewRecorder() 1163 healthI, err := s.Server.HealthRequest(respW, req) 1164 require.Nil(err) 1165 require.Equal(http.StatusOK, respW.Code) 1166 require.NotNil(healthI) 1167 health := healthI.(*healthResponse) 1168 require.NotNil(health.Client) 1169 require.True(health.Client.Ok) 1170 require.Equal("ok", health.Client.Message) 1171 require.Nil(health.Server) 1172 } 1173 1174 // type=server means server is considered unhealthy 1175 { 1176 req, err := http.NewRequest("GET", "/v1/agent/health?type=server", nil) 1177 require.Nil(err) 1178 1179 respW := httptest.NewRecorder() 1180 _, err = s.Server.HealthRequest(respW, req) 1181 require.NotNil(err) 1182 httpErr, ok := err.(HTTPCodedError) 1183 require.True(ok) 1184 require.Equal(500, httpErr.Code()) 1185 require.Equal(`{"server":{"ok":false,"message":"server not enabled"}}`, err.Error()) 1186 } 1187 }) 1188 } 1189 1190 func TestHTTP_AgentHealth_BadClient(t *testing.T) { 1191 t.Parallel() 1192 require := require.New(t) 1193 1194 // Enable ACLs to ensure they're not enforced 1195 httpACLTest(t, nil, func(s *TestAgent) { 1196 1197 // Set s.Agent.client=nil to make server unhealthy if requested 1198 s.Agent.client = nil 1199 1200 // No ?type= means client is just skipped 1201 { 1202 req, err := http.NewRequest("GET", "/v1/agent/health", nil) 1203 require.Nil(err) 1204 1205 respW := httptest.NewRecorder() 1206 healthI, err := s.Server.HealthRequest(respW, req) 1207 require.Nil(err) 1208 require.Equal(http.StatusOK, respW.Code) 1209 require.NotNil(healthI) 1210 health := healthI.(*healthResponse) 1211 require.NotNil(health.Server) 1212 require.True(health.Server.Ok) 1213 require.Equal("ok", health.Server.Message) 1214 require.Nil(health.Client) 1215 } 1216 1217 // type=client means client is considered unhealthy 1218 { 1219 req, err := http.NewRequest("GET", "/v1/agent/health?type=client", nil) 1220 require.Nil(err) 1221 1222 respW := httptest.NewRecorder() 1223 _, err = s.Server.HealthRequest(respW, req) 1224 require.NotNil(err) 1225 httpErr, ok := err.(HTTPCodedError) 1226 require.True(ok) 1227 require.Equal(500, httpErr.Code()) 1228 require.Equal(`{"client":{"ok":false,"message":"client not enabled"}}`, err.Error()) 1229 } 1230 }) 1231 }