github.com/bigcommerce/nomad@v0.9.3-bc/command/agent/alloc_endpoint_test.go (about) 1 package agent 2 3 import ( 4 "archive/tar" 5 "fmt" 6 "io" 7 "io/ioutil" 8 "net/http" 9 "net/http/httptest" 10 "os" 11 "reflect" 12 "strings" 13 "testing" 14 15 "github.com/golang/snappy" 16 "github.com/hashicorp/nomad/acl" 17 "github.com/hashicorp/nomad/client/allocdir" 18 "github.com/hashicorp/nomad/helper" 19 "github.com/hashicorp/nomad/helper/uuid" 20 "github.com/hashicorp/nomad/nomad/mock" 21 "github.com/hashicorp/nomad/nomad/structs" 22 "github.com/hashicorp/nomad/testutil" 23 "github.com/stretchr/testify/require" 24 ) 25 26 func TestHTTP_AllocsList(t *testing.T) { 27 t.Parallel() 28 httpTest(t, nil, func(s *TestAgent) { 29 // Directly manipulate the state 30 state := s.Agent.server.State() 31 alloc1 := mock.Alloc() 32 testEvent := structs.NewTaskEvent(structs.TaskSiblingFailed) 33 var events1 []*structs.TaskEvent 34 events1 = append(events1, testEvent) 35 taskState := &structs.TaskState{Events: events1} 36 alloc1.TaskStates = make(map[string]*structs.TaskState) 37 alloc1.TaskStates["test"] = taskState 38 39 alloc2 := mock.Alloc() 40 alloc2.TaskStates = make(map[string]*structs.TaskState) 41 alloc2.TaskStates["test"] = taskState 42 43 state.UpsertJobSummary(998, mock.JobSummary(alloc1.JobID)) 44 state.UpsertJobSummary(999, mock.JobSummary(alloc2.JobID)) 45 err := state.UpsertAllocs(1000, 46 []*structs.Allocation{alloc1, alloc2}) 47 if err != nil { 48 t.Fatalf("err: %v", err) 49 } 50 51 // Make the HTTP request 52 req, err := http.NewRequest("GET", "/v1/allocations", nil) 53 if err != nil { 54 t.Fatalf("err: %v", err) 55 } 56 respW := httptest.NewRecorder() 57 58 // Make the request 59 obj, err := s.Server.AllocsRequest(respW, req) 60 if err != nil { 61 t.Fatalf("err: %v", err) 62 } 63 64 // Check for the index 65 if respW.HeaderMap.Get("X-Nomad-Index") == "" { 66 t.Fatalf("missing index") 67 } 68 if respW.HeaderMap.Get("X-Nomad-KnownLeader") != "true" { 69 t.Fatalf("missing known leader") 70 } 71 if respW.HeaderMap.Get("X-Nomad-LastContact") == "" { 72 t.Fatalf("missing last contact") 73 } 74 75 // Check the alloc 76 allocs := obj.([]*structs.AllocListStub) 77 if len(allocs) != 2 { 78 t.Fatalf("bad: %#v", allocs) 79 } 80 expectedMsg := "Task's sibling failed" 81 displayMsg1 := allocs[0].TaskStates["test"].Events[0].DisplayMessage 82 require.Equal(t, expectedMsg, displayMsg1, "DisplayMessage should be set") 83 displayMsg2 := allocs[0].TaskStates["test"].Events[0].DisplayMessage 84 require.Equal(t, expectedMsg, displayMsg2, "DisplayMessage should be set") 85 }) 86 } 87 88 func TestHTTP_AllocsPrefixList(t *testing.T) { 89 t.Parallel() 90 httpTest(t, nil, func(s *TestAgent) { 91 // Directly manipulate the state 92 state := s.Agent.server.State() 93 94 alloc1 := mock.Alloc() 95 alloc1.ID = "aaaaaaaa-e8f7-fd38-c855-ab94ceb89706" 96 alloc2 := mock.Alloc() 97 alloc2.ID = "aaabbbbb-e8f7-fd38-c855-ab94ceb89706" 98 99 testEvent := structs.NewTaskEvent(structs.TaskSiblingFailed) 100 var events1 []*structs.TaskEvent 101 events1 = append(events1, testEvent) 102 taskState := &structs.TaskState{Events: events1} 103 alloc2.TaskStates = make(map[string]*structs.TaskState) 104 alloc2.TaskStates["test"] = taskState 105 106 summary1 := mock.JobSummary(alloc1.JobID) 107 summary2 := mock.JobSummary(alloc2.JobID) 108 if err := state.UpsertJobSummary(998, summary1); err != nil { 109 t.Fatal(err) 110 } 111 if err := state.UpsertJobSummary(999, summary2); err != nil { 112 t.Fatal(err) 113 } 114 if err := state.UpsertAllocs(1000, 115 []*structs.Allocation{alloc1, alloc2}); err != nil { 116 t.Fatalf("err: %v", err) 117 } 118 119 // Make the HTTP request 120 req, err := http.NewRequest("GET", "/v1/allocations?prefix=aaab", nil) 121 if err != nil { 122 t.Fatalf("err: %v", err) 123 } 124 respW := httptest.NewRecorder() 125 126 // Make the request 127 obj, err := s.Server.AllocsRequest(respW, req) 128 if err != nil { 129 t.Fatalf("err: %v", err) 130 } 131 132 // Check for the index 133 if respW.HeaderMap.Get("X-Nomad-Index") == "" { 134 t.Fatalf("missing index") 135 } 136 if respW.HeaderMap.Get("X-Nomad-KnownLeader") != "true" { 137 t.Fatalf("missing known leader") 138 } 139 if respW.HeaderMap.Get("X-Nomad-LastContact") == "" { 140 t.Fatalf("missing last contact") 141 } 142 143 // Check the alloc 144 n := obj.([]*structs.AllocListStub) 145 if len(n) != 1 { 146 t.Fatalf("bad: %#v", n) 147 } 148 149 // Check the identifier 150 if n[0].ID != alloc2.ID { 151 t.Fatalf("expected alloc ID: %v, Actual: %v", alloc2.ID, n[0].ID) 152 } 153 expectedMsg := "Task's sibling failed" 154 displayMsg1 := n[0].TaskStates["test"].Events[0].DisplayMessage 155 require.Equal(t, expectedMsg, displayMsg1, "DisplayMessage should be set") 156 157 }) 158 } 159 160 func TestHTTP_AllocQuery(t *testing.T) { 161 t.Parallel() 162 httpTest(t, nil, func(s *TestAgent) { 163 // Directly manipulate the state 164 state := s.Agent.server.State() 165 alloc := mock.Alloc() 166 if err := state.UpsertJobSummary(999, mock.JobSummary(alloc.JobID)); err != nil { 167 t.Fatal(err) 168 } 169 err := state.UpsertAllocs(1000, 170 []*structs.Allocation{alloc}) 171 if err != nil { 172 t.Fatalf("err: %v", err) 173 } 174 175 // Make the HTTP request 176 req, err := http.NewRequest("GET", "/v1/allocation/"+alloc.ID, nil) 177 if err != nil { 178 t.Fatalf("err: %v", err) 179 } 180 respW := httptest.NewRecorder() 181 182 // Make the request 183 obj, err := s.Server.AllocSpecificRequest(respW, req) 184 if err != nil { 185 t.Fatalf("err: %v", err) 186 } 187 188 // Check for the index 189 if respW.HeaderMap.Get("X-Nomad-Index") == "" { 190 t.Fatalf("missing index") 191 } 192 if respW.HeaderMap.Get("X-Nomad-KnownLeader") != "true" { 193 t.Fatalf("missing known leader") 194 } 195 if respW.HeaderMap.Get("X-Nomad-LastContact") == "" { 196 t.Fatalf("missing last contact") 197 } 198 199 // Check the job 200 a := obj.(*structs.Allocation) 201 if a.ID != alloc.ID { 202 t.Fatalf("bad: %#v", a) 203 } 204 }) 205 } 206 207 func TestHTTP_AllocQuery_Payload(t *testing.T) { 208 t.Parallel() 209 httpTest(t, nil, func(s *TestAgent) { 210 // Directly manipulate the state 211 state := s.Agent.server.State() 212 alloc := mock.Alloc() 213 if err := state.UpsertJobSummary(999, mock.JobSummary(alloc.JobID)); err != nil { 214 t.Fatal(err) 215 } 216 217 // Insert Payload compressed 218 expected := []byte("hello world") 219 compressed := snappy.Encode(nil, expected) 220 alloc.Job.Payload = compressed 221 222 err := state.UpsertAllocs(1000, []*structs.Allocation{alloc}) 223 if err != nil { 224 t.Fatalf("err: %v", err) 225 } 226 227 // Make the HTTP request 228 req, err := http.NewRequest("GET", "/v1/allocation/"+alloc.ID, nil) 229 if err != nil { 230 t.Fatalf("err: %v", err) 231 } 232 respW := httptest.NewRecorder() 233 234 // Make the request 235 obj, err := s.Server.AllocSpecificRequest(respW, req) 236 if err != nil { 237 t.Fatalf("err: %v", err) 238 } 239 240 // Check for the index 241 if respW.HeaderMap.Get("X-Nomad-Index") == "" { 242 t.Fatalf("missing index") 243 } 244 if respW.HeaderMap.Get("X-Nomad-KnownLeader") != "true" { 245 t.Fatalf("missing known leader") 246 } 247 if respW.HeaderMap.Get("X-Nomad-LastContact") == "" { 248 t.Fatalf("missing last contact") 249 } 250 251 // Check the job 252 a := obj.(*structs.Allocation) 253 if a.ID != alloc.ID { 254 t.Fatalf("bad: %#v", a) 255 } 256 257 // Check the payload is decompressed 258 if !reflect.DeepEqual(a.Job.Payload, expected) { 259 t.Fatalf("Payload not decompressed properly; got %#v; want %#v", a.Job.Payload, expected) 260 } 261 }) 262 } 263 264 func TestHTTP_AllocRestart(t *testing.T) { 265 t.Parallel() 266 require := require.New(t) 267 268 // Validates that all methods of forwarding the request are processed correctly 269 httpTest(t, nil, func(s *TestAgent) { 270 // Local node, local resp 271 { 272 // Make the HTTP request 273 buf := encodeReq(map[string]string{}) 274 req, err := http.NewRequest("GET", fmt.Sprintf("/v1/client/allocation/%s/restart", uuid.Generate()), buf) 275 if err != nil { 276 t.Fatalf("err: %v", err) 277 } 278 respW := httptest.NewRecorder() 279 280 // Make the request 281 _, err = s.Server.ClientAllocRequest(respW, req) 282 require.NotNil(err) 283 require.True(structs.IsErrUnknownAllocation(err)) 284 } 285 286 // Local node, server resp 287 { 288 srv := s.server 289 s.server = nil 290 291 buf := encodeReq(map[string]string{}) 292 req, err := http.NewRequest("GET", fmt.Sprintf("/v1/client/allocation/%s/restart", uuid.Generate()), buf) 293 require.Nil(err) 294 295 respW := httptest.NewRecorder() 296 _, err = s.Server.ClientAllocRequest(respW, req) 297 require.NotNil(err) 298 require.True(structs.IsErrUnknownAllocation(err)) 299 300 s.server = srv 301 } 302 303 // no client, server resp 304 { 305 c := s.client 306 s.client = nil 307 308 testutil.WaitForResult(func() (bool, error) { 309 n, err := s.server.State().NodeByID(nil, c.NodeID()) 310 if err != nil { 311 return false, err 312 } 313 return n != nil, nil 314 }, func(err error) { 315 t.Fatalf("should have client: %v", err) 316 }) 317 318 buf := encodeReq(map[string]string{}) 319 req, err := http.NewRequest("GET", fmt.Sprintf("/v1/client/allocation/%s/restart", uuid.Generate()), buf) 320 require.Nil(err) 321 322 respW := httptest.NewRecorder() 323 _, err = s.Server.ClientAllocRequest(respW, req) 324 require.NotNil(err) 325 require.True(structs.IsErrUnknownAllocation(err)) 326 327 s.client = c 328 } 329 }) 330 } 331 332 func TestHTTP_AllocRestart_ACL(t *testing.T) { 333 t.Parallel() 334 require := require.New(t) 335 336 httpACLTest(t, nil, func(s *TestAgent) { 337 state := s.Agent.server.State() 338 339 // If there's no token, we expect the request to fail. 340 { 341 buf := encodeReq(map[string]string{}) 342 req, err := http.NewRequest("GET", fmt.Sprintf("/v1/client/allocation/%s/restart", uuid.Generate()), buf) 343 require.NoError(err) 344 345 respW := httptest.NewRecorder() 346 _, err = s.Server.ClientAllocRequest(respW, req) 347 require.NotNil(err) 348 require.Equal(err.Error(), structs.ErrPermissionDenied.Error()) 349 } 350 351 // Try request with an invalid token and expect it to fail 352 { 353 buf := encodeReq(map[string]string{}) 354 req, err := http.NewRequest("GET", fmt.Sprintf("/v1/client/allocation/%s/restart", uuid.Generate()), buf) 355 require.NoError(err) 356 357 respW := httptest.NewRecorder() 358 token := mock.CreatePolicyAndToken(t, state, 1005, "invalid", mock.NodePolicy(acl.PolicyWrite)) 359 setToken(req, token) 360 _, err = s.Server.ClientAllocRequest(respW, req) 361 require.NotNil(err) 362 require.Equal(err.Error(), structs.ErrPermissionDenied.Error()) 363 } 364 365 // Try request with a valid token 366 // Still returns an error because the alloc does not exist 367 { 368 buf := encodeReq(map[string]string{}) 369 req, err := http.NewRequest("GET", fmt.Sprintf("/v1/client/allocation/%s/restart", uuid.Generate()), buf) 370 require.NoError(err) 371 372 respW := httptest.NewRecorder() 373 policy := mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityAllocLifecycle}) 374 token := mock.CreatePolicyAndToken(t, state, 1007, "valid", policy) 375 setToken(req, token) 376 _, err = s.Server.ClientAllocRequest(respW, req) 377 require.NotNil(err) 378 require.True(structs.IsErrUnknownAllocation(err)) 379 } 380 381 // Try request with a management token 382 // Still returns an error because the alloc does not exist 383 { 384 buf := encodeReq(map[string]string{}) 385 req, err := http.NewRequest("GET", fmt.Sprintf("/v1/client/allocation/%s/restart", uuid.Generate()), buf) 386 require.NoError(err) 387 388 respW := httptest.NewRecorder() 389 setToken(req, s.RootToken) 390 _, err = s.Server.ClientAllocRequest(respW, req) 391 require.NotNil(err) 392 require.True(structs.IsErrUnknownAllocation(err)) 393 } 394 }) 395 } 396 397 func TestHTTP_AllocStop(t *testing.T) { 398 t.Parallel() 399 httpTest(t, nil, func(s *TestAgent) { 400 // Directly manipulate the state 401 state := s.Agent.server.State() 402 alloc := mock.Alloc() 403 require := require.New(t) 404 require.NoError(state.UpsertJobSummary(999, mock.JobSummary(alloc.JobID))) 405 406 require.NoError(state.UpsertAllocs(1000, []*structs.Allocation{alloc})) 407 408 // Test that the happy path works 409 { 410 // Make the HTTP request 411 req, err := http.NewRequest("POST", "/v1/allocation/"+alloc.ID+"/stop", nil) 412 require.NoError(err) 413 respW := httptest.NewRecorder() 414 415 // Make the request 416 obj, err := s.Server.AllocSpecificRequest(respW, req) 417 require.NoError(err) 418 419 a := obj.(*structs.AllocStopResponse) 420 require.NotEmpty(a.EvalID, "missing eval") 421 require.NotEmpty(a.Index, "missing index") 422 } 423 424 // Test that we 404 when the allocid is invalid 425 { 426 // Make the HTTP request 427 req, err := http.NewRequest("POST", "/v1/allocation/"+uuid.Generate()+"/stop", nil) 428 require.NoError(err) 429 respW := httptest.NewRecorder() 430 431 // Make the request 432 _, err = s.Server.AllocSpecificRequest(respW, req) 433 require.NotNil(err) 434 if !strings.Contains(err.Error(), allocNotFoundErr) { 435 t.Fatalf("err: %v", err) 436 } 437 } 438 }) 439 } 440 441 func TestHTTP_AllocStats(t *testing.T) { 442 t.Parallel() 443 require := require.New(t) 444 445 httpTest(t, nil, func(s *TestAgent) { 446 // Local node, local resp 447 { 448 // Make the HTTP request 449 req, err := http.NewRequest("GET", fmt.Sprintf("/v1/client/allocation/%s/stats", uuid.Generate()), nil) 450 if err != nil { 451 t.Fatalf("err: %v", err) 452 } 453 respW := httptest.NewRecorder() 454 455 // Make the request 456 _, err = s.Server.ClientAllocRequest(respW, req) 457 require.NotNil(err) 458 require.True(structs.IsErrUnknownAllocation(err)) 459 } 460 461 // Local node, server resp 462 { 463 srv := s.server 464 s.server = nil 465 466 req, err := http.NewRequest("GET", fmt.Sprintf("/v1/client/allocation/%s/stats", uuid.Generate()), nil) 467 require.Nil(err) 468 469 respW := httptest.NewRecorder() 470 _, err = s.Server.ClientAllocRequest(respW, req) 471 require.NotNil(err) 472 require.True(structs.IsErrUnknownAllocation(err)) 473 474 s.server = srv 475 } 476 477 // no client, server resp 478 { 479 c := s.client 480 s.client = nil 481 482 testutil.WaitForResult(func() (bool, error) { 483 n, err := s.server.State().NodeByID(nil, c.NodeID()) 484 if err != nil { 485 return false, err 486 } 487 return n != nil, nil 488 }, func(err error) { 489 t.Fatalf("should have client: %v", err) 490 }) 491 492 req, err := http.NewRequest("GET", fmt.Sprintf("/v1/client/allocation/%s/stats", uuid.Generate()), nil) 493 require.Nil(err) 494 495 respW := httptest.NewRecorder() 496 _, err = s.Server.ClientAllocRequest(respW, req) 497 require.NotNil(err) 498 require.True(structs.IsErrUnknownAllocation(err)) 499 500 s.client = c 501 } 502 }) 503 } 504 505 func TestHTTP_AllocStats_ACL(t *testing.T) { 506 t.Parallel() 507 require := require.New(t) 508 509 httpACLTest(t, nil, func(s *TestAgent) { 510 state := s.Agent.server.State() 511 512 // Make the HTTP request 513 req, err := http.NewRequest("GET", fmt.Sprintf("/v1/client/allocation/%s/stats", uuid.Generate()), nil) 514 if err != nil { 515 t.Fatalf("err: %v", err) 516 } 517 518 // Try request without a token and expect failure 519 { 520 respW := httptest.NewRecorder() 521 _, err := s.Server.ClientAllocRequest(respW, req) 522 require.NotNil(err) 523 require.Equal(err.Error(), structs.ErrPermissionDenied.Error()) 524 } 525 526 // Try request with an invalid token and expect failure 527 { 528 respW := httptest.NewRecorder() 529 token := mock.CreatePolicyAndToken(t, state, 1005, "invalid", mock.NodePolicy(acl.PolicyWrite)) 530 setToken(req, token) 531 _, err := s.Server.ClientAllocRequest(respW, req) 532 require.NotNil(err) 533 require.Equal(err.Error(), structs.ErrPermissionDenied.Error()) 534 } 535 536 // Try request with a valid token 537 // Still returns an error because the alloc does not exist 538 { 539 respW := httptest.NewRecorder() 540 policy := mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob}) 541 token := mock.CreatePolicyAndToken(t, state, 1007, "valid", policy) 542 setToken(req, token) 543 _, err := s.Server.ClientAllocRequest(respW, req) 544 require.NotNil(err) 545 require.True(structs.IsErrUnknownAllocation(err)) 546 } 547 548 // Try request with a management token 549 // Still returns an error because the alloc does not exist 550 { 551 respW := httptest.NewRecorder() 552 setToken(req, s.RootToken) 553 _, err := s.Server.ClientAllocRequest(respW, req) 554 require.NotNil(err) 555 require.True(structs.IsErrUnknownAllocation(err)) 556 } 557 }) 558 } 559 560 func TestHTTP_AllocSnapshot(t *testing.T) { 561 t.Parallel() 562 httpTest(t, nil, func(s *TestAgent) { 563 // Make the HTTP request 564 req, err := http.NewRequest("GET", "/v1/client/allocation/123/snapshot", nil) 565 if err != nil { 566 t.Fatalf("err: %v", err) 567 } 568 respW := httptest.NewRecorder() 569 570 // Make the request 571 _, err = s.Server.ClientAllocRequest(respW, req) 572 if !strings.Contains(err.Error(), allocNotFoundErr) { 573 t.Fatalf("err: %v", err) 574 } 575 }) 576 } 577 578 func TestHTTP_AllocSnapshot_WithMigrateToken(t *testing.T) { 579 t.Parallel() 580 require := require.New(t) 581 httpACLTest(t, nil, func(s *TestAgent) { 582 // Request without a token fails 583 req, err := http.NewRequest("GET", "/v1/client/allocation/123/snapshot", nil) 584 require.Nil(err) 585 586 // Make the unauthorized request 587 respW := httptest.NewRecorder() 588 _, err = s.Server.ClientAllocRequest(respW, req) 589 require.NotNil(err) 590 require.EqualError(err, structs.ErrPermissionDenied.Error()) 591 592 // Create an allocation 593 alloc := mock.Alloc() 594 595 validMigrateToken, err := structs.GenerateMigrateToken(alloc.ID, s.Agent.Client().Node().SecretID) 596 require.Nil(err) 597 598 // Request with a token succeeds 599 url := fmt.Sprintf("/v1/client/allocation/%s/snapshot", alloc.ID) 600 req, err = http.NewRequest("GET", url, nil) 601 require.Nil(err) 602 603 req.Header.Set("X-Nomad-Token", validMigrateToken) 604 605 // Make the unauthorized request 606 respW = httptest.NewRecorder() 607 _, err = s.Server.ClientAllocRequest(respW, req) 608 require.NotContains(err.Error(), structs.ErrPermissionDenied.Error()) 609 }) 610 } 611 612 // TestHTTP_AllocSnapshot_Atomic ensures that when a client encounters an error 613 // snapshotting a valid tar is not returned. 614 func TestHTTP_AllocSnapshot_Atomic(t *testing.T) { 615 t.Parallel() 616 httpTest(t, func(c *Config) { 617 // Disable the schedulers 618 c.Server.NumSchedulers = helper.IntToPtr(0) 619 }, func(s *TestAgent) { 620 // Create an alloc 621 state := s.server.State() 622 alloc := mock.Alloc() 623 alloc.Job.TaskGroups[0].Tasks[0].Driver = "mock_driver" 624 alloc.Job.TaskGroups[0].Tasks[0].Config = map[string]interface{}{ 625 "run_for": "30s", 626 } 627 alloc.NodeID = s.client.NodeID() 628 state.UpsertJobSummary(998, mock.JobSummary(alloc.JobID)) 629 if err := state.UpsertAllocs(1000, []*structs.Allocation{alloc.Copy()}); err != nil { 630 t.Fatalf("error upserting alloc: %v", err) 631 } 632 633 // Wait for the client to run it 634 testutil.WaitForResult(func() (bool, error) { 635 if _, err := s.client.GetAllocState(alloc.ID); err != nil { 636 return false, err 637 } 638 639 serverAlloc, err := state.AllocByID(nil, alloc.ID) 640 if err != nil { 641 return false, err 642 } 643 644 return serverAlloc.ClientStatus == structs.AllocClientStatusRunning, fmt.Errorf(serverAlloc.ClientStatus) 645 }, func(err error) { 646 t.Fatalf("client not running alloc: %v", err) 647 }) 648 649 // Now write to its shared dir 650 allocDirI, err := s.client.GetAllocFS(alloc.ID) 651 if err != nil { 652 t.Fatalf("unable to find alloc dir: %v", err) 653 } 654 allocDir := allocDirI.(*allocdir.AllocDir) 655 656 // Remove the task dir to break Snapshot 657 os.RemoveAll(allocDir.TaskDirs["web"].LocalDir) 658 659 // require Snapshot fails 660 if err := allocDir.Snapshot(ioutil.Discard); err != nil { 661 t.Logf("[DEBUG] agent.test: snapshot returned error: %v", err) 662 } else { 663 t.Errorf("expected Snapshot() to fail but it did not") 664 } 665 666 // Make the HTTP request to ensure the Snapshot error is 667 // propagated through to the HTTP layer. Since the tar is 668 // streamed over a 200 HTTP response the only way to signal an 669 // error is by writing a marker file. 670 respW := httptest.NewRecorder() 671 req, err := http.NewRequest("GET", fmt.Sprintf("/v1/client/allocation/%s/snapshot", alloc.ID), nil) 672 if err != nil { 673 t.Fatalf("err: %v", err) 674 } 675 676 // Make the request via the mux to make sure the error returned 677 // by Snapshot is properly propagated via HTTP 678 s.Server.mux.ServeHTTP(respW, req) 679 resp := respW.Result() 680 r := tar.NewReader(resp.Body) 681 errorFilename := allocdir.SnapshotErrorFilename(alloc.ID) 682 markerFound := false 683 markerContents := "" 684 for { 685 header, err := r.Next() 686 if err != nil { 687 if err != io.EOF { 688 // Huh, I wonder how a non-EOF error can happen? 689 t.Errorf("Unexpected error while streaming: %v", err) 690 } 691 break 692 } 693 694 if markerFound { 695 // No more files should be found after the failure marker 696 t.Errorf("Next file found after error marker: %s", header.Name) 697 break 698 } 699 700 if header.Name == errorFilename { 701 // Found it! 702 markerFound = true 703 buf := make([]byte, int(header.Size)) 704 if _, err := r.Read(buf); err != nil && err != io.EOF { 705 t.Errorf("Unexpected error reading error marker %s: %v", errorFilename, err) 706 } else { 707 markerContents = string(buf) 708 } 709 } 710 } 711 712 if !markerFound { 713 t.Fatalf("marker file %s not written; bad tar will be treated as good!", errorFilename) 714 } 715 if markerContents == "" { 716 t.Fatalf("marker file %s empty", markerContents) 717 } else { 718 t.Logf("EXPECTED snapshot error: %s", markerContents) 719 } 720 }) 721 } 722 723 func TestHTTP_AllocGC(t *testing.T) { 724 t.Parallel() 725 require := require.New(t) 726 path := fmt.Sprintf("/v1/client/allocation/%s/gc", uuid.Generate()) 727 httpTest(t, nil, func(s *TestAgent) { 728 // Local node, local resp 729 { 730 req, err := http.NewRequest("GET", path, nil) 731 if err != nil { 732 t.Fatalf("err: %v", err) 733 } 734 735 respW := httptest.NewRecorder() 736 _, err = s.Server.ClientAllocRequest(respW, req) 737 if !structs.IsErrUnknownAllocation(err) { 738 t.Fatalf("unexpected err: %v", err) 739 } 740 } 741 742 // Local node, server resp 743 { 744 srv := s.server 745 s.server = nil 746 747 req, err := http.NewRequest("GET", path, nil) 748 if err != nil { 749 t.Fatalf("err: %v", err) 750 } 751 752 respW := httptest.NewRecorder() 753 _, err = s.Server.ClientAllocRequest(respW, req) 754 if !structs.IsErrUnknownAllocation(err) { 755 t.Fatalf("unexpected err: %v", err) 756 } 757 758 s.server = srv 759 } 760 761 // no client, server resp 762 { 763 c := s.client 764 s.client = nil 765 766 testutil.WaitForResult(func() (bool, error) { 767 n, err := s.server.State().NodeByID(nil, c.NodeID()) 768 if err != nil { 769 return false, err 770 } 771 return n != nil, nil 772 }, func(err error) { 773 t.Fatalf("should have client: %v", err) 774 }) 775 776 req, err := http.NewRequest("GET", path, nil) 777 if err != nil { 778 t.Fatalf("err: %v", err) 779 } 780 781 respW := httptest.NewRecorder() 782 _, err = s.Server.ClientAllocRequest(respW, req) 783 require.NotNil(err) 784 if !structs.IsErrUnknownAllocation(err) { 785 t.Fatalf("unexpected err: %v", err) 786 } 787 788 s.client = c 789 } 790 }) 791 } 792 793 func TestHTTP_AllocGC_ACL(t *testing.T) { 794 t.Parallel() 795 require := require.New(t) 796 path := fmt.Sprintf("/v1/client/allocation/%s/gc", uuid.Generate()) 797 798 httpACLTest(t, nil, func(s *TestAgent) { 799 state := s.Agent.server.State() 800 801 // Make the HTTP request 802 req, err := http.NewRequest("GET", path, nil) 803 if err != nil { 804 t.Fatalf("err: %v", err) 805 } 806 807 // Try request without a token and expect failure 808 { 809 respW := httptest.NewRecorder() 810 _, err := s.Server.ClientAllocRequest(respW, req) 811 require.NotNil(err) 812 require.Equal(err.Error(), structs.ErrPermissionDenied.Error()) 813 } 814 815 // Try request with an invalid token and expect failure 816 { 817 respW := httptest.NewRecorder() 818 token := mock.CreatePolicyAndToken(t, state, 1005, "invalid", mock.NodePolicy(acl.PolicyWrite)) 819 setToken(req, token) 820 _, err := s.Server.ClientAllocRequest(respW, req) 821 require.NotNil(err) 822 require.Equal(err.Error(), structs.ErrPermissionDenied.Error()) 823 } 824 825 // Try request with a valid token 826 // Still returns an error because the alloc does not exist 827 { 828 respW := httptest.NewRecorder() 829 policy := mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilitySubmitJob}) 830 token := mock.CreatePolicyAndToken(t, state, 1007, "valid", policy) 831 setToken(req, token) 832 _, err := s.Server.ClientAllocRequest(respW, req) 833 require.NotNil(err) 834 require.True(structs.IsErrUnknownAllocation(err)) 835 } 836 837 // Try request with a management token 838 // Still returns an error because the alloc does not exist 839 { 840 respW := httptest.NewRecorder() 841 setToken(req, s.RootToken) 842 _, err := s.Server.ClientAllocRequest(respW, req) 843 require.NotNil(err) 844 require.True(structs.IsErrUnknownAllocation(err)) 845 } 846 }) 847 } 848 849 func TestHTTP_AllocAllGC(t *testing.T) { 850 t.Parallel() 851 require := require.New(t) 852 httpTest(t, nil, func(s *TestAgent) { 853 // Local node, local resp 854 { 855 req, err := http.NewRequest("GET", "/v1/client/gc", nil) 856 if err != nil { 857 t.Fatalf("err: %v", err) 858 } 859 860 respW := httptest.NewRecorder() 861 _, err = s.Server.ClientGCRequest(respW, req) 862 if err != nil { 863 t.Fatalf("unexpected err: %v", err) 864 } 865 } 866 867 // Local node, server resp 868 { 869 srv := s.server 870 s.server = nil 871 872 req, err := http.NewRequest("GET", fmt.Sprintf("/v1/client/gc?node_id=%s", uuid.Generate()), nil) 873 require.Nil(err) 874 875 respW := httptest.NewRecorder() 876 _, err = s.Server.ClientGCRequest(respW, req) 877 require.NotNil(err) 878 require.Contains(err.Error(), "Unknown node") 879 880 s.server = srv 881 } 882 883 // client stats from server, should not error 884 { 885 c := s.client 886 s.client = nil 887 888 testutil.WaitForResult(func() (bool, error) { 889 n, err := s.server.State().NodeByID(nil, c.NodeID()) 890 if err != nil { 891 return false, err 892 } 893 return n != nil, nil 894 }, func(err error) { 895 t.Fatalf("should have client: %v", err) 896 }) 897 898 req, err := http.NewRequest("GET", fmt.Sprintf("/v1/client/gc?node_id=%s", c.NodeID()), nil) 899 require.Nil(err) 900 901 respW := httptest.NewRecorder() 902 _, err = s.Server.ClientGCRequest(respW, req) 903 require.Nil(err) 904 905 s.client = c 906 } 907 }) 908 909 } 910 911 func TestHTTP_AllocAllGC_ACL(t *testing.T) { 912 t.Parallel() 913 require := require.New(t) 914 httpACLTest(t, nil, func(s *TestAgent) { 915 state := s.Agent.server.State() 916 917 // Make the HTTP request 918 req, err := http.NewRequest("GET", "/v1/client/gc", nil) 919 require.Nil(err) 920 921 // Try request without a token and expect failure 922 { 923 respW := httptest.NewRecorder() 924 _, err := s.Server.ClientGCRequest(respW, req) 925 require.NotNil(err) 926 require.Equal(err.Error(), structs.ErrPermissionDenied.Error()) 927 } 928 929 // Try request with an invalid token and expect failure 930 { 931 respW := httptest.NewRecorder() 932 token := mock.CreatePolicyAndToken(t, state, 1005, "invalid", mock.NodePolicy(acl.PolicyRead)) 933 setToken(req, token) 934 _, err := s.Server.ClientGCRequest(respW, req) 935 require.NotNil(err) 936 require.Equal(err.Error(), structs.ErrPermissionDenied.Error()) 937 } 938 939 // Try request with a valid token 940 { 941 respW := httptest.NewRecorder() 942 token := mock.CreatePolicyAndToken(t, state, 1007, "valid", mock.NodePolicy(acl.PolicyWrite)) 943 setToken(req, token) 944 _, err := s.Server.ClientGCRequest(respW, req) 945 require.Nil(err) 946 require.Equal(http.StatusOK, respW.Code) 947 } 948 949 // Try request with a management token 950 { 951 respW := httptest.NewRecorder() 952 setToken(req, s.RootToken) 953 _, err := s.Server.ClientGCRequest(respW, req) 954 require.Nil(err) 955 require.Equal(http.StatusOK, respW.Code) 956 } 957 }) 958 }