github.com/ferranbt/nomad@v0.9.3-0.20190607002617-85c449b7667c/command/agent/alloc_endpoint_test.go (about) 1 package agent 2 3 import ( 4 "archive/tar" 5 "fmt" 6 "io" 7 "io/ioutil" 8 "net/http" 9 "net/http/httptest" 10 "os" 11 "reflect" 12 "strings" 13 "testing" 14 15 "github.com/golang/snappy" 16 "github.com/hashicorp/nomad/acl" 17 "github.com/hashicorp/nomad/client/allocdir" 18 "github.com/hashicorp/nomad/helper" 19 "github.com/hashicorp/nomad/helper/uuid" 20 "github.com/hashicorp/nomad/nomad/mock" 21 "github.com/hashicorp/nomad/nomad/structs" 22 "github.com/hashicorp/nomad/testutil" 23 "github.com/stretchr/testify/require" 24 ) 25 26 func TestHTTP_AllocsList(t *testing.T) { 27 t.Parallel() 28 httpTest(t, nil, func(s *TestAgent) { 29 // Directly manipulate the state 30 state := s.Agent.server.State() 31 alloc1 := mock.Alloc() 32 testEvent := structs.NewTaskEvent(structs.TaskSiblingFailed) 33 var events1 []*structs.TaskEvent 34 events1 = append(events1, testEvent) 35 taskState := &structs.TaskState{Events: events1} 36 alloc1.TaskStates = make(map[string]*structs.TaskState) 37 alloc1.TaskStates["test"] = taskState 38 39 alloc2 := mock.Alloc() 40 alloc2.TaskStates = make(map[string]*structs.TaskState) 41 alloc2.TaskStates["test"] = taskState 42 43 state.UpsertJobSummary(998, mock.JobSummary(alloc1.JobID)) 44 state.UpsertJobSummary(999, mock.JobSummary(alloc2.JobID)) 45 err := state.UpsertAllocs(1000, 46 []*structs.Allocation{alloc1, alloc2}) 47 if err != nil { 48 t.Fatalf("err: %v", err) 49 } 50 51 // Make the HTTP request 52 req, err := http.NewRequest("GET", "/v1/allocations", nil) 53 if err != nil { 54 t.Fatalf("err: %v", err) 55 } 56 respW := httptest.NewRecorder() 57 58 // Make the request 59 obj, err := s.Server.AllocsRequest(respW, req) 60 if err != nil { 61 t.Fatalf("err: %v", err) 62 } 63 64 // Check for the index 65 if respW.HeaderMap.Get("X-Nomad-Index") == "" { 66 t.Fatalf("missing index") 67 } 68 if respW.HeaderMap.Get("X-Nomad-KnownLeader") != "true" { 69 t.Fatalf("missing known leader") 70 } 71 if respW.HeaderMap.Get("X-Nomad-LastContact") == "" { 72 t.Fatalf("missing last contact") 73 } 74 75 // Check the alloc 76 allocs := obj.([]*structs.AllocListStub) 77 if len(allocs) != 2 { 78 t.Fatalf("bad: %#v", allocs) 79 } 80 expectedMsg := "Task's sibling failed" 81 displayMsg1 := allocs[0].TaskStates["test"].Events[0].DisplayMessage 82 require.Equal(t, expectedMsg, displayMsg1, "DisplayMessage should be set") 83 displayMsg2 := allocs[0].TaskStates["test"].Events[0].DisplayMessage 84 require.Equal(t, expectedMsg, displayMsg2, "DisplayMessage should be set") 85 }) 86 } 87 88 func TestHTTP_AllocsPrefixList(t *testing.T) { 89 t.Parallel() 90 httpTest(t, nil, func(s *TestAgent) { 91 // Directly manipulate the state 92 state := s.Agent.server.State() 93 94 alloc1 := mock.Alloc() 95 alloc1.ID = "aaaaaaaa-e8f7-fd38-c855-ab94ceb89706" 96 alloc2 := mock.Alloc() 97 alloc2.ID = "aaabbbbb-e8f7-fd38-c855-ab94ceb89706" 98 99 testEvent := structs.NewTaskEvent(structs.TaskSiblingFailed) 100 var events1 []*structs.TaskEvent 101 events1 = append(events1, testEvent) 102 taskState := &structs.TaskState{Events: events1} 103 alloc2.TaskStates = make(map[string]*structs.TaskState) 104 alloc2.TaskStates["test"] = taskState 105 106 summary1 := mock.JobSummary(alloc1.JobID) 107 summary2 := mock.JobSummary(alloc2.JobID) 108 if err := state.UpsertJobSummary(998, summary1); err != nil { 109 t.Fatal(err) 110 } 111 if err := state.UpsertJobSummary(999, summary2); err != nil { 112 t.Fatal(err) 113 } 114 if err := state.UpsertAllocs(1000, 115 []*structs.Allocation{alloc1, alloc2}); err != nil { 116 t.Fatalf("err: %v", err) 117 } 118 119 // Make the HTTP request 120 req, err := http.NewRequest("GET", "/v1/allocations?prefix=aaab", nil) 121 if err != nil { 122 t.Fatalf("err: %v", err) 123 } 124 respW := httptest.NewRecorder() 125 126 // Make the request 127 obj, err := s.Server.AllocsRequest(respW, req) 128 if err != nil { 129 t.Fatalf("err: %v", err) 130 } 131 132 // Check for the index 133 if respW.HeaderMap.Get("X-Nomad-Index") == "" { 134 t.Fatalf("missing index") 135 } 136 if respW.HeaderMap.Get("X-Nomad-KnownLeader") != "true" { 137 t.Fatalf("missing known leader") 138 } 139 if respW.HeaderMap.Get("X-Nomad-LastContact") == "" { 140 t.Fatalf("missing last contact") 141 } 142 143 // Check the alloc 144 n := obj.([]*structs.AllocListStub) 145 if len(n) != 1 { 146 t.Fatalf("bad: %#v", n) 147 } 148 149 // Check the identifier 150 if n[0].ID != alloc2.ID { 151 t.Fatalf("expected alloc ID: %v, Actual: %v", alloc2.ID, n[0].ID) 152 } 153 expectedMsg := "Task's sibling failed" 154 displayMsg1 := n[0].TaskStates["test"].Events[0].DisplayMessage 155 require.Equal(t, expectedMsg, displayMsg1, "DisplayMessage should be set") 156 157 }) 158 } 159 160 func TestHTTP_AllocQuery(t *testing.T) { 161 t.Parallel() 162 httpTest(t, nil, func(s *TestAgent) { 163 // Directly manipulate the state 164 state := s.Agent.server.State() 165 alloc := mock.Alloc() 166 if err := state.UpsertJobSummary(999, mock.JobSummary(alloc.JobID)); err != nil { 167 t.Fatal(err) 168 } 169 err := state.UpsertAllocs(1000, 170 []*structs.Allocation{alloc}) 171 if err != nil { 172 t.Fatalf("err: %v", err) 173 } 174 175 // Make the HTTP request 176 req, err := http.NewRequest("GET", "/v1/allocation/"+alloc.ID, nil) 177 if err != nil { 178 t.Fatalf("err: %v", err) 179 } 180 respW := httptest.NewRecorder() 181 182 // Make the request 183 obj, err := s.Server.AllocSpecificRequest(respW, req) 184 if err != nil { 185 t.Fatalf("err: %v", err) 186 } 187 188 // Check for the index 189 if respW.HeaderMap.Get("X-Nomad-Index") == "" { 190 t.Fatalf("missing index") 191 } 192 if respW.HeaderMap.Get("X-Nomad-KnownLeader") != "true" { 193 t.Fatalf("missing known leader") 194 } 195 if respW.HeaderMap.Get("X-Nomad-LastContact") == "" { 196 t.Fatalf("missing last contact") 197 } 198 199 // Check the job 200 a := obj.(*structs.Allocation) 201 if a.ID != alloc.ID { 202 t.Fatalf("bad: %#v", a) 203 } 204 }) 205 } 206 207 func TestHTTP_AllocQuery_Payload(t *testing.T) { 208 t.Parallel() 209 httpTest(t, nil, func(s *TestAgent) { 210 // Directly manipulate the state 211 state := s.Agent.server.State() 212 alloc := mock.Alloc() 213 if err := state.UpsertJobSummary(999, mock.JobSummary(alloc.JobID)); err != nil { 214 t.Fatal(err) 215 } 216 217 // Insert Payload compressed 218 expected := []byte("hello world") 219 compressed := snappy.Encode(nil, expected) 220 alloc.Job.Payload = compressed 221 222 err := state.UpsertAllocs(1000, []*structs.Allocation{alloc}) 223 if err != nil { 224 t.Fatalf("err: %v", err) 225 } 226 227 // Make the HTTP request 228 req, err := http.NewRequest("GET", "/v1/allocation/"+alloc.ID, nil) 229 if err != nil { 230 t.Fatalf("err: %v", err) 231 } 232 respW := httptest.NewRecorder() 233 234 // Make the request 235 obj, err := s.Server.AllocSpecificRequest(respW, req) 236 if err != nil { 237 t.Fatalf("err: %v", err) 238 } 239 240 // Check for the index 241 if respW.HeaderMap.Get("X-Nomad-Index") == "" { 242 t.Fatalf("missing index") 243 } 244 if respW.HeaderMap.Get("X-Nomad-KnownLeader") != "true" { 245 t.Fatalf("missing known leader") 246 } 247 if respW.HeaderMap.Get("X-Nomad-LastContact") == "" { 248 t.Fatalf("missing last contact") 249 } 250 251 // Check the job 252 a := obj.(*structs.Allocation) 253 if a.ID != alloc.ID { 254 t.Fatalf("bad: %#v", a) 255 } 256 257 // Check the payload is decompressed 258 if !reflect.DeepEqual(a.Job.Payload, expected) { 259 t.Fatalf("Payload not decompressed properly; got %#v; want %#v", a.Job.Payload, expected) 260 } 261 }) 262 } 263 264 func TestHTTP_AllocRestart(t *testing.T) { 265 t.Parallel() 266 require := require.New(t) 267 268 // Validates that all methods of forwarding the request are processed correctly 269 httpTest(t, nil, func(s *TestAgent) { 270 // Local node, local resp 271 { 272 // Make the HTTP request 273 buf := encodeReq(map[string]string{}) 274 req, err := http.NewRequest("GET", fmt.Sprintf("/v1/client/allocation/%s/restart", uuid.Generate()), buf) 275 if err != nil { 276 t.Fatalf("err: %v", err) 277 } 278 respW := httptest.NewRecorder() 279 280 // Make the request 281 _, err = s.Server.ClientAllocRequest(respW, req) 282 require.NotNil(err) 283 require.True(structs.IsErrUnknownAllocation(err)) 284 } 285 286 // Local node, server resp 287 { 288 srv := s.server 289 s.server = nil 290 291 buf := encodeReq(map[string]string{}) 292 req, err := http.NewRequest("GET", fmt.Sprintf("/v1/client/allocation/%s/restart", uuid.Generate()), buf) 293 require.Nil(err) 294 295 respW := httptest.NewRecorder() 296 _, err = s.Server.ClientAllocRequest(respW, req) 297 require.NotNil(err) 298 require.True(structs.IsErrUnknownAllocation(err)) 299 300 s.server = srv 301 } 302 303 // no client, server resp 304 { 305 c := s.client 306 s.client = nil 307 308 testutil.WaitForResult(func() (bool, error) { 309 n, err := s.server.State().NodeByID(nil, c.NodeID()) 310 if err != nil { 311 return false, err 312 } 313 return n != nil, nil 314 }, func(err error) { 315 t.Fatalf("should have client: %v", err) 316 }) 317 318 buf := encodeReq(map[string]string{}) 319 req, err := http.NewRequest("GET", fmt.Sprintf("/v1/client/allocation/%s/restart", uuid.Generate()), buf) 320 require.Nil(err) 321 322 respW := httptest.NewRecorder() 323 _, err = s.Server.ClientAllocRequest(respW, req) 324 require.NotNil(err) 325 require.True(structs.IsErrUnknownAllocation(err)) 326 327 s.client = c 328 } 329 }) 330 } 331 332 func TestHTTP_AllocRestart_ACL(t *testing.T) { 333 t.Parallel() 334 require := require.New(t) 335 336 httpACLTest(t, nil, func(s *TestAgent) { 337 state := s.Agent.server.State() 338 339 // If there's no token, we expect the request to fail. 340 { 341 buf := encodeReq(map[string]string{}) 342 req, err := http.NewRequest("GET", fmt.Sprintf("/v1/client/allocation/%s/restart", uuid.Generate()), buf) 343 require.NoError(err) 344 345 respW := httptest.NewRecorder() 346 _, err = s.Server.ClientAllocRequest(respW, req) 347 require.NotNil(err) 348 require.Equal(err.Error(), structs.ErrPermissionDenied.Error()) 349 } 350 351 // Try request with an invalid token and expect it to fail 352 { 353 buf := encodeReq(map[string]string{}) 354 req, err := http.NewRequest("GET", fmt.Sprintf("/v1/client/allocation/%s/restart", uuid.Generate()), buf) 355 require.NoError(err) 356 357 respW := httptest.NewRecorder() 358 token := mock.CreatePolicyAndToken(t, state, 1005, "invalid", mock.NodePolicy(acl.PolicyWrite)) 359 setToken(req, token) 360 _, err = s.Server.ClientAllocRequest(respW, req) 361 require.NotNil(err) 362 require.Equal(err.Error(), structs.ErrPermissionDenied.Error()) 363 } 364 365 // Try request with a valid token 366 // Still returns an error because the alloc does not exist 367 { 368 buf := encodeReq(map[string]string{}) 369 req, err := http.NewRequest("GET", fmt.Sprintf("/v1/client/allocation/%s/restart", uuid.Generate()), buf) 370 require.NoError(err) 371 372 respW := httptest.NewRecorder() 373 policy := mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityAllocLifecycle}) 374 token := mock.CreatePolicyAndToken(t, state, 1007, "valid", policy) 375 setToken(req, token) 376 _, err = s.Server.ClientAllocRequest(respW, req) 377 require.NotNil(err) 378 require.True(structs.IsErrUnknownAllocation(err)) 379 } 380 381 // Try request with a management token 382 // Still returns an error because the alloc does not exist 383 { 384 buf := encodeReq(map[string]string{}) 385 req, err := http.NewRequest("GET", fmt.Sprintf("/v1/client/allocation/%s/restart", uuid.Generate()), buf) 386 require.NoError(err) 387 388 respW := httptest.NewRecorder() 389 setToken(req, s.RootToken) 390 _, err = s.Server.ClientAllocRequest(respW, req) 391 require.NotNil(err) 392 require.True(structs.IsErrUnknownAllocation(err)) 393 } 394 }) 395 } 396 397 func TestHTTP_AllocStop(t *testing.T) { 398 t.Parallel() 399 httpTest(t, nil, func(s *TestAgent) { 400 // Directly manipulate the state 401 state := s.Agent.server.State() 402 alloc := mock.Alloc() 403 require := require.New(t) 404 require.NoError(state.UpsertJobSummary(999, mock.JobSummary(alloc.JobID))) 405 406 require.NoError(state.UpsertAllocs(1000, []*structs.Allocation{alloc})) 407 408 // Test that the happy path works 409 { 410 // Make the HTTP request 411 req, err := http.NewRequest("POST", "/v1/allocation/"+alloc.ID+"/stop", nil) 412 require.NoError(err) 413 respW := httptest.NewRecorder() 414 415 // Make the request 416 obj, err := s.Server.AllocSpecificRequest(respW, req) 417 require.NoError(err) 418 419 a := obj.(*structs.AllocStopResponse) 420 require.NotEmpty(a.EvalID, "missing eval") 421 require.NotEmpty(a.Index, "missing index") 422 } 423 424 // Test that we 404 when the allocid is invalid 425 { 426 // Make the HTTP request 427 req, err := http.NewRequest("POST", "/v1/allocation/"+alloc.ID+"/stop", nil) 428 require.NoError(err) 429 respW := httptest.NewRecorder() 430 431 // Make the request 432 obj, err := s.Server.AllocSpecificRequest(respW, req) 433 require.NoError(err) 434 435 a := obj.(*structs.AllocStopResponse) 436 require.NotEmpty(a.EvalID, "missing eval") 437 require.NotEmpty(a.Index, "missing index") 438 } 439 }) 440 } 441 442 func TestHTTP_AllocStats(t *testing.T) { 443 t.Parallel() 444 require := require.New(t) 445 446 httpTest(t, nil, func(s *TestAgent) { 447 // Local node, local resp 448 { 449 // Make the HTTP request 450 req, err := http.NewRequest("GET", fmt.Sprintf("/v1/client/allocation/%s/stats", uuid.Generate()), nil) 451 if err != nil { 452 t.Fatalf("err: %v", err) 453 } 454 respW := httptest.NewRecorder() 455 456 // Make the request 457 _, err = s.Server.ClientAllocRequest(respW, req) 458 require.NotNil(err) 459 require.True(structs.IsErrUnknownAllocation(err)) 460 } 461 462 // Local node, server resp 463 { 464 srv := s.server 465 s.server = nil 466 467 req, err := http.NewRequest("GET", fmt.Sprintf("/v1/client/allocation/%s/stats", uuid.Generate()), nil) 468 require.Nil(err) 469 470 respW := httptest.NewRecorder() 471 _, err = s.Server.ClientAllocRequest(respW, req) 472 require.NotNil(err) 473 require.True(structs.IsErrUnknownAllocation(err)) 474 475 s.server = srv 476 } 477 478 // no client, server resp 479 { 480 c := s.client 481 s.client = nil 482 483 testutil.WaitForResult(func() (bool, error) { 484 n, err := s.server.State().NodeByID(nil, c.NodeID()) 485 if err != nil { 486 return false, err 487 } 488 return n != nil, nil 489 }, func(err error) { 490 t.Fatalf("should have client: %v", err) 491 }) 492 493 req, err := http.NewRequest("GET", fmt.Sprintf("/v1/client/allocation/%s/stats", uuid.Generate()), nil) 494 require.Nil(err) 495 496 respW := httptest.NewRecorder() 497 _, err = s.Server.ClientAllocRequest(respW, req) 498 require.NotNil(err) 499 require.True(structs.IsErrUnknownAllocation(err)) 500 501 s.client = c 502 } 503 }) 504 } 505 506 func TestHTTP_AllocStats_ACL(t *testing.T) { 507 t.Parallel() 508 require := require.New(t) 509 510 httpACLTest(t, nil, func(s *TestAgent) { 511 state := s.Agent.server.State() 512 513 // Make the HTTP request 514 req, err := http.NewRequest("GET", fmt.Sprintf("/v1/client/allocation/%s/stats", uuid.Generate()), nil) 515 if err != nil { 516 t.Fatalf("err: %v", err) 517 } 518 519 // Try request without a token and expect failure 520 { 521 respW := httptest.NewRecorder() 522 _, err := s.Server.ClientAllocRequest(respW, req) 523 require.NotNil(err) 524 require.Equal(err.Error(), structs.ErrPermissionDenied.Error()) 525 } 526 527 // Try request with an invalid token and expect failure 528 { 529 respW := httptest.NewRecorder() 530 token := mock.CreatePolicyAndToken(t, state, 1005, "invalid", mock.NodePolicy(acl.PolicyWrite)) 531 setToken(req, token) 532 _, err := s.Server.ClientAllocRequest(respW, req) 533 require.NotNil(err) 534 require.Equal(err.Error(), structs.ErrPermissionDenied.Error()) 535 } 536 537 // Try request with a valid token 538 // Still returns an error because the alloc does not exist 539 { 540 respW := httptest.NewRecorder() 541 policy := mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob}) 542 token := mock.CreatePolicyAndToken(t, state, 1007, "valid", policy) 543 setToken(req, token) 544 _, err := s.Server.ClientAllocRequest(respW, req) 545 require.NotNil(err) 546 require.True(structs.IsErrUnknownAllocation(err)) 547 } 548 549 // Try request with a management token 550 // Still returns an error because the alloc does not exist 551 { 552 respW := httptest.NewRecorder() 553 setToken(req, s.RootToken) 554 _, err := s.Server.ClientAllocRequest(respW, req) 555 require.NotNil(err) 556 require.True(structs.IsErrUnknownAllocation(err)) 557 } 558 }) 559 } 560 561 func TestHTTP_AllocSnapshot(t *testing.T) { 562 t.Parallel() 563 httpTest(t, nil, func(s *TestAgent) { 564 // Make the HTTP request 565 req, err := http.NewRequest("GET", "/v1/client/allocation/123/snapshot", nil) 566 if err != nil { 567 t.Fatalf("err: %v", err) 568 } 569 respW := httptest.NewRecorder() 570 571 // Make the request 572 _, err = s.Server.ClientAllocRequest(respW, req) 573 if !strings.Contains(err.Error(), allocNotFoundErr) { 574 t.Fatalf("err: %v", err) 575 } 576 }) 577 } 578 579 func TestHTTP_AllocSnapshot_WithMigrateToken(t *testing.T) { 580 t.Parallel() 581 require := require.New(t) 582 httpACLTest(t, nil, func(s *TestAgent) { 583 // Request without a token fails 584 req, err := http.NewRequest("GET", "/v1/client/allocation/123/snapshot", nil) 585 require.Nil(err) 586 587 // Make the unauthorized request 588 respW := httptest.NewRecorder() 589 _, err = s.Server.ClientAllocRequest(respW, req) 590 require.NotNil(err) 591 require.EqualError(err, structs.ErrPermissionDenied.Error()) 592 593 // Create an allocation 594 alloc := mock.Alloc() 595 596 validMigrateToken, err := structs.GenerateMigrateToken(alloc.ID, s.Agent.Client().Node().SecretID) 597 require.Nil(err) 598 599 // Request with a token succeeds 600 url := fmt.Sprintf("/v1/client/allocation/%s/snapshot", alloc.ID) 601 req, err = http.NewRequest("GET", url, nil) 602 require.Nil(err) 603 604 req.Header.Set("X-Nomad-Token", validMigrateToken) 605 606 // Make the unauthorized request 607 respW = httptest.NewRecorder() 608 _, err = s.Server.ClientAllocRequest(respW, req) 609 require.NotContains(err.Error(), structs.ErrPermissionDenied.Error()) 610 }) 611 } 612 613 // TestHTTP_AllocSnapshot_Atomic ensures that when a client encounters an error 614 // snapshotting a valid tar is not returned. 615 func TestHTTP_AllocSnapshot_Atomic(t *testing.T) { 616 t.Parallel() 617 httpTest(t, func(c *Config) { 618 // Disable the schedulers 619 c.Server.NumSchedulers = helper.IntToPtr(0) 620 }, func(s *TestAgent) { 621 // Create an alloc 622 state := s.server.State() 623 alloc := mock.Alloc() 624 alloc.Job.TaskGroups[0].Tasks[0].Driver = "mock_driver" 625 alloc.Job.TaskGroups[0].Tasks[0].Config = map[string]interface{}{ 626 "run_for": "30s", 627 } 628 alloc.NodeID = s.client.NodeID() 629 state.UpsertJobSummary(998, mock.JobSummary(alloc.JobID)) 630 if err := state.UpsertAllocs(1000, []*structs.Allocation{alloc.Copy()}); err != nil { 631 t.Fatalf("error upserting alloc: %v", err) 632 } 633 634 // Wait for the client to run it 635 testutil.WaitForResult(func() (bool, error) { 636 if _, err := s.client.GetAllocState(alloc.ID); err != nil { 637 return false, err 638 } 639 640 serverAlloc, err := state.AllocByID(nil, alloc.ID) 641 if err != nil { 642 return false, err 643 } 644 645 return serverAlloc.ClientStatus == structs.AllocClientStatusRunning, fmt.Errorf(serverAlloc.ClientStatus) 646 }, func(err error) { 647 t.Fatalf("client not running alloc: %v", err) 648 }) 649 650 // Now write to its shared dir 651 allocDirI, err := s.client.GetAllocFS(alloc.ID) 652 if err != nil { 653 t.Fatalf("unable to find alloc dir: %v", err) 654 } 655 allocDir := allocDirI.(*allocdir.AllocDir) 656 657 // Remove the task dir to break Snapshot 658 os.RemoveAll(allocDir.TaskDirs["web"].LocalDir) 659 660 // require Snapshot fails 661 if err := allocDir.Snapshot(ioutil.Discard); err != nil { 662 t.Logf("[DEBUG] agent.test: snapshot returned error: %v", err) 663 } else { 664 t.Errorf("expected Snapshot() to fail but it did not") 665 } 666 667 // Make the HTTP request to ensure the Snapshot error is 668 // propagated through to the HTTP layer. Since the tar is 669 // streamed over a 200 HTTP response the only way to signal an 670 // error is by writing a marker file. 671 respW := httptest.NewRecorder() 672 req, err := http.NewRequest("GET", fmt.Sprintf("/v1/client/allocation/%s/snapshot", alloc.ID), nil) 673 if err != nil { 674 t.Fatalf("err: %v", err) 675 } 676 677 // Make the request via the mux to make sure the error returned 678 // by Snapshot is properly propagated via HTTP 679 s.Server.mux.ServeHTTP(respW, req) 680 resp := respW.Result() 681 r := tar.NewReader(resp.Body) 682 errorFilename := allocdir.SnapshotErrorFilename(alloc.ID) 683 markerFound := false 684 markerContents := "" 685 for { 686 header, err := r.Next() 687 if err != nil { 688 if err != io.EOF { 689 // Huh, I wonder how a non-EOF error can happen? 690 t.Errorf("Unexpected error while streaming: %v", err) 691 } 692 break 693 } 694 695 if markerFound { 696 // No more files should be found after the failure marker 697 t.Errorf("Next file found after error marker: %s", header.Name) 698 break 699 } 700 701 if header.Name == errorFilename { 702 // Found it! 703 markerFound = true 704 buf := make([]byte, int(header.Size)) 705 if _, err := r.Read(buf); err != nil && err != io.EOF { 706 t.Errorf("Unexpected error reading error marker %s: %v", errorFilename, err) 707 } else { 708 markerContents = string(buf) 709 } 710 } 711 } 712 713 if !markerFound { 714 t.Fatalf("marker file %s not written; bad tar will be treated as good!", errorFilename) 715 } 716 if markerContents == "" { 717 t.Fatalf("marker file %s empty", markerContents) 718 } else { 719 t.Logf("EXPECTED snapshot error: %s", markerContents) 720 } 721 }) 722 } 723 724 func TestHTTP_AllocGC(t *testing.T) { 725 t.Parallel() 726 require := require.New(t) 727 path := fmt.Sprintf("/v1/client/allocation/%s/gc", uuid.Generate()) 728 httpTest(t, nil, func(s *TestAgent) { 729 // Local node, local resp 730 { 731 req, err := http.NewRequest("GET", path, nil) 732 if err != nil { 733 t.Fatalf("err: %v", err) 734 } 735 736 respW := httptest.NewRecorder() 737 _, err = s.Server.ClientAllocRequest(respW, req) 738 if !structs.IsErrUnknownAllocation(err) { 739 t.Fatalf("unexpected err: %v", err) 740 } 741 } 742 743 // Local node, server resp 744 { 745 srv := s.server 746 s.server = nil 747 748 req, err := http.NewRequest("GET", path, nil) 749 if err != nil { 750 t.Fatalf("err: %v", err) 751 } 752 753 respW := httptest.NewRecorder() 754 _, err = s.Server.ClientAllocRequest(respW, req) 755 if !structs.IsErrUnknownAllocation(err) { 756 t.Fatalf("unexpected err: %v", err) 757 } 758 759 s.server = srv 760 } 761 762 // no client, server resp 763 { 764 c := s.client 765 s.client = nil 766 767 testutil.WaitForResult(func() (bool, error) { 768 n, err := s.server.State().NodeByID(nil, c.NodeID()) 769 if err != nil { 770 return false, err 771 } 772 return n != nil, nil 773 }, func(err error) { 774 t.Fatalf("should have client: %v", err) 775 }) 776 777 req, err := http.NewRequest("GET", path, nil) 778 if err != nil { 779 t.Fatalf("err: %v", err) 780 } 781 782 respW := httptest.NewRecorder() 783 _, err = s.Server.ClientAllocRequest(respW, req) 784 require.NotNil(err) 785 if !structs.IsErrUnknownAllocation(err) { 786 t.Fatalf("unexpected err: %v", err) 787 } 788 789 s.client = c 790 } 791 }) 792 } 793 794 func TestHTTP_AllocGC_ACL(t *testing.T) { 795 t.Parallel() 796 require := require.New(t) 797 path := fmt.Sprintf("/v1/client/allocation/%s/gc", uuid.Generate()) 798 799 httpACLTest(t, nil, func(s *TestAgent) { 800 state := s.Agent.server.State() 801 802 // Make the HTTP request 803 req, err := http.NewRequest("GET", path, nil) 804 if err != nil { 805 t.Fatalf("err: %v", err) 806 } 807 808 // Try request without a token and expect failure 809 { 810 respW := httptest.NewRecorder() 811 _, err := s.Server.ClientAllocRequest(respW, req) 812 require.NotNil(err) 813 require.Equal(err.Error(), structs.ErrPermissionDenied.Error()) 814 } 815 816 // Try request with an invalid token and expect failure 817 { 818 respW := httptest.NewRecorder() 819 token := mock.CreatePolicyAndToken(t, state, 1005, "invalid", mock.NodePolicy(acl.PolicyWrite)) 820 setToken(req, token) 821 _, err := s.Server.ClientAllocRequest(respW, req) 822 require.NotNil(err) 823 require.Equal(err.Error(), structs.ErrPermissionDenied.Error()) 824 } 825 826 // Try request with a valid token 827 // Still returns an error because the alloc does not exist 828 { 829 respW := httptest.NewRecorder() 830 policy := mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilitySubmitJob}) 831 token := mock.CreatePolicyAndToken(t, state, 1007, "valid", policy) 832 setToken(req, token) 833 _, err := s.Server.ClientAllocRequest(respW, req) 834 require.NotNil(err) 835 require.True(structs.IsErrUnknownAllocation(err)) 836 } 837 838 // Try request with a management token 839 // Still returns an error because the alloc does not exist 840 { 841 respW := httptest.NewRecorder() 842 setToken(req, s.RootToken) 843 _, err := s.Server.ClientAllocRequest(respW, req) 844 require.NotNil(err) 845 require.True(structs.IsErrUnknownAllocation(err)) 846 } 847 }) 848 } 849 850 func TestHTTP_AllocAllGC(t *testing.T) { 851 t.Parallel() 852 require := require.New(t) 853 httpTest(t, nil, func(s *TestAgent) { 854 // Local node, local resp 855 { 856 req, err := http.NewRequest("GET", "/v1/client/gc", nil) 857 if err != nil { 858 t.Fatalf("err: %v", err) 859 } 860 861 respW := httptest.NewRecorder() 862 _, err = s.Server.ClientGCRequest(respW, req) 863 if err != nil { 864 t.Fatalf("unexpected err: %v", err) 865 } 866 } 867 868 // Local node, server resp 869 { 870 srv := s.server 871 s.server = nil 872 873 req, err := http.NewRequest("GET", fmt.Sprintf("/v1/client/gc?node_id=%s", uuid.Generate()), nil) 874 require.Nil(err) 875 876 respW := httptest.NewRecorder() 877 _, err = s.Server.ClientGCRequest(respW, req) 878 require.NotNil(err) 879 require.Contains(err.Error(), "Unknown node") 880 881 s.server = srv 882 } 883 884 // client stats from server, should not error 885 { 886 c := s.client 887 s.client = nil 888 889 testutil.WaitForResult(func() (bool, error) { 890 n, err := s.server.State().NodeByID(nil, c.NodeID()) 891 if err != nil { 892 return false, err 893 } 894 return n != nil, nil 895 }, func(err error) { 896 t.Fatalf("should have client: %v", err) 897 }) 898 899 req, err := http.NewRequest("GET", fmt.Sprintf("/v1/client/gc?node_id=%s", c.NodeID()), nil) 900 require.Nil(err) 901 902 respW := httptest.NewRecorder() 903 _, err = s.Server.ClientGCRequest(respW, req) 904 require.Nil(err) 905 906 s.client = c 907 } 908 }) 909 910 } 911 912 func TestHTTP_AllocAllGC_ACL(t *testing.T) { 913 t.Parallel() 914 require := require.New(t) 915 httpACLTest(t, nil, func(s *TestAgent) { 916 state := s.Agent.server.State() 917 918 // Make the HTTP request 919 req, err := http.NewRequest("GET", "/v1/client/gc", nil) 920 require.Nil(err) 921 922 // Try request without a token and expect failure 923 { 924 respW := httptest.NewRecorder() 925 _, err := s.Server.ClientGCRequest(respW, req) 926 require.NotNil(err) 927 require.Equal(err.Error(), structs.ErrPermissionDenied.Error()) 928 } 929 930 // Try request with an invalid token and expect failure 931 { 932 respW := httptest.NewRecorder() 933 token := mock.CreatePolicyAndToken(t, state, 1005, "invalid", mock.NodePolicy(acl.PolicyRead)) 934 setToken(req, token) 935 _, err := s.Server.ClientGCRequest(respW, req) 936 require.NotNil(err) 937 require.Equal(err.Error(), structs.ErrPermissionDenied.Error()) 938 } 939 940 // Try request with a valid token 941 { 942 respW := httptest.NewRecorder() 943 token := mock.CreatePolicyAndToken(t, state, 1007, "valid", mock.NodePolicy(acl.PolicyWrite)) 944 setToken(req, token) 945 _, err := s.Server.ClientGCRequest(respW, req) 946 require.Nil(err) 947 require.Equal(http.StatusOK, respW.Code) 948 } 949 950 // Try request with a management token 951 { 952 respW := httptest.NewRecorder() 953 setToken(req, s.RootToken) 954 _, err := s.Server.ClientGCRequest(respW, req) 955 require.Nil(err) 956 require.Equal(http.StatusOK, respW.Code) 957 } 958 }) 959 }