github.com/manicqin/nomad@v0.9.5/command/agent/alloc_endpoint_test.go (about) 1 package agent 2 3 import ( 4 "archive/tar" 5 "fmt" 6 "io" 7 "io/ioutil" 8 "net/http" 9 "net/http/httptest" 10 "os" 11 "reflect" 12 "strconv" 13 "strings" 14 "testing" 15 16 "github.com/golang/snappy" 17 "github.com/hashicorp/nomad/acl" 18 "github.com/hashicorp/nomad/client/allocdir" 19 "github.com/hashicorp/nomad/helper" 20 "github.com/hashicorp/nomad/helper/uuid" 21 "github.com/hashicorp/nomad/nomad/mock" 22 "github.com/hashicorp/nomad/nomad/structs" 23 "github.com/hashicorp/nomad/testutil" 24 "github.com/stretchr/testify/require" 25 ) 26 27 func TestHTTP_AllocsList(t *testing.T) { 28 t.Parallel() 29 httpTest(t, nil, func(s *TestAgent) { 30 // Directly manipulate the state 31 state := s.Agent.server.State() 32 alloc1 := mock.Alloc() 33 testEvent := structs.NewTaskEvent(structs.TaskSiblingFailed) 34 var events1 []*structs.TaskEvent 35 events1 = append(events1, testEvent) 36 taskState := &structs.TaskState{Events: events1} 37 alloc1.TaskStates = make(map[string]*structs.TaskState) 38 alloc1.TaskStates["test"] = taskState 39 40 alloc2 := mock.Alloc() 41 alloc2.TaskStates = make(map[string]*structs.TaskState) 42 alloc2.TaskStates["test"] = taskState 43 44 state.UpsertJobSummary(998, mock.JobSummary(alloc1.JobID)) 45 state.UpsertJobSummary(999, mock.JobSummary(alloc2.JobID)) 46 err := state.UpsertAllocs(1000, 47 []*structs.Allocation{alloc1, alloc2}) 48 if err != nil { 49 t.Fatalf("err: %v", err) 50 } 51 52 // Make the HTTP request 53 req, err := http.NewRequest("GET", "/v1/allocations", nil) 54 if err != nil { 55 t.Fatalf("err: %v", err) 56 } 57 respW := httptest.NewRecorder() 58 59 // Make the request 60 obj, err := s.Server.AllocsRequest(respW, req) 61 if err != nil { 62 t.Fatalf("err: %v", err) 63 } 64 65 // Check for the index 66 if respW.HeaderMap.Get("X-Nomad-Index") == "" { 67 t.Fatalf("missing index") 68 } 69 if respW.HeaderMap.Get("X-Nomad-KnownLeader") != "true" { 70 t.Fatalf("missing known leader") 71 } 72 if respW.HeaderMap.Get("X-Nomad-LastContact") == "" { 73 t.Fatalf("missing last contact") 74 } 75 76 // Check the alloc 77 allocs := obj.([]*structs.AllocListStub) 78 if len(allocs) != 2 { 79 t.Fatalf("bad: %#v", allocs) 80 } 81 expectedMsg := "Task's sibling failed" 82 displayMsg1 := allocs[0].TaskStates["test"].Events[0].DisplayMessage 83 require.Equal(t, expectedMsg, displayMsg1, "DisplayMessage should be set") 84 displayMsg2 := allocs[0].TaskStates["test"].Events[0].DisplayMessage 85 require.Equal(t, expectedMsg, displayMsg2, "DisplayMessage should be set") 86 }) 87 } 88 89 func TestHTTP_AllocsPrefixList(t *testing.T) { 90 t.Parallel() 91 httpTest(t, nil, func(s *TestAgent) { 92 // Directly manipulate the state 93 state := s.Agent.server.State() 94 95 alloc1 := mock.Alloc() 96 alloc1.ID = "aaaaaaaa-e8f7-fd38-c855-ab94ceb89706" 97 alloc2 := mock.Alloc() 98 alloc2.ID = "aaabbbbb-e8f7-fd38-c855-ab94ceb89706" 99 100 testEvent := structs.NewTaskEvent(structs.TaskSiblingFailed) 101 var events1 []*structs.TaskEvent 102 events1 = append(events1, testEvent) 103 taskState := &structs.TaskState{Events: events1} 104 alloc2.TaskStates = make(map[string]*structs.TaskState) 105 alloc2.TaskStates["test"] = taskState 106 107 summary1 := mock.JobSummary(alloc1.JobID) 108 summary2 := mock.JobSummary(alloc2.JobID) 109 if err := state.UpsertJobSummary(998, summary1); err != nil { 110 t.Fatal(err) 111 } 112 if err := state.UpsertJobSummary(999, summary2); err != nil { 113 t.Fatal(err) 114 } 115 if err := state.UpsertAllocs(1000, 116 []*structs.Allocation{alloc1, alloc2}); err != nil { 117 t.Fatalf("err: %v", err) 118 } 119 120 // Make the HTTP request 121 req, err := http.NewRequest("GET", "/v1/allocations?prefix=aaab", nil) 122 if err != nil { 123 t.Fatalf("err: %v", err) 124 } 125 respW := httptest.NewRecorder() 126 127 // Make the request 128 obj, err := s.Server.AllocsRequest(respW, req) 129 if err != nil { 130 t.Fatalf("err: %v", err) 131 } 132 133 // Check for the index 134 if respW.HeaderMap.Get("X-Nomad-Index") == "" { 135 t.Fatalf("missing index") 136 } 137 if respW.HeaderMap.Get("X-Nomad-KnownLeader") != "true" { 138 t.Fatalf("missing known leader") 139 } 140 if respW.HeaderMap.Get("X-Nomad-LastContact") == "" { 141 t.Fatalf("missing last contact") 142 } 143 144 // Check the alloc 145 n := obj.([]*structs.AllocListStub) 146 if len(n) != 1 { 147 t.Fatalf("bad: %#v", n) 148 } 149 150 // Check the identifier 151 if n[0].ID != alloc2.ID { 152 t.Fatalf("expected alloc ID: %v, Actual: %v", alloc2.ID, n[0].ID) 153 } 154 expectedMsg := "Task's sibling failed" 155 displayMsg1 := n[0].TaskStates["test"].Events[0].DisplayMessage 156 require.Equal(t, expectedMsg, displayMsg1, "DisplayMessage should be set") 157 158 }) 159 } 160 161 func TestHTTP_AllocQuery(t *testing.T) { 162 t.Parallel() 163 httpTest(t, nil, func(s *TestAgent) { 164 // Directly manipulate the state 165 state := s.Agent.server.State() 166 alloc := mock.Alloc() 167 if err := state.UpsertJobSummary(999, mock.JobSummary(alloc.JobID)); err != nil { 168 t.Fatal(err) 169 } 170 err := state.UpsertAllocs(1000, 171 []*structs.Allocation{alloc}) 172 if err != nil { 173 t.Fatalf("err: %v", err) 174 } 175 176 // Make the HTTP request 177 req, err := http.NewRequest("GET", "/v1/allocation/"+alloc.ID, nil) 178 if err != nil { 179 t.Fatalf("err: %v", err) 180 } 181 respW := httptest.NewRecorder() 182 183 // Make the request 184 obj, err := s.Server.AllocSpecificRequest(respW, req) 185 if err != nil { 186 t.Fatalf("err: %v", err) 187 } 188 189 // Check for the index 190 if respW.HeaderMap.Get("X-Nomad-Index") == "" { 191 t.Fatalf("missing index") 192 } 193 if respW.HeaderMap.Get("X-Nomad-KnownLeader") != "true" { 194 t.Fatalf("missing known leader") 195 } 196 if respW.HeaderMap.Get("X-Nomad-LastContact") == "" { 197 t.Fatalf("missing last contact") 198 } 199 200 // Check the job 201 a := obj.(*structs.Allocation) 202 if a.ID != alloc.ID { 203 t.Fatalf("bad: %#v", a) 204 } 205 }) 206 } 207 208 func TestHTTP_AllocQuery_Payload(t *testing.T) { 209 t.Parallel() 210 httpTest(t, nil, func(s *TestAgent) { 211 // Directly manipulate the state 212 state := s.Agent.server.State() 213 alloc := mock.Alloc() 214 if err := state.UpsertJobSummary(999, mock.JobSummary(alloc.JobID)); err != nil { 215 t.Fatal(err) 216 } 217 218 // Insert Payload compressed 219 expected := []byte("hello world") 220 compressed := snappy.Encode(nil, expected) 221 alloc.Job.Payload = compressed 222 223 err := state.UpsertAllocs(1000, []*structs.Allocation{alloc}) 224 if err != nil { 225 t.Fatalf("err: %v", err) 226 } 227 228 // Make the HTTP request 229 req, err := http.NewRequest("GET", "/v1/allocation/"+alloc.ID, nil) 230 if err != nil { 231 t.Fatalf("err: %v", err) 232 } 233 respW := httptest.NewRecorder() 234 235 // Make the request 236 obj, err := s.Server.AllocSpecificRequest(respW, req) 237 if err != nil { 238 t.Fatalf("err: %v", err) 239 } 240 241 // Check for the index 242 if respW.HeaderMap.Get("X-Nomad-Index") == "" { 243 t.Fatalf("missing index") 244 } 245 if respW.HeaderMap.Get("X-Nomad-KnownLeader") != "true" { 246 t.Fatalf("missing known leader") 247 } 248 if respW.HeaderMap.Get("X-Nomad-LastContact") == "" { 249 t.Fatalf("missing last contact") 250 } 251 252 // Check the job 253 a := obj.(*structs.Allocation) 254 if a.ID != alloc.ID { 255 t.Fatalf("bad: %#v", a) 256 } 257 258 // Check the payload is decompressed 259 if !reflect.DeepEqual(a.Job.Payload, expected) { 260 t.Fatalf("Payload not decompressed properly; got %#v; want %#v", a.Job.Payload, expected) 261 } 262 }) 263 } 264 265 func TestHTTP_AllocRestart(t *testing.T) { 266 t.Parallel() 267 require := require.New(t) 268 269 // Validates that all methods of forwarding the request are processed correctly 270 httpTest(t, nil, func(s *TestAgent) { 271 // Local node, local resp 272 { 273 // Make the HTTP request 274 buf := encodeReq(map[string]string{}) 275 req, err := http.NewRequest("GET", fmt.Sprintf("/v1/client/allocation/%s/restart", uuid.Generate()), buf) 276 if err != nil { 277 t.Fatalf("err: %v", err) 278 } 279 respW := httptest.NewRecorder() 280 281 // Make the request 282 _, err = s.Server.ClientAllocRequest(respW, req) 283 require.NotNil(err) 284 require.True(structs.IsErrUnknownAllocation(err)) 285 } 286 287 // Local node, server resp 288 { 289 srv := s.server 290 s.server = nil 291 292 buf := encodeReq(map[string]string{}) 293 req, err := http.NewRequest("GET", fmt.Sprintf("/v1/client/allocation/%s/restart", uuid.Generate()), buf) 294 require.Nil(err) 295 296 respW := httptest.NewRecorder() 297 _, err = s.Server.ClientAllocRequest(respW, req) 298 require.NotNil(err) 299 require.True(structs.IsErrUnknownAllocation(err)) 300 301 s.server = srv 302 } 303 304 // no client, server resp 305 { 306 c := s.client 307 s.client = nil 308 309 testutil.WaitForResult(func() (bool, error) { 310 n, err := s.server.State().NodeByID(nil, c.NodeID()) 311 if err != nil { 312 return false, err 313 } 314 return n != nil, nil 315 }, func(err error) { 316 t.Fatalf("should have client: %v", err) 317 }) 318 319 buf := encodeReq(map[string]string{}) 320 req, err := http.NewRequest("GET", fmt.Sprintf("/v1/client/allocation/%s/restart", uuid.Generate()), buf) 321 require.Nil(err) 322 323 respW := httptest.NewRecorder() 324 _, err = s.Server.ClientAllocRequest(respW, req) 325 require.NotNil(err) 326 require.True(structs.IsErrUnknownAllocation(err)) 327 328 s.client = c 329 } 330 }) 331 } 332 333 func TestHTTP_AllocRestart_ACL(t *testing.T) { 334 t.Parallel() 335 require := require.New(t) 336 337 httpACLTest(t, nil, func(s *TestAgent) { 338 state := s.Agent.server.State() 339 340 // If there's no token, we expect the request to fail. 341 { 342 buf := encodeReq(map[string]string{}) 343 req, err := http.NewRequest("GET", fmt.Sprintf("/v1/client/allocation/%s/restart", uuid.Generate()), buf) 344 require.NoError(err) 345 346 respW := httptest.NewRecorder() 347 _, err = s.Server.ClientAllocRequest(respW, req) 348 require.NotNil(err) 349 require.True(structs.IsErrUnknownAllocation(err), "(%T) %v", err, err) 350 } 351 352 // Try request with an invalid token and expect it to fail 353 { 354 buf := encodeReq(map[string]string{}) 355 req, err := http.NewRequest("GET", fmt.Sprintf("/v1/client/allocation/%s/restart", uuid.Generate()), buf) 356 require.NoError(err) 357 358 respW := httptest.NewRecorder() 359 token := mock.CreatePolicyAndToken(t, state, 1005, "invalid", mock.NodePolicy(acl.PolicyWrite)) 360 setToken(req, token) 361 _, err = s.Server.ClientAllocRequest(respW, req) 362 require.NotNil(err) 363 require.True(structs.IsErrUnknownAllocation(err), "(%T) %v", err, err) 364 } 365 366 // Try request with a valid token 367 // Still returns an error because the alloc does not exist 368 { 369 buf := encodeReq(map[string]string{}) 370 req, err := http.NewRequest("GET", fmt.Sprintf("/v1/client/allocation/%s/restart", uuid.Generate()), buf) 371 require.NoError(err) 372 373 respW := httptest.NewRecorder() 374 policy := mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityAllocLifecycle}) 375 token := mock.CreatePolicyAndToken(t, state, 1007, "valid", policy) 376 setToken(req, token) 377 _, err = s.Server.ClientAllocRequest(respW, req) 378 require.NotNil(err) 379 require.True(structs.IsErrUnknownAllocation(err), "(%T) %v", err, err) 380 } 381 382 // Try request with a management token 383 // Still returns an error because the alloc does not exist 384 { 385 buf := encodeReq(map[string]string{}) 386 req, err := http.NewRequest("GET", fmt.Sprintf("/v1/client/allocation/%s/restart", uuid.Generate()), buf) 387 require.NoError(err) 388 389 respW := httptest.NewRecorder() 390 setToken(req, s.RootToken) 391 _, err = s.Server.ClientAllocRequest(respW, req) 392 require.NotNil(err) 393 require.True(structs.IsErrUnknownAllocation(err)) 394 } 395 }) 396 } 397 398 func TestHTTP_AllocStop(t *testing.T) { 399 t.Parallel() 400 httpTest(t, nil, func(s *TestAgent) { 401 // Directly manipulate the state 402 state := s.Agent.server.State() 403 alloc := mock.Alloc() 404 require := require.New(t) 405 require.NoError(state.UpsertJobSummary(999, mock.JobSummary(alloc.JobID))) 406 407 require.NoError(state.UpsertAllocs(1000, []*structs.Allocation{alloc})) 408 409 // Test that the happy path works 410 { 411 // Make the HTTP request 412 req, err := http.NewRequest("POST", "/v1/allocation/"+alloc.ID+"/stop", nil) 413 require.NoError(err) 414 respW := httptest.NewRecorder() 415 416 // Make the request 417 obj, err := s.Server.AllocSpecificRequest(respW, req) 418 require.NoError(err) 419 420 a := obj.(*structs.AllocStopResponse) 421 require.NotEmpty(a.EvalID, "missing eval") 422 require.NotEmpty(a.Index, "missing index") 423 headerIndex, _ := strconv.ParseUint(respW.Header().Get("X-Nomad-Index"), 10, 64) 424 require.Equal(a.Index, headerIndex) 425 } 426 427 // Test that we 404 when the allocid is invalid 428 { 429 // Make the HTTP request 430 req, err := http.NewRequest("POST", "/v1/allocation/"+uuid.Generate()+"/stop", nil) 431 require.NoError(err) 432 respW := httptest.NewRecorder() 433 434 // Make the request 435 _, err = s.Server.AllocSpecificRequest(respW, req) 436 require.NotNil(err) 437 if !strings.Contains(err.Error(), allocNotFoundErr) { 438 t.Fatalf("err: %v", err) 439 } 440 } 441 }) 442 } 443 444 func TestHTTP_AllocStats(t *testing.T) { 445 t.Parallel() 446 require := require.New(t) 447 448 httpTest(t, nil, func(s *TestAgent) { 449 // Local node, local resp 450 { 451 // Make the HTTP request 452 req, err := http.NewRequest("GET", fmt.Sprintf("/v1/client/allocation/%s/stats", uuid.Generate()), nil) 453 if err != nil { 454 t.Fatalf("err: %v", err) 455 } 456 respW := httptest.NewRecorder() 457 458 // Make the request 459 _, err = s.Server.ClientAllocRequest(respW, req) 460 require.NotNil(err) 461 require.True(structs.IsErrUnknownAllocation(err)) 462 } 463 464 // Local node, server resp 465 { 466 srv := s.server 467 s.server = nil 468 469 req, err := http.NewRequest("GET", fmt.Sprintf("/v1/client/allocation/%s/stats", uuid.Generate()), nil) 470 require.Nil(err) 471 472 respW := httptest.NewRecorder() 473 _, err = s.Server.ClientAllocRequest(respW, req) 474 require.NotNil(err) 475 require.True(structs.IsErrUnknownAllocation(err)) 476 477 s.server = srv 478 } 479 480 // no client, server resp 481 { 482 c := s.client 483 s.client = nil 484 485 testutil.WaitForResult(func() (bool, error) { 486 n, err := s.server.State().NodeByID(nil, c.NodeID()) 487 if err != nil { 488 return false, err 489 } 490 return n != nil, nil 491 }, func(err error) { 492 t.Fatalf("should have client: %v", err) 493 }) 494 495 req, err := http.NewRequest("GET", fmt.Sprintf("/v1/client/allocation/%s/stats", uuid.Generate()), nil) 496 require.Nil(err) 497 498 respW := httptest.NewRecorder() 499 _, err = s.Server.ClientAllocRequest(respW, req) 500 require.NotNil(err) 501 require.True(structs.IsErrUnknownAllocation(err)) 502 503 s.client = c 504 } 505 }) 506 } 507 508 func TestHTTP_AllocStats_ACL(t *testing.T) { 509 t.Parallel() 510 require := require.New(t) 511 512 httpACLTest(t, nil, func(s *TestAgent) { 513 state := s.Agent.server.State() 514 515 // Make the HTTP request 516 req, err := http.NewRequest("GET", fmt.Sprintf("/v1/client/allocation/%s/stats", uuid.Generate()), nil) 517 if err != nil { 518 t.Fatalf("err: %v", err) 519 } 520 521 // Try request without a token and expect failure 522 { 523 respW := httptest.NewRecorder() 524 _, err := s.Server.ClientAllocRequest(respW, req) 525 require.NotNil(err) 526 require.True(structs.IsErrUnknownAllocation(err), "(%T) %v", err, err) 527 } 528 529 // Try request with an invalid token and expect failure 530 { 531 respW := httptest.NewRecorder() 532 token := mock.CreatePolicyAndToken(t, state, 1005, "invalid", mock.NodePolicy(acl.PolicyWrite)) 533 setToken(req, token) 534 _, err := s.Server.ClientAllocRequest(respW, req) 535 require.NotNil(err) 536 require.True(structs.IsErrUnknownAllocation(err), "(%T) %v", err, err) 537 } 538 539 // Try request with a valid token 540 // Still returns an error because the alloc does not exist 541 { 542 respW := httptest.NewRecorder() 543 policy := mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob}) 544 token := mock.CreatePolicyAndToken(t, state, 1007, "valid", policy) 545 setToken(req, token) 546 _, err := s.Server.ClientAllocRequest(respW, req) 547 require.NotNil(err) 548 require.True(structs.IsErrUnknownAllocation(err), "(%T) %v", err, err) 549 } 550 551 // Try request with a management token 552 // Still returns an error because the alloc does not exist 553 { 554 respW := httptest.NewRecorder() 555 setToken(req, s.RootToken) 556 _, err := s.Server.ClientAllocRequest(respW, req) 557 require.NotNil(err) 558 require.True(structs.IsErrUnknownAllocation(err)) 559 } 560 }) 561 } 562 563 func TestHTTP_AllocSnapshot(t *testing.T) { 564 t.Parallel() 565 httpTest(t, nil, func(s *TestAgent) { 566 // Make the HTTP request 567 req, err := http.NewRequest("GET", "/v1/client/allocation/123/snapshot", nil) 568 if err != nil { 569 t.Fatalf("err: %v", err) 570 } 571 respW := httptest.NewRecorder() 572 573 // Make the request 574 _, err = s.Server.ClientAllocRequest(respW, req) 575 if !strings.Contains(err.Error(), allocNotFoundErr) { 576 t.Fatalf("err: %v", err) 577 } 578 }) 579 } 580 581 func TestHTTP_AllocSnapshot_WithMigrateToken(t *testing.T) { 582 t.Parallel() 583 require := require.New(t) 584 httpACLTest(t, nil, func(s *TestAgent) { 585 // Request without a token fails 586 req, err := http.NewRequest("GET", "/v1/client/allocation/123/snapshot", nil) 587 require.Nil(err) 588 589 // Make the unauthorized request 590 respW := httptest.NewRecorder() 591 _, err = s.Server.ClientAllocRequest(respW, req) 592 require.NotNil(err) 593 require.EqualError(err, structs.ErrPermissionDenied.Error()) 594 595 // Create an allocation 596 alloc := mock.Alloc() 597 598 validMigrateToken, err := structs.GenerateMigrateToken(alloc.ID, s.Agent.Client().Node().SecretID) 599 require.Nil(err) 600 601 // Request with a token succeeds 602 url := fmt.Sprintf("/v1/client/allocation/%s/snapshot", alloc.ID) 603 req, err = http.NewRequest("GET", url, nil) 604 require.Nil(err) 605 606 req.Header.Set("X-Nomad-Token", validMigrateToken) 607 608 // Make the unauthorized request 609 respW = httptest.NewRecorder() 610 _, err = s.Server.ClientAllocRequest(respW, req) 611 require.NotContains(err.Error(), structs.ErrPermissionDenied.Error()) 612 }) 613 } 614 615 // TestHTTP_AllocSnapshot_Atomic ensures that when a client encounters an error 616 // snapshotting a valid tar is not returned. 617 func TestHTTP_AllocSnapshot_Atomic(t *testing.T) { 618 t.Parallel() 619 httpTest(t, func(c *Config) { 620 // Disable the schedulers 621 c.Server.NumSchedulers = helper.IntToPtr(0) 622 }, func(s *TestAgent) { 623 // Create an alloc 624 state := s.server.State() 625 alloc := mock.Alloc() 626 alloc.Job.TaskGroups[0].Tasks[0].Driver = "mock_driver" 627 alloc.Job.TaskGroups[0].Tasks[0].Config = map[string]interface{}{ 628 "run_for": "30s", 629 } 630 alloc.NodeID = s.client.NodeID() 631 state.UpsertJobSummary(998, mock.JobSummary(alloc.JobID)) 632 if err := state.UpsertAllocs(1000, []*structs.Allocation{alloc.Copy()}); err != nil { 633 t.Fatalf("error upserting alloc: %v", err) 634 } 635 636 // Wait for the client to run it 637 testutil.WaitForResult(func() (bool, error) { 638 if _, err := s.client.GetAllocState(alloc.ID); err != nil { 639 return false, err 640 } 641 642 serverAlloc, err := state.AllocByID(nil, alloc.ID) 643 if err != nil { 644 return false, err 645 } 646 647 return serverAlloc.ClientStatus == structs.AllocClientStatusRunning, fmt.Errorf(serverAlloc.ClientStatus) 648 }, func(err error) { 649 t.Fatalf("client not running alloc: %v", err) 650 }) 651 652 // Now write to its shared dir 653 allocDirI, err := s.client.GetAllocFS(alloc.ID) 654 if err != nil { 655 t.Fatalf("unable to find alloc dir: %v", err) 656 } 657 allocDir := allocDirI.(*allocdir.AllocDir) 658 659 // Remove the task dir to break Snapshot 660 os.RemoveAll(allocDir.TaskDirs["web"].LocalDir) 661 662 // require Snapshot fails 663 if err := allocDir.Snapshot(ioutil.Discard); err != nil { 664 t.Logf("[DEBUG] agent.test: snapshot returned error: %v", err) 665 } else { 666 t.Errorf("expected Snapshot() to fail but it did not") 667 } 668 669 // Make the HTTP request to ensure the Snapshot error is 670 // propagated through to the HTTP layer. Since the tar is 671 // streamed over a 200 HTTP response the only way to signal an 672 // error is by writing a marker file. 673 respW := httptest.NewRecorder() 674 req, err := http.NewRequest("GET", fmt.Sprintf("/v1/client/allocation/%s/snapshot", alloc.ID), nil) 675 if err != nil { 676 t.Fatalf("err: %v", err) 677 } 678 679 // Make the request via the mux to make sure the error returned 680 // by Snapshot is properly propagated via HTTP 681 s.Server.mux.ServeHTTP(respW, req) 682 resp := respW.Result() 683 r := tar.NewReader(resp.Body) 684 errorFilename := allocdir.SnapshotErrorFilename(alloc.ID) 685 markerFound := false 686 markerContents := "" 687 for { 688 header, err := r.Next() 689 if err != nil { 690 if err != io.EOF { 691 // Huh, I wonder how a non-EOF error can happen? 692 t.Errorf("Unexpected error while streaming: %v", err) 693 } 694 break 695 } 696 697 if markerFound { 698 // No more files should be found after the failure marker 699 t.Errorf("Next file found after error marker: %s", header.Name) 700 break 701 } 702 703 if header.Name == errorFilename { 704 // Found it! 705 markerFound = true 706 buf := make([]byte, int(header.Size)) 707 if _, err := r.Read(buf); err != nil && err != io.EOF { 708 t.Errorf("Unexpected error reading error marker %s: %v", errorFilename, err) 709 } else { 710 markerContents = string(buf) 711 } 712 } 713 } 714 715 if !markerFound { 716 t.Fatalf("marker file %s not written; bad tar will be treated as good!", errorFilename) 717 } 718 if markerContents == "" { 719 t.Fatalf("marker file %s empty", markerContents) 720 } else { 721 t.Logf("EXPECTED snapshot error: %s", markerContents) 722 } 723 }) 724 } 725 726 func TestHTTP_AllocGC(t *testing.T) { 727 t.Parallel() 728 require := require.New(t) 729 path := fmt.Sprintf("/v1/client/allocation/%s/gc", uuid.Generate()) 730 httpTest(t, nil, func(s *TestAgent) { 731 // Local node, local resp 732 { 733 req, err := http.NewRequest("GET", path, nil) 734 if err != nil { 735 t.Fatalf("err: %v", err) 736 } 737 738 respW := httptest.NewRecorder() 739 _, err = s.Server.ClientAllocRequest(respW, req) 740 if !structs.IsErrUnknownAllocation(err) { 741 t.Fatalf("unexpected err: %v", err) 742 } 743 } 744 745 // Local node, server resp 746 { 747 srv := s.server 748 s.server = nil 749 750 req, err := http.NewRequest("GET", path, nil) 751 if err != nil { 752 t.Fatalf("err: %v", err) 753 } 754 755 respW := httptest.NewRecorder() 756 _, err = s.Server.ClientAllocRequest(respW, req) 757 if !structs.IsErrUnknownAllocation(err) { 758 t.Fatalf("unexpected err: %v", err) 759 } 760 761 s.server = srv 762 } 763 764 // no client, server resp 765 { 766 c := s.client 767 s.client = nil 768 769 testutil.WaitForResult(func() (bool, error) { 770 n, err := s.server.State().NodeByID(nil, c.NodeID()) 771 if err != nil { 772 return false, err 773 } 774 return n != nil, nil 775 }, func(err error) { 776 t.Fatalf("should have client: %v", err) 777 }) 778 779 req, err := http.NewRequest("GET", path, nil) 780 if err != nil { 781 t.Fatalf("err: %v", err) 782 } 783 784 respW := httptest.NewRecorder() 785 _, err = s.Server.ClientAllocRequest(respW, req) 786 require.NotNil(err) 787 if !structs.IsErrUnknownAllocation(err) { 788 t.Fatalf("unexpected err: %v", err) 789 } 790 791 s.client = c 792 } 793 }) 794 } 795 796 func TestHTTP_AllocGC_ACL(t *testing.T) { 797 t.Parallel() 798 require := require.New(t) 799 path := fmt.Sprintf("/v1/client/allocation/%s/gc", uuid.Generate()) 800 801 httpACLTest(t, nil, func(s *TestAgent) { 802 state := s.Agent.server.State() 803 804 // Make the HTTP request 805 req, err := http.NewRequest("GET", path, nil) 806 if err != nil { 807 t.Fatalf("err: %v", err) 808 } 809 810 // Try request without a token and expect failure 811 { 812 respW := httptest.NewRecorder() 813 _, err := s.Server.ClientAllocRequest(respW, req) 814 require.NotNil(err) 815 require.True(structs.IsErrUnknownAllocation(err), "(%T) %v", err, err) 816 } 817 818 // Try request with an invalid token and expect failure 819 { 820 respW := httptest.NewRecorder() 821 token := mock.CreatePolicyAndToken(t, state, 1005, "invalid", mock.NodePolicy(acl.PolicyWrite)) 822 setToken(req, token) 823 _, err := s.Server.ClientAllocRequest(respW, req) 824 require.NotNil(err) 825 require.True(structs.IsErrUnknownAllocation(err), "(%T) %v", err, err) 826 } 827 828 // Try request with a valid token 829 // Still returns an error because the alloc does not exist 830 { 831 respW := httptest.NewRecorder() 832 policy := mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilitySubmitJob}) 833 token := mock.CreatePolicyAndToken(t, state, 1007, "valid", policy) 834 setToken(req, token) 835 _, err := s.Server.ClientAllocRequest(respW, req) 836 require.NotNil(err) 837 require.True(structs.IsErrUnknownAllocation(err), "(%T) %v", err, err) 838 } 839 840 // Try request with a management token 841 // Still returns an error because the alloc does not exist 842 { 843 respW := httptest.NewRecorder() 844 setToken(req, s.RootToken) 845 _, err := s.Server.ClientAllocRequest(respW, req) 846 require.NotNil(err) 847 require.True(structs.IsErrUnknownAllocation(err)) 848 } 849 }) 850 } 851 852 func TestHTTP_AllocAllGC(t *testing.T) { 853 t.Parallel() 854 require := require.New(t) 855 httpTest(t, nil, func(s *TestAgent) { 856 // Local node, local resp 857 { 858 req, err := http.NewRequest("GET", "/v1/client/gc", nil) 859 if err != nil { 860 t.Fatalf("err: %v", err) 861 } 862 863 respW := httptest.NewRecorder() 864 _, err = s.Server.ClientGCRequest(respW, req) 865 if err != nil { 866 t.Fatalf("unexpected err: %v", err) 867 } 868 } 869 870 // Local node, server resp 871 { 872 srv := s.server 873 s.server = nil 874 875 req, err := http.NewRequest("GET", fmt.Sprintf("/v1/client/gc?node_id=%s", uuid.Generate()), nil) 876 require.Nil(err) 877 878 respW := httptest.NewRecorder() 879 _, err = s.Server.ClientGCRequest(respW, req) 880 require.NotNil(err) 881 require.Contains(err.Error(), "Unknown node") 882 883 s.server = srv 884 } 885 886 // client stats from server, should not error 887 { 888 c := s.client 889 s.client = nil 890 891 testutil.WaitForResult(func() (bool, error) { 892 n, err := s.server.State().NodeByID(nil, c.NodeID()) 893 if err != nil { 894 return false, err 895 } 896 return n != nil, nil 897 }, func(err error) { 898 t.Fatalf("should have client: %v", err) 899 }) 900 901 req, err := http.NewRequest("GET", fmt.Sprintf("/v1/client/gc?node_id=%s", c.NodeID()), nil) 902 require.Nil(err) 903 904 respW := httptest.NewRecorder() 905 _, err = s.Server.ClientGCRequest(respW, req) 906 require.Nil(err) 907 908 s.client = c 909 } 910 }) 911 912 } 913 914 func TestHTTP_AllocAllGC_ACL(t *testing.T) { 915 t.Parallel() 916 require := require.New(t) 917 httpACLTest(t, nil, func(s *TestAgent) { 918 state := s.Agent.server.State() 919 920 // Make the HTTP request 921 req, err := http.NewRequest("GET", "/v1/client/gc", nil) 922 require.Nil(err) 923 924 // Try request without a token and expect failure 925 { 926 respW := httptest.NewRecorder() 927 _, err := s.Server.ClientGCRequest(respW, req) 928 require.NotNil(err) 929 require.Equal(err.Error(), structs.ErrPermissionDenied.Error()) 930 } 931 932 // Try request with an invalid token and expect failure 933 { 934 respW := httptest.NewRecorder() 935 token := mock.CreatePolicyAndToken(t, state, 1005, "invalid", mock.NodePolicy(acl.PolicyRead)) 936 setToken(req, token) 937 _, err := s.Server.ClientGCRequest(respW, req) 938 require.NotNil(err) 939 require.Equal(err.Error(), structs.ErrPermissionDenied.Error()) 940 } 941 942 // Try request with a valid token 943 { 944 respW := httptest.NewRecorder() 945 token := mock.CreatePolicyAndToken(t, state, 1007, "valid", mock.NodePolicy(acl.PolicyWrite)) 946 setToken(req, token) 947 _, err := s.Server.ClientGCRequest(respW, req) 948 require.Nil(err) 949 require.Equal(http.StatusOK, respW.Code) 950 } 951 952 // Try request with a management token 953 { 954 respW := httptest.NewRecorder() 955 setToken(req, s.RootToken) 956 _, err := s.Server.ClientGCRequest(respW, req) 957 require.Nil(err) 958 require.Equal(http.StatusOK, respW.Code) 959 } 960 }) 961 }