github.com/dkerwin/nomad@v0.3.3-0.20160525181927-74554135514b/scheduler/system_sched_test.go (about) 1 package scheduler 2 3 import ( 4 "reflect" 5 "testing" 6 "time" 7 8 "github.com/hashicorp/nomad/nomad/mock" 9 "github.com/hashicorp/nomad/nomad/structs" 10 ) 11 12 func TestSystemSched_JobRegister(t *testing.T) { 13 h := NewHarness(t) 14 15 // Create some nodes 16 for i := 0; i < 10; i++ { 17 node := mock.Node() 18 noErr(t, h.State.UpsertNode(h.NextIndex(), node)) 19 } 20 21 // Create a job 22 job := mock.SystemJob() 23 noErr(t, h.State.UpsertJob(h.NextIndex(), job)) 24 25 // Create a mock evaluation to deregister the job 26 eval := &structs.Evaluation{ 27 ID: structs.GenerateUUID(), 28 Priority: job.Priority, 29 TriggeredBy: structs.EvalTriggerJobRegister, 30 JobID: job.ID, 31 } 32 33 // Process the evaluation 34 err := h.Process(NewSystemScheduler, eval) 35 if err != nil { 36 t.Fatalf("err: %v", err) 37 } 38 39 // Ensure a single plan 40 if len(h.Plans) != 1 { 41 t.Fatalf("bad: %#v", h.Plans) 42 } 43 plan := h.Plans[0] 44 45 // Ensure the plan doesn't have annotations. 46 if plan.Annotations != nil { 47 t.Fatalf("expected no annotations") 48 } 49 50 // Ensure the plan allocated 51 var planned []*structs.Allocation 52 for _, allocList := range plan.NodeAllocation { 53 planned = append(planned, allocList...) 54 } 55 if len(planned) != 10 { 56 t.Fatalf("bad: %#v", plan) 57 } 58 59 // Lookup the allocations by JobID 60 out, err := h.State.AllocsByJob(job.ID) 61 noErr(t, err) 62 63 // Ensure all allocations placed 64 if len(out) != 10 { 65 t.Fatalf("bad: %#v", out) 66 } 67 68 // Check the available nodes 69 if count, ok := out[0].Metrics.NodesAvailable["dc1"]; !ok || count != 10 { 70 t.Fatalf("bad: %#v", out[0].Metrics) 71 } 72 73 h.AssertEvalStatus(t, structs.EvalStatusComplete) 74 } 75 76 func TestSystemSched_JobRegister_Annotate(t *testing.T) { 77 h := NewHarness(t) 78 79 // Create some nodes 80 for i := 0; i < 10; i++ { 81 node := mock.Node() 82 noErr(t, h.State.UpsertNode(h.NextIndex(), node)) 83 } 84 85 // Create a job 86 job := mock.SystemJob() 87 noErr(t, h.State.UpsertJob(h.NextIndex(), job)) 88 89 // Create a mock evaluation to deregister the job 90 eval := &structs.Evaluation{ 91 ID: structs.GenerateUUID(), 92 Priority: job.Priority, 93 TriggeredBy: structs.EvalTriggerJobRegister, 94 JobID: job.ID, 95 AnnotatePlan: true, 96 } 97 98 // Process the evaluation 99 err := h.Process(NewSystemScheduler, eval) 100 if err != nil { 101 t.Fatalf("err: %v", err) 102 } 103 104 // Ensure a single plan 105 if len(h.Plans) != 1 { 106 t.Fatalf("bad: %#v", h.Plans) 107 } 108 plan := h.Plans[0] 109 110 // Ensure the plan allocated 111 var planned []*structs.Allocation 112 for _, allocList := range plan.NodeAllocation { 113 planned = append(planned, allocList...) 114 } 115 if len(planned) != 10 { 116 t.Fatalf("bad: %#v", plan) 117 } 118 119 // Lookup the allocations by JobID 120 out, err := h.State.AllocsByJob(job.ID) 121 noErr(t, err) 122 123 // Ensure all allocations placed 124 if len(out) != 10 { 125 t.Fatalf("bad: %#v", out) 126 } 127 128 // Check the available nodes 129 if count, ok := out[0].Metrics.NodesAvailable["dc1"]; !ok || count != 10 { 130 t.Fatalf("bad: %#v", out[0].Metrics) 131 } 132 133 h.AssertEvalStatus(t, structs.EvalStatusComplete) 134 135 // Ensure the plan had annotations. 136 if plan.Annotations == nil { 137 t.Fatalf("expected annotations") 138 } 139 140 desiredTGs := plan.Annotations.DesiredTGUpdates 141 if l := len(desiredTGs); l != 1 { 142 t.Fatalf("incorrect number of task groups; got %v; want %v", l, 1) 143 } 144 145 desiredChanges, ok := desiredTGs["web"] 146 if !ok { 147 t.Fatalf("expected task group web to have desired changes") 148 } 149 150 expected := &structs.DesiredUpdates{Place: 10} 151 if !reflect.DeepEqual(desiredChanges, expected) { 152 t.Fatalf("Unexpected desired updates; got %#v; want %#v", desiredChanges, expected) 153 } 154 } 155 156 func TestSystemSched_JobRegister_AddNode(t *testing.T) { 157 h := NewHarness(t) 158 159 // Create some nodes 160 var nodes []*structs.Node 161 for i := 0; i < 10; i++ { 162 node := mock.Node() 163 nodes = append(nodes, node) 164 noErr(t, h.State.UpsertNode(h.NextIndex(), node)) 165 } 166 167 // Generate a fake job with allocations 168 job := mock.SystemJob() 169 noErr(t, h.State.UpsertJob(h.NextIndex(), job)) 170 171 var allocs []*structs.Allocation 172 for _, node := range nodes { 173 alloc := mock.Alloc() 174 alloc.Job = job 175 alloc.JobID = job.ID 176 alloc.NodeID = node.ID 177 alloc.Name = "my-job.web[0]" 178 allocs = append(allocs, alloc) 179 } 180 noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) 181 182 // Add a new node. 183 node := mock.Node() 184 noErr(t, h.State.UpsertNode(h.NextIndex(), node)) 185 186 // Create a mock evaluation to deal with the node update 187 eval := &structs.Evaluation{ 188 ID: structs.GenerateUUID(), 189 Priority: 50, 190 TriggeredBy: structs.EvalTriggerNodeUpdate, 191 JobID: job.ID, 192 } 193 194 // Process the evaluation 195 err := h.Process(NewSystemScheduler, eval) 196 if err != nil { 197 t.Fatalf("err: %v", err) 198 } 199 200 // Ensure a single plan 201 if len(h.Plans) != 1 { 202 t.Fatalf("bad: %#v", h.Plans) 203 } 204 plan := h.Plans[0] 205 206 // Ensure the plan had no node updates 207 var update []*structs.Allocation 208 for _, updateList := range plan.NodeUpdate { 209 update = append(update, updateList...) 210 } 211 if len(update) != 0 { 212 t.Log(len(update)) 213 t.Fatalf("bad: %#v", plan) 214 } 215 216 // Ensure the plan allocated on the new node 217 var planned []*structs.Allocation 218 for _, allocList := range plan.NodeAllocation { 219 planned = append(planned, allocList...) 220 } 221 if len(planned) != 1 { 222 t.Fatalf("bad: %#v", plan) 223 } 224 225 // Ensure it allocated on the right node 226 if _, ok := plan.NodeAllocation[node.ID]; !ok { 227 t.Fatalf("allocated on wrong node: %#v", plan) 228 } 229 230 // Lookup the allocations by JobID 231 out, err := h.State.AllocsByJob(job.ID) 232 noErr(t, err) 233 234 // Ensure all allocations placed 235 out = structs.FilterTerminalAllocs(out) 236 if len(out) != 11 { 237 t.Fatalf("bad: %#v", out) 238 } 239 240 h.AssertEvalStatus(t, structs.EvalStatusComplete) 241 } 242 243 func TestSystemSched_JobRegister_AllocFail(t *testing.T) { 244 h := NewHarness(t) 245 246 // Create NO nodes 247 // Create a job 248 job := mock.SystemJob() 249 noErr(t, h.State.UpsertJob(h.NextIndex(), job)) 250 251 // Create a mock evaluation to register the job 252 eval := &structs.Evaluation{ 253 ID: structs.GenerateUUID(), 254 Priority: job.Priority, 255 TriggeredBy: structs.EvalTriggerJobRegister, 256 JobID: job.ID, 257 } 258 259 // Process the evaluation 260 err := h.Process(NewSystemScheduler, eval) 261 if err != nil { 262 t.Fatalf("err: %v", err) 263 } 264 265 // Ensure no plan as this should be a no-op. 266 if len(h.Plans) != 0 { 267 t.Fatalf("bad: %#v", h.Plans) 268 } 269 270 h.AssertEvalStatus(t, structs.EvalStatusComplete) 271 } 272 273 func TestSystemSched_JobModify(t *testing.T) { 274 h := NewHarness(t) 275 276 // Create some nodes 277 var nodes []*structs.Node 278 for i := 0; i < 10; i++ { 279 node := mock.Node() 280 nodes = append(nodes, node) 281 noErr(t, h.State.UpsertNode(h.NextIndex(), node)) 282 } 283 284 // Generate a fake job with allocations 285 job := mock.SystemJob() 286 noErr(t, h.State.UpsertJob(h.NextIndex(), job)) 287 288 var allocs []*structs.Allocation 289 for _, node := range nodes { 290 alloc := mock.Alloc() 291 alloc.Job = job 292 alloc.JobID = job.ID 293 alloc.NodeID = node.ID 294 alloc.Name = "my-job.web[0]" 295 allocs = append(allocs, alloc) 296 } 297 noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) 298 299 // Add a few terminal status allocations, these should be ignored 300 var terminal []*structs.Allocation 301 for i := 0; i < 5; i++ { 302 alloc := mock.Alloc() 303 alloc.Job = job 304 alloc.JobID = job.ID 305 alloc.NodeID = nodes[i].ID 306 alloc.Name = "my-job.web[0]" 307 alloc.DesiredStatus = structs.AllocDesiredStatusFailed 308 terminal = append(terminal, alloc) 309 } 310 noErr(t, h.State.UpsertAllocs(h.NextIndex(), terminal)) 311 312 // Update the job 313 job2 := mock.SystemJob() 314 job2.ID = job.ID 315 316 // Update the task, such that it cannot be done in-place 317 job2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other" 318 noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) 319 320 // Create a mock evaluation to deal with drain 321 eval := &structs.Evaluation{ 322 ID: structs.GenerateUUID(), 323 Priority: 50, 324 TriggeredBy: structs.EvalTriggerJobRegister, 325 JobID: job.ID, 326 } 327 328 // Process the evaluation 329 err := h.Process(NewSystemScheduler, eval) 330 if err != nil { 331 t.Fatalf("err: %v", err) 332 } 333 334 // Ensure a single plan 335 if len(h.Plans) != 1 { 336 t.Fatalf("bad: %#v", h.Plans) 337 } 338 plan := h.Plans[0] 339 340 // Ensure the plan evicted all allocs 341 var update []*structs.Allocation 342 for _, updateList := range plan.NodeUpdate { 343 update = append(update, updateList...) 344 } 345 if len(update) != len(allocs) { 346 t.Fatalf("bad: %#v", plan) 347 } 348 349 // Ensure the plan allocated 350 var planned []*structs.Allocation 351 for _, allocList := range plan.NodeAllocation { 352 planned = append(planned, allocList...) 353 } 354 if len(planned) != 10 { 355 t.Fatalf("bad: %#v", plan) 356 } 357 358 // Lookup the allocations by JobID 359 out, err := h.State.AllocsByJob(job.ID) 360 noErr(t, err) 361 362 // Ensure all allocations placed 363 out = structs.FilterTerminalAllocs(out) 364 if len(out) != 10 { 365 t.Fatalf("bad: %#v", out) 366 } 367 368 h.AssertEvalStatus(t, structs.EvalStatusComplete) 369 } 370 371 func TestSystemSched_JobModify_Rolling(t *testing.T) { 372 h := NewHarness(t) 373 374 // Create some nodes 375 var nodes []*structs.Node 376 for i := 0; i < 10; i++ { 377 node := mock.Node() 378 nodes = append(nodes, node) 379 noErr(t, h.State.UpsertNode(h.NextIndex(), node)) 380 } 381 382 // Generate a fake job with allocations 383 job := mock.SystemJob() 384 noErr(t, h.State.UpsertJob(h.NextIndex(), job)) 385 386 var allocs []*structs.Allocation 387 for _, node := range nodes { 388 alloc := mock.Alloc() 389 alloc.Job = job 390 alloc.JobID = job.ID 391 alloc.NodeID = node.ID 392 alloc.Name = "my-job.web[0]" 393 allocs = append(allocs, alloc) 394 } 395 noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) 396 397 // Update the job 398 job2 := mock.SystemJob() 399 job2.ID = job.ID 400 job2.Update = structs.UpdateStrategy{ 401 Stagger: 30 * time.Second, 402 MaxParallel: 5, 403 } 404 405 // Update the task, such that it cannot be done in-place 406 job2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other" 407 noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) 408 409 // Create a mock evaluation to deal with drain 410 eval := &structs.Evaluation{ 411 ID: structs.GenerateUUID(), 412 Priority: 50, 413 TriggeredBy: structs.EvalTriggerJobRegister, 414 JobID: job.ID, 415 } 416 417 // Process the evaluation 418 err := h.Process(NewSystemScheduler, eval) 419 if err != nil { 420 t.Fatalf("err: %v", err) 421 } 422 423 // Ensure a single plan 424 if len(h.Plans) != 1 { 425 t.Fatalf("bad: %#v", h.Plans) 426 } 427 plan := h.Plans[0] 428 429 // Ensure the plan evicted only MaxParallel 430 var update []*structs.Allocation 431 for _, updateList := range plan.NodeUpdate { 432 update = append(update, updateList...) 433 } 434 if len(update) != job2.Update.MaxParallel { 435 t.Fatalf("bad: %#v", plan) 436 } 437 438 // Ensure the plan allocated 439 var planned []*structs.Allocation 440 for _, allocList := range plan.NodeAllocation { 441 planned = append(planned, allocList...) 442 } 443 if len(planned) != job2.Update.MaxParallel { 444 t.Fatalf("bad: %#v", plan) 445 } 446 447 h.AssertEvalStatus(t, structs.EvalStatusComplete) 448 449 // Ensure a follow up eval was created 450 eval = h.Evals[0] 451 if eval.NextEval == "" { 452 t.Fatalf("missing next eval") 453 } 454 455 // Check for create 456 if len(h.CreateEvals) == 0 { 457 t.Fatalf("missing created eval") 458 } 459 create := h.CreateEvals[0] 460 if eval.NextEval != create.ID { 461 t.Fatalf("ID mismatch") 462 } 463 if create.PreviousEval != eval.ID { 464 t.Fatalf("missing previous eval") 465 } 466 467 if create.TriggeredBy != structs.EvalTriggerRollingUpdate { 468 t.Fatalf("bad: %#v", create) 469 } 470 } 471 472 func TestSystemSched_JobModify_InPlace(t *testing.T) { 473 h := NewHarness(t) 474 475 // Create some nodes 476 var nodes []*structs.Node 477 for i := 0; i < 10; i++ { 478 node := mock.Node() 479 nodes = append(nodes, node) 480 noErr(t, h.State.UpsertNode(h.NextIndex(), node)) 481 } 482 483 // Generate a fake job with allocations 484 job := mock.SystemJob() 485 noErr(t, h.State.UpsertJob(h.NextIndex(), job)) 486 487 var allocs []*structs.Allocation 488 for _, node := range nodes { 489 alloc := mock.Alloc() 490 alloc.Job = job 491 alloc.JobID = job.ID 492 alloc.NodeID = node.ID 493 alloc.Name = "my-job.web[0]" 494 allocs = append(allocs, alloc) 495 } 496 noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) 497 498 // Update the job 499 job2 := mock.SystemJob() 500 job2.ID = job.ID 501 noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) 502 503 // Create a mock evaluation to deal with drain 504 eval := &structs.Evaluation{ 505 ID: structs.GenerateUUID(), 506 Priority: 50, 507 TriggeredBy: structs.EvalTriggerJobRegister, 508 JobID: job.ID, 509 } 510 511 // Process the evaluation 512 err := h.Process(NewSystemScheduler, eval) 513 if err != nil { 514 t.Fatalf("err: %v", err) 515 } 516 517 // Ensure a single plan 518 if len(h.Plans) != 1 { 519 t.Fatalf("bad: %#v", h.Plans) 520 } 521 plan := h.Plans[0] 522 523 // Ensure the plan did not evict any allocs 524 var update []*structs.Allocation 525 for _, updateList := range plan.NodeUpdate { 526 update = append(update, updateList...) 527 } 528 if len(update) != 0 { 529 t.Fatalf("bad: %#v", plan) 530 } 531 532 // Ensure the plan updated the existing allocs 533 var planned []*structs.Allocation 534 for _, allocList := range plan.NodeAllocation { 535 planned = append(planned, allocList...) 536 } 537 if len(planned) != 10 { 538 t.Fatalf("bad: %#v", plan) 539 } 540 for _, p := range planned { 541 if p.Job != job2 { 542 t.Fatalf("should update job") 543 } 544 } 545 546 // Lookup the allocations by JobID 547 out, err := h.State.AllocsByJob(job.ID) 548 noErr(t, err) 549 550 // Ensure all allocations placed 551 if len(out) != 10 { 552 t.Fatalf("bad: %#v", out) 553 } 554 h.AssertEvalStatus(t, structs.EvalStatusComplete) 555 556 // Verify the network did not change 557 rp := structs.Port{Label: "main", Value: 5000} 558 for _, alloc := range out { 559 for _, resources := range alloc.TaskResources { 560 if resources.Networks[0].ReservedPorts[0] != rp { 561 t.Fatalf("bad: %#v", alloc) 562 } 563 } 564 } 565 } 566 567 func TestSystemSched_JobDeregister(t *testing.T) { 568 h := NewHarness(t) 569 570 // Create some nodes 571 var nodes []*structs.Node 572 for i := 0; i < 10; i++ { 573 node := mock.Node() 574 nodes = append(nodes, node) 575 noErr(t, h.State.UpsertNode(h.NextIndex(), node)) 576 } 577 578 // Generate a fake job with allocations 579 job := mock.SystemJob() 580 581 var allocs []*structs.Allocation 582 for _, node := range nodes { 583 alloc := mock.Alloc() 584 alloc.Job = job 585 alloc.JobID = job.ID 586 alloc.NodeID = node.ID 587 alloc.Name = "my-job.web[0]" 588 allocs = append(allocs, alloc) 589 } 590 noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) 591 592 // Create a mock evaluation to deregister the job 593 eval := &structs.Evaluation{ 594 ID: structs.GenerateUUID(), 595 Priority: 50, 596 TriggeredBy: structs.EvalTriggerJobDeregister, 597 JobID: job.ID, 598 } 599 600 // Process the evaluation 601 err := h.Process(NewSystemScheduler, eval) 602 if err != nil { 603 t.Fatalf("err: %v", err) 604 } 605 606 // Ensure a single plan 607 if len(h.Plans) != 1 { 608 t.Fatalf("bad: %#v", h.Plans) 609 } 610 plan := h.Plans[0] 611 612 // Ensure the plan evicted the job from all nodes. 613 for _, node := range nodes { 614 if len(plan.NodeUpdate[node.ID]) != 1 { 615 t.Fatalf("bad: %#v", plan) 616 } 617 } 618 619 // Lookup the allocations by JobID 620 out, err := h.State.AllocsByJob(job.ID) 621 noErr(t, err) 622 623 // Ensure no remaining allocations 624 out = structs.FilterTerminalAllocs(out) 625 if len(out) != 0 { 626 t.Fatalf("bad: %#v", out) 627 } 628 629 h.AssertEvalStatus(t, structs.EvalStatusComplete) 630 } 631 632 func TestSystemSched_NodeDrain(t *testing.T) { 633 h := NewHarness(t) 634 635 // Register a draining node 636 node := mock.Node() 637 node.Drain = true 638 noErr(t, h.State.UpsertNode(h.NextIndex(), node)) 639 640 // Generate a fake job allocated on that node. 641 job := mock.SystemJob() 642 noErr(t, h.State.UpsertJob(h.NextIndex(), job)) 643 644 alloc := mock.Alloc() 645 alloc.Job = job 646 alloc.JobID = job.ID 647 alloc.NodeID = node.ID 648 alloc.Name = "my-job.web[0]" 649 noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) 650 651 // Create a mock evaluation to deal with drain 652 eval := &structs.Evaluation{ 653 ID: structs.GenerateUUID(), 654 Priority: 50, 655 TriggeredBy: structs.EvalTriggerNodeUpdate, 656 JobID: job.ID, 657 NodeID: node.ID, 658 } 659 660 // Process the evaluation 661 err := h.Process(NewSystemScheduler, eval) 662 if err != nil { 663 t.Fatalf("err: %v", err) 664 } 665 666 // Ensure a single plan 667 if len(h.Plans) != 1 { 668 t.Fatalf("bad: %#v", h.Plans) 669 } 670 plan := h.Plans[0] 671 672 // Ensure the plan evicted all allocs 673 if len(plan.NodeUpdate[node.ID]) != 1 { 674 t.Fatalf("bad: %#v", plan) 675 } 676 677 // Ensure the plan updated the allocation. 678 var planned []*structs.Allocation 679 for _, allocList := range plan.NodeUpdate { 680 planned = append(planned, allocList...) 681 } 682 if len(planned) != 1 { 683 t.Log(len(planned)) 684 t.Fatalf("bad: %#v", plan) 685 } 686 687 // Lookup the allocations by JobID 688 out, err := h.State.AllocsByJob(job.ID) 689 noErr(t, err) 690 691 // Ensure the allocations is stopped 692 if planned[0].DesiredStatus != structs.AllocDesiredStatusStop { 693 t.Fatalf("bad: %#v", out) 694 } 695 696 h.AssertEvalStatus(t, structs.EvalStatusComplete) 697 } 698 699 func TestSystemSched_RetryLimit(t *testing.T) { 700 h := NewHarness(t) 701 h.Planner = &RejectPlan{h} 702 703 // Create some nodes 704 for i := 0; i < 10; i++ { 705 node := mock.Node() 706 noErr(t, h.State.UpsertNode(h.NextIndex(), node)) 707 } 708 709 // Create a job 710 job := mock.SystemJob() 711 noErr(t, h.State.UpsertJob(h.NextIndex(), job)) 712 713 // Create a mock evaluation to deregister the job 714 eval := &structs.Evaluation{ 715 ID: structs.GenerateUUID(), 716 Priority: job.Priority, 717 TriggeredBy: structs.EvalTriggerJobRegister, 718 JobID: job.ID, 719 } 720 721 // Process the evaluation 722 err := h.Process(NewSystemScheduler, eval) 723 if err != nil { 724 t.Fatalf("err: %v", err) 725 } 726 727 // Ensure multiple plans 728 if len(h.Plans) == 0 { 729 t.Fatalf("bad: %#v", h.Plans) 730 } 731 732 // Lookup the allocations by JobID 733 out, err := h.State.AllocsByJob(job.ID) 734 noErr(t, err) 735 736 // Ensure no allocations placed 737 if len(out) != 0 { 738 t.Fatalf("bad: %#v", out) 739 } 740 741 // Should hit the retry limit 742 h.AssertEvalStatus(t, structs.EvalStatusFailed) 743 }