github.com/ranjib/nomad@v0.1.1-0.20160225204057-97751b02f70b/scheduler/generic_sched_test.go (about) 1 package scheduler 2 3 import ( 4 "fmt" 5 "testing" 6 "time" 7 8 "github.com/hashicorp/nomad/nomad/mock" 9 "github.com/hashicorp/nomad/nomad/structs" 10 ) 11 12 func TestServiceSched_JobRegister(t *testing.T) { 13 h := NewHarness(t) 14 15 // Create some nodes 16 for i := 0; i < 10; i++ { 17 node := mock.Node() 18 noErr(t, h.State.UpsertNode(h.NextIndex(), node)) 19 } 20 21 // Create a job 22 job := mock.Job() 23 noErr(t, h.State.UpsertJob(h.NextIndex(), job)) 24 25 // Create a mock evaluation to register the job 26 eval := &structs.Evaluation{ 27 ID: structs.GenerateUUID(), 28 Priority: job.Priority, 29 TriggeredBy: structs.EvalTriggerJobRegister, 30 JobID: job.ID, 31 } 32 33 // Process the evaluation 34 err := h.Process(NewServiceScheduler, eval) 35 if err != nil { 36 t.Fatalf("err: %v", err) 37 } 38 39 // Ensure a single plan 40 if len(h.Plans) != 1 { 41 t.Fatalf("bad: %#v", h.Plans) 42 } 43 plan := h.Plans[0] 44 45 // Ensure the plan allocated 46 var planned []*structs.Allocation 47 for _, allocList := range plan.NodeAllocation { 48 planned = append(planned, allocList...) 49 } 50 if len(planned) != 10 { 51 t.Fatalf("bad: %#v", plan) 52 } 53 54 // Lookup the allocations by JobID 55 out, err := h.State.AllocsByJob(job.ID) 56 noErr(t, err) 57 58 // Ensure all allocations placed 59 if len(out) != 10 { 60 t.Fatalf("bad: %#v", out) 61 } 62 63 // Ensure different ports were used. 64 used := make(map[int]struct{}) 65 for _, alloc := range out { 66 for _, resource := range alloc.TaskResources { 67 for _, port := range resource.Networks[0].DynamicPorts { 68 if _, ok := used[port.Value]; ok { 69 t.Fatalf("Port collision %v", port.Value) 70 } 71 used[port.Value] = struct{}{} 72 } 73 } 74 } 75 76 h.AssertEvalStatus(t, structs.EvalStatusComplete) 77 } 78 79 func TestServiceSched_JobRegister_AllocFail(t *testing.T) { 80 h := NewHarness(t) 81 82 // Create NO nodes 83 // Create a job 84 job := mock.Job() 85 noErr(t, h.State.UpsertJob(h.NextIndex(), job)) 86 87 // Create a mock evaluation to register the job 88 eval := &structs.Evaluation{ 89 ID: structs.GenerateUUID(), 90 Priority: job.Priority, 91 TriggeredBy: structs.EvalTriggerJobRegister, 92 JobID: job.ID, 93 } 94 95 // Process the evaluation 96 err := h.Process(NewServiceScheduler, eval) 97 if err != nil { 98 t.Fatalf("err: %v", err) 99 } 100 101 // Ensure a single plan 102 if len(h.Plans) != 1 { 103 t.Fatalf("bad: %#v", h.Plans) 104 } 105 plan := h.Plans[0] 106 107 // Ensure the plan has created a follow up eval. 108 if len(h.CreateEvals) != 1 || h.CreateEvals[0].Status != structs.EvalStatusBlocked { 109 t.Fatalf("bad: %#v", h.CreateEvals) 110 } 111 112 // Ensure the plan failed to alloc 113 if len(plan.FailedAllocs) != 1 { 114 t.Fatalf("bad: %#v", plan) 115 } 116 117 // Lookup the allocations by JobID 118 out, err := h.State.AllocsByJob(job.ID) 119 noErr(t, err) 120 121 // Ensure all allocations placed 122 if len(out) != 1 { 123 t.Fatalf("bad: %#v", out) 124 } 125 126 // Check the coalesced failures 127 if out[0].Metrics.CoalescedFailures != 9 { 128 t.Fatalf("bad: %#v", out[0].Metrics) 129 } 130 131 // Check the available nodes 132 if count, ok := out[0].Metrics.NodesAvailable["dc1"]; !ok || count != 0 { 133 t.Fatalf("bad: %#v", out[0].Metrics) 134 } 135 136 h.AssertEvalStatus(t, structs.EvalStatusComplete) 137 } 138 139 func TestServiceSched_JobRegister_BlockedEval(t *testing.T) { 140 h := NewHarness(t) 141 142 // Create a full node 143 node := mock.Node() 144 node.Reserved = node.Resources 145 node.ComputeClass() 146 noErr(t, h.State.UpsertNode(h.NextIndex(), node)) 147 148 // Create an ineligible node 149 node2 := mock.Node() 150 node2.Attributes["kernel.name"] = "windows" 151 node2.ComputeClass() 152 noErr(t, h.State.UpsertNode(h.NextIndex(), node2)) 153 154 // Create a jobs 155 job := mock.Job() 156 noErr(t, h.State.UpsertJob(h.NextIndex(), job)) 157 158 // Create a mock evaluation to register the job 159 eval := &structs.Evaluation{ 160 ID: structs.GenerateUUID(), 161 Priority: job.Priority, 162 TriggeredBy: structs.EvalTriggerJobRegister, 163 JobID: job.ID, 164 } 165 166 // Process the evaluation 167 err := h.Process(NewServiceScheduler, eval) 168 if err != nil { 169 t.Fatalf("err: %v", err) 170 } 171 172 // Ensure a single plan 173 if len(h.Plans) != 1 { 174 t.Fatalf("bad: %#v", h.Plans) 175 } 176 plan := h.Plans[0] 177 178 // Ensure the plan has created a follow up eval. 179 if len(h.CreateEvals) != 1 { 180 t.Fatalf("bad: %#v", h.CreateEvals) 181 } 182 183 created := h.CreateEvals[0] 184 if created.Status != structs.EvalStatusBlocked { 185 t.Fatalf("bad: %#v", created) 186 } 187 188 classes := created.ClassEligibility 189 if len(classes) != 2 || !classes[node.ComputedClass] || classes[node2.ComputedClass] { 190 t.Fatalf("bad: %#v", classes) 191 } 192 193 if created.EscapedComputedClass { 194 t.Fatalf("bad: %#v", created) 195 } 196 197 // Ensure the plan failed to alloc 198 if len(plan.FailedAllocs) != 1 { 199 t.Fatalf("bad: %#v", plan) 200 } 201 202 // Lookup the allocations by JobID 203 out, err := h.State.AllocsByJob(job.ID) 204 noErr(t, err) 205 206 // Ensure all allocations placed 207 if len(out) != 1 { 208 for _, a := range out { 209 t.Logf("%#v", a) 210 } 211 t.Fatalf("bad: %#v", out) 212 } 213 214 // Check the coalesced failures 215 if out[0].Metrics.CoalescedFailures != 9 { 216 t.Fatalf("bad: %#v", out[0].Metrics) 217 } 218 219 // Check the available nodes 220 if count, ok := out[0].Metrics.NodesAvailable["dc1"]; !ok || count != 2 { 221 t.Fatalf("bad: %#v", out[0].Metrics) 222 } 223 224 h.AssertEvalStatus(t, structs.EvalStatusComplete) 225 } 226 227 func TestServiceSched_JobRegister_FeasibleAndInfeasibleTG(t *testing.T) { 228 h := NewHarness(t) 229 230 // Create one node 231 node := mock.Node() 232 node.NodeClass = "class_0" 233 noErr(t, node.ComputeClass()) 234 noErr(t, h.State.UpsertNode(h.NextIndex(), node)) 235 236 // Create a job that constrains on a node class 237 job := mock.Job() 238 job.TaskGroups[0].Count = 2 239 job.TaskGroups[0].Constraints = append(job.Constraints, 240 &structs.Constraint{ 241 LTarget: "${node.class}", 242 RTarget: "class_0", 243 Operand: "=", 244 }, 245 ) 246 tg2 := job.TaskGroups[0].Copy() 247 tg2.Name = "web2" 248 tg2.Constraints[1].RTarget = "class_1" 249 job.TaskGroups = append(job.TaskGroups, tg2) 250 noErr(t, h.State.UpsertJob(h.NextIndex(), job)) 251 252 // Create a mock evaluation to register the job 253 eval := &structs.Evaluation{ 254 ID: structs.GenerateUUID(), 255 Priority: job.Priority, 256 TriggeredBy: structs.EvalTriggerJobRegister, 257 JobID: job.ID, 258 } 259 260 // Process the evaluation 261 err := h.Process(NewServiceScheduler, eval) 262 if err != nil { 263 t.Fatalf("err: %v", err) 264 } 265 266 // Ensure a single plan 267 if len(h.Plans) != 1 { 268 t.Fatalf("bad: %#v", h.Plans) 269 } 270 plan := h.Plans[0] 271 272 // Ensure the plan allocated 273 var planned []*structs.Allocation 274 for _, allocList := range plan.NodeAllocation { 275 planned = append(planned, allocList...) 276 } 277 if len(planned) != 2 { 278 t.Fatalf("bad: %#v", plan) 279 } 280 if len(plan.FailedAllocs) != 1 { 281 t.Fatalf("bad: %#v", plan) 282 } 283 284 // Lookup the allocations by JobID 285 out, err := h.State.AllocsByJob(job.ID) 286 noErr(t, err) 287 288 // Ensure all allocations placed 289 if len(out) != 3 { 290 t.Fatalf("bad: %#v", out) 291 } 292 293 h.AssertEvalStatus(t, structs.EvalStatusComplete) 294 } 295 296 func TestServiceSched_JobModify(t *testing.T) { 297 h := NewHarness(t) 298 299 // Create some nodes 300 var nodes []*structs.Node 301 for i := 0; i < 10; i++ { 302 node := mock.Node() 303 nodes = append(nodes, node) 304 noErr(t, h.State.UpsertNode(h.NextIndex(), node)) 305 } 306 307 // Generate a fake job with allocations 308 job := mock.Job() 309 noErr(t, h.State.UpsertJob(h.NextIndex(), job)) 310 311 var allocs []*structs.Allocation 312 for i := 0; i < 10; i++ { 313 alloc := mock.Alloc() 314 alloc.Job = job 315 alloc.JobID = job.ID 316 alloc.NodeID = nodes[i].ID 317 alloc.Name = fmt.Sprintf("my-job.web[%d]", i) 318 allocs = append(allocs, alloc) 319 } 320 noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) 321 322 // Add a few terminal status allocations, these should be ignored 323 var terminal []*structs.Allocation 324 for i := 0; i < 5; i++ { 325 alloc := mock.Alloc() 326 alloc.Job = job 327 alloc.JobID = job.ID 328 alloc.NodeID = nodes[i].ID 329 alloc.Name = fmt.Sprintf("my-job.web[%d]", i) 330 alloc.DesiredStatus = structs.AllocDesiredStatusFailed 331 terminal = append(terminal, alloc) 332 } 333 noErr(t, h.State.UpsertAllocs(h.NextIndex(), terminal)) 334 335 // Update the job 336 job2 := mock.Job() 337 job2.ID = job.ID 338 339 // Update the task, such that it cannot be done in-place 340 job2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other" 341 noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) 342 343 // Create a mock evaluation to deal with drain 344 eval := &structs.Evaluation{ 345 ID: structs.GenerateUUID(), 346 Priority: 50, 347 TriggeredBy: structs.EvalTriggerJobRegister, 348 JobID: job.ID, 349 } 350 351 // Process the evaluation 352 err := h.Process(NewServiceScheduler, eval) 353 if err != nil { 354 t.Fatalf("err: %v", err) 355 } 356 357 // Ensure a single plan 358 if len(h.Plans) != 1 { 359 t.Fatalf("bad: %#v", h.Plans) 360 } 361 plan := h.Plans[0] 362 363 // Ensure the plan evicted all allocs 364 var update []*structs.Allocation 365 for _, updateList := range plan.NodeUpdate { 366 update = append(update, updateList...) 367 } 368 if len(update) != len(allocs) { 369 t.Fatalf("bad: %#v", plan) 370 } 371 372 // Ensure the plan allocated 373 var planned []*structs.Allocation 374 for _, allocList := range plan.NodeAllocation { 375 planned = append(planned, allocList...) 376 } 377 if len(planned) != 10 { 378 t.Fatalf("bad: %#v", plan) 379 } 380 381 // Lookup the allocations by JobID 382 out, err := h.State.AllocsByJob(job.ID) 383 noErr(t, err) 384 385 // Ensure all allocations placed 386 out = structs.FilterTerminalAllocs(out) 387 if len(out) != 10 { 388 t.Fatalf("bad: %#v", out) 389 } 390 391 h.AssertEvalStatus(t, structs.EvalStatusComplete) 392 } 393 394 func TestServiceSched_JobModify_Rolling(t *testing.T) { 395 h := NewHarness(t) 396 397 // Create some nodes 398 var nodes []*structs.Node 399 for i := 0; i < 10; i++ { 400 node := mock.Node() 401 nodes = append(nodes, node) 402 noErr(t, h.State.UpsertNode(h.NextIndex(), node)) 403 } 404 405 // Generate a fake job with allocations 406 job := mock.Job() 407 noErr(t, h.State.UpsertJob(h.NextIndex(), job)) 408 409 var allocs []*structs.Allocation 410 for i := 0; i < 10; i++ { 411 alloc := mock.Alloc() 412 alloc.Job = job 413 alloc.JobID = job.ID 414 alloc.NodeID = nodes[i].ID 415 alloc.Name = fmt.Sprintf("my-job.web[%d]", i) 416 allocs = append(allocs, alloc) 417 } 418 noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) 419 420 // Update the job 421 job2 := mock.Job() 422 job2.ID = job.ID 423 job2.Update = structs.UpdateStrategy{ 424 Stagger: 30 * time.Second, 425 MaxParallel: 5, 426 } 427 428 // Update the task, such that it cannot be done in-place 429 job2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other" 430 noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) 431 432 // Create a mock evaluation to deal with drain 433 eval := &structs.Evaluation{ 434 ID: structs.GenerateUUID(), 435 Priority: 50, 436 TriggeredBy: structs.EvalTriggerJobRegister, 437 JobID: job.ID, 438 } 439 440 // Process the evaluation 441 err := h.Process(NewServiceScheduler, eval) 442 if err != nil { 443 t.Fatalf("err: %v", err) 444 } 445 446 // Ensure a single plan 447 if len(h.Plans) != 1 { 448 t.Fatalf("bad: %#v", h.Plans) 449 } 450 plan := h.Plans[0] 451 452 // Ensure the plan evicted only MaxParallel 453 var update []*structs.Allocation 454 for _, updateList := range plan.NodeUpdate { 455 update = append(update, updateList...) 456 } 457 if len(update) != job2.Update.MaxParallel { 458 t.Fatalf("bad: %#v", plan) 459 } 460 461 // Ensure the plan allocated 462 var planned []*structs.Allocation 463 for _, allocList := range plan.NodeAllocation { 464 planned = append(planned, allocList...) 465 } 466 if len(planned) != job2.Update.MaxParallel { 467 t.Fatalf("bad: %#v", plan) 468 } 469 470 h.AssertEvalStatus(t, structs.EvalStatusComplete) 471 472 // Ensure a follow up eval was created 473 eval = h.Evals[0] 474 if eval.NextEval == "" { 475 t.Fatalf("missing next eval") 476 } 477 478 // Check for create 479 if len(h.CreateEvals) == 0 { 480 t.Fatalf("missing created eval") 481 } 482 create := h.CreateEvals[0] 483 if eval.NextEval != create.ID { 484 t.Fatalf("ID mismatch") 485 } 486 if create.PreviousEval != eval.ID { 487 t.Fatalf("missing previous eval") 488 } 489 490 if create.TriggeredBy != structs.EvalTriggerRollingUpdate { 491 t.Fatalf("bad: %#v", create) 492 } 493 } 494 495 func TestServiceSched_JobModify_InPlace(t *testing.T) { 496 h := NewHarness(t) 497 498 // Create some nodes 499 var nodes []*structs.Node 500 for i := 0; i < 10; i++ { 501 node := mock.Node() 502 nodes = append(nodes, node) 503 noErr(t, h.State.UpsertNode(h.NextIndex(), node)) 504 } 505 506 // Generate a fake job with allocations 507 job := mock.Job() 508 noErr(t, h.State.UpsertJob(h.NextIndex(), job)) 509 510 var allocs []*structs.Allocation 511 for i := 0; i < 10; i++ { 512 alloc := mock.Alloc() 513 alloc.Job = job 514 alloc.JobID = job.ID 515 alloc.NodeID = nodes[i].ID 516 alloc.Name = fmt.Sprintf("my-job.web[%d]", i) 517 allocs = append(allocs, alloc) 518 } 519 noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) 520 521 // Update the job 522 job2 := mock.Job() 523 job2.ID = job.ID 524 noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) 525 526 // Create a mock evaluation to deal with drain 527 eval := &structs.Evaluation{ 528 ID: structs.GenerateUUID(), 529 Priority: 50, 530 TriggeredBy: structs.EvalTriggerJobRegister, 531 JobID: job.ID, 532 } 533 534 // Process the evaluation 535 err := h.Process(NewServiceScheduler, eval) 536 if err != nil { 537 t.Fatalf("err: %v", err) 538 } 539 540 // Ensure a single plan 541 if len(h.Plans) != 1 { 542 t.Fatalf("bad: %#v", h.Plans) 543 } 544 plan := h.Plans[0] 545 546 // Ensure the plan did not evict any allocs 547 var update []*structs.Allocation 548 for _, updateList := range plan.NodeUpdate { 549 update = append(update, updateList...) 550 } 551 if len(update) != 0 { 552 t.Fatalf("bad: %#v", plan) 553 } 554 555 // Ensure the plan updated the existing allocs 556 var planned []*structs.Allocation 557 for _, allocList := range plan.NodeAllocation { 558 planned = append(planned, allocList...) 559 } 560 if len(planned) != 10 { 561 t.Fatalf("bad: %#v", plan) 562 } 563 for _, p := range planned { 564 if p.Job != job2 { 565 t.Fatalf("should update job") 566 } 567 } 568 569 // Lookup the allocations by JobID 570 out, err := h.State.AllocsByJob(job.ID) 571 noErr(t, err) 572 573 // Ensure all allocations placed 574 if len(out) != 10 { 575 for _, alloc := range out { 576 t.Logf("%#v", alloc) 577 } 578 t.Fatalf("bad: %#v", out) 579 } 580 h.AssertEvalStatus(t, structs.EvalStatusComplete) 581 582 // Verify the network did not change 583 rp := structs.Port{Label: "main", Value: 5000} 584 for _, alloc := range out { 585 for _, resources := range alloc.TaskResources { 586 if resources.Networks[0].ReservedPorts[0] != rp { 587 t.Fatalf("bad: %#v", alloc) 588 } 589 } 590 } 591 } 592 593 func TestServiceSched_JobDeregister(t *testing.T) { 594 h := NewHarness(t) 595 596 // Generate a fake job with allocations 597 job := mock.Job() 598 599 var allocs []*structs.Allocation 600 for i := 0; i < 10; i++ { 601 alloc := mock.Alloc() 602 alloc.Job = job 603 alloc.JobID = job.ID 604 allocs = append(allocs, alloc) 605 } 606 noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) 607 608 // Create a mock evaluation to deregister the job 609 eval := &structs.Evaluation{ 610 ID: structs.GenerateUUID(), 611 Priority: 50, 612 TriggeredBy: structs.EvalTriggerJobDeregister, 613 JobID: job.ID, 614 } 615 616 // Process the evaluation 617 err := h.Process(NewServiceScheduler, eval) 618 if err != nil { 619 t.Fatalf("err: %v", err) 620 } 621 622 // Ensure a single plan 623 if len(h.Plans) != 1 { 624 t.Fatalf("bad: %#v", h.Plans) 625 } 626 plan := h.Plans[0] 627 628 // Ensure the plan evicted all nodes 629 if len(plan.NodeUpdate["12345678-abcd-efab-cdef-123456789abc"]) != len(allocs) { 630 t.Fatalf("bad: %#v", plan) 631 } 632 633 // Lookup the allocations by JobID 634 out, err := h.State.AllocsByJob(job.ID) 635 noErr(t, err) 636 637 // Ensure that the job field on the allocation is still populated 638 for _, alloc := range out { 639 if alloc.Job == nil { 640 t.Fatalf("bad: %#v", alloc) 641 } 642 } 643 644 // Ensure no remaining allocations 645 out = structs.FilterTerminalAllocs(out) 646 if len(out) != 0 { 647 t.Fatalf("bad: %#v", out) 648 } 649 650 h.AssertEvalStatus(t, structs.EvalStatusComplete) 651 } 652 653 func TestServiceSched_NodeDrain(t *testing.T) { 654 h := NewHarness(t) 655 656 // Register a draining node 657 node := mock.Node() 658 node.Drain = true 659 noErr(t, h.State.UpsertNode(h.NextIndex(), node)) 660 661 // Create some nodes 662 for i := 0; i < 10; i++ { 663 node := mock.Node() 664 noErr(t, h.State.UpsertNode(h.NextIndex(), node)) 665 } 666 667 // Generate a fake job with allocations and an update policy. 668 job := mock.Job() 669 noErr(t, h.State.UpsertJob(h.NextIndex(), job)) 670 671 var allocs []*structs.Allocation 672 for i := 0; i < 10; i++ { 673 alloc := mock.Alloc() 674 alloc.Job = job 675 alloc.JobID = job.ID 676 alloc.NodeID = node.ID 677 alloc.Name = fmt.Sprintf("my-job.web[%d]", i) 678 allocs = append(allocs, alloc) 679 } 680 noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) 681 682 // Create a mock evaluation to deal with drain 683 eval := &structs.Evaluation{ 684 ID: structs.GenerateUUID(), 685 Priority: 50, 686 TriggeredBy: structs.EvalTriggerNodeUpdate, 687 JobID: job.ID, 688 NodeID: node.ID, 689 } 690 691 // Process the evaluation 692 err := h.Process(NewServiceScheduler, eval) 693 if err != nil { 694 t.Fatalf("err: %v", err) 695 } 696 697 // Ensure a single plan 698 if len(h.Plans) != 1 { 699 t.Fatalf("bad: %#v", h.Plans) 700 } 701 plan := h.Plans[0] 702 703 // Ensure the plan evicted all allocs 704 if len(plan.NodeUpdate[node.ID]) != len(allocs) { 705 t.Fatalf("bad: %#v", plan) 706 } 707 708 // Ensure the plan allocated 709 var planned []*structs.Allocation 710 for _, allocList := range plan.NodeAllocation { 711 planned = append(planned, allocList...) 712 } 713 if len(planned) != 10 { 714 t.Fatalf("bad: %#v", plan) 715 } 716 717 // Lookup the allocations by JobID 718 out, err := h.State.AllocsByJob(job.ID) 719 noErr(t, err) 720 721 // Ensure all allocations placed 722 out = structs.FilterTerminalAllocs(out) 723 if len(out) != 10 { 724 t.Fatalf("bad: %#v", out) 725 } 726 727 h.AssertEvalStatus(t, structs.EvalStatusComplete) 728 } 729 730 func TestServiceSched_NodeDrain_UpdateStrategy(t *testing.T) { 731 h := NewHarness(t) 732 733 // Register a draining node 734 node := mock.Node() 735 node.Drain = true 736 noErr(t, h.State.UpsertNode(h.NextIndex(), node)) 737 738 // Create some nodes 739 for i := 0; i < 10; i++ { 740 node := mock.Node() 741 noErr(t, h.State.UpsertNode(h.NextIndex(), node)) 742 } 743 744 // Generate a fake job with allocations and an update policy. 745 job := mock.Job() 746 mp := 5 747 job.Update = structs.UpdateStrategy{ 748 Stagger: time.Second, 749 MaxParallel: mp, 750 } 751 noErr(t, h.State.UpsertJob(h.NextIndex(), job)) 752 753 var allocs []*structs.Allocation 754 for i := 0; i < 10; i++ { 755 alloc := mock.Alloc() 756 alloc.Job = job 757 alloc.JobID = job.ID 758 alloc.NodeID = node.ID 759 alloc.Name = fmt.Sprintf("my-job.web[%d]", i) 760 allocs = append(allocs, alloc) 761 } 762 noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) 763 764 // Create a mock evaluation to deal with drain 765 eval := &structs.Evaluation{ 766 ID: structs.GenerateUUID(), 767 Priority: 50, 768 TriggeredBy: structs.EvalTriggerNodeUpdate, 769 JobID: job.ID, 770 NodeID: node.ID, 771 } 772 773 // Process the evaluation 774 err := h.Process(NewServiceScheduler, eval) 775 if err != nil { 776 t.Fatalf("err: %v", err) 777 } 778 779 // Ensure a single plan 780 if len(h.Plans) != 1 { 781 t.Fatalf("bad: %#v", h.Plans) 782 } 783 plan := h.Plans[0] 784 785 // Ensure the plan evicted all allocs 786 if len(plan.NodeUpdate[node.ID]) != mp { 787 t.Fatalf("bad: %#v", plan) 788 } 789 790 // Ensure the plan allocated 791 var planned []*structs.Allocation 792 for _, allocList := range plan.NodeAllocation { 793 planned = append(planned, allocList...) 794 } 795 if len(planned) != mp { 796 t.Fatalf("bad: %#v", plan) 797 } 798 799 // Ensure there is a followup eval. 800 if len(h.CreateEvals) != 1 || 801 h.CreateEvals[0].TriggeredBy != structs.EvalTriggerRollingUpdate { 802 t.Fatalf("bad: %#v", h.CreateEvals) 803 } 804 805 h.AssertEvalStatus(t, structs.EvalStatusComplete) 806 } 807 808 func TestServiceSched_RetryLimit(t *testing.T) { 809 h := NewHarness(t) 810 h.Planner = &RejectPlan{h} 811 812 // Create some nodes 813 for i := 0; i < 10; i++ { 814 node := mock.Node() 815 noErr(t, h.State.UpsertNode(h.NextIndex(), node)) 816 } 817 818 // Create a job 819 job := mock.Job() 820 noErr(t, h.State.UpsertJob(h.NextIndex(), job)) 821 822 // Create a mock evaluation to register the job 823 eval := &structs.Evaluation{ 824 ID: structs.GenerateUUID(), 825 Priority: job.Priority, 826 TriggeredBy: structs.EvalTriggerJobRegister, 827 JobID: job.ID, 828 } 829 830 // Process the evaluation 831 err := h.Process(NewServiceScheduler, eval) 832 if err != nil { 833 t.Fatalf("err: %v", err) 834 } 835 836 // Ensure multiple plans 837 if len(h.Plans) == 0 { 838 t.Fatalf("bad: %#v", h.Plans) 839 } 840 841 // Lookup the allocations by JobID 842 out, err := h.State.AllocsByJob(job.ID) 843 noErr(t, err) 844 845 // Ensure no allocations placed 846 if len(out) != 0 { 847 t.Fatalf("bad: %#v", out) 848 } 849 850 // Should hit the retry limit 851 h.AssertEvalStatus(t, structs.EvalStatusFailed) 852 } 853 854 func TestBatchSched_Run_DeadAlloc(t *testing.T) { 855 h := NewHarness(t) 856 857 // Create a node 858 node := mock.Node() 859 noErr(t, h.State.UpsertNode(h.NextIndex(), node)) 860 861 // Create a job 862 job := mock.Job() 863 job.TaskGroups[0].Count = 1 864 noErr(t, h.State.UpsertJob(h.NextIndex(), job)) 865 866 // Create a failed alloc 867 alloc := mock.Alloc() 868 alloc.Job = job 869 alloc.JobID = job.ID 870 alloc.NodeID = node.ID 871 alloc.Name = "my-job.web[0]" 872 alloc.ClientStatus = structs.AllocClientStatusDead 873 noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) 874 875 // Create a mock evaluation to register the job 876 eval := &structs.Evaluation{ 877 ID: structs.GenerateUUID(), 878 Priority: job.Priority, 879 TriggeredBy: structs.EvalTriggerJobRegister, 880 JobID: job.ID, 881 } 882 883 // Process the evaluation 884 err := h.Process(NewBatchScheduler, eval) 885 if err != nil { 886 t.Fatalf("err: %v", err) 887 } 888 889 // Ensure no plan as it should be a no-op 890 if len(h.Plans) != 0 { 891 t.Fatalf("bad: %#v", h.Plans) 892 } 893 894 // Lookup the allocations by JobID 895 out, err := h.State.AllocsByJob(job.ID) 896 noErr(t, err) 897 898 // Ensure no allocations placed 899 if len(out) != 1 { 900 t.Fatalf("bad: %#v", out) 901 } 902 903 h.AssertEvalStatus(t, structs.EvalStatusComplete) 904 } 905 906 func TestBatchSched_Run_FailedAlloc(t *testing.T) { 907 h := NewHarness(t) 908 909 // Create a node 910 node := mock.Node() 911 noErr(t, h.State.UpsertNode(h.NextIndex(), node)) 912 913 // Create a job 914 job := mock.Job() 915 job.TaskGroups[0].Count = 1 916 noErr(t, h.State.UpsertJob(h.NextIndex(), job)) 917 918 // Create a failed alloc 919 alloc := mock.Alloc() 920 alloc.Job = job 921 alloc.JobID = job.ID 922 alloc.NodeID = node.ID 923 alloc.Name = "my-job.web[0]" 924 alloc.ClientStatus = structs.AllocClientStatusFailed 925 noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) 926 927 // Create a mock evaluation to register the job 928 eval := &structs.Evaluation{ 929 ID: structs.GenerateUUID(), 930 Priority: job.Priority, 931 TriggeredBy: structs.EvalTriggerJobRegister, 932 JobID: job.ID, 933 } 934 935 // Process the evaluation 936 err := h.Process(NewBatchScheduler, eval) 937 if err != nil { 938 t.Fatalf("err: %v", err) 939 } 940 941 // Ensure no plan as it should be a no-op 942 if len(h.Plans) != 1 { 943 t.Fatalf("bad: %#v", h.Plans) 944 } 945 946 // Lookup the allocations by JobID 947 out, err := h.State.AllocsByJob(job.ID) 948 noErr(t, err) 949 950 // Ensure a replacement alloc was placed. 951 if len(out) != 2 { 952 t.Fatalf("bad: %#v", out) 953 } 954 955 h.AssertEvalStatus(t, structs.EvalStatusComplete) 956 }