github.com/uchennaokeke444/nomad@v0.11.8/scheduler/system_sched_test.go (about) 1 package scheduler 2 3 import ( 4 "fmt" 5 "reflect" 6 "sort" 7 "testing" 8 "time" 9 10 memdb "github.com/hashicorp/go-memdb" 11 "github.com/hashicorp/nomad/helper" 12 "github.com/hashicorp/nomad/helper/uuid" 13 "github.com/hashicorp/nomad/nomad/mock" 14 "github.com/hashicorp/nomad/nomad/structs" 15 "github.com/stretchr/testify/require" 16 ) 17 18 func TestSystemSched_JobRegister(t *testing.T) { 19 h := NewHarness(t) 20 21 // Create some nodes 22 for i := 0; i < 10; i++ { 23 node := mock.Node() 24 require.NoError(t, h.State.UpsertNode(h.NextIndex(), node)) 25 } 26 27 // Create a job 28 job := mock.SystemJob() 29 require.NoError(t, h.State.UpsertJob(h.NextIndex(), job)) 30 31 // Create a mock evaluation to deregister the job 32 eval := &structs.Evaluation{ 33 Namespace: structs.DefaultNamespace, 34 ID: uuid.Generate(), 35 Priority: job.Priority, 36 TriggeredBy: structs.EvalTriggerJobRegister, 37 JobID: job.ID, 38 Status: structs.EvalStatusPending, 39 } 40 require.NoError(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) 41 42 // Process the evaluation 43 err := h.Process(NewSystemScheduler, eval) 44 if err != nil { 45 t.Fatalf("err: %v", err) 46 } 47 48 // Ensure a single plan 49 if len(h.Plans) != 1 { 50 t.Fatalf("bad: %#v", h.Plans) 51 } 52 plan := h.Plans[0] 53 54 // Ensure the plan doesn't have annotations. 55 if plan.Annotations != nil { 56 t.Fatalf("expected no annotations") 57 } 58 59 // Ensure the plan allocated 60 var planned []*structs.Allocation 61 for _, allocList := range plan.NodeAllocation { 62 planned = append(planned, allocList...) 63 } 64 if len(planned) != 10 { 65 t.Fatalf("bad: %#v", plan) 66 } 67 68 // Lookup the allocations by JobID 69 ws := memdb.NewWatchSet() 70 out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) 71 require.NoError(t, err) 72 73 // Ensure all allocations placed 74 if len(out) != 10 { 75 t.Fatalf("bad: %#v", out) 76 } 77 78 // Check the available nodes 79 if count, ok := out[0].Metrics.NodesAvailable["dc1"]; !ok || count != 10 { 80 t.Fatalf("bad: %#v", out[0].Metrics) 81 } 82 83 // Ensure no allocations are queued 84 queued := h.Evals[0].QueuedAllocations["web"] 85 if queued != 0 { 86 t.Fatalf("expected queued allocations: %v, actual: %v", 0, queued) 87 } 88 89 h.AssertEvalStatus(t, structs.EvalStatusComplete) 90 } 91 92 func TestSystemSched_JobRegister_StickyAllocs(t *testing.T) { 93 h := NewHarness(t) 94 95 // Create some nodes 96 for i := 0; i < 10; i++ { 97 node := mock.Node() 98 require.NoError(t, h.State.UpsertNode(h.NextIndex(), node)) 99 } 100 101 // Create a job 102 job := mock.SystemJob() 103 job.TaskGroups[0].EphemeralDisk.Sticky = true 104 require.NoError(t, h.State.UpsertJob(h.NextIndex(), job)) 105 106 // Create a mock evaluation to register the job 107 eval := &structs.Evaluation{ 108 Namespace: structs.DefaultNamespace, 109 ID: uuid.Generate(), 110 Priority: job.Priority, 111 TriggeredBy: structs.EvalTriggerJobRegister, 112 JobID: job.ID, 113 Status: structs.EvalStatusPending, 114 } 115 require.NoError(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) 116 117 // Process the evaluation 118 if err := h.Process(NewSystemScheduler, eval); err != nil { 119 t.Fatalf("err: %v", err) 120 } 121 122 // Ensure the plan allocated 123 plan := h.Plans[0] 124 var planned []*structs.Allocation 125 for _, allocList := range plan.NodeAllocation { 126 planned = append(planned, allocList...) 127 } 128 if len(planned) != 10 { 129 t.Fatalf("bad: %#v", plan) 130 } 131 132 // Get an allocation and mark it as failed 133 alloc := planned[4].Copy() 134 alloc.ClientStatus = structs.AllocClientStatusFailed 135 require.NoError(t, h.State.UpdateAllocsFromClient(h.NextIndex(), []*structs.Allocation{alloc})) 136 137 // Create a mock evaluation to handle the update 138 eval = &structs.Evaluation{ 139 Namespace: structs.DefaultNamespace, 140 ID: uuid.Generate(), 141 Priority: job.Priority, 142 TriggeredBy: structs.EvalTriggerNodeUpdate, 143 JobID: job.ID, 144 Status: structs.EvalStatusPending, 145 } 146 require.NoError(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) 147 h1 := NewHarnessWithState(t, h.State) 148 if err := h1.Process(NewSystemScheduler, eval); err != nil { 149 t.Fatalf("err: %v", err) 150 } 151 152 // Ensure we have created only one new allocation 153 plan = h1.Plans[0] 154 var newPlanned []*structs.Allocation 155 for _, allocList := range plan.NodeAllocation { 156 newPlanned = append(newPlanned, allocList...) 157 } 158 if len(newPlanned) != 1 { 159 t.Fatalf("bad plan: %#v", plan) 160 } 161 // Ensure that the new allocation was placed on the same node as the older 162 // one 163 if newPlanned[0].NodeID != alloc.NodeID || newPlanned[0].PreviousAllocation != alloc.ID { 164 t.Fatalf("expected: %#v, actual: %#v", alloc, newPlanned[0]) 165 } 166 } 167 168 func TestSystemSched_JobRegister_EphemeralDiskConstraint(t *testing.T) { 169 h := NewHarness(t) 170 171 // Create a nodes 172 node := mock.Node() 173 require.NoError(t, h.State.UpsertNode(h.NextIndex(), node)) 174 175 // Create a job 176 job := mock.SystemJob() 177 job.TaskGroups[0].EphemeralDisk.SizeMB = 60 * 1024 178 require.NoError(t, h.State.UpsertJob(h.NextIndex(), job)) 179 180 // Create another job with a lot of disk resource ask so that it doesn't fit 181 // the node 182 job1 := mock.SystemJob() 183 job1.TaskGroups[0].EphemeralDisk.SizeMB = 60 * 1024 184 require.NoError(t, h.State.UpsertJob(h.NextIndex(), job1)) 185 186 // Create a mock evaluation to register the job 187 eval := &structs.Evaluation{ 188 Namespace: structs.DefaultNamespace, 189 ID: uuid.Generate(), 190 Priority: job.Priority, 191 TriggeredBy: structs.EvalTriggerJobRegister, 192 JobID: job.ID, 193 Status: structs.EvalStatusPending, 194 } 195 require.NoError(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) 196 197 // Process the evaluation 198 if err := h.Process(NewSystemScheduler, eval); err != nil { 199 t.Fatalf("err: %v", err) 200 } 201 202 // Lookup the allocations by JobID 203 ws := memdb.NewWatchSet() 204 out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) 205 require.NoError(t, err) 206 207 // Ensure all allocations placed 208 if len(out) != 1 { 209 t.Fatalf("bad: %#v", out) 210 } 211 212 // Create a new harness to test the scheduling result for the second job 213 h1 := NewHarnessWithState(t, h.State) 214 // Create a mock evaluation to register the job 215 eval1 := &structs.Evaluation{ 216 Namespace: structs.DefaultNamespace, 217 ID: uuid.Generate(), 218 Priority: job1.Priority, 219 TriggeredBy: structs.EvalTriggerJobRegister, 220 JobID: job1.ID, 221 Status: structs.EvalStatusPending, 222 } 223 require.NoError(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval1})) 224 225 // Process the evaluation 226 if err := h1.Process(NewSystemScheduler, eval1); err != nil { 227 t.Fatalf("err: %v", err) 228 } 229 230 out, err = h1.State.AllocsByJob(ws, job.Namespace, job1.ID, false) 231 require.NoError(t, err) 232 if len(out) != 0 { 233 t.Fatalf("bad: %#v", out) 234 } 235 } 236 237 func TestSystemSched_ExhaustResources(t *testing.T) { 238 h := NewHarness(t) 239 240 // Create a nodes 241 node := mock.Node() 242 require.NoError(t, h.State.UpsertNode(h.NextIndex(), node)) 243 244 // Enable Preemption 245 h.State.SchedulerSetConfig(h.NextIndex(), &structs.SchedulerConfiguration{ 246 PreemptionConfig: structs.PreemptionConfig{ 247 SystemSchedulerEnabled: true, 248 }, 249 }) 250 251 // Create a service job which consumes most of the system resources 252 svcJob := mock.Job() 253 svcJob.TaskGroups[0].Count = 1 254 svcJob.TaskGroups[0].Tasks[0].Resources.CPU = 3600 255 require.NoError(t, h.State.UpsertJob(h.NextIndex(), svcJob)) 256 257 // Create a mock evaluation to register the job 258 eval := &structs.Evaluation{ 259 Namespace: structs.DefaultNamespace, 260 ID: uuid.Generate(), 261 Priority: svcJob.Priority, 262 TriggeredBy: structs.EvalTriggerJobRegister, 263 JobID: svcJob.ID, 264 Status: structs.EvalStatusPending, 265 } 266 require.NoError(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) 267 // Process the evaluation 268 err := h.Process(NewServiceScheduler, eval) 269 if err != nil { 270 t.Fatalf("err: %v", err) 271 } 272 273 // Create a system job 274 job := mock.SystemJob() 275 require.NoError(t, h.State.UpsertJob(h.NextIndex(), job)) 276 277 // Create a mock evaluation to register the job 278 eval1 := &structs.Evaluation{ 279 Namespace: structs.DefaultNamespace, 280 ID: uuid.Generate(), 281 Priority: job.Priority, 282 TriggeredBy: structs.EvalTriggerJobRegister, 283 JobID: job.ID, 284 Status: structs.EvalStatusPending, 285 } 286 require.NoError(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval1})) 287 // Process the evaluation 288 if err := h.Process(NewSystemScheduler, eval1); err != nil { 289 t.Fatalf("err: %v", err) 290 } 291 292 // System scheduler will preempt the service job and would have placed eval1 293 require := require.New(t) 294 295 newPlan := h.Plans[1] 296 require.Len(newPlan.NodeAllocation, 1) 297 require.Len(newPlan.NodePreemptions, 1) 298 299 for _, allocList := range newPlan.NodeAllocation { 300 require.Len(allocList, 1) 301 require.Equal(job.ID, allocList[0].JobID) 302 } 303 304 for _, allocList := range newPlan.NodePreemptions { 305 require.Len(allocList, 1) 306 require.Equal(svcJob.ID, allocList[0].JobID) 307 } 308 // Ensure that we have no queued allocations on the second eval 309 queued := h.Evals[1].QueuedAllocations["web"] 310 if queued != 0 { 311 t.Fatalf("expected: %v, actual: %v", 1, queued) 312 } 313 } 314 315 func TestSystemSched_JobRegister_Annotate(t *testing.T) { 316 h := NewHarness(t) 317 318 // Create some nodes 319 for i := 0; i < 10; i++ { 320 node := mock.Node() 321 if i < 9 { 322 node.NodeClass = "foo" 323 } else { 324 node.NodeClass = "bar" 325 } 326 node.ComputeClass() 327 require.NoError(t, h.State.UpsertNode(h.NextIndex(), node)) 328 } 329 330 // Create a job constraining on node class 331 job := mock.SystemJob() 332 fooConstraint := &structs.Constraint{ 333 LTarget: "${node.class}", 334 RTarget: "foo", 335 Operand: "==", 336 } 337 job.Constraints = append(job.Constraints, fooConstraint) 338 require.NoError(t, h.State.UpsertJob(h.NextIndex(), job)) 339 340 // Create a mock evaluation to deregister the job 341 eval := &structs.Evaluation{ 342 Namespace: structs.DefaultNamespace, 343 ID: uuid.Generate(), 344 Priority: job.Priority, 345 TriggeredBy: structs.EvalTriggerJobRegister, 346 JobID: job.ID, 347 AnnotatePlan: true, 348 Status: structs.EvalStatusPending, 349 } 350 require.NoError(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) 351 352 // Process the evaluation 353 err := h.Process(NewSystemScheduler, eval) 354 if err != nil { 355 t.Fatalf("err: %v", err) 356 } 357 358 // Ensure a single plan 359 if len(h.Plans) != 1 { 360 t.Fatalf("bad: %#v", h.Plans) 361 } 362 plan := h.Plans[0] 363 364 // Ensure the plan allocated 365 var planned []*structs.Allocation 366 for _, allocList := range plan.NodeAllocation { 367 planned = append(planned, allocList...) 368 } 369 if len(planned) != 9 { 370 t.Fatalf("bad: %#v %d", planned, len(planned)) 371 } 372 373 // Lookup the allocations by JobID 374 ws := memdb.NewWatchSet() 375 out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) 376 require.NoError(t, err) 377 378 // Ensure all allocations placed 379 if len(out) != 9 { 380 t.Fatalf("bad: %#v", out) 381 } 382 383 // Check the available nodes 384 if count, ok := out[0].Metrics.NodesAvailable["dc1"]; !ok || count != 10 { 385 t.Fatalf("bad: %#v", out[0].Metrics) 386 } 387 388 h.AssertEvalStatus(t, structs.EvalStatusComplete) 389 390 // Ensure the plan had annotations. 391 if plan.Annotations == nil { 392 t.Fatalf("expected annotations") 393 } 394 395 desiredTGs := plan.Annotations.DesiredTGUpdates 396 if l := len(desiredTGs); l != 1 { 397 t.Fatalf("incorrect number of task groups; got %v; want %v", l, 1) 398 } 399 400 desiredChanges, ok := desiredTGs["web"] 401 if !ok { 402 t.Fatalf("expected task group web to have desired changes") 403 } 404 405 expected := &structs.DesiredUpdates{Place: 9} 406 if !reflect.DeepEqual(desiredChanges, expected) { 407 t.Fatalf("Unexpected desired updates; got %#v; want %#v", desiredChanges, expected) 408 } 409 } 410 411 func TestSystemSched_JobRegister_AddNode(t *testing.T) { 412 h := NewHarness(t) 413 414 // Create some nodes 415 var nodes []*structs.Node 416 for i := 0; i < 10; i++ { 417 node := mock.Node() 418 nodes = append(nodes, node) 419 require.NoError(t, h.State.UpsertNode(h.NextIndex(), node)) 420 } 421 422 // Generate a fake job with allocations 423 job := mock.SystemJob() 424 require.NoError(t, h.State.UpsertJob(h.NextIndex(), job)) 425 426 var allocs []*structs.Allocation 427 for _, node := range nodes { 428 alloc := mock.Alloc() 429 alloc.Job = job 430 alloc.JobID = job.ID 431 alloc.NodeID = node.ID 432 alloc.Name = "my-job.web[0]" 433 allocs = append(allocs, alloc) 434 } 435 require.NoError(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) 436 437 // Add a new node. 438 node := mock.Node() 439 require.NoError(t, h.State.UpsertNode(h.NextIndex(), node)) 440 441 // Create a mock evaluation to deal with the node update 442 eval := &structs.Evaluation{ 443 Namespace: structs.DefaultNamespace, 444 ID: uuid.Generate(), 445 Priority: 50, 446 TriggeredBy: structs.EvalTriggerNodeUpdate, 447 JobID: job.ID, 448 Status: structs.EvalStatusPending, 449 } 450 require.NoError(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) 451 // Process the evaluation 452 err := h.Process(NewSystemScheduler, eval) 453 if err != nil { 454 t.Fatalf("err: %v", err) 455 } 456 457 // Ensure a single plan 458 if len(h.Plans) != 1 { 459 t.Fatalf("bad: %#v", h.Plans) 460 } 461 plan := h.Plans[0] 462 463 // Ensure the plan had no node updates 464 var update []*structs.Allocation 465 for _, updateList := range plan.NodeUpdate { 466 update = append(update, updateList...) 467 } 468 if len(update) != 0 { 469 t.Log(len(update)) 470 t.Fatalf("bad: %#v", plan) 471 } 472 473 // Ensure the plan allocated on the new node 474 var planned []*structs.Allocation 475 for _, allocList := range plan.NodeAllocation { 476 planned = append(planned, allocList...) 477 } 478 if len(planned) != 1 { 479 t.Fatalf("bad: %#v", plan) 480 } 481 482 // Ensure it allocated on the right node 483 if _, ok := plan.NodeAllocation[node.ID]; !ok { 484 t.Fatalf("allocated on wrong node: %#v", plan) 485 } 486 487 // Lookup the allocations by JobID 488 ws := memdb.NewWatchSet() 489 out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) 490 require.NoError(t, err) 491 492 // Ensure all allocations placed 493 out, _ = structs.FilterTerminalAllocs(out) 494 if len(out) != 11 { 495 t.Fatalf("bad: %#v", out) 496 } 497 498 h.AssertEvalStatus(t, structs.EvalStatusComplete) 499 } 500 501 func TestSystemSched_JobRegister_AllocFail(t *testing.T) { 502 h := NewHarness(t) 503 504 // Create NO nodes 505 // Create a job 506 job := mock.SystemJob() 507 require.NoError(t, h.State.UpsertJob(h.NextIndex(), job)) 508 509 // Create a mock evaluation to register the job 510 eval := &structs.Evaluation{ 511 Namespace: structs.DefaultNamespace, 512 ID: uuid.Generate(), 513 Priority: job.Priority, 514 TriggeredBy: structs.EvalTriggerJobRegister, 515 JobID: job.ID, 516 Status: structs.EvalStatusPending, 517 } 518 require.NoError(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) 519 // Process the evaluation 520 err := h.Process(NewSystemScheduler, eval) 521 if err != nil { 522 t.Fatalf("err: %v", err) 523 } 524 525 // Ensure no plan as this should be a no-op. 526 if len(h.Plans) != 0 { 527 t.Fatalf("bad: %#v", h.Plans) 528 } 529 530 h.AssertEvalStatus(t, structs.EvalStatusComplete) 531 } 532 533 func TestSystemSched_JobModify(t *testing.T) { 534 h := NewHarness(t) 535 536 // Create some nodes 537 var nodes []*structs.Node 538 for i := 0; i < 10; i++ { 539 node := mock.Node() 540 nodes = append(nodes, node) 541 require.NoError(t, h.State.UpsertNode(h.NextIndex(), node)) 542 } 543 544 // Generate a fake job with allocations 545 job := mock.SystemJob() 546 require.NoError(t, h.State.UpsertJob(h.NextIndex(), job)) 547 548 var allocs []*structs.Allocation 549 for _, node := range nodes { 550 alloc := mock.Alloc() 551 alloc.Job = job 552 alloc.JobID = job.ID 553 alloc.NodeID = node.ID 554 alloc.Name = "my-job.web[0]" 555 allocs = append(allocs, alloc) 556 } 557 require.NoError(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) 558 559 // Add a few terminal status allocations, these should be ignored 560 var terminal []*structs.Allocation 561 for i := 0; i < 5; i++ { 562 alloc := mock.Alloc() 563 alloc.Job = job 564 alloc.JobID = job.ID 565 alloc.NodeID = nodes[i].ID 566 alloc.Name = "my-job.web[0]" 567 alloc.DesiredStatus = structs.AllocDesiredStatusStop 568 terminal = append(terminal, alloc) 569 } 570 require.NoError(t, h.State.UpsertAllocs(h.NextIndex(), terminal)) 571 572 // Update the job 573 job2 := mock.SystemJob() 574 job2.ID = job.ID 575 576 // Update the task, such that it cannot be done in-place 577 job2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other" 578 require.NoError(t, h.State.UpsertJob(h.NextIndex(), job2)) 579 580 // Create a mock evaluation to deal with drain 581 eval := &structs.Evaluation{ 582 Namespace: structs.DefaultNamespace, 583 ID: uuid.Generate(), 584 Priority: 50, 585 TriggeredBy: structs.EvalTriggerJobRegister, 586 JobID: job.ID, 587 Status: structs.EvalStatusPending, 588 } 589 require.NoError(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) 590 591 // Process the evaluation 592 err := h.Process(NewSystemScheduler, eval) 593 if err != nil { 594 t.Fatalf("err: %v", err) 595 } 596 597 // Ensure a single plan 598 if len(h.Plans) != 1 { 599 t.Fatalf("bad: %#v", h.Plans) 600 } 601 plan := h.Plans[0] 602 603 // Ensure the plan evicted all allocs 604 var update []*structs.Allocation 605 for _, updateList := range plan.NodeUpdate { 606 update = append(update, updateList...) 607 } 608 if len(update) != len(allocs) { 609 t.Fatalf("bad: %#v", plan) 610 } 611 612 // Ensure the plan allocated 613 var planned []*structs.Allocation 614 for _, allocList := range plan.NodeAllocation { 615 planned = append(planned, allocList...) 616 } 617 if len(planned) != 10 { 618 t.Fatalf("bad: %#v", plan) 619 } 620 621 // Lookup the allocations by JobID 622 ws := memdb.NewWatchSet() 623 out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) 624 require.NoError(t, err) 625 626 // Ensure all allocations placed 627 out, _ = structs.FilterTerminalAllocs(out) 628 if len(out) != 10 { 629 t.Fatalf("bad: %#v", out) 630 } 631 632 h.AssertEvalStatus(t, structs.EvalStatusComplete) 633 } 634 635 func TestSystemSched_JobModify_Rolling(t *testing.T) { 636 h := NewHarness(t) 637 638 // Create some nodes 639 var nodes []*structs.Node 640 for i := 0; i < 10; i++ { 641 node := mock.Node() 642 nodes = append(nodes, node) 643 require.NoError(t, h.State.UpsertNode(h.NextIndex(), node)) 644 } 645 646 // Generate a fake job with allocations 647 job := mock.SystemJob() 648 require.NoError(t, h.State.UpsertJob(h.NextIndex(), job)) 649 650 var allocs []*structs.Allocation 651 for _, node := range nodes { 652 alloc := mock.Alloc() 653 alloc.Job = job 654 alloc.JobID = job.ID 655 alloc.NodeID = node.ID 656 alloc.Name = "my-job.web[0]" 657 allocs = append(allocs, alloc) 658 } 659 require.NoError(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) 660 661 // Update the job 662 job2 := mock.SystemJob() 663 job2.ID = job.ID 664 job2.Update = structs.UpdateStrategy{ 665 Stagger: 30 * time.Second, 666 MaxParallel: 5, 667 } 668 669 // Update the task, such that it cannot be done in-place 670 job2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other" 671 require.NoError(t, h.State.UpsertJob(h.NextIndex(), job2)) 672 673 // Create a mock evaluation to deal with drain 674 eval := &structs.Evaluation{ 675 Namespace: structs.DefaultNamespace, 676 ID: uuid.Generate(), 677 Priority: 50, 678 TriggeredBy: structs.EvalTriggerJobRegister, 679 JobID: job.ID, 680 Status: structs.EvalStatusPending, 681 } 682 require.NoError(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) 683 // Process the evaluation 684 err := h.Process(NewSystemScheduler, eval) 685 if err != nil { 686 t.Fatalf("err: %v", err) 687 } 688 689 // Ensure a single plan 690 if len(h.Plans) != 1 { 691 t.Fatalf("bad: %#v", h.Plans) 692 } 693 plan := h.Plans[0] 694 695 // Ensure the plan evicted only MaxParallel 696 var update []*structs.Allocation 697 for _, updateList := range plan.NodeUpdate { 698 update = append(update, updateList...) 699 } 700 if len(update) != job2.Update.MaxParallel { 701 t.Fatalf("bad: %#v", plan) 702 } 703 704 // Ensure the plan allocated 705 var planned []*structs.Allocation 706 for _, allocList := range plan.NodeAllocation { 707 planned = append(planned, allocList...) 708 } 709 if len(planned) != job2.Update.MaxParallel { 710 t.Fatalf("bad: %#v", plan) 711 } 712 713 h.AssertEvalStatus(t, structs.EvalStatusComplete) 714 715 // Ensure a follow up eval was created 716 eval = h.Evals[0] 717 if eval.NextEval == "" { 718 t.Fatalf("missing next eval") 719 } 720 721 // Check for create 722 if len(h.CreateEvals) == 0 { 723 t.Fatalf("missing created eval") 724 } 725 create := h.CreateEvals[0] 726 if eval.NextEval != create.ID { 727 t.Fatalf("ID mismatch") 728 } 729 if create.PreviousEval != eval.ID { 730 t.Fatalf("missing previous eval") 731 } 732 733 if create.TriggeredBy != structs.EvalTriggerRollingUpdate { 734 t.Fatalf("bad: %#v", create) 735 } 736 } 737 738 func TestSystemSched_JobModify_InPlace(t *testing.T) { 739 h := NewHarness(t) 740 741 // Create some nodes 742 var nodes []*structs.Node 743 for i := 0; i < 10; i++ { 744 node := mock.Node() 745 nodes = append(nodes, node) 746 require.NoError(t, h.State.UpsertNode(h.NextIndex(), node)) 747 } 748 749 // Generate a fake job with allocations 750 job := mock.SystemJob() 751 require.NoError(t, h.State.UpsertJob(h.NextIndex(), job)) 752 753 var allocs []*structs.Allocation 754 for _, node := range nodes { 755 alloc := mock.Alloc() 756 alloc.Job = job 757 alloc.JobID = job.ID 758 alloc.NodeID = node.ID 759 alloc.Name = "my-job.web[0]" 760 allocs = append(allocs, alloc) 761 } 762 require.NoError(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) 763 764 // Update the job 765 job2 := mock.SystemJob() 766 job2.ID = job.ID 767 require.NoError(t, h.State.UpsertJob(h.NextIndex(), job2)) 768 769 // Create a mock evaluation to deal with drain 770 eval := &structs.Evaluation{ 771 Namespace: structs.DefaultNamespace, 772 ID: uuid.Generate(), 773 Priority: 50, 774 TriggeredBy: structs.EvalTriggerJobRegister, 775 JobID: job.ID, 776 Status: structs.EvalStatusPending, 777 } 778 require.NoError(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) 779 780 // Process the evaluation 781 err := h.Process(NewSystemScheduler, eval) 782 if err != nil { 783 t.Fatalf("err: %v", err) 784 } 785 786 // Ensure a single plan 787 if len(h.Plans) != 1 { 788 t.Fatalf("bad: %#v", h.Plans) 789 } 790 plan := h.Plans[0] 791 792 // Ensure the plan did not evict any allocs 793 var update []*structs.Allocation 794 for _, updateList := range plan.NodeUpdate { 795 update = append(update, updateList...) 796 } 797 if len(update) != 0 { 798 t.Fatalf("bad: %#v", plan) 799 } 800 801 // Ensure the plan updated the existing allocs 802 var planned []*structs.Allocation 803 for _, allocList := range plan.NodeAllocation { 804 planned = append(planned, allocList...) 805 } 806 if len(planned) != 10 { 807 t.Fatalf("bad: %#v", plan) 808 } 809 for _, p := range planned { 810 if p.Job != job2 { 811 t.Fatalf("should update job") 812 } 813 } 814 815 // Lookup the allocations by JobID 816 ws := memdb.NewWatchSet() 817 out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) 818 require.NoError(t, err) 819 820 // Ensure all allocations placed 821 if len(out) != 10 { 822 t.Fatalf("bad: %#v", out) 823 } 824 h.AssertEvalStatus(t, structs.EvalStatusComplete) 825 826 // Verify the network did not change 827 rp := structs.Port{Label: "admin", Value: 5000} 828 for _, alloc := range out { 829 for _, resources := range alloc.TaskResources { 830 if resources.Networks[0].ReservedPorts[0] != rp { 831 t.Fatalf("bad: %#v", alloc) 832 } 833 } 834 } 835 } 836 837 func TestSystemSched_JobDeregister_Purged(t *testing.T) { 838 h := NewHarness(t) 839 840 // Create some nodes 841 var nodes []*structs.Node 842 for i := 0; i < 10; i++ { 843 node := mock.Node() 844 nodes = append(nodes, node) 845 require.NoError(t, h.State.UpsertNode(h.NextIndex(), node)) 846 } 847 848 // Generate a fake job with allocations 849 job := mock.SystemJob() 850 851 var allocs []*structs.Allocation 852 for _, node := range nodes { 853 alloc := mock.Alloc() 854 alloc.Job = job 855 alloc.JobID = job.ID 856 alloc.NodeID = node.ID 857 alloc.Name = "my-job.web[0]" 858 allocs = append(allocs, alloc) 859 } 860 for _, alloc := range allocs { 861 require.NoError(t, h.State.UpsertJobSummary(h.NextIndex(), mock.JobSummary(alloc.JobID))) 862 } 863 require.NoError(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) 864 865 // Create a mock evaluation to deregister the job 866 eval := &structs.Evaluation{ 867 Namespace: structs.DefaultNamespace, 868 ID: uuid.Generate(), 869 Priority: 50, 870 TriggeredBy: structs.EvalTriggerJobDeregister, 871 JobID: job.ID, 872 Status: structs.EvalStatusPending, 873 } 874 require.NoError(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) 875 876 // Process the evaluation 877 err := h.Process(NewSystemScheduler, eval) 878 if err != nil { 879 t.Fatalf("err: %v", err) 880 } 881 882 // Ensure a single plan 883 if len(h.Plans) != 1 { 884 t.Fatalf("bad: %#v", h.Plans) 885 } 886 plan := h.Plans[0] 887 888 // Ensure the plan evicted the job from all nodes. 889 for _, node := range nodes { 890 if len(plan.NodeUpdate[node.ID]) != 1 { 891 t.Fatalf("bad: %#v", plan) 892 } 893 } 894 895 // Lookup the allocations by JobID 896 ws := memdb.NewWatchSet() 897 out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) 898 require.NoError(t, err) 899 900 // Ensure no remaining allocations 901 out, _ = structs.FilterTerminalAllocs(out) 902 if len(out) != 0 { 903 t.Fatalf("bad: %#v", out) 904 } 905 906 h.AssertEvalStatus(t, structs.EvalStatusComplete) 907 } 908 909 func TestSystemSched_JobDeregister_Stopped(t *testing.T) { 910 h := NewHarness(t) 911 912 // Create some nodes 913 var nodes []*structs.Node 914 for i := 0; i < 10; i++ { 915 node := mock.Node() 916 nodes = append(nodes, node) 917 require.NoError(t, h.State.UpsertNode(h.NextIndex(), node)) 918 } 919 920 // Generate a fake job with allocations 921 job := mock.SystemJob() 922 job.Stop = true 923 require.NoError(t, h.State.UpsertJob(h.NextIndex(), job)) 924 925 var allocs []*structs.Allocation 926 for _, node := range nodes { 927 alloc := mock.Alloc() 928 alloc.Job = job 929 alloc.JobID = job.ID 930 alloc.NodeID = node.ID 931 alloc.Name = "my-job.web[0]" 932 allocs = append(allocs, alloc) 933 } 934 for _, alloc := range allocs { 935 require.NoError(t, h.State.UpsertJobSummary(h.NextIndex(), mock.JobSummary(alloc.JobID))) 936 } 937 require.NoError(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) 938 939 // Create a mock evaluation to deregister the job 940 eval := &structs.Evaluation{ 941 Namespace: structs.DefaultNamespace, 942 ID: uuid.Generate(), 943 Priority: 50, 944 TriggeredBy: structs.EvalTriggerJobDeregister, 945 JobID: job.ID, 946 Status: structs.EvalStatusPending, 947 } 948 require.NoError(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) 949 950 // Process the evaluation 951 err := h.Process(NewSystemScheduler, eval) 952 if err != nil { 953 t.Fatalf("err: %v", err) 954 } 955 956 // Ensure a single plan 957 if len(h.Plans) != 1 { 958 t.Fatalf("bad: %#v", h.Plans) 959 } 960 plan := h.Plans[0] 961 962 // Ensure the plan evicted the job from all nodes. 963 for _, node := range nodes { 964 if len(plan.NodeUpdate[node.ID]) != 1 { 965 t.Fatalf("bad: %#v", plan) 966 } 967 } 968 969 // Lookup the allocations by JobID 970 ws := memdb.NewWatchSet() 971 out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) 972 require.NoError(t, err) 973 974 // Ensure no remaining allocations 975 out, _ = structs.FilterTerminalAllocs(out) 976 if len(out) != 0 { 977 t.Fatalf("bad: %#v", out) 978 } 979 980 h.AssertEvalStatus(t, structs.EvalStatusComplete) 981 } 982 983 func TestSystemSched_NodeDown(t *testing.T) { 984 h := NewHarness(t) 985 986 // Register a down node 987 node := mock.Node() 988 node.Status = structs.NodeStatusDown 989 require.NoError(t, h.State.UpsertNode(h.NextIndex(), node)) 990 991 // Generate a fake job allocated on that node. 992 job := mock.SystemJob() 993 require.NoError(t, h.State.UpsertJob(h.NextIndex(), job)) 994 995 alloc := mock.Alloc() 996 alloc.Job = job 997 alloc.JobID = job.ID 998 alloc.NodeID = node.ID 999 alloc.Name = "my-job.web[0]" 1000 alloc.DesiredTransition.Migrate = helper.BoolToPtr(true) 1001 require.NoError(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) 1002 1003 // Create a mock evaluation to deal with drain 1004 eval := &structs.Evaluation{ 1005 Namespace: structs.DefaultNamespace, 1006 ID: uuid.Generate(), 1007 Priority: 50, 1008 TriggeredBy: structs.EvalTriggerNodeUpdate, 1009 JobID: job.ID, 1010 NodeID: node.ID, 1011 Status: structs.EvalStatusPending, 1012 } 1013 require.NoError(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) 1014 1015 // Process the evaluation 1016 err := h.Process(NewSystemScheduler, eval) 1017 if err != nil { 1018 t.Fatalf("err: %v", err) 1019 } 1020 1021 // Ensure a single plan 1022 if len(h.Plans) != 1 { 1023 t.Fatalf("bad: %#v", h.Plans) 1024 } 1025 plan := h.Plans[0] 1026 1027 // Ensure the plan evicted all allocs 1028 if len(plan.NodeUpdate[node.ID]) != 1 { 1029 t.Fatalf("bad: %#v", plan) 1030 } 1031 1032 // Ensure the plan updated the allocation. 1033 var planned []*structs.Allocation 1034 for _, allocList := range plan.NodeUpdate { 1035 planned = append(planned, allocList...) 1036 } 1037 if len(planned) != 1 { 1038 t.Fatalf("bad: %#v", plan) 1039 } 1040 1041 // Ensure the allocations is stopped 1042 if p := planned[0]; p.DesiredStatus != structs.AllocDesiredStatusStop && 1043 p.ClientStatus != structs.AllocClientStatusLost { 1044 t.Fatalf("bad: %#v", planned[0]) 1045 } 1046 1047 h.AssertEvalStatus(t, structs.EvalStatusComplete) 1048 } 1049 1050 func TestSystemSched_NodeDrain_Down(t *testing.T) { 1051 h := NewHarness(t) 1052 1053 // Register a draining node 1054 node := mock.Node() 1055 node.Drain = true 1056 node.Status = structs.NodeStatusDown 1057 require.NoError(t, h.State.UpsertNode(h.NextIndex(), node)) 1058 1059 // Generate a fake job allocated on that node. 1060 job := mock.SystemJob() 1061 require.NoError(t, h.State.UpsertJob(h.NextIndex(), job)) 1062 1063 alloc := mock.Alloc() 1064 alloc.Job = job 1065 alloc.JobID = job.ID 1066 alloc.NodeID = node.ID 1067 alloc.Name = "my-job.web[0]" 1068 require.NoError(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) 1069 1070 // Create a mock evaluation to deal with the node update 1071 eval := &structs.Evaluation{ 1072 Namespace: structs.DefaultNamespace, 1073 ID: uuid.Generate(), 1074 Priority: 50, 1075 TriggeredBy: structs.EvalTriggerNodeUpdate, 1076 JobID: job.ID, 1077 NodeID: node.ID, 1078 Status: structs.EvalStatusPending, 1079 } 1080 require.NoError(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) 1081 1082 // Process the evaluation 1083 err := h.Process(NewServiceScheduler, eval) 1084 if err != nil { 1085 t.Fatalf("err: %v", err) 1086 } 1087 1088 // Ensure a single plan 1089 if len(h.Plans) != 1 { 1090 t.Fatalf("bad: %#v", h.Plans) 1091 } 1092 plan := h.Plans[0] 1093 1094 // Ensure the plan evicted non terminal allocs 1095 if len(plan.NodeUpdate[node.ID]) != 1 { 1096 t.Fatalf("bad: %#v", plan) 1097 } 1098 1099 // Ensure that the allocation is marked as lost 1100 var lostAllocs []string 1101 for _, alloc := range plan.NodeUpdate[node.ID] { 1102 lostAllocs = append(lostAllocs, alloc.ID) 1103 } 1104 expected := []string{alloc.ID} 1105 1106 if !reflect.DeepEqual(lostAllocs, expected) { 1107 t.Fatalf("expected: %v, actual: %v", expected, lostAllocs) 1108 } 1109 h.AssertEvalStatus(t, structs.EvalStatusComplete) 1110 } 1111 1112 func TestSystemSched_NodeDrain(t *testing.T) { 1113 h := NewHarness(t) 1114 1115 // Register a draining node 1116 node := mock.Node() 1117 node.Drain = true 1118 require.NoError(t, h.State.UpsertNode(h.NextIndex(), node)) 1119 1120 // Generate a fake job allocated on that node. 1121 job := mock.SystemJob() 1122 require.NoError(t, h.State.UpsertJob(h.NextIndex(), job)) 1123 1124 alloc := mock.Alloc() 1125 alloc.Job = job 1126 alloc.JobID = job.ID 1127 alloc.NodeID = node.ID 1128 alloc.Name = "my-job.web[0]" 1129 alloc.DesiredTransition.Migrate = helper.BoolToPtr(true) 1130 require.NoError(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) 1131 1132 // Create a mock evaluation to deal with drain 1133 eval := &structs.Evaluation{ 1134 Namespace: structs.DefaultNamespace, 1135 ID: uuid.Generate(), 1136 Priority: 50, 1137 TriggeredBy: structs.EvalTriggerNodeUpdate, 1138 JobID: job.ID, 1139 NodeID: node.ID, 1140 Status: structs.EvalStatusPending, 1141 } 1142 require.NoError(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) 1143 1144 // Process the evaluation 1145 err := h.Process(NewSystemScheduler, eval) 1146 if err != nil { 1147 t.Fatalf("err: %v", err) 1148 } 1149 1150 // Ensure a single plan 1151 if len(h.Plans) != 1 { 1152 t.Fatalf("bad: %#v", h.Plans) 1153 } 1154 plan := h.Plans[0] 1155 1156 // Ensure the plan evicted all allocs 1157 if len(plan.NodeUpdate[node.ID]) != 1 { 1158 t.Fatalf("bad: %#v", plan) 1159 } 1160 1161 // Ensure the plan updated the allocation. 1162 var planned []*structs.Allocation 1163 for _, allocList := range plan.NodeUpdate { 1164 planned = append(planned, allocList...) 1165 } 1166 if len(planned) != 1 { 1167 t.Log(len(planned)) 1168 t.Fatalf("bad: %#v", plan) 1169 } 1170 1171 // Ensure the allocations is stopped 1172 if planned[0].DesiredStatus != structs.AllocDesiredStatusStop { 1173 t.Fatalf("bad: %#v", planned[0]) 1174 } 1175 1176 h.AssertEvalStatus(t, structs.EvalStatusComplete) 1177 } 1178 1179 func TestSystemSched_NodeUpdate(t *testing.T) { 1180 h := NewHarness(t) 1181 1182 // Register a node 1183 node := mock.Node() 1184 require.NoError(t, h.State.UpsertNode(h.NextIndex(), node)) 1185 1186 // Generate a fake job allocated on that node. 1187 job := mock.SystemJob() 1188 require.NoError(t, h.State.UpsertJob(h.NextIndex(), job)) 1189 1190 alloc := mock.Alloc() 1191 alloc.Job = job 1192 alloc.JobID = job.ID 1193 alloc.NodeID = node.ID 1194 alloc.Name = "my-job.web[0]" 1195 require.NoError(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) 1196 1197 // Create a mock evaluation to deal 1198 eval := &structs.Evaluation{ 1199 Namespace: structs.DefaultNamespace, 1200 ID: uuid.Generate(), 1201 Priority: 50, 1202 TriggeredBy: structs.EvalTriggerNodeUpdate, 1203 JobID: job.ID, 1204 NodeID: node.ID, 1205 Status: structs.EvalStatusPending, 1206 } 1207 require.NoError(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) 1208 1209 // Process the evaluation 1210 err := h.Process(NewSystemScheduler, eval) 1211 if err != nil { 1212 t.Fatalf("err: %v", err) 1213 } 1214 1215 // Ensure that queued allocations is zero 1216 if val, ok := h.Evals[0].QueuedAllocations["web"]; !ok || val != 0 { 1217 t.Fatalf("bad queued allocations: %#v", h.Evals[0].QueuedAllocations) 1218 } 1219 1220 h.AssertEvalStatus(t, structs.EvalStatusComplete) 1221 } 1222 1223 func TestSystemSched_RetryLimit(t *testing.T) { 1224 h := NewHarness(t) 1225 h.Planner = &RejectPlan{h} 1226 1227 // Create some nodes 1228 for i := 0; i < 10; i++ { 1229 node := mock.Node() 1230 require.NoError(t, h.State.UpsertNode(h.NextIndex(), node)) 1231 } 1232 1233 // Create a job 1234 job := mock.SystemJob() 1235 require.NoError(t, h.State.UpsertJob(h.NextIndex(), job)) 1236 1237 // Create a mock evaluation to deregister the job 1238 eval := &structs.Evaluation{ 1239 Namespace: structs.DefaultNamespace, 1240 ID: uuid.Generate(), 1241 Priority: job.Priority, 1242 TriggeredBy: structs.EvalTriggerJobRegister, 1243 JobID: job.ID, 1244 Status: structs.EvalStatusPending, 1245 } 1246 require.NoError(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) 1247 1248 // Process the evaluation 1249 err := h.Process(NewSystemScheduler, eval) 1250 if err != nil { 1251 t.Fatalf("err: %v", err) 1252 } 1253 1254 // Ensure multiple plans 1255 if len(h.Plans) == 0 { 1256 t.Fatalf("bad: %#v", h.Plans) 1257 } 1258 1259 // Lookup the allocations by JobID 1260 ws := memdb.NewWatchSet() 1261 out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) 1262 require.NoError(t, err) 1263 1264 // Ensure no allocations placed 1265 if len(out) != 0 { 1266 t.Fatalf("bad: %#v", out) 1267 } 1268 1269 // Should hit the retry limit 1270 h.AssertEvalStatus(t, structs.EvalStatusFailed) 1271 } 1272 1273 // This test ensures that the scheduler doesn't increment the queued allocation 1274 // count for a task group when allocations can't be created on currently 1275 // available nodes because of constrain mismatches. 1276 func TestSystemSched_Queued_With_Constraints(t *testing.T) { 1277 h := NewHarness(t) 1278 1279 // Register a node 1280 node := mock.Node() 1281 node.Attributes["kernel.name"] = "darwin" 1282 require.NoError(t, h.State.UpsertNode(h.NextIndex(), node)) 1283 1284 // Generate a system job which can't be placed on the node 1285 job := mock.SystemJob() 1286 require.NoError(t, h.State.UpsertJob(h.NextIndex(), job)) 1287 1288 // Create a mock evaluation to deal 1289 eval := &structs.Evaluation{ 1290 Namespace: structs.DefaultNamespace, 1291 ID: uuid.Generate(), 1292 Priority: 50, 1293 TriggeredBy: structs.EvalTriggerNodeUpdate, 1294 JobID: job.ID, 1295 NodeID: node.ID, 1296 Status: structs.EvalStatusPending, 1297 } 1298 require.NoError(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) 1299 1300 // Process the evaluation 1301 err := h.Process(NewSystemScheduler, eval) 1302 if err != nil { 1303 t.Fatalf("err: %v", err) 1304 } 1305 1306 // Ensure that queued allocations is zero 1307 if val, ok := h.Evals[0].QueuedAllocations["web"]; !ok || val != 0 { 1308 t.Fatalf("bad queued allocations: %#v", h.Evals[0].QueuedAllocations) 1309 } 1310 1311 } 1312 1313 // This test ensures that the scheduler correctly ignores ineligible 1314 // nodes when scheduling due to a new node being added. The job has two 1315 // task groups contrained to a particular node class. The desired behavior 1316 // should be that the TaskGroup constrained to the newly added node class is 1317 // added and that the TaskGroup constrained to the ineligible node is ignored. 1318 func TestSystemSched_JobConstraint_AddNode(t *testing.T) { 1319 h := NewHarness(t) 1320 1321 // Create two nodes 1322 var node *structs.Node 1323 node = mock.Node() 1324 node.NodeClass = "Class-A" 1325 node.ComputeClass() 1326 require.Nil(t, h.State.UpsertNode(h.NextIndex(), node)) 1327 1328 var nodeB *structs.Node 1329 nodeB = mock.Node() 1330 nodeB.NodeClass = "Class-B" 1331 nodeB.ComputeClass() 1332 require.Nil(t, h.State.UpsertNode(h.NextIndex(), nodeB)) 1333 1334 // Make a job with two task groups, each constraint to a node class 1335 job := mock.SystemJob() 1336 tgA := job.TaskGroups[0] 1337 tgA.Name = "groupA" 1338 tgA.Constraints = []*structs.Constraint{ 1339 { 1340 LTarget: "${node.class}", 1341 RTarget: node.NodeClass, 1342 Operand: "=", 1343 }, 1344 } 1345 tgB := job.TaskGroups[0].Copy() 1346 tgB.Name = "groupB" 1347 tgB.Constraints = []*structs.Constraint{ 1348 { 1349 LTarget: "${node.class}", 1350 RTarget: nodeB.NodeClass, 1351 Operand: "=", 1352 }, 1353 } 1354 1355 // Upsert Job 1356 job.TaskGroups = []*structs.TaskGroup{tgA, tgB} 1357 require.Nil(t, h.State.UpsertJob(h.NextIndex(), job)) 1358 1359 // Evaluate the job 1360 eval := &structs.Evaluation{ 1361 Namespace: structs.DefaultNamespace, 1362 ID: uuid.Generate(), 1363 Priority: job.Priority, 1364 TriggeredBy: structs.EvalTriggerJobRegister, 1365 JobID: job.ID, 1366 Status: structs.EvalStatusPending, 1367 } 1368 1369 require.Nil(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) 1370 1371 require.Nil(t, h.Process(NewSystemScheduler, eval)) 1372 require.Equal(t, "complete", h.Evals[0].Status) 1373 1374 // QueuedAllocations is drained 1375 val, ok := h.Evals[0].QueuedAllocations["groupA"] 1376 require.True(t, ok) 1377 require.Equal(t, 0, val) 1378 1379 val, ok = h.Evals[0].QueuedAllocations["groupB"] 1380 require.True(t, ok) 1381 require.Equal(t, 0, val) 1382 1383 // Single plan with two NodeAllocations 1384 require.Len(t, h.Plans, 1) 1385 require.Len(t, h.Plans[0].NodeAllocation, 2) 1386 1387 // Mark the node as ineligible 1388 node.SchedulingEligibility = structs.NodeSchedulingIneligible 1389 1390 // Evaluate the node update 1391 eval2 := &structs.Evaluation{ 1392 Namespace: structs.DefaultNamespace, 1393 ID: uuid.Generate(), 1394 Priority: job.Priority, 1395 TriggeredBy: structs.EvalTriggerNodeUpdate, 1396 NodeID: node.ID, 1397 JobID: job.ID, 1398 Status: structs.EvalStatusPending, 1399 } 1400 1401 require.Nil(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval2})) 1402 require.Nil(t, h.Process(NewSystemScheduler, eval2)) 1403 require.Equal(t, "complete", h.Evals[1].Status) 1404 1405 // Ensure no new plans 1406 require.Equal(t, 1, len(h.Plans)) 1407 1408 // Ensure all NodeAllocations are from first Eval 1409 for _, allocs := range h.Plans[0].NodeAllocation { 1410 require.Len(t, allocs, 1) 1411 require.Equal(t, eval.ID, allocs[0].EvalID) 1412 } 1413 1414 // Add a new node Class-B 1415 var nodeBTwo *structs.Node 1416 nodeBTwo = mock.Node() 1417 nodeBTwo.ComputeClass() 1418 nodeBTwo.NodeClass = "Class-B" 1419 require.Nil(t, h.State.UpsertNode(h.NextIndex(), nodeBTwo)) 1420 1421 // Evaluate the new node 1422 eval3 := &structs.Evaluation{ 1423 Namespace: structs.DefaultNamespace, 1424 ID: uuid.Generate(), 1425 Priority: 50, 1426 TriggeredBy: structs.EvalTriggerNodeUpdate, 1427 NodeID: nodeBTwo.ID, 1428 JobID: job.ID, 1429 Status: structs.EvalStatusPending, 1430 } 1431 1432 // Ensure New eval is complete 1433 require.Nil(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval3})) 1434 require.Nil(t, h.Process(NewSystemScheduler, eval3)) 1435 require.Equal(t, "complete", h.Evals[2].Status) 1436 1437 // Ensure no failed TG allocs 1438 require.Equal(t, 0, len(h.Evals[2].FailedTGAllocs)) 1439 1440 require.Len(t, h.Plans, 2) 1441 require.Len(t, h.Plans[1].NodeAllocation, 1) 1442 // Ensure all NodeAllocations are from first Eval 1443 for _, allocs := range h.Plans[1].NodeAllocation { 1444 require.Len(t, allocs, 1) 1445 require.Equal(t, eval3.ID, allocs[0].EvalID) 1446 } 1447 1448 ws := memdb.NewWatchSet() 1449 1450 allocsNodeOne, err := h.State.AllocsByNode(ws, node.ID) 1451 require.NoError(t, err) 1452 require.Len(t, allocsNodeOne, 1) 1453 1454 allocsNodeTwo, err := h.State.AllocsByNode(ws, nodeB.ID) 1455 require.NoError(t, err) 1456 require.Len(t, allocsNodeTwo, 1) 1457 1458 allocsNodeThree, err := h.State.AllocsByNode(ws, nodeBTwo.ID) 1459 require.NoError(t, err) 1460 require.Len(t, allocsNodeThree, 1) 1461 } 1462 1463 // No errors reported when no available nodes prevent placement 1464 func TestSystemSched_ExistingAllocNoNodes(t *testing.T) { 1465 h := NewHarness(t) 1466 1467 var node *structs.Node 1468 // Create a node 1469 node = mock.Node() 1470 node.ComputeClass() 1471 require.Nil(t, h.State.UpsertNode(h.NextIndex(), node)) 1472 1473 // Make a job 1474 job := mock.SystemJob() 1475 require.Nil(t, h.State.UpsertJob(h.NextIndex(), job)) 1476 1477 // Evaluate the job 1478 eval := &structs.Evaluation{ 1479 Namespace: structs.DefaultNamespace, 1480 ID: uuid.Generate(), 1481 Priority: job.Priority, 1482 TriggeredBy: structs.EvalTriggerJobRegister, 1483 JobID: job.ID, 1484 Status: structs.EvalStatusPending, 1485 } 1486 1487 require.Nil(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) 1488 require.Nil(t, h.Process(NewSystemScheduler, eval)) 1489 require.Equal(t, "complete", h.Evals[0].Status) 1490 1491 // QueuedAllocations is drained 1492 val, ok := h.Evals[0].QueuedAllocations["web"] 1493 require.True(t, ok) 1494 require.Equal(t, 0, val) 1495 1496 // The plan has one NodeAllocations 1497 require.Equal(t, 1, len(h.Plans)) 1498 1499 // Mark the node as ineligible 1500 node.SchedulingEligibility = structs.NodeSchedulingIneligible 1501 // Evaluate the job 1502 eval2 := &structs.Evaluation{ 1503 Namespace: structs.DefaultNamespace, 1504 ID: uuid.Generate(), 1505 Priority: job.Priority, 1506 TriggeredBy: structs.EvalTriggerNodeUpdate, 1507 JobID: job.ID, 1508 NodeID: node.ID, 1509 Status: structs.EvalStatusPending, 1510 } 1511 require.Nil(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval2})) 1512 require.Nil(t, h.Process(NewSystemScheduler, eval2)) 1513 require.Equal(t, "complete", h.Evals[1].Status) 1514 1515 // Create a new job version, deploy 1516 job2 := job.Copy() 1517 job2.Meta["version"] = "2" 1518 require.Nil(t, h.State.UpsertJob(h.NextIndex(), job2)) 1519 1520 // Run evaluation as a plan 1521 eval3 := &structs.Evaluation{ 1522 Namespace: structs.DefaultNamespace, 1523 ID: uuid.Generate(), 1524 Priority: job2.Priority, 1525 TriggeredBy: structs.EvalTriggerJobRegister, 1526 JobID: job2.ID, 1527 Status: structs.EvalStatusPending, 1528 AnnotatePlan: true, 1529 } 1530 1531 // Ensure New eval is complete 1532 require.Nil(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval3})) 1533 require.Nil(t, h.Process(NewSystemScheduler, eval3)) 1534 require.Equal(t, "complete", h.Evals[2].Status) 1535 1536 // Ensure there are no FailedTGAllocs 1537 require.Equal(t, 0, len(h.Evals[2].FailedTGAllocs)) 1538 require.Equal(t, 0, h.Evals[2].QueuedAllocations[job2.Name]) 1539 } 1540 1541 // No errors reported when constraints prevent placement 1542 func TestSystemSched_ConstraintErrors(t *testing.T) { 1543 h := NewHarness(t) 1544 1545 var node *structs.Node 1546 // Register some nodes 1547 // the tag "aaaaaa" is hashed so that the nodes are processed 1548 // in an order other than good, good, bad 1549 for _, tag := range []string{"aaaaaa", "foo", "foo", "foo"} { 1550 node = mock.Node() 1551 node.Meta["tag"] = tag 1552 node.ComputeClass() 1553 require.Nil(t, h.State.UpsertNode(h.NextIndex(), node)) 1554 } 1555 1556 // Mark the last node as ineligible 1557 node.SchedulingEligibility = structs.NodeSchedulingIneligible 1558 1559 // Make a job with a constraint that matches a subset of the nodes 1560 job := mock.SystemJob() 1561 job.Constraints = append(job.Constraints, 1562 &structs.Constraint{ 1563 LTarget: "${meta.tag}", 1564 RTarget: "foo", 1565 Operand: "=", 1566 }) 1567 1568 require.Nil(t, h.State.UpsertJob(h.NextIndex(), job)) 1569 1570 // Evaluate the job 1571 eval := &structs.Evaluation{ 1572 Namespace: structs.DefaultNamespace, 1573 ID: uuid.Generate(), 1574 Priority: job.Priority, 1575 TriggeredBy: structs.EvalTriggerJobRegister, 1576 JobID: job.ID, 1577 Status: structs.EvalStatusPending, 1578 } 1579 1580 require.Nil(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) 1581 require.Nil(t, h.Process(NewSystemScheduler, eval)) 1582 require.Equal(t, "complete", h.Evals[0].Status) 1583 1584 // QueuedAllocations is drained 1585 val, ok := h.Evals[0].QueuedAllocations["web"] 1586 require.True(t, ok) 1587 require.Equal(t, 0, val) 1588 1589 // The plan has two NodeAllocations 1590 require.Equal(t, 1, len(h.Plans)) 1591 require.Nil(t, h.Plans[0].Annotations) 1592 require.Equal(t, 2, len(h.Plans[0].NodeAllocation)) 1593 1594 // Two nodes were allocated and are running 1595 ws := memdb.NewWatchSet() 1596 as, err := h.State.AllocsByJob(ws, structs.DefaultNamespace, job.ID, false) 1597 require.Nil(t, err) 1598 1599 running := 0 1600 for _, a := range as { 1601 if "running" == a.Job.Status { 1602 running++ 1603 } 1604 } 1605 1606 require.Equal(t, 2, len(as)) 1607 require.Equal(t, 2, running) 1608 1609 // Failed allocations is empty 1610 require.Equal(t, 0, len(h.Evals[0].FailedTGAllocs)) 1611 } 1612 1613 func TestSystemSched_ChainedAlloc(t *testing.T) { 1614 h := NewHarness(t) 1615 1616 // Create some nodes 1617 for i := 0; i < 10; i++ { 1618 node := mock.Node() 1619 require.NoError(t, h.State.UpsertNode(h.NextIndex(), node)) 1620 } 1621 1622 // Create a job 1623 job := mock.SystemJob() 1624 require.NoError(t, h.State.UpsertJob(h.NextIndex(), job)) 1625 1626 // Create a mock evaluation to register the job 1627 eval := &structs.Evaluation{ 1628 Namespace: structs.DefaultNamespace, 1629 ID: uuid.Generate(), 1630 Priority: job.Priority, 1631 TriggeredBy: structs.EvalTriggerJobRegister, 1632 JobID: job.ID, 1633 Status: structs.EvalStatusPending, 1634 } 1635 require.NoError(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) 1636 // Process the evaluation 1637 if err := h.Process(NewSystemScheduler, eval); err != nil { 1638 t.Fatalf("err: %v", err) 1639 } 1640 1641 var allocIDs []string 1642 for _, allocList := range h.Plans[0].NodeAllocation { 1643 for _, alloc := range allocList { 1644 allocIDs = append(allocIDs, alloc.ID) 1645 } 1646 } 1647 sort.Strings(allocIDs) 1648 1649 // Create a new harness to invoke the scheduler again 1650 h1 := NewHarnessWithState(t, h.State) 1651 job1 := mock.SystemJob() 1652 job1.ID = job.ID 1653 job1.TaskGroups[0].Tasks[0].Env = make(map[string]string) 1654 job1.TaskGroups[0].Tasks[0].Env["foo"] = "bar" 1655 require.NoError(t, h1.State.UpsertJob(h1.NextIndex(), job1)) 1656 1657 // Insert two more nodes 1658 for i := 0; i < 2; i++ { 1659 node := mock.Node() 1660 require.NoError(t, h.State.UpsertNode(h.NextIndex(), node)) 1661 } 1662 1663 // Create a mock evaluation to update the job 1664 eval1 := &structs.Evaluation{ 1665 Namespace: structs.DefaultNamespace, 1666 ID: uuid.Generate(), 1667 Priority: job1.Priority, 1668 TriggeredBy: structs.EvalTriggerJobRegister, 1669 JobID: job1.ID, 1670 Status: structs.EvalStatusPending, 1671 } 1672 require.NoError(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval1})) 1673 // Process the evaluation 1674 if err := h1.Process(NewSystemScheduler, eval1); err != nil { 1675 t.Fatalf("err: %v", err) 1676 } 1677 1678 plan := h1.Plans[0] 1679 1680 // Collect all the chained allocation ids and the new allocations which 1681 // don't have any chained allocations 1682 var prevAllocs []string 1683 var newAllocs []string 1684 for _, allocList := range plan.NodeAllocation { 1685 for _, alloc := range allocList { 1686 if alloc.PreviousAllocation == "" { 1687 newAllocs = append(newAllocs, alloc.ID) 1688 continue 1689 } 1690 prevAllocs = append(prevAllocs, alloc.PreviousAllocation) 1691 } 1692 } 1693 sort.Strings(prevAllocs) 1694 1695 // Ensure that the new allocations has their corresponding original 1696 // allocation ids 1697 if !reflect.DeepEqual(prevAllocs, allocIDs) { 1698 t.Fatalf("expected: %v, actual: %v", len(allocIDs), len(prevAllocs)) 1699 } 1700 1701 // Ensuring two new allocations don't have any chained allocations 1702 if len(newAllocs) != 2 { 1703 t.Fatalf("expected: %v, actual: %v", 2, len(newAllocs)) 1704 } 1705 } 1706 1707 func TestSystemSched_PlanWithDrainedNode(t *testing.T) { 1708 h := NewHarness(t) 1709 1710 // Register two nodes with two different classes 1711 node := mock.Node() 1712 node.NodeClass = "green" 1713 node.Drain = true 1714 node.ComputeClass() 1715 require.NoError(t, h.State.UpsertNode(h.NextIndex(), node)) 1716 1717 node2 := mock.Node() 1718 node2.NodeClass = "blue" 1719 node2.ComputeClass() 1720 require.NoError(t, h.State.UpsertNode(h.NextIndex(), node2)) 1721 1722 // Create a Job with two task groups, each constrained on node class 1723 job := mock.SystemJob() 1724 tg1 := job.TaskGroups[0] 1725 tg1.Constraints = append(tg1.Constraints, 1726 &structs.Constraint{ 1727 LTarget: "${node.class}", 1728 RTarget: "green", 1729 Operand: "==", 1730 }) 1731 1732 tg2 := tg1.Copy() 1733 tg2.Name = "web2" 1734 tg2.Constraints[0].RTarget = "blue" 1735 job.TaskGroups = append(job.TaskGroups, tg2) 1736 require.NoError(t, h.State.UpsertJob(h.NextIndex(), job)) 1737 1738 // Create an allocation on each node 1739 alloc := mock.Alloc() 1740 alloc.Job = job 1741 alloc.JobID = job.ID 1742 alloc.NodeID = node.ID 1743 alloc.Name = "my-job.web[0]" 1744 alloc.DesiredTransition.Migrate = helper.BoolToPtr(true) 1745 alloc.TaskGroup = "web" 1746 1747 alloc2 := mock.Alloc() 1748 alloc2.Job = job 1749 alloc2.JobID = job.ID 1750 alloc2.NodeID = node2.ID 1751 alloc2.Name = "my-job.web2[0]" 1752 alloc2.TaskGroup = "web2" 1753 require.NoError(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc, alloc2})) 1754 1755 // Create a mock evaluation to deal with drain 1756 eval := &structs.Evaluation{ 1757 Namespace: structs.DefaultNamespace, 1758 ID: uuid.Generate(), 1759 Priority: 50, 1760 TriggeredBy: structs.EvalTriggerNodeUpdate, 1761 JobID: job.ID, 1762 NodeID: node.ID, 1763 Status: structs.EvalStatusPending, 1764 } 1765 require.NoError(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) 1766 1767 // Process the evaluation 1768 err := h.Process(NewSystemScheduler, eval) 1769 if err != nil { 1770 t.Fatalf("err: %v", err) 1771 } 1772 1773 // Ensure a single plan 1774 if len(h.Plans) != 1 { 1775 t.Fatalf("bad: %#v", h.Plans) 1776 } 1777 plan := h.Plans[0] 1778 1779 // Ensure the plan evicted the alloc on the failed node 1780 planned := plan.NodeUpdate[node.ID] 1781 if len(planned) != 1 { 1782 t.Fatalf("bad: %#v", plan) 1783 } 1784 1785 // Ensure the plan didn't place 1786 if len(plan.NodeAllocation) != 0 { 1787 t.Fatalf("bad: %#v", plan) 1788 } 1789 1790 // Ensure the allocations is stopped 1791 if planned[0].DesiredStatus != structs.AllocDesiredStatusStop { 1792 t.Fatalf("bad: %#v", planned[0]) 1793 } 1794 1795 h.AssertEvalStatus(t, structs.EvalStatusComplete) 1796 } 1797 1798 func TestSystemSched_QueuedAllocsMultTG(t *testing.T) { 1799 h := NewHarness(t) 1800 1801 // Register two nodes with two different classes 1802 node := mock.Node() 1803 node.NodeClass = "green" 1804 node.ComputeClass() 1805 require.NoError(t, h.State.UpsertNode(h.NextIndex(), node)) 1806 1807 node2 := mock.Node() 1808 node2.NodeClass = "blue" 1809 node2.ComputeClass() 1810 require.NoError(t, h.State.UpsertNode(h.NextIndex(), node2)) 1811 1812 // Create a Job with two task groups, each constrained on node class 1813 job := mock.SystemJob() 1814 tg1 := job.TaskGroups[0] 1815 tg1.Constraints = append(tg1.Constraints, 1816 &structs.Constraint{ 1817 LTarget: "${node.class}", 1818 RTarget: "green", 1819 Operand: "==", 1820 }) 1821 1822 tg2 := tg1.Copy() 1823 tg2.Name = "web2" 1824 tg2.Constraints[0].RTarget = "blue" 1825 job.TaskGroups = append(job.TaskGroups, tg2) 1826 require.NoError(t, h.State.UpsertJob(h.NextIndex(), job)) 1827 1828 // Create a mock evaluation to deal with drain 1829 eval := &structs.Evaluation{ 1830 Namespace: structs.DefaultNamespace, 1831 ID: uuid.Generate(), 1832 Priority: 50, 1833 TriggeredBy: structs.EvalTriggerNodeUpdate, 1834 JobID: job.ID, 1835 NodeID: node.ID, 1836 Status: structs.EvalStatusPending, 1837 } 1838 require.NoError(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) 1839 1840 // Process the evaluation 1841 err := h.Process(NewSystemScheduler, eval) 1842 if err != nil { 1843 t.Fatalf("err: %v", err) 1844 } 1845 1846 // Ensure a single plan 1847 if len(h.Plans) != 1 { 1848 t.Fatalf("bad: %#v", h.Plans) 1849 } 1850 1851 qa := h.Evals[0].QueuedAllocations 1852 if qa["web"] != 0 || qa["web2"] != 0 { 1853 t.Fatalf("bad queued allocations %#v", qa) 1854 } 1855 1856 h.AssertEvalStatus(t, structs.EvalStatusComplete) 1857 } 1858 1859 func TestSystemSched_Preemption(t *testing.T) { 1860 h := NewHarness(t) 1861 1862 // Create nodes 1863 var nodes []*structs.Node 1864 for i := 0; i < 2; i++ { 1865 node := mock.Node() 1866 // TODO(preetha): remove in 0.11 1867 node.Resources = &structs.Resources{ 1868 CPU: 3072, 1869 MemoryMB: 5034, 1870 DiskMB: 20 * 1024, 1871 Networks: []*structs.NetworkResource{ 1872 { 1873 Device: "eth0", 1874 CIDR: "192.168.0.100/32", 1875 MBits: 1000, 1876 }, 1877 }, 1878 } 1879 node.NodeResources = &structs.NodeResources{ 1880 Cpu: structs.NodeCpuResources{ 1881 CpuShares: 3072, 1882 }, 1883 Memory: structs.NodeMemoryResources{ 1884 MemoryMB: 5034, 1885 }, 1886 Disk: structs.NodeDiskResources{ 1887 DiskMB: 20 * 1024, 1888 }, 1889 Networks: []*structs.NetworkResource{ 1890 { 1891 Device: "eth0", 1892 CIDR: "192.168.0.100/32", 1893 MBits: 1000, 1894 }, 1895 }, 1896 } 1897 require.NoError(t, h.State.UpsertNode(h.NextIndex(), node)) 1898 nodes = append(nodes, node) 1899 } 1900 1901 // Enable Preemption 1902 h.State.SchedulerSetConfig(h.NextIndex(), &structs.SchedulerConfiguration{ 1903 PreemptionConfig: structs.PreemptionConfig{ 1904 SystemSchedulerEnabled: true, 1905 }, 1906 }) 1907 1908 // Create some low priority batch jobs and allocations for them 1909 // One job uses a reserved port 1910 job1 := mock.BatchJob() 1911 job1.Type = structs.JobTypeBatch 1912 job1.Priority = 20 1913 job1.TaskGroups[0].Tasks[0].Resources = &structs.Resources{ 1914 CPU: 512, 1915 MemoryMB: 1024, 1916 Networks: []*structs.NetworkResource{ 1917 { 1918 MBits: 200, 1919 ReservedPorts: []structs.Port{ 1920 { 1921 Label: "web", 1922 Value: 80, 1923 }, 1924 }, 1925 }, 1926 }, 1927 } 1928 1929 alloc1 := mock.Alloc() 1930 alloc1.Job = job1 1931 alloc1.JobID = job1.ID 1932 alloc1.NodeID = nodes[0].ID 1933 alloc1.Name = "my-job[0]" 1934 alloc1.TaskGroup = job1.TaskGroups[0].Name 1935 alloc1.AllocatedResources = &structs.AllocatedResources{ 1936 Tasks: map[string]*structs.AllocatedTaskResources{ 1937 "web": { 1938 Cpu: structs.AllocatedCpuResources{ 1939 CpuShares: 512, 1940 }, 1941 Memory: structs.AllocatedMemoryResources{ 1942 MemoryMB: 1024, 1943 }, 1944 Networks: []*structs.NetworkResource{ 1945 { 1946 Device: "eth0", 1947 IP: "192.168.0.100", 1948 ReservedPorts: []structs.Port{{Label: "web", Value: 80}}, 1949 MBits: 200, 1950 }, 1951 }, 1952 }, 1953 }, 1954 Shared: structs.AllocatedSharedResources{ 1955 DiskMB: 5 * 1024, 1956 }, 1957 } 1958 1959 require.NoError(t, h.State.UpsertJob(h.NextIndex(), job1)) 1960 1961 job2 := mock.BatchJob() 1962 job2.Type = structs.JobTypeBatch 1963 job2.Priority = 20 1964 job2.TaskGroups[0].Tasks[0].Resources = &structs.Resources{ 1965 CPU: 512, 1966 MemoryMB: 1024, 1967 Networks: []*structs.NetworkResource{ 1968 { 1969 MBits: 200, 1970 }, 1971 }, 1972 } 1973 1974 alloc2 := mock.Alloc() 1975 alloc2.Job = job2 1976 alloc2.JobID = job2.ID 1977 alloc2.NodeID = nodes[0].ID 1978 alloc2.Name = "my-job[2]" 1979 alloc2.TaskGroup = job2.TaskGroups[0].Name 1980 alloc2.AllocatedResources = &structs.AllocatedResources{ 1981 Tasks: map[string]*structs.AllocatedTaskResources{ 1982 "web": { 1983 Cpu: structs.AllocatedCpuResources{ 1984 CpuShares: 512, 1985 }, 1986 Memory: structs.AllocatedMemoryResources{ 1987 MemoryMB: 1024, 1988 }, 1989 Networks: []*structs.NetworkResource{ 1990 { 1991 Device: "eth0", 1992 IP: "192.168.0.100", 1993 MBits: 200, 1994 }, 1995 }, 1996 }, 1997 }, 1998 Shared: structs.AllocatedSharedResources{ 1999 DiskMB: 5 * 1024, 2000 }, 2001 } 2002 require.NoError(t, h.State.UpsertJob(h.NextIndex(), job2)) 2003 2004 job3 := mock.Job() 2005 job3.Type = structs.JobTypeBatch 2006 job3.Priority = 40 2007 job3.TaskGroups[0].Tasks[0].Resources = &structs.Resources{ 2008 CPU: 1024, 2009 MemoryMB: 2048, 2010 Networks: []*structs.NetworkResource{ 2011 { 2012 Device: "eth0", 2013 MBits: 400, 2014 }, 2015 }, 2016 } 2017 2018 alloc3 := mock.Alloc() 2019 alloc3.Job = job3 2020 alloc3.JobID = job3.ID 2021 alloc3.NodeID = nodes[0].ID 2022 alloc3.Name = "my-job[0]" 2023 alloc3.TaskGroup = job3.TaskGroups[0].Name 2024 alloc3.AllocatedResources = &structs.AllocatedResources{ 2025 Tasks: map[string]*structs.AllocatedTaskResources{ 2026 "web": { 2027 Cpu: structs.AllocatedCpuResources{ 2028 CpuShares: 1024, 2029 }, 2030 Memory: structs.AllocatedMemoryResources{ 2031 MemoryMB: 25, 2032 }, 2033 Networks: []*structs.NetworkResource{ 2034 { 2035 Device: "eth0", 2036 IP: "192.168.0.100", 2037 ReservedPorts: []structs.Port{{Label: "web", Value: 80}}, 2038 MBits: 400, 2039 }, 2040 }, 2041 }, 2042 }, 2043 Shared: structs.AllocatedSharedResources{ 2044 DiskMB: 5 * 1024, 2045 }, 2046 } 2047 require.NoError(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc1, alloc2, alloc3})) 2048 2049 // Create a high priority job and allocs for it 2050 // These allocs should not be preempted 2051 2052 job4 := mock.BatchJob() 2053 job4.Type = structs.JobTypeBatch 2054 job4.Priority = 100 2055 job4.TaskGroups[0].Tasks[0].Resources = &structs.Resources{ 2056 CPU: 1024, 2057 MemoryMB: 2048, 2058 Networks: []*structs.NetworkResource{ 2059 { 2060 MBits: 100, 2061 }, 2062 }, 2063 } 2064 2065 alloc4 := mock.Alloc() 2066 alloc4.Job = job4 2067 alloc4.JobID = job4.ID 2068 alloc4.NodeID = nodes[0].ID 2069 alloc4.Name = "my-job4[0]" 2070 alloc4.TaskGroup = job4.TaskGroups[0].Name 2071 alloc4.AllocatedResources = &structs.AllocatedResources{ 2072 Tasks: map[string]*structs.AllocatedTaskResources{ 2073 "web": { 2074 Cpu: structs.AllocatedCpuResources{ 2075 CpuShares: 1024, 2076 }, 2077 Memory: structs.AllocatedMemoryResources{ 2078 MemoryMB: 2048, 2079 }, 2080 Networks: []*structs.NetworkResource{ 2081 { 2082 Device: "eth0", 2083 IP: "192.168.0.100", 2084 ReservedPorts: []structs.Port{{Label: "web", Value: 80}}, 2085 MBits: 100, 2086 }, 2087 }, 2088 }, 2089 }, 2090 Shared: structs.AllocatedSharedResources{ 2091 DiskMB: 2 * 1024, 2092 }, 2093 } 2094 require.NoError(t, h.State.UpsertJob(h.NextIndex(), job4)) 2095 require.NoError(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc4})) 2096 2097 // Create a system job such that it would need to preempt both allocs to succeed 2098 job := mock.SystemJob() 2099 job.TaskGroups[0].Tasks[0].Resources = &structs.Resources{ 2100 CPU: 1948, 2101 MemoryMB: 256, 2102 Networks: []*structs.NetworkResource{ 2103 { 2104 MBits: 800, 2105 DynamicPorts: []structs.Port{{Label: "http"}}, 2106 }, 2107 }, 2108 } 2109 require.NoError(t, h.State.UpsertJob(h.NextIndex(), job)) 2110 2111 // Create a mock evaluation to register the job 2112 eval := &structs.Evaluation{ 2113 Namespace: structs.DefaultNamespace, 2114 ID: uuid.Generate(), 2115 Priority: job.Priority, 2116 TriggeredBy: structs.EvalTriggerJobRegister, 2117 JobID: job.ID, 2118 Status: structs.EvalStatusPending, 2119 } 2120 require.NoError(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) 2121 2122 // Process the evaluation 2123 err := h.Process(NewSystemScheduler, eval) 2124 require := require.New(t) 2125 require.Nil(err) 2126 2127 // Ensure a single plan 2128 require.Equal(1, len(h.Plans)) 2129 plan := h.Plans[0] 2130 2131 // Ensure the plan doesn't have annotations. 2132 require.Nil(plan.Annotations) 2133 2134 // Ensure the plan allocated on both nodes 2135 var planned []*structs.Allocation 2136 preemptingAllocId := "" 2137 require.Equal(2, len(plan.NodeAllocation)) 2138 2139 // The alloc that got placed on node 1 is the preemptor 2140 for _, allocList := range plan.NodeAllocation { 2141 planned = append(planned, allocList...) 2142 for _, alloc := range allocList { 2143 if alloc.NodeID == nodes[0].ID { 2144 preemptingAllocId = alloc.ID 2145 } 2146 } 2147 } 2148 2149 // Lookup the allocations by JobID 2150 ws := memdb.NewWatchSet() 2151 out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) 2152 require.NoError(err) 2153 2154 // Ensure all allocations placed 2155 require.Equal(2, len(out)) 2156 2157 // Verify that one node has preempted allocs 2158 require.NotNil(plan.NodePreemptions[nodes[0].ID]) 2159 preemptedAllocs := plan.NodePreemptions[nodes[0].ID] 2160 2161 // Verify that three jobs have preempted allocs 2162 require.Equal(3, len(preemptedAllocs)) 2163 2164 expectedPreemptedJobIDs := []string{job1.ID, job2.ID, job3.ID} 2165 2166 // We expect job1, job2 and job3 to have preempted allocations 2167 // job4 should not have any allocs preempted 2168 for _, alloc := range preemptedAllocs { 2169 require.Contains(expectedPreemptedJobIDs, alloc.JobID) 2170 } 2171 // Look up the preempted allocs by job ID 2172 ws = memdb.NewWatchSet() 2173 2174 for _, jobId := range expectedPreemptedJobIDs { 2175 out, err = h.State.AllocsByJob(ws, structs.DefaultNamespace, jobId, false) 2176 require.NoError(err) 2177 for _, alloc := range out { 2178 require.Equal(structs.AllocDesiredStatusEvict, alloc.DesiredStatus) 2179 require.Equal(fmt.Sprintf("Preempted by alloc ID %v", preemptingAllocId), alloc.DesiredDescription) 2180 } 2181 } 2182 2183 h.AssertEvalStatus(t, structs.EvalStatusComplete) 2184 2185 }