github.com/djenriquez/nomad-1@v0.8.1/scheduler/util_test.go (about) 1 package scheduler 2 3 import ( 4 "fmt" 5 "log" 6 "os" 7 "reflect" 8 "testing" 9 10 "github.com/hashicorp/nomad/helper" 11 "github.com/hashicorp/nomad/helper/uuid" 12 "github.com/hashicorp/nomad/nomad/mock" 13 "github.com/hashicorp/nomad/nomad/state" 14 "github.com/hashicorp/nomad/nomad/structs" 15 ) 16 17 // noErr is used to assert there are no errors 18 func noErr(t *testing.T, err error) { 19 if err != nil { 20 t.Fatalf("err: %v", err) 21 } 22 } 23 24 func TestMaterializeTaskGroups(t *testing.T) { 25 job := mock.Job() 26 index := materializeTaskGroups(job) 27 if len(index) != 10 { 28 t.Fatalf("Bad: %#v", index) 29 } 30 31 for i := 0; i < 10; i++ { 32 name := fmt.Sprintf("my-job.web[%d]", i) 33 tg, ok := index[name] 34 if !ok { 35 t.Fatalf("bad") 36 } 37 if tg != job.TaskGroups[0] { 38 t.Fatalf("bad") 39 } 40 } 41 } 42 43 func TestDiffAllocs(t *testing.T) { 44 job := mock.Job() 45 required := materializeTaskGroups(job) 46 47 // The "old" job has a previous modify index 48 oldJob := new(structs.Job) 49 *oldJob = *job 50 oldJob.JobModifyIndex -= 1 51 52 drainNode := mock.Node() 53 drainNode.Drain = true 54 55 deadNode := mock.Node() 56 deadNode.Status = structs.NodeStatusDown 57 58 tainted := map[string]*structs.Node{ 59 "dead": deadNode, 60 "drainNode": drainNode, 61 } 62 63 allocs := []*structs.Allocation{ 64 // Update the 1st 65 { 66 ID: uuid.Generate(), 67 NodeID: "zip", 68 Name: "my-job.web[0]", 69 Job: oldJob, 70 }, 71 72 // Ignore the 2rd 73 { 74 ID: uuid.Generate(), 75 NodeID: "zip", 76 Name: "my-job.web[1]", 77 Job: job, 78 }, 79 80 // Evict 11th 81 { 82 ID: uuid.Generate(), 83 NodeID: "zip", 84 Name: "my-job.web[10]", 85 Job: oldJob, 86 }, 87 88 // Migrate the 3rd 89 { 90 ID: uuid.Generate(), 91 NodeID: "drainNode", 92 Name: "my-job.web[2]", 93 Job: oldJob, 94 DesiredTransition: structs.DesiredTransition{ 95 Migrate: helper.BoolToPtr(true), 96 }, 97 }, 98 // Mark the 4th lost 99 { 100 ID: uuid.Generate(), 101 NodeID: "dead", 102 Name: "my-job.web[3]", 103 Job: oldJob, 104 }, 105 } 106 107 // Have three terminal allocs 108 terminalAllocs := map[string]*structs.Allocation{ 109 "my-job.web[4]": { 110 ID: uuid.Generate(), 111 NodeID: "zip", 112 Name: "my-job.web[4]", 113 Job: job, 114 }, 115 "my-job.web[5]": { 116 ID: uuid.Generate(), 117 NodeID: "zip", 118 Name: "my-job.web[5]", 119 Job: job, 120 }, 121 "my-job.web[6]": { 122 ID: uuid.Generate(), 123 NodeID: "zip", 124 Name: "my-job.web[6]", 125 Job: job, 126 }, 127 } 128 129 diff := diffAllocs(job, tainted, required, allocs, terminalAllocs) 130 place := diff.place 131 update := diff.update 132 migrate := diff.migrate 133 stop := diff.stop 134 ignore := diff.ignore 135 lost := diff.lost 136 137 // We should update the first alloc 138 if len(update) != 1 || update[0].Alloc != allocs[0] { 139 t.Fatalf("bad: %#v", update) 140 } 141 142 // We should ignore the second alloc 143 if len(ignore) != 1 || ignore[0].Alloc != allocs[1] { 144 t.Fatalf("bad: %#v", ignore) 145 } 146 147 // We should stop the 3rd alloc 148 if len(stop) != 1 || stop[0].Alloc != allocs[2] { 149 t.Fatalf("bad: %#v", stop) 150 } 151 152 // We should migrate the 4rd alloc 153 if len(migrate) != 1 || migrate[0].Alloc != allocs[3] { 154 t.Fatalf("bad: %#v", migrate) 155 } 156 157 // We should mark the 5th alloc as lost 158 if len(lost) != 1 || lost[0].Alloc != allocs[4] { 159 t.Fatalf("bad: %#v", migrate) 160 } 161 162 // We should place 6 163 if len(place) != 6 { 164 t.Fatalf("bad: %#v", place) 165 } 166 167 // Ensure that the allocations which are replacements of terminal allocs are 168 // annotated 169 for name, alloc := range terminalAllocs { 170 for _, allocTuple := range diff.place { 171 if name == allocTuple.Name { 172 if !reflect.DeepEqual(alloc, allocTuple.Alloc) { 173 t.Fatalf("expected: %#v, actual: %#v", alloc, allocTuple.Alloc) 174 } 175 } 176 } 177 } 178 } 179 180 func TestDiffSystemAllocs(t *testing.T) { 181 job := mock.SystemJob() 182 183 drainNode := mock.Node() 184 drainNode.Drain = true 185 186 deadNode := mock.Node() 187 deadNode.Status = structs.NodeStatusDown 188 189 tainted := map[string]*structs.Node{ 190 deadNode.ID: deadNode, 191 drainNode.ID: drainNode, 192 } 193 194 // Create three alive nodes. 195 nodes := []*structs.Node{{ID: "foo"}, {ID: "bar"}, {ID: "baz"}, 196 {ID: "pipe"}, {ID: drainNode.ID}, {ID: deadNode.ID}} 197 198 // The "old" job has a previous modify index 199 oldJob := new(structs.Job) 200 *oldJob = *job 201 oldJob.JobModifyIndex -= 1 202 203 allocs := []*structs.Allocation{ 204 // Update allocation on baz 205 { 206 ID: uuid.Generate(), 207 NodeID: "baz", 208 Name: "my-job.web[0]", 209 Job: oldJob, 210 }, 211 212 // Ignore allocation on bar 213 { 214 ID: uuid.Generate(), 215 NodeID: "bar", 216 Name: "my-job.web[0]", 217 Job: job, 218 }, 219 220 // Stop allocation on draining node. 221 { 222 ID: uuid.Generate(), 223 NodeID: drainNode.ID, 224 Name: "my-job.web[0]", 225 Job: oldJob, 226 DesiredTransition: structs.DesiredTransition{ 227 Migrate: helper.BoolToPtr(true), 228 }, 229 }, 230 // Mark as lost on a dead node 231 { 232 ID: uuid.Generate(), 233 NodeID: deadNode.ID, 234 Name: "my-job.web[0]", 235 Job: oldJob, 236 }, 237 } 238 239 // Have three terminal allocs 240 terminalAllocs := map[string]*structs.Allocation{ 241 "my-job.web[0]": { 242 ID: uuid.Generate(), 243 NodeID: "pipe", 244 Name: "my-job.web[0]", 245 Job: job, 246 }, 247 } 248 249 diff := diffSystemAllocs(job, nodes, tainted, allocs, terminalAllocs) 250 place := diff.place 251 update := diff.update 252 migrate := diff.migrate 253 stop := diff.stop 254 ignore := diff.ignore 255 lost := diff.lost 256 257 // We should update the first alloc 258 if len(update) != 1 || update[0].Alloc != allocs[0] { 259 t.Fatalf("bad: %#v", update) 260 } 261 262 // We should ignore the second alloc 263 if len(ignore) != 1 || ignore[0].Alloc != allocs[1] { 264 t.Fatalf("bad: %#v", ignore) 265 } 266 267 // We should stop the third alloc 268 if len(stop) != 0 { 269 t.Fatalf("bad: %#v", stop) 270 } 271 272 // There should be no migrates. 273 if len(migrate) != 1 || migrate[0].Alloc != allocs[2] { 274 t.Fatalf("bad: %#v", migrate) 275 } 276 277 // We should mark the 5th alloc as lost 278 if len(lost) != 1 || lost[0].Alloc != allocs[3] { 279 t.Fatalf("bad: %#v", migrate) 280 } 281 282 // We should place 1 283 if l := len(place); l != 2 { 284 t.Fatalf("bad: %#v", l) 285 } 286 287 // Ensure that the allocations which are replacements of terminal allocs are 288 // annotated 289 for _, alloc := range terminalAllocs { 290 for _, allocTuple := range diff.place { 291 if alloc.NodeID == allocTuple.Alloc.NodeID { 292 if !reflect.DeepEqual(alloc, allocTuple.Alloc) { 293 t.Fatalf("expected: %#v, actual: %#v", alloc, allocTuple.Alloc) 294 } 295 } 296 } 297 } 298 } 299 300 func TestReadyNodesInDCs(t *testing.T) { 301 state := state.TestStateStore(t) 302 node1 := mock.Node() 303 node2 := mock.Node() 304 node2.Datacenter = "dc2" 305 node3 := mock.Node() 306 node3.Datacenter = "dc2" 307 node3.Status = structs.NodeStatusDown 308 node4 := mock.Node() 309 node4.Drain = true 310 311 noErr(t, state.UpsertNode(1000, node1)) 312 noErr(t, state.UpsertNode(1001, node2)) 313 noErr(t, state.UpsertNode(1002, node3)) 314 noErr(t, state.UpsertNode(1003, node4)) 315 316 nodes, dc, err := readyNodesInDCs(state, []string{"dc1", "dc2"}) 317 if err != nil { 318 t.Fatalf("err: %v", err) 319 } 320 321 if len(nodes) != 2 { 322 t.Fatalf("bad: %v", nodes) 323 } 324 if nodes[0].ID == node3.ID || nodes[1].ID == node3.ID { 325 t.Fatalf("Bad: %#v", nodes) 326 } 327 if count, ok := dc["dc1"]; !ok || count != 1 { 328 t.Fatalf("Bad: dc1 count %v", count) 329 } 330 if count, ok := dc["dc2"]; !ok || count != 1 { 331 t.Fatalf("Bad: dc2 count %v", count) 332 } 333 } 334 335 func TestRetryMax(t *testing.T) { 336 calls := 0 337 bad := func() (bool, error) { 338 calls += 1 339 return false, nil 340 } 341 err := retryMax(3, bad, nil) 342 if err == nil { 343 t.Fatalf("should fail") 344 } 345 if calls != 3 { 346 t.Fatalf("mis match") 347 } 348 349 calls = 0 350 first := true 351 reset := func() bool { 352 if calls == 3 && first { 353 first = false 354 return true 355 } 356 return false 357 } 358 err = retryMax(3, bad, reset) 359 if err == nil { 360 t.Fatalf("should fail") 361 } 362 if calls != 6 { 363 t.Fatalf("mis match") 364 } 365 366 calls = 0 367 good := func() (bool, error) { 368 calls += 1 369 return true, nil 370 } 371 err = retryMax(3, good, nil) 372 if err != nil { 373 t.Fatalf("err: %v", err) 374 } 375 if calls != 1 { 376 t.Fatalf("mis match") 377 } 378 } 379 380 func TestTaintedNodes(t *testing.T) { 381 state := state.TestStateStore(t) 382 node1 := mock.Node() 383 node2 := mock.Node() 384 node2.Datacenter = "dc2" 385 node3 := mock.Node() 386 node3.Datacenter = "dc2" 387 node3.Status = structs.NodeStatusDown 388 node4 := mock.Node() 389 node4.Drain = true 390 noErr(t, state.UpsertNode(1000, node1)) 391 noErr(t, state.UpsertNode(1001, node2)) 392 noErr(t, state.UpsertNode(1002, node3)) 393 noErr(t, state.UpsertNode(1003, node4)) 394 395 allocs := []*structs.Allocation{ 396 {NodeID: node1.ID}, 397 {NodeID: node2.ID}, 398 {NodeID: node3.ID}, 399 {NodeID: node4.ID}, 400 {NodeID: "12345678-abcd-efab-cdef-123456789abc"}, 401 } 402 tainted, err := taintedNodes(state, allocs) 403 if err != nil { 404 t.Fatalf("err: %v", err) 405 } 406 407 if len(tainted) != 3 { 408 t.Fatalf("bad: %v", tainted) 409 } 410 411 if _, ok := tainted[node1.ID]; ok { 412 t.Fatalf("Bad: %v", tainted) 413 } 414 if _, ok := tainted[node2.ID]; ok { 415 t.Fatalf("Bad: %v", tainted) 416 } 417 418 if node, ok := tainted[node3.ID]; !ok || node == nil { 419 t.Fatalf("Bad: %v", tainted) 420 } 421 422 if node, ok := tainted[node4.ID]; !ok || node == nil { 423 t.Fatalf("Bad: %v", tainted) 424 } 425 426 if node, ok := tainted["12345678-abcd-efab-cdef-123456789abc"]; !ok || node != nil { 427 t.Fatalf("Bad: %v", tainted) 428 } 429 } 430 431 func TestShuffleNodes(t *testing.T) { 432 // Use a large number of nodes to make the probability of shuffling to the 433 // original order very low. 434 nodes := []*structs.Node{ 435 mock.Node(), 436 mock.Node(), 437 mock.Node(), 438 mock.Node(), 439 mock.Node(), 440 mock.Node(), 441 mock.Node(), 442 mock.Node(), 443 mock.Node(), 444 mock.Node(), 445 } 446 orig := make([]*structs.Node, len(nodes)) 447 copy(orig, nodes) 448 shuffleNodes(nodes) 449 if reflect.DeepEqual(nodes, orig) { 450 t.Fatalf("should not match") 451 } 452 } 453 454 func TestTasksUpdated(t *testing.T) { 455 j1 := mock.Job() 456 j2 := mock.Job() 457 name := j1.TaskGroups[0].Name 458 459 if tasksUpdated(j1, j2, name) { 460 t.Fatalf("bad") 461 } 462 463 j2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other" 464 if !tasksUpdated(j1, j2, name) { 465 t.Fatalf("bad") 466 } 467 468 j3 := mock.Job() 469 j3.TaskGroups[0].Tasks[0].Name = "foo" 470 if !tasksUpdated(j1, j3, name) { 471 t.Fatalf("bad") 472 } 473 474 j4 := mock.Job() 475 j4.TaskGroups[0].Tasks[0].Driver = "foo" 476 if !tasksUpdated(j1, j4, name) { 477 t.Fatalf("bad") 478 } 479 480 j5 := mock.Job() 481 j5.TaskGroups[0].Tasks = append(j5.TaskGroups[0].Tasks, 482 j5.TaskGroups[0].Tasks[0]) 483 if !tasksUpdated(j1, j5, name) { 484 t.Fatalf("bad") 485 } 486 487 j6 := mock.Job() 488 j6.TaskGroups[0].Tasks[0].Resources.Networks[0].DynamicPorts = []structs.Port{ 489 {Label: "http", Value: 0}, 490 {Label: "https", Value: 0}, 491 {Label: "admin", Value: 0}, 492 } 493 if !tasksUpdated(j1, j6, name) { 494 t.Fatalf("bad") 495 } 496 497 j7 := mock.Job() 498 j7.TaskGroups[0].Tasks[0].Env["NEW_ENV"] = "NEW_VALUE" 499 if !tasksUpdated(j1, j7, name) { 500 t.Fatalf("bad") 501 } 502 503 j8 := mock.Job() 504 j8.TaskGroups[0].Tasks[0].User = "foo" 505 if !tasksUpdated(j1, j8, name) { 506 t.Fatalf("bad") 507 } 508 509 j9 := mock.Job() 510 j9.TaskGroups[0].Tasks[0].Artifacts = []*structs.TaskArtifact{ 511 { 512 GetterSource: "http://foo.com/bar", 513 }, 514 } 515 if !tasksUpdated(j1, j9, name) { 516 t.Fatalf("bad") 517 } 518 519 j10 := mock.Job() 520 j10.TaskGroups[0].Tasks[0].Meta["baz"] = "boom" 521 if !tasksUpdated(j1, j10, name) { 522 t.Fatalf("bad") 523 } 524 525 j11 := mock.Job() 526 j11.TaskGroups[0].Tasks[0].Resources.CPU = 1337 527 if !tasksUpdated(j1, j11, name) { 528 t.Fatalf("bad") 529 } 530 531 j12 := mock.Job() 532 j12.TaskGroups[0].Tasks[0].Resources.Networks[0].MBits = 100 533 if !tasksUpdated(j1, j12, name) { 534 t.Fatalf("bad") 535 } 536 537 j13 := mock.Job() 538 j13.TaskGroups[0].Tasks[0].Resources.Networks[0].DynamicPorts[0].Label = "foobar" 539 if !tasksUpdated(j1, j13, name) { 540 t.Fatalf("bad") 541 } 542 543 j14 := mock.Job() 544 j14.TaskGroups[0].Tasks[0].Resources.Networks[0].ReservedPorts = []structs.Port{{Label: "foo", Value: 1312}} 545 if !tasksUpdated(j1, j14, name) { 546 t.Fatalf("bad") 547 } 548 549 j15 := mock.Job() 550 j15.TaskGroups[0].Tasks[0].Vault = &structs.Vault{Policies: []string{"foo"}} 551 if !tasksUpdated(j1, j15, name) { 552 t.Fatalf("bad") 553 } 554 555 j16 := mock.Job() 556 j16.TaskGroups[0].EphemeralDisk.Sticky = true 557 if !tasksUpdated(j1, j16, name) { 558 t.Fatal("bad") 559 } 560 561 // Change group meta 562 j17 := mock.Job() 563 j17.TaskGroups[0].Meta["j17_test"] = "roll_baby_roll" 564 if !tasksUpdated(j1, j17, name) { 565 t.Fatal("bad") 566 } 567 568 // Change job meta 569 j18 := mock.Job() 570 j18.Meta["j18_test"] = "roll_baby_roll" 571 if !tasksUpdated(j1, j18, name) { 572 t.Fatal("bad") 573 } 574 } 575 576 func TestEvictAndPlace_LimitLessThanAllocs(t *testing.T) { 577 _, ctx := testContext(t) 578 allocs := []allocTuple{ 579 {Alloc: &structs.Allocation{ID: uuid.Generate()}}, 580 {Alloc: &structs.Allocation{ID: uuid.Generate()}}, 581 {Alloc: &structs.Allocation{ID: uuid.Generate()}}, 582 {Alloc: &structs.Allocation{ID: uuid.Generate()}}, 583 } 584 diff := &diffResult{} 585 586 limit := 2 587 if !evictAndPlace(ctx, diff, allocs, "", &limit) { 588 t.Fatal("evictAndReplace() should have returned true") 589 } 590 591 if limit != 0 { 592 t.Fatalf("evictAndReplace() should decremented limit; got %v; want 0", limit) 593 } 594 595 if len(diff.place) != 2 { 596 t.Fatalf("evictAndReplace() didn't insert into diffResult properly: %v", diff.place) 597 } 598 } 599 600 func TestEvictAndPlace_LimitEqualToAllocs(t *testing.T) { 601 _, ctx := testContext(t) 602 allocs := []allocTuple{ 603 {Alloc: &structs.Allocation{ID: uuid.Generate()}}, 604 {Alloc: &structs.Allocation{ID: uuid.Generate()}}, 605 {Alloc: &structs.Allocation{ID: uuid.Generate()}}, 606 {Alloc: &structs.Allocation{ID: uuid.Generate()}}, 607 } 608 diff := &diffResult{} 609 610 limit := 4 611 if evictAndPlace(ctx, diff, allocs, "", &limit) { 612 t.Fatal("evictAndReplace() should have returned false") 613 } 614 615 if limit != 0 { 616 t.Fatalf("evictAndReplace() should decremented limit; got %v; want 0", limit) 617 } 618 619 if len(diff.place) != 4 { 620 t.Fatalf("evictAndReplace() didn't insert into diffResult properly: %v", diff.place) 621 } 622 } 623 624 func TestSetStatus(t *testing.T) { 625 h := NewHarness(t) 626 logger := log.New(os.Stderr, "", log.LstdFlags) 627 eval := mock.Eval() 628 status := "a" 629 desc := "b" 630 if err := setStatus(logger, h, eval, nil, nil, nil, status, desc, nil, ""); err != nil { 631 t.Fatalf("setStatus() failed: %v", err) 632 } 633 634 if len(h.Evals) != 1 { 635 t.Fatalf("setStatus() didn't update plan: %v", h.Evals) 636 } 637 638 newEval := h.Evals[0] 639 if newEval.ID != eval.ID || newEval.Status != status || newEval.StatusDescription != desc { 640 t.Fatalf("setStatus() submited invalid eval: %v", newEval) 641 } 642 643 // Test next evals 644 h = NewHarness(t) 645 next := mock.Eval() 646 if err := setStatus(logger, h, eval, next, nil, nil, status, desc, nil, ""); err != nil { 647 t.Fatalf("setStatus() failed: %v", err) 648 } 649 650 if len(h.Evals) != 1 { 651 t.Fatalf("setStatus() didn't update plan: %v", h.Evals) 652 } 653 654 newEval = h.Evals[0] 655 if newEval.NextEval != next.ID { 656 t.Fatalf("setStatus() didn't set nextEval correctly: %v", newEval) 657 } 658 659 // Test blocked evals 660 h = NewHarness(t) 661 blocked := mock.Eval() 662 if err := setStatus(logger, h, eval, nil, blocked, nil, status, desc, nil, ""); err != nil { 663 t.Fatalf("setStatus() failed: %v", err) 664 } 665 666 if len(h.Evals) != 1 { 667 t.Fatalf("setStatus() didn't update plan: %v", h.Evals) 668 } 669 670 newEval = h.Evals[0] 671 if newEval.BlockedEval != blocked.ID { 672 t.Fatalf("setStatus() didn't set BlockedEval correctly: %v", newEval) 673 } 674 675 // Test metrics 676 h = NewHarness(t) 677 metrics := map[string]*structs.AllocMetric{"foo": nil} 678 if err := setStatus(logger, h, eval, nil, nil, metrics, status, desc, nil, ""); err != nil { 679 t.Fatalf("setStatus() failed: %v", err) 680 } 681 682 if len(h.Evals) != 1 { 683 t.Fatalf("setStatus() didn't update plan: %v", h.Evals) 684 } 685 686 newEval = h.Evals[0] 687 if !reflect.DeepEqual(newEval.FailedTGAllocs, metrics) { 688 t.Fatalf("setStatus() didn't set failed task group metrics correctly: %v", newEval) 689 } 690 691 // Test queued allocations 692 h = NewHarness(t) 693 queuedAllocs := map[string]int{"web": 1} 694 695 if err := setStatus(logger, h, eval, nil, nil, metrics, status, desc, queuedAllocs, ""); err != nil { 696 t.Fatalf("setStatus() failed: %v", err) 697 } 698 699 if len(h.Evals) != 1 { 700 t.Fatalf("setStatus() didn't update plan: %v", h.Evals) 701 } 702 703 newEval = h.Evals[0] 704 if !reflect.DeepEqual(newEval.QueuedAllocations, queuedAllocs) { 705 t.Fatalf("setStatus() didn't set failed task group metrics correctly: %v", newEval) 706 } 707 708 h = NewHarness(t) 709 dID := uuid.Generate() 710 if err := setStatus(logger, h, eval, nil, nil, metrics, status, desc, queuedAllocs, dID); err != nil { 711 t.Fatalf("setStatus() failed: %v", err) 712 } 713 714 if len(h.Evals) != 1 { 715 t.Fatalf("setStatus() didn't update plan: %v", h.Evals) 716 } 717 718 newEval = h.Evals[0] 719 if newEval.DeploymentID != dID { 720 t.Fatalf("setStatus() didn't set deployment id correctly: %v", newEval) 721 } 722 } 723 724 func TestInplaceUpdate_ChangedTaskGroup(t *testing.T) { 725 state, ctx := testContext(t) 726 eval := mock.Eval() 727 job := mock.Job() 728 729 node := mock.Node() 730 noErr(t, state.UpsertNode(900, node)) 731 732 // Register an alloc 733 alloc := &structs.Allocation{ 734 Namespace: structs.DefaultNamespace, 735 ID: uuid.Generate(), 736 EvalID: eval.ID, 737 NodeID: node.ID, 738 JobID: job.ID, 739 Job: job, 740 Resources: &structs.Resources{ 741 CPU: 2048, 742 MemoryMB: 2048, 743 }, 744 DesiredStatus: structs.AllocDesiredStatusRun, 745 TaskGroup: "web", 746 } 747 alloc.TaskResources = map[string]*structs.Resources{"web": alloc.Resources} 748 noErr(t, state.UpsertJobSummary(1000, mock.JobSummary(alloc.JobID))) 749 noErr(t, state.UpsertAllocs(1001, []*structs.Allocation{alloc})) 750 751 // Create a new task group that prevents in-place updates. 752 tg := &structs.TaskGroup{} 753 *tg = *job.TaskGroups[0] 754 task := &structs.Task{Name: "FOO"} 755 tg.Tasks = nil 756 tg.Tasks = append(tg.Tasks, task) 757 758 updates := []allocTuple{{Alloc: alloc, TaskGroup: tg}} 759 stack := NewGenericStack(false, ctx) 760 761 // Do the inplace update. 762 unplaced, inplace := inplaceUpdate(ctx, eval, job, stack, updates) 763 764 if len(unplaced) != 1 || len(inplace) != 0 { 765 t.Fatal("inplaceUpdate incorrectly did an inplace update") 766 } 767 768 if len(ctx.plan.NodeAllocation) != 0 { 769 t.Fatal("inplaceUpdate incorrectly did an inplace update") 770 } 771 } 772 773 func TestInplaceUpdate_NoMatch(t *testing.T) { 774 state, ctx := testContext(t) 775 eval := mock.Eval() 776 job := mock.Job() 777 778 node := mock.Node() 779 noErr(t, state.UpsertNode(900, node)) 780 781 // Register an alloc 782 alloc := &structs.Allocation{ 783 Namespace: structs.DefaultNamespace, 784 ID: uuid.Generate(), 785 EvalID: eval.ID, 786 NodeID: node.ID, 787 JobID: job.ID, 788 Job: job, 789 Resources: &structs.Resources{ 790 CPU: 2048, 791 MemoryMB: 2048, 792 }, 793 DesiredStatus: structs.AllocDesiredStatusRun, 794 TaskGroup: "web", 795 } 796 alloc.TaskResources = map[string]*structs.Resources{"web": alloc.Resources} 797 noErr(t, state.UpsertJobSummary(1000, mock.JobSummary(alloc.JobID))) 798 noErr(t, state.UpsertAllocs(1001, []*structs.Allocation{alloc})) 799 800 // Create a new task group that requires too much resources. 801 tg := &structs.TaskGroup{} 802 *tg = *job.TaskGroups[0] 803 resource := &structs.Resources{CPU: 9999} 804 tg.Tasks[0].Resources = resource 805 806 updates := []allocTuple{{Alloc: alloc, TaskGroup: tg}} 807 stack := NewGenericStack(false, ctx) 808 809 // Do the inplace update. 810 unplaced, inplace := inplaceUpdate(ctx, eval, job, stack, updates) 811 812 if len(unplaced) != 1 || len(inplace) != 0 { 813 t.Fatal("inplaceUpdate incorrectly did an inplace update") 814 } 815 816 if len(ctx.plan.NodeAllocation) != 0 { 817 t.Fatal("inplaceUpdate incorrectly did an inplace update") 818 } 819 } 820 821 func TestInplaceUpdate_Success(t *testing.T) { 822 state, ctx := testContext(t) 823 eval := mock.Eval() 824 job := mock.Job() 825 826 node := mock.Node() 827 noErr(t, state.UpsertNode(900, node)) 828 829 // Register an alloc 830 alloc := &structs.Allocation{ 831 Namespace: structs.DefaultNamespace, 832 ID: uuid.Generate(), 833 EvalID: eval.ID, 834 NodeID: node.ID, 835 JobID: job.ID, 836 Job: job, 837 TaskGroup: job.TaskGroups[0].Name, 838 Resources: &structs.Resources{ 839 CPU: 2048, 840 MemoryMB: 2048, 841 }, 842 DesiredStatus: structs.AllocDesiredStatusRun, 843 } 844 alloc.TaskResources = map[string]*structs.Resources{"web": alloc.Resources} 845 noErr(t, state.UpsertJobSummary(999, mock.JobSummary(alloc.JobID))) 846 noErr(t, state.UpsertAllocs(1001, []*structs.Allocation{alloc})) 847 848 // Create a new task group that updates the resources. 849 tg := &structs.TaskGroup{} 850 *tg = *job.TaskGroups[0] 851 resource := &structs.Resources{CPU: 737} 852 tg.Tasks[0].Resources = resource 853 newServices := []*structs.Service{ 854 { 855 Name: "dummy-service", 856 PortLabel: "http", 857 }, 858 { 859 Name: "dummy-service2", 860 PortLabel: "http", 861 }, 862 } 863 864 // Delete service 2 865 tg.Tasks[0].Services = tg.Tasks[0].Services[:1] 866 867 // Add the new services 868 tg.Tasks[0].Services = append(tg.Tasks[0].Services, newServices...) 869 870 updates := []allocTuple{{Alloc: alloc, TaskGroup: tg}} 871 stack := NewGenericStack(false, ctx) 872 stack.SetJob(job) 873 874 // Do the inplace update. 875 unplaced, inplace := inplaceUpdate(ctx, eval, job, stack, updates) 876 877 if len(unplaced) != 0 || len(inplace) != 1 { 878 t.Fatal("inplaceUpdate did not do an inplace update") 879 } 880 881 if len(ctx.plan.NodeAllocation) != 1 { 882 t.Fatal("inplaceUpdate did not do an inplace update") 883 } 884 885 if inplace[0].Alloc.ID != alloc.ID { 886 t.Fatalf("inplaceUpdate returned the wrong, inplace updated alloc: %#v", inplace) 887 } 888 889 // Get the alloc we inserted. 890 a := inplace[0].Alloc // TODO(sean@): Verify this is correct vs: ctx.plan.NodeAllocation[alloc.NodeID][0] 891 if a.Job == nil { 892 t.Fatalf("bad") 893 } 894 895 if len(a.Job.TaskGroups) != 1 { 896 t.Fatalf("bad") 897 } 898 899 if len(a.Job.TaskGroups[0].Tasks) != 1 { 900 t.Fatalf("bad") 901 } 902 903 if len(a.Job.TaskGroups[0].Tasks[0].Services) != 3 { 904 t.Fatalf("Expected number of services: %v, Actual: %v", 3, len(a.Job.TaskGroups[0].Tasks[0].Services)) 905 } 906 907 serviceNames := make(map[string]struct{}, 3) 908 for _, consulService := range a.Job.TaskGroups[0].Tasks[0].Services { 909 serviceNames[consulService.Name] = struct{}{} 910 } 911 if len(serviceNames) != 3 { 912 t.Fatalf("bad") 913 } 914 915 for _, name := range []string{"dummy-service", "dummy-service2", "web-frontend"} { 916 if _, found := serviceNames[name]; !found { 917 t.Errorf("Expected consul service name missing: %v", name) 918 } 919 } 920 } 921 922 func TestEvictAndPlace_LimitGreaterThanAllocs(t *testing.T) { 923 _, ctx := testContext(t) 924 allocs := []allocTuple{ 925 {Alloc: &structs.Allocation{ID: uuid.Generate()}}, 926 {Alloc: &structs.Allocation{ID: uuid.Generate()}}, 927 {Alloc: &structs.Allocation{ID: uuid.Generate()}}, 928 {Alloc: &structs.Allocation{ID: uuid.Generate()}}, 929 } 930 diff := &diffResult{} 931 932 limit := 6 933 if evictAndPlace(ctx, diff, allocs, "", &limit) { 934 t.Fatal("evictAndReplace() should have returned false") 935 } 936 937 if limit != 2 { 938 t.Fatalf("evictAndReplace() should decremented limit; got %v; want 2", limit) 939 } 940 941 if len(diff.place) != 4 { 942 t.Fatalf("evictAndReplace() didn't insert into diffResult properly: %v", diff.place) 943 } 944 } 945 946 func TestTaskGroupConstraints(t *testing.T) { 947 constr := &structs.Constraint{RTarget: "bar"} 948 constr2 := &structs.Constraint{LTarget: "foo"} 949 constr3 := &structs.Constraint{Operand: "<"} 950 951 tg := &structs.TaskGroup{ 952 Name: "web", 953 Count: 10, 954 Constraints: []*structs.Constraint{constr}, 955 EphemeralDisk: &structs.EphemeralDisk{}, 956 Tasks: []*structs.Task{ 957 { 958 Driver: "exec", 959 Resources: &structs.Resources{ 960 CPU: 500, 961 MemoryMB: 256, 962 }, 963 Constraints: []*structs.Constraint{constr2}, 964 }, 965 { 966 Driver: "docker", 967 Resources: &structs.Resources{ 968 CPU: 500, 969 MemoryMB: 256, 970 }, 971 Constraints: []*structs.Constraint{constr3}, 972 }, 973 }, 974 } 975 976 // Build the expected values. 977 expConstr := []*structs.Constraint{constr, constr2, constr3} 978 expDrivers := map[string]struct{}{"exec": {}, "docker": {}} 979 expSize := &structs.Resources{ 980 CPU: 1000, 981 MemoryMB: 512, 982 } 983 984 actConstrains := taskGroupConstraints(tg) 985 if !reflect.DeepEqual(actConstrains.constraints, expConstr) { 986 t.Fatalf("taskGroupConstraints(%v) returned %v; want %v", tg, actConstrains.constraints, expConstr) 987 } 988 if !reflect.DeepEqual(actConstrains.drivers, expDrivers) { 989 t.Fatalf("taskGroupConstraints(%v) returned %v; want %v", tg, actConstrains.drivers, expDrivers) 990 } 991 if !reflect.DeepEqual(actConstrains.size, expSize) { 992 t.Fatalf("taskGroupConstraints(%v) returned %v; want %v", tg, actConstrains.size, expSize) 993 } 994 995 } 996 997 func TestProgressMade(t *testing.T) { 998 noopPlan := &structs.PlanResult{} 999 if progressMade(nil) || progressMade(noopPlan) { 1000 t.Fatal("no progress plan marked as making progress") 1001 } 1002 1003 m := map[string][]*structs.Allocation{ 1004 "foo": {mock.Alloc()}, 1005 } 1006 both := &structs.PlanResult{ 1007 NodeAllocation: m, 1008 NodeUpdate: m, 1009 } 1010 update := &structs.PlanResult{NodeUpdate: m} 1011 alloc := &structs.PlanResult{NodeAllocation: m} 1012 deployment := &structs.PlanResult{Deployment: mock.Deployment()} 1013 deploymentUpdates := &structs.PlanResult{ 1014 DeploymentUpdates: []*structs.DeploymentStatusUpdate{ 1015 {DeploymentID: uuid.Generate()}, 1016 }, 1017 } 1018 if !(progressMade(both) && progressMade(update) && progressMade(alloc) && 1019 progressMade(deployment) && progressMade(deploymentUpdates)) { 1020 t.Fatal("bad") 1021 } 1022 } 1023 1024 func TestDesiredUpdates(t *testing.T) { 1025 tg1 := &structs.TaskGroup{Name: "foo"} 1026 tg2 := &structs.TaskGroup{Name: "bar"} 1027 a2 := &structs.Allocation{TaskGroup: "bar"} 1028 1029 place := []allocTuple{ 1030 {TaskGroup: tg1}, 1031 {TaskGroup: tg1}, 1032 {TaskGroup: tg1}, 1033 {TaskGroup: tg2}, 1034 } 1035 stop := []allocTuple{ 1036 {TaskGroup: tg2, Alloc: a2}, 1037 {TaskGroup: tg2, Alloc: a2}, 1038 } 1039 ignore := []allocTuple{ 1040 {TaskGroup: tg1}, 1041 } 1042 migrate := []allocTuple{ 1043 {TaskGroup: tg2}, 1044 } 1045 inplace := []allocTuple{ 1046 {TaskGroup: tg1}, 1047 {TaskGroup: tg1}, 1048 } 1049 destructive := []allocTuple{ 1050 {TaskGroup: tg1}, 1051 {TaskGroup: tg2}, 1052 {TaskGroup: tg2}, 1053 } 1054 diff := &diffResult{ 1055 place: place, 1056 stop: stop, 1057 ignore: ignore, 1058 migrate: migrate, 1059 } 1060 1061 expected := map[string]*structs.DesiredUpdates{ 1062 "foo": { 1063 Place: 3, 1064 Ignore: 1, 1065 InPlaceUpdate: 2, 1066 DestructiveUpdate: 1, 1067 }, 1068 "bar": { 1069 Place: 1, 1070 Stop: 2, 1071 Migrate: 1, 1072 DestructiveUpdate: 2, 1073 }, 1074 } 1075 1076 desired := desiredUpdates(diff, inplace, destructive) 1077 if !reflect.DeepEqual(desired, expected) { 1078 t.Fatalf("desiredUpdates() returned %#v; want %#v", desired, expected) 1079 } 1080 } 1081 1082 func TestUtil_AdjustQueuedAllocations(t *testing.T) { 1083 logger := log.New(os.Stderr, "", log.LstdFlags) 1084 alloc1 := mock.Alloc() 1085 alloc2 := mock.Alloc() 1086 alloc2.CreateIndex = 4 1087 alloc2.ModifyIndex = 4 1088 alloc3 := mock.Alloc() 1089 alloc3.CreateIndex = 3 1090 alloc3.ModifyIndex = 5 1091 alloc4 := mock.Alloc() 1092 alloc4.CreateIndex = 6 1093 alloc4.ModifyIndex = 8 1094 1095 planResult := structs.PlanResult{ 1096 NodeUpdate: map[string][]*structs.Allocation{ 1097 "node-1": {alloc1}, 1098 }, 1099 NodeAllocation: map[string][]*structs.Allocation{ 1100 "node-1": { 1101 alloc2, 1102 }, 1103 "node-2": { 1104 alloc3, alloc4, 1105 }, 1106 }, 1107 RefreshIndex: 3, 1108 AllocIndex: 16, // Should not be considered 1109 } 1110 1111 queuedAllocs := map[string]int{"web": 2} 1112 adjustQueuedAllocations(logger, &planResult, queuedAllocs) 1113 1114 if queuedAllocs["web"] != 1 { 1115 t.Fatalf("expected: %v, actual: %v", 1, queuedAllocs["web"]) 1116 } 1117 } 1118 1119 func TestUtil_UpdateNonTerminalAllocsToLost(t *testing.T) { 1120 node := mock.Node() 1121 node.Status = structs.NodeStatusDown 1122 alloc1 := mock.Alloc() 1123 alloc1.NodeID = node.ID 1124 alloc1.DesiredStatus = structs.AllocDesiredStatusStop 1125 1126 alloc2 := mock.Alloc() 1127 alloc2.NodeID = node.ID 1128 alloc2.DesiredStatus = structs.AllocDesiredStatusStop 1129 alloc2.ClientStatus = structs.AllocClientStatusRunning 1130 1131 alloc3 := mock.Alloc() 1132 alloc3.NodeID = node.ID 1133 alloc3.DesiredStatus = structs.AllocDesiredStatusStop 1134 alloc3.ClientStatus = structs.AllocClientStatusComplete 1135 1136 alloc4 := mock.Alloc() 1137 alloc4.NodeID = node.ID 1138 alloc4.DesiredStatus = structs.AllocDesiredStatusStop 1139 alloc4.ClientStatus = structs.AllocClientStatusFailed 1140 1141 allocs := []*structs.Allocation{alloc1, alloc2, alloc3, alloc4} 1142 plan := structs.Plan{ 1143 NodeUpdate: make(map[string][]*structs.Allocation), 1144 } 1145 tainted := map[string]*structs.Node{node.ID: node} 1146 1147 updateNonTerminalAllocsToLost(&plan, tainted, allocs) 1148 1149 allocsLost := make([]string, 0, 2) 1150 for _, alloc := range plan.NodeUpdate[node.ID] { 1151 allocsLost = append(allocsLost, alloc.ID) 1152 } 1153 expected := []string{alloc1.ID, alloc2.ID} 1154 if !reflect.DeepEqual(allocsLost, expected) { 1155 t.Fatalf("actual: %v, expected: %v", allocsLost, expected) 1156 } 1157 1158 // Update the node status to ready and try again 1159 plan = structs.Plan{ 1160 NodeUpdate: make(map[string][]*structs.Allocation), 1161 } 1162 node.Status = structs.NodeStatusReady 1163 updateNonTerminalAllocsToLost(&plan, tainted, allocs) 1164 1165 allocsLost = make([]string, 0, 2) 1166 for _, alloc := range plan.NodeUpdate[node.ID] { 1167 allocsLost = append(allocsLost, alloc.ID) 1168 } 1169 expected = []string{} 1170 if !reflect.DeepEqual(allocsLost, expected) { 1171 t.Fatalf("actual: %v, expected: %v", allocsLost, expected) 1172 } 1173 }