github.com/jrxfive/nomad@v0.6.1-0.20170802162750-1fef470e89bf/scheduler/reconcile_test.go (about) 1 package scheduler 2 3 import ( 4 "fmt" 5 "log" 6 "os" 7 "reflect" 8 "regexp" 9 "strconv" 10 "testing" 11 "time" 12 13 "github.com/hashicorp/nomad/helper" 14 "github.com/hashicorp/nomad/nomad/mock" 15 "github.com/hashicorp/nomad/nomad/structs" 16 "github.com/kr/pretty" 17 ) 18 19 /* 20 Basic Tests: 21 √ Place when there is nothing in the cluster 22 √ Place remainder when there is some in the cluster 23 √ Scale down from n to n-m where n != m 24 √ Scale down from n to zero 25 √ Inplace upgrade test 26 √ Inplace upgrade and scale up test 27 √ Inplace upgrade and scale down test 28 √ Destructive upgrade 29 √ Destructive upgrade and scale up test 30 √ Destructive upgrade and scale down test 31 √ Handle lost nodes 32 √ Handle lost nodes and scale up 33 √ Handle lost nodes and scale down 34 √ Handle draining nodes 35 √ Handle draining nodes and scale up 36 √ Handle draining nodes and scale down 37 √ Handle task group being removed 38 √ Handle job being stopped both as .Stopped and nil 39 √ Place more that one group 40 41 Update stanza Tests: 42 √ Stopped job cancels any active deployment 43 √ Stopped job doesn't cancel terminal deployment 44 √ JobIndex change cancels any active deployment 45 √ JobIndex change doens't cancels any terminal deployment 46 √ Destructive changes create deployment and get rolled out via max_parallelism 47 √ Don't create a deployment if there are no changes 48 √ Deployment created by all inplace updates 49 √ Paused or failed deployment doesn't create any more canaries 50 √ Paused or failed deployment doesn't do any placements 51 √ Paused or failed deployment doesn't do destructive updates 52 √ Paused does do migrations 53 √ Failed deployment doesn't do migrations 54 √ Canary that is on a draining node 55 √ Canary that is on a lost node 56 √ Stop old canaries 57 √ Create new canaries on job change 58 √ Create new canaries on job change while scaling up 59 √ Create new canaries on job change while scaling down 60 √ Fill canaries if partial placement 61 √ Promote canaries unblocks max_parallel 62 √ Promote canaries when canaries == count 63 √ Only place as many as are healthy in deployment 64 √ Limit calculation accounts for healthy allocs on migrating/lost nodes 65 √ Failed deployment should not place anything 66 √ Run after canaries have been promoted, new allocs have been rolled out and there is no deployment 67 √ Failed deployment cancels non-promoted task groups 68 √ Failed deployment and updated job works 69 √ Finished deployment gets marked as complete 70 √ The stagger is correctly calculated when it is applied across multiple task groups. 71 √ Change job change while scaling up 72 */ 73 74 var ( 75 canaryUpdate = &structs.UpdateStrategy{ 76 Canary: 2, 77 MaxParallel: 2, 78 HealthCheck: structs.UpdateStrategyHealthCheck_Checks, 79 MinHealthyTime: 10 * time.Second, 80 HealthyDeadline: 10 * time.Minute, 81 Stagger: 31 * time.Second, 82 } 83 84 noCanaryUpdate = &structs.UpdateStrategy{ 85 MaxParallel: 4, 86 HealthCheck: structs.UpdateStrategyHealthCheck_Checks, 87 MinHealthyTime: 10 * time.Second, 88 HealthyDeadline: 10 * time.Minute, 89 Stagger: 31 * time.Second, 90 } 91 ) 92 93 func testLogger() *log.Logger { 94 return log.New(os.Stderr, "", log.LstdFlags) 95 } 96 97 func allocUpdateFnIgnore(*structs.Allocation, *structs.Job, *structs.TaskGroup) (bool, bool, *structs.Allocation) { 98 return true, false, nil 99 } 100 101 func allocUpdateFnDestructive(*structs.Allocation, *structs.Job, *structs.TaskGroup) (bool, bool, *structs.Allocation) { 102 return false, true, nil 103 } 104 105 func allocUpdateFnInplace(existing *structs.Allocation, _ *structs.Job, newTG *structs.TaskGroup) (bool, bool, *structs.Allocation) { 106 // Create a shallow copy 107 newAlloc := new(structs.Allocation) 108 *newAlloc = *existing 109 newAlloc.TaskResources = make(map[string]*structs.Resources) 110 111 // Use the new task resources but keep the network from the old 112 for _, task := range newTG.Tasks { 113 r := task.Resources.Copy() 114 r.Networks = existing.TaskResources[task.Name].Networks 115 newAlloc.TaskResources[task.Name] = r 116 } 117 118 return false, false, newAlloc 119 } 120 121 func allocUpdateFnMock(handled map[string]allocUpdateType, unhandled allocUpdateType) allocUpdateType { 122 return func(existing *structs.Allocation, newJob *structs.Job, newTG *structs.TaskGroup) (bool, bool, *structs.Allocation) { 123 if fn, ok := handled[existing.ID]; ok { 124 return fn(existing, newJob, newTG) 125 } 126 127 return unhandled(existing, newJob, newTG) 128 } 129 } 130 131 var ( 132 // AllocationIndexRegex is a regular expression to find the allocation index. 133 allocationIndexRegex = regexp.MustCompile(".+\\[(\\d+)\\]$") 134 ) 135 136 // allocNameToIndex returns the index of the allocation. 137 func allocNameToIndex(name string) uint { 138 matches := allocationIndexRegex.FindStringSubmatch(name) 139 if len(matches) != 2 { 140 return 0 141 } 142 143 index, err := strconv.Atoi(matches[1]) 144 if err != nil { 145 return 0 146 } 147 148 return uint(index) 149 } 150 151 func assertNamesHaveIndexes(t *testing.T, indexes []int, names []string) { 152 m := make(map[uint]int) 153 for _, i := range indexes { 154 m[uint(i)] += 1 155 } 156 157 for _, n := range names { 158 index := allocNameToIndex(n) 159 val, contained := m[index] 160 if !contained { 161 t.Fatalf("Unexpected index %d from name %s\nAll names: %v", index, n, names) 162 } 163 164 val-- 165 if val < 0 { 166 t.Fatalf("Index %d repeated too many times\nAll names: %v", index, names) 167 } 168 m[index] = val 169 } 170 171 for k, remainder := range m { 172 if remainder != 0 { 173 t.Fatalf("Index %d has %d remaining uses expected\nAll names: %v", k, remainder, names) 174 } 175 } 176 } 177 178 func assertNoCanariesStopped(t *testing.T, d *structs.Deployment, stop []allocStopResult) { 179 canaryIndex := make(map[string]struct{}) 180 for _, state := range d.TaskGroups { 181 for _, c := range state.PlacedCanaries { 182 canaryIndex[c] = struct{}{} 183 } 184 } 185 186 for _, s := range stop { 187 if _, ok := canaryIndex[s.alloc.ID]; ok { 188 t.Fatalf("Stopping canary alloc %q %q", s.alloc.ID, s.alloc.Name) 189 } 190 } 191 } 192 193 func assertPlaceResultsHavePreviousAllocs(t *testing.T, numPrevious int, place []allocPlaceResult) { 194 names := make(map[string]struct{}, numPrevious) 195 196 found := 0 197 for _, p := range place { 198 if _, ok := names[p.name]; ok { 199 t.Fatalf("Name %q already placed", p.name) 200 } 201 names[p.name] = struct{}{} 202 203 if p.previousAlloc == nil { 204 continue 205 } 206 207 if act := p.previousAlloc.Name; p.name != act { 208 t.Fatalf("Name mismatch on previous alloc; got %q; want %q", act, p.name) 209 } 210 found++ 211 } 212 if numPrevious != found { 213 t.Fatalf("wanted %d; got %d placements with previous allocs", numPrevious, found) 214 } 215 } 216 217 func intRange(pairs ...int) []int { 218 if len(pairs)%2 != 0 { 219 return nil 220 } 221 222 var r []int 223 for i := 0; i < len(pairs); i += 2 { 224 for j := pairs[i]; j <= pairs[i+1]; j++ { 225 r = append(r, j) 226 } 227 } 228 return r 229 } 230 231 func placeResultsToNames(place []allocPlaceResult) []string { 232 names := make([]string, 0, len(place)) 233 for _, p := range place { 234 names = append(names, p.name) 235 } 236 return names 237 } 238 239 func destructiveResultsToNames(destructive []allocDestructiveResult) []string { 240 names := make([]string, 0, len(destructive)) 241 for _, d := range destructive { 242 names = append(names, d.placeName) 243 } 244 return names 245 } 246 247 func stopResultsToNames(stop []allocStopResult) []string { 248 names := make([]string, 0, len(stop)) 249 for _, s := range stop { 250 names = append(names, s.alloc.Name) 251 } 252 return names 253 } 254 255 func allocsToNames(allocs []*structs.Allocation) []string { 256 names := make([]string, 0, len(allocs)) 257 for _, a := range allocs { 258 names = append(names, a.Name) 259 } 260 return names 261 } 262 263 type resultExpectation struct { 264 createDeployment *structs.Deployment 265 deploymentUpdates []*structs.DeploymentStatusUpdate 266 place int 267 destructive int 268 inplace int 269 stop int 270 desiredTGUpdates map[string]*structs.DesiredUpdates 271 followupEvalWait time.Duration 272 } 273 274 func assertResults(t *testing.T, r *reconcileResults, exp *resultExpectation) { 275 276 if exp.createDeployment != nil && r.deployment == nil { 277 t.Fatalf("Expect a created deployment got none") 278 } else if exp.createDeployment == nil && r.deployment != nil { 279 t.Fatalf("Expect no created deployment; got %#v", r.deployment) 280 } else if exp.createDeployment != nil && r.deployment != nil { 281 // Clear the deployment ID 282 r.deployment.ID, exp.createDeployment.ID = "", "" 283 if !reflect.DeepEqual(r.deployment, exp.createDeployment) { 284 t.Fatalf("Unexpected createdDeployment; got\n %#v\nwant\n%#v\nDiff: %v", 285 r.deployment, exp.createDeployment, pretty.Diff(r.deployment, exp.createDeployment)) 286 } 287 } 288 289 if !reflect.DeepEqual(r.deploymentUpdates, exp.deploymentUpdates) { 290 t.Fatalf("Unexpected deploymentUpdates: %v", pretty.Diff(r.deploymentUpdates, exp.deploymentUpdates)) 291 } 292 if l := len(r.place); l != exp.place { 293 t.Fatalf("Expected %d placements; got %d", exp.place, l) 294 } 295 if l := len(r.destructiveUpdate); l != exp.destructive { 296 t.Fatalf("Expected %d destructive; got %d", exp.destructive, l) 297 } 298 if l := len(r.inplaceUpdate); l != exp.inplace { 299 t.Fatalf("Expected %d inplaceUpdate; got %d", exp.inplace, l) 300 } 301 if l := len(r.stop); l != exp.stop { 302 t.Fatalf("Expected %d stops; got %d", exp.stop, l) 303 } 304 if l := len(r.desiredTGUpdates); l != len(exp.desiredTGUpdates) { 305 t.Fatalf("Expected %d task group desired tg updates annotations; got %d", len(exp.desiredTGUpdates), l) 306 } 307 if r.followupEvalWait != exp.followupEvalWait { 308 t.Fatalf("Unexpected followup eval wait time. Got %v; want %v", r.followupEvalWait, exp.followupEvalWait) 309 } 310 311 // Check the desired updates happened 312 for group, desired := range exp.desiredTGUpdates { 313 act, ok := r.desiredTGUpdates[group] 314 if !ok { 315 t.Fatalf("Expected desired updates for group %q", group) 316 } 317 318 if !reflect.DeepEqual(act, desired) { 319 t.Fatalf("Unexpected annotations for group %q: %v", group, pretty.Diff(act, desired)) 320 } 321 } 322 } 323 324 // Tests the reconciler properly handles placements for a job that has no 325 // existing allocations 326 func TestReconciler_Place_NoExisting(t *testing.T) { 327 job := mock.Job() 328 reconciler := NewAllocReconciler(testLogger(), allocUpdateFnIgnore, false, job.ID, job, nil, nil, nil) 329 r := reconciler.Compute() 330 331 // Assert the correct results 332 assertResults(t, r, &resultExpectation{ 333 createDeployment: nil, 334 deploymentUpdates: nil, 335 place: 10, 336 inplace: 0, 337 stop: 0, 338 desiredTGUpdates: map[string]*structs.DesiredUpdates{ 339 job.TaskGroups[0].Name: { 340 Place: 10, 341 }, 342 }, 343 }) 344 345 assertNamesHaveIndexes(t, intRange(0, 9), placeResultsToNames(r.place)) 346 } 347 348 // Tests the reconciler properly handles placements for a job that has some 349 // existing allocations 350 func TestReconciler_Place_Existing(t *testing.T) { 351 job := mock.Job() 352 353 // Create 3 existing allocations 354 var allocs []*structs.Allocation 355 for i := 0; i < 5; i++ { 356 alloc := mock.Alloc() 357 alloc.Job = job 358 alloc.JobID = job.ID 359 alloc.NodeID = structs.GenerateUUID() 360 alloc.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i)) 361 allocs = append(allocs, alloc) 362 } 363 364 reconciler := NewAllocReconciler(testLogger(), allocUpdateFnIgnore, false, job.ID, job, nil, allocs, nil) 365 r := reconciler.Compute() 366 367 // Assert the correct results 368 assertResults(t, r, &resultExpectation{ 369 createDeployment: nil, 370 deploymentUpdates: nil, 371 place: 5, 372 inplace: 0, 373 stop: 0, 374 desiredTGUpdates: map[string]*structs.DesiredUpdates{ 375 job.TaskGroups[0].Name: { 376 Place: 5, 377 Ignore: 5, 378 }, 379 }, 380 }) 381 382 assertNamesHaveIndexes(t, intRange(5, 9), placeResultsToNames(r.place)) 383 } 384 385 // Tests the reconciler properly handles stopping allocations for a job that has 386 // scaled down 387 func TestReconciler_ScaleDown_Partial(t *testing.T) { 388 // Has desired 10 389 job := mock.Job() 390 391 // Create 20 existing allocations 392 var allocs []*structs.Allocation 393 for i := 0; i < 20; i++ { 394 alloc := mock.Alloc() 395 alloc.Job = job 396 alloc.JobID = job.ID 397 alloc.NodeID = structs.GenerateUUID() 398 alloc.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i)) 399 allocs = append(allocs, alloc) 400 } 401 402 reconciler := NewAllocReconciler(testLogger(), allocUpdateFnIgnore, false, job.ID, job, nil, allocs, nil) 403 r := reconciler.Compute() 404 405 // Assert the correct results 406 assertResults(t, r, &resultExpectation{ 407 createDeployment: nil, 408 deploymentUpdates: nil, 409 place: 0, 410 inplace: 0, 411 stop: 10, 412 desiredTGUpdates: map[string]*structs.DesiredUpdates{ 413 job.TaskGroups[0].Name: { 414 Ignore: 10, 415 Stop: 10, 416 }, 417 }, 418 }) 419 420 assertNamesHaveIndexes(t, intRange(10, 19), stopResultsToNames(r.stop)) 421 } 422 423 // Tests the reconciler properly handles stopping allocations for a job that has 424 // scaled down to zero desired 425 func TestReconciler_ScaleDown_Zero(t *testing.T) { 426 // Set desired 0 427 job := mock.Job() 428 job.TaskGroups[0].Count = 0 429 430 // Create 20 existing allocations 431 var allocs []*structs.Allocation 432 for i := 0; i < 20; i++ { 433 alloc := mock.Alloc() 434 alloc.Job = job 435 alloc.JobID = job.ID 436 alloc.NodeID = structs.GenerateUUID() 437 alloc.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i)) 438 allocs = append(allocs, alloc) 439 } 440 441 reconciler := NewAllocReconciler(testLogger(), allocUpdateFnIgnore, false, job.ID, job, nil, allocs, nil) 442 r := reconciler.Compute() 443 444 // Assert the correct results 445 assertResults(t, r, &resultExpectation{ 446 createDeployment: nil, 447 deploymentUpdates: nil, 448 place: 0, 449 inplace: 0, 450 stop: 20, 451 desiredTGUpdates: map[string]*structs.DesiredUpdates{ 452 job.TaskGroups[0].Name: { 453 Stop: 20, 454 }, 455 }, 456 }) 457 458 assertNamesHaveIndexes(t, intRange(0, 19), stopResultsToNames(r.stop)) 459 } 460 461 // Tests the reconciler properly handles inplace upgrading allocations 462 func TestReconciler_Inplace(t *testing.T) { 463 job := mock.Job() 464 465 // Create 10 existing allocations 466 var allocs []*structs.Allocation 467 for i := 0; i < 10; i++ { 468 alloc := mock.Alloc() 469 alloc.Job = job 470 alloc.JobID = job.ID 471 alloc.NodeID = structs.GenerateUUID() 472 alloc.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i)) 473 allocs = append(allocs, alloc) 474 } 475 476 reconciler := NewAllocReconciler(testLogger(), allocUpdateFnInplace, false, job.ID, job, nil, allocs, nil) 477 r := reconciler.Compute() 478 479 // Assert the correct results 480 assertResults(t, r, &resultExpectation{ 481 createDeployment: nil, 482 deploymentUpdates: nil, 483 place: 0, 484 inplace: 10, 485 stop: 0, 486 desiredTGUpdates: map[string]*structs.DesiredUpdates{ 487 job.TaskGroups[0].Name: { 488 InPlaceUpdate: 10, 489 }, 490 }, 491 }) 492 493 assertNamesHaveIndexes(t, intRange(0, 9), allocsToNames(r.inplaceUpdate)) 494 } 495 496 // Tests the reconciler properly handles inplace upgrading allocations while 497 // scaling up 498 func TestReconciler_Inplace_ScaleUp(t *testing.T) { 499 // Set desired 15 500 job := mock.Job() 501 job.TaskGroups[0].Count = 15 502 503 // Create 10 existing allocations 504 var allocs []*structs.Allocation 505 for i := 0; i < 10; i++ { 506 alloc := mock.Alloc() 507 alloc.Job = job 508 alloc.JobID = job.ID 509 alloc.NodeID = structs.GenerateUUID() 510 alloc.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i)) 511 allocs = append(allocs, alloc) 512 } 513 514 reconciler := NewAllocReconciler(testLogger(), allocUpdateFnInplace, false, job.ID, job, nil, allocs, nil) 515 r := reconciler.Compute() 516 517 // Assert the correct results 518 assertResults(t, r, &resultExpectation{ 519 createDeployment: nil, 520 deploymentUpdates: nil, 521 place: 5, 522 inplace: 10, 523 stop: 0, 524 desiredTGUpdates: map[string]*structs.DesiredUpdates{ 525 job.TaskGroups[0].Name: { 526 Place: 5, 527 InPlaceUpdate: 10, 528 }, 529 }, 530 }) 531 532 assertNamesHaveIndexes(t, intRange(0, 9), allocsToNames(r.inplaceUpdate)) 533 assertNamesHaveIndexes(t, intRange(10, 14), placeResultsToNames(r.place)) 534 } 535 536 // Tests the reconciler properly handles inplace upgrading allocations while 537 // scaling down 538 func TestReconciler_Inplace_ScaleDown(t *testing.T) { 539 // Set desired 5 540 job := mock.Job() 541 job.TaskGroups[0].Count = 5 542 543 // Create 10 existing allocations 544 var allocs []*structs.Allocation 545 for i := 0; i < 10; i++ { 546 alloc := mock.Alloc() 547 alloc.Job = job 548 alloc.JobID = job.ID 549 alloc.NodeID = structs.GenerateUUID() 550 alloc.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i)) 551 allocs = append(allocs, alloc) 552 } 553 554 reconciler := NewAllocReconciler(testLogger(), allocUpdateFnInplace, false, job.ID, job, nil, allocs, nil) 555 r := reconciler.Compute() 556 557 // Assert the correct results 558 assertResults(t, r, &resultExpectation{ 559 createDeployment: nil, 560 deploymentUpdates: nil, 561 place: 0, 562 inplace: 5, 563 stop: 5, 564 desiredTGUpdates: map[string]*structs.DesiredUpdates{ 565 job.TaskGroups[0].Name: { 566 Stop: 5, 567 InPlaceUpdate: 5, 568 }, 569 }, 570 }) 571 572 assertNamesHaveIndexes(t, intRange(0, 4), allocsToNames(r.inplaceUpdate)) 573 assertNamesHaveIndexes(t, intRange(5, 9), stopResultsToNames(r.stop)) 574 } 575 576 // Tests the reconciler properly handles destructive upgrading allocations 577 func TestReconciler_Destructive(t *testing.T) { 578 job := mock.Job() 579 580 // Create 10 existing allocations 581 var allocs []*structs.Allocation 582 for i := 0; i < 10; i++ { 583 alloc := mock.Alloc() 584 alloc.Job = job 585 alloc.JobID = job.ID 586 alloc.NodeID = structs.GenerateUUID() 587 alloc.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i)) 588 allocs = append(allocs, alloc) 589 } 590 591 reconciler := NewAllocReconciler(testLogger(), allocUpdateFnDestructive, false, job.ID, job, nil, allocs, nil) 592 r := reconciler.Compute() 593 594 // Assert the correct results 595 assertResults(t, r, &resultExpectation{ 596 createDeployment: nil, 597 deploymentUpdates: nil, 598 destructive: 10, 599 desiredTGUpdates: map[string]*structs.DesiredUpdates{ 600 job.TaskGroups[0].Name: { 601 DestructiveUpdate: 10, 602 }, 603 }, 604 }) 605 606 assertNamesHaveIndexes(t, intRange(0, 9), destructiveResultsToNames(r.destructiveUpdate)) 607 } 608 609 // Tests the reconciler properly handles destructive upgrading allocations while 610 // scaling up 611 func TestReconciler_Destructive_ScaleUp(t *testing.T) { 612 // Set desired 15 613 job := mock.Job() 614 job.TaskGroups[0].Count = 15 615 616 // Create 10 existing allocations 617 var allocs []*structs.Allocation 618 for i := 0; i < 10; i++ { 619 alloc := mock.Alloc() 620 alloc.Job = job 621 alloc.JobID = job.ID 622 alloc.NodeID = structs.GenerateUUID() 623 alloc.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i)) 624 allocs = append(allocs, alloc) 625 } 626 627 reconciler := NewAllocReconciler(testLogger(), allocUpdateFnDestructive, false, job.ID, job, nil, allocs, nil) 628 r := reconciler.Compute() 629 630 // Assert the correct results 631 assertResults(t, r, &resultExpectation{ 632 createDeployment: nil, 633 deploymentUpdates: nil, 634 place: 5, 635 destructive: 10, 636 desiredTGUpdates: map[string]*structs.DesiredUpdates{ 637 job.TaskGroups[0].Name: { 638 Place: 5, 639 DestructiveUpdate: 10, 640 }, 641 }, 642 }) 643 644 assertNamesHaveIndexes(t, intRange(0, 9), destructiveResultsToNames(r.destructiveUpdate)) 645 assertNamesHaveIndexes(t, intRange(10, 14), placeResultsToNames(r.place)) 646 } 647 648 // Tests the reconciler properly handles destructive upgrading allocations while 649 // scaling down 650 func TestReconciler_Destructive_ScaleDown(t *testing.T) { 651 // Set desired 5 652 job := mock.Job() 653 job.TaskGroups[0].Count = 5 654 655 // Create 10 existing allocations 656 var allocs []*structs.Allocation 657 for i := 0; i < 10; i++ { 658 alloc := mock.Alloc() 659 alloc.Job = job 660 alloc.JobID = job.ID 661 alloc.NodeID = structs.GenerateUUID() 662 alloc.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i)) 663 allocs = append(allocs, alloc) 664 } 665 666 reconciler := NewAllocReconciler(testLogger(), allocUpdateFnDestructive, false, job.ID, job, nil, allocs, nil) 667 r := reconciler.Compute() 668 669 // Assert the correct results 670 assertResults(t, r, &resultExpectation{ 671 createDeployment: nil, 672 deploymentUpdates: nil, 673 destructive: 5, 674 stop: 5, 675 desiredTGUpdates: map[string]*structs.DesiredUpdates{ 676 job.TaskGroups[0].Name: { 677 Stop: 5, 678 DestructiveUpdate: 5, 679 }, 680 }, 681 }) 682 683 assertNamesHaveIndexes(t, intRange(5, 9), stopResultsToNames(r.stop)) 684 assertNamesHaveIndexes(t, intRange(0, 4), destructiveResultsToNames(r.destructiveUpdate)) 685 } 686 687 // Tests the reconciler properly handles lost nodes with allocations 688 func TestReconciler_LostNode(t *testing.T) { 689 job := mock.Job() 690 691 // Create 10 existing allocations 692 var allocs []*structs.Allocation 693 for i := 0; i < 10; i++ { 694 alloc := mock.Alloc() 695 alloc.Job = job 696 alloc.JobID = job.ID 697 alloc.NodeID = structs.GenerateUUID() 698 alloc.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i)) 699 allocs = append(allocs, alloc) 700 } 701 702 // Build a map of tainted nodes 703 tainted := make(map[string]*structs.Node, 2) 704 for i := 0; i < 2; i++ { 705 n := mock.Node() 706 n.ID = allocs[i].NodeID 707 n.Status = structs.NodeStatusDown 708 tainted[n.ID] = n 709 } 710 711 reconciler := NewAllocReconciler(testLogger(), allocUpdateFnIgnore, false, job.ID, job, nil, allocs, tainted) 712 r := reconciler.Compute() 713 714 // Assert the correct results 715 assertResults(t, r, &resultExpectation{ 716 createDeployment: nil, 717 deploymentUpdates: nil, 718 place: 2, 719 inplace: 0, 720 stop: 2, 721 desiredTGUpdates: map[string]*structs.DesiredUpdates{ 722 job.TaskGroups[0].Name: { 723 Place: 2, 724 Stop: 2, 725 Ignore: 8, 726 }, 727 }, 728 }) 729 730 assertNamesHaveIndexes(t, intRange(0, 1), stopResultsToNames(r.stop)) 731 assertNamesHaveIndexes(t, intRange(0, 1), placeResultsToNames(r.place)) 732 } 733 734 // Tests the reconciler properly handles lost nodes with allocations while 735 // scaling up 736 func TestReconciler_LostNode_ScaleUp(t *testing.T) { 737 // Set desired 15 738 job := mock.Job() 739 job.TaskGroups[0].Count = 15 740 741 // Create 10 existing allocations 742 var allocs []*structs.Allocation 743 for i := 0; i < 10; i++ { 744 alloc := mock.Alloc() 745 alloc.Job = job 746 alloc.JobID = job.ID 747 alloc.NodeID = structs.GenerateUUID() 748 alloc.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i)) 749 allocs = append(allocs, alloc) 750 } 751 752 // Build a map of tainted nodes 753 tainted := make(map[string]*structs.Node, 2) 754 for i := 0; i < 2; i++ { 755 n := mock.Node() 756 n.ID = allocs[i].NodeID 757 n.Status = structs.NodeStatusDown 758 tainted[n.ID] = n 759 } 760 761 reconciler := NewAllocReconciler(testLogger(), allocUpdateFnIgnore, false, job.ID, job, nil, allocs, tainted) 762 r := reconciler.Compute() 763 764 // Assert the correct results 765 assertResults(t, r, &resultExpectation{ 766 createDeployment: nil, 767 deploymentUpdates: nil, 768 place: 7, 769 inplace: 0, 770 stop: 2, 771 desiredTGUpdates: map[string]*structs.DesiredUpdates{ 772 job.TaskGroups[0].Name: { 773 Place: 7, 774 Stop: 2, 775 Ignore: 8, 776 }, 777 }, 778 }) 779 780 assertNamesHaveIndexes(t, intRange(0, 1), stopResultsToNames(r.stop)) 781 assertNamesHaveIndexes(t, intRange(0, 1, 10, 14), placeResultsToNames(r.place)) 782 } 783 784 // Tests the reconciler properly handles lost nodes with allocations while 785 // scaling down 786 func TestReconciler_LostNode_ScaleDown(t *testing.T) { 787 // Set desired 5 788 job := mock.Job() 789 job.TaskGroups[0].Count = 5 790 791 // Create 10 existing allocations 792 var allocs []*structs.Allocation 793 for i := 0; i < 10; i++ { 794 alloc := mock.Alloc() 795 alloc.Job = job 796 alloc.JobID = job.ID 797 alloc.NodeID = structs.GenerateUUID() 798 alloc.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i)) 799 allocs = append(allocs, alloc) 800 } 801 802 // Build a map of tainted nodes 803 tainted := make(map[string]*structs.Node, 2) 804 for i := 0; i < 2; i++ { 805 n := mock.Node() 806 n.ID = allocs[i].NodeID 807 n.Status = structs.NodeStatusDown 808 tainted[n.ID] = n 809 } 810 811 reconciler := NewAllocReconciler(testLogger(), allocUpdateFnIgnore, false, job.ID, job, nil, allocs, tainted) 812 r := reconciler.Compute() 813 814 // Assert the correct results 815 assertResults(t, r, &resultExpectation{ 816 createDeployment: nil, 817 deploymentUpdates: nil, 818 place: 0, 819 inplace: 0, 820 stop: 5, 821 desiredTGUpdates: map[string]*structs.DesiredUpdates{ 822 job.TaskGroups[0].Name: { 823 Stop: 5, 824 Ignore: 5, 825 }, 826 }, 827 }) 828 829 assertNamesHaveIndexes(t, intRange(0, 1, 7, 9), stopResultsToNames(r.stop)) 830 } 831 832 // Tests the reconciler properly handles draining nodes with allocations 833 func TestReconciler_DrainNode(t *testing.T) { 834 job := mock.Job() 835 836 // Create 10 existing allocations 837 var allocs []*structs.Allocation 838 for i := 0; i < 10; i++ { 839 alloc := mock.Alloc() 840 alloc.Job = job 841 alloc.JobID = job.ID 842 alloc.NodeID = structs.GenerateUUID() 843 alloc.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i)) 844 allocs = append(allocs, alloc) 845 } 846 847 // Build a map of tainted nodes 848 tainted := make(map[string]*structs.Node, 2) 849 for i := 0; i < 2; i++ { 850 n := mock.Node() 851 n.ID = allocs[i].NodeID 852 n.Drain = true 853 tainted[n.ID] = n 854 } 855 856 reconciler := NewAllocReconciler(testLogger(), allocUpdateFnIgnore, false, job.ID, job, nil, allocs, tainted) 857 r := reconciler.Compute() 858 859 // Assert the correct results 860 assertResults(t, r, &resultExpectation{ 861 createDeployment: nil, 862 deploymentUpdates: nil, 863 place: 2, 864 inplace: 0, 865 stop: 2, 866 desiredTGUpdates: map[string]*structs.DesiredUpdates{ 867 job.TaskGroups[0].Name: { 868 Migrate: 2, 869 Ignore: 8, 870 }, 871 }, 872 }) 873 874 assertNamesHaveIndexes(t, intRange(0, 1), stopResultsToNames(r.stop)) 875 assertNamesHaveIndexes(t, intRange(0, 1), placeResultsToNames(r.place)) 876 assertPlaceResultsHavePreviousAllocs(t, 2, r.place) 877 } 878 879 // Tests the reconciler properly handles draining nodes with allocations while 880 // scaling up 881 func TestReconciler_DrainNode_ScaleUp(t *testing.T) { 882 // Set desired 15 883 job := mock.Job() 884 job.TaskGroups[0].Count = 15 885 886 // Create 10 existing allocations 887 var allocs []*structs.Allocation 888 for i := 0; i < 10; i++ { 889 alloc := mock.Alloc() 890 alloc.Job = job 891 alloc.JobID = job.ID 892 alloc.NodeID = structs.GenerateUUID() 893 alloc.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i)) 894 allocs = append(allocs, alloc) 895 } 896 897 // Build a map of tainted nodes 898 tainted := make(map[string]*structs.Node, 2) 899 for i := 0; i < 2; i++ { 900 n := mock.Node() 901 n.ID = allocs[i].NodeID 902 n.Drain = true 903 tainted[n.ID] = n 904 } 905 906 reconciler := NewAllocReconciler(testLogger(), allocUpdateFnIgnore, false, job.ID, job, nil, allocs, tainted) 907 r := reconciler.Compute() 908 909 // Assert the correct results 910 assertResults(t, r, &resultExpectation{ 911 createDeployment: nil, 912 deploymentUpdates: nil, 913 place: 7, 914 inplace: 0, 915 stop: 2, 916 desiredTGUpdates: map[string]*structs.DesiredUpdates{ 917 job.TaskGroups[0].Name: { 918 Place: 5, 919 Migrate: 2, 920 Ignore: 8, 921 }, 922 }, 923 }) 924 925 assertNamesHaveIndexes(t, intRange(0, 1), stopResultsToNames(r.stop)) 926 assertNamesHaveIndexes(t, intRange(0, 1, 10, 14), placeResultsToNames(r.place)) 927 assertPlaceResultsHavePreviousAllocs(t, 2, r.place) 928 } 929 930 // Tests the reconciler properly handles draining nodes with allocations while 931 // scaling down 932 func TestReconciler_DrainNode_ScaleDown(t *testing.T) { 933 // Set desired 8 934 job := mock.Job() 935 job.TaskGroups[0].Count = 8 936 937 // Create 10 existing allocations 938 var allocs []*structs.Allocation 939 for i := 0; i < 10; i++ { 940 alloc := mock.Alloc() 941 alloc.Job = job 942 alloc.JobID = job.ID 943 alloc.NodeID = structs.GenerateUUID() 944 alloc.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i)) 945 allocs = append(allocs, alloc) 946 } 947 948 // Build a map of tainted nodes 949 tainted := make(map[string]*structs.Node, 3) 950 for i := 0; i < 3; i++ { 951 n := mock.Node() 952 n.ID = allocs[i].NodeID 953 n.Drain = true 954 tainted[n.ID] = n 955 } 956 957 reconciler := NewAllocReconciler(testLogger(), allocUpdateFnIgnore, false, job.ID, job, nil, allocs, tainted) 958 r := reconciler.Compute() 959 960 // Assert the correct results 961 assertResults(t, r, &resultExpectation{ 962 createDeployment: nil, 963 deploymentUpdates: nil, 964 place: 1, 965 inplace: 0, 966 stop: 3, 967 desiredTGUpdates: map[string]*structs.DesiredUpdates{ 968 job.TaskGroups[0].Name: { 969 Migrate: 1, 970 Stop: 2, 971 Ignore: 7, 972 }, 973 }, 974 }) 975 976 assertNamesHaveIndexes(t, intRange(0, 2), stopResultsToNames(r.stop)) 977 assertNamesHaveIndexes(t, intRange(0, 0), placeResultsToNames(r.place)) 978 assertPlaceResultsHavePreviousAllocs(t, 1, r.place) 979 } 980 981 // Tests the reconciler properly handles a task group being removed 982 func TestReconciler_RemovedTG(t *testing.T) { 983 job := mock.Job() 984 985 // Create 10 allocations for a tg that no longer exists 986 var allocs []*structs.Allocation 987 for i := 0; i < 10; i++ { 988 alloc := mock.Alloc() 989 alloc.Job = job 990 alloc.JobID = job.ID 991 alloc.NodeID = structs.GenerateUUID() 992 alloc.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i)) 993 allocs = append(allocs, alloc) 994 } 995 996 oldName := job.TaskGroups[0].Name 997 newName := "different" 998 job.TaskGroups[0].Name = newName 999 1000 reconciler := NewAllocReconciler(testLogger(), allocUpdateFnIgnore, false, job.ID, job, nil, allocs, nil) 1001 r := reconciler.Compute() 1002 1003 // Assert the correct results 1004 assertResults(t, r, &resultExpectation{ 1005 createDeployment: nil, 1006 deploymentUpdates: nil, 1007 place: 10, 1008 inplace: 0, 1009 stop: 10, 1010 desiredTGUpdates: map[string]*structs.DesiredUpdates{ 1011 oldName: { 1012 Stop: 10, 1013 }, 1014 newName: { 1015 Place: 10, 1016 }, 1017 }, 1018 }) 1019 1020 assertNamesHaveIndexes(t, intRange(0, 9), stopResultsToNames(r.stop)) 1021 assertNamesHaveIndexes(t, intRange(0, 9), placeResultsToNames(r.place)) 1022 } 1023 1024 // Tests the reconciler properly handles a job in stopped states 1025 func TestReconciler_JobStopped(t *testing.T) { 1026 job := mock.Job() 1027 job.Stop = true 1028 1029 cases := []struct { 1030 name string 1031 job *structs.Job 1032 jobID, taskGroup string 1033 }{ 1034 { 1035 name: "stopped job", 1036 job: job, 1037 jobID: job.ID, 1038 taskGroup: job.TaskGroups[0].Name, 1039 }, 1040 { 1041 name: "nil job", 1042 job: nil, 1043 jobID: "foo", 1044 taskGroup: "bar", 1045 }, 1046 } 1047 1048 for _, c := range cases { 1049 t.Run(c.name, func(t *testing.T) { 1050 // Create 10 allocations 1051 var allocs []*structs.Allocation 1052 for i := 0; i < 10; i++ { 1053 alloc := mock.Alloc() 1054 alloc.Job = c.job 1055 alloc.JobID = c.jobID 1056 alloc.NodeID = structs.GenerateUUID() 1057 alloc.Name = structs.AllocName(c.jobID, c.taskGroup, uint(i)) 1058 alloc.TaskGroup = c.taskGroup 1059 allocs = append(allocs, alloc) 1060 } 1061 1062 reconciler := NewAllocReconciler(testLogger(), allocUpdateFnIgnore, false, c.jobID, c.job, nil, allocs, nil) 1063 r := reconciler.Compute() 1064 1065 // Assert the correct results 1066 assertResults(t, r, &resultExpectation{ 1067 createDeployment: nil, 1068 deploymentUpdates: nil, 1069 place: 0, 1070 inplace: 0, 1071 stop: 10, 1072 desiredTGUpdates: map[string]*structs.DesiredUpdates{ 1073 c.taskGroup: { 1074 Stop: 10, 1075 }, 1076 }, 1077 }) 1078 1079 assertNamesHaveIndexes(t, intRange(0, 9), stopResultsToNames(r.stop)) 1080 }) 1081 } 1082 } 1083 1084 // Tests the reconciler properly handles jobs with multiple task groups 1085 func TestReconciler_MultiTG(t *testing.T) { 1086 job := mock.Job() 1087 tg2 := job.TaskGroups[0].Copy() 1088 tg2.Name = "foo" 1089 job.TaskGroups = append(job.TaskGroups, tg2) 1090 1091 // Create 2 existing allocations for the first tg 1092 var allocs []*structs.Allocation 1093 for i := 0; i < 2; i++ { 1094 alloc := mock.Alloc() 1095 alloc.Job = job 1096 alloc.JobID = job.ID 1097 alloc.NodeID = structs.GenerateUUID() 1098 alloc.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i)) 1099 allocs = append(allocs, alloc) 1100 } 1101 1102 reconciler := NewAllocReconciler(testLogger(), allocUpdateFnIgnore, false, job.ID, job, nil, allocs, nil) 1103 r := reconciler.Compute() 1104 1105 // Assert the correct results 1106 assertResults(t, r, &resultExpectation{ 1107 createDeployment: nil, 1108 deploymentUpdates: nil, 1109 place: 18, 1110 inplace: 0, 1111 stop: 0, 1112 desiredTGUpdates: map[string]*structs.DesiredUpdates{ 1113 job.TaskGroups[0].Name: { 1114 Place: 8, 1115 Ignore: 2, 1116 }, 1117 tg2.Name: { 1118 Place: 10, 1119 }, 1120 }, 1121 }) 1122 1123 assertNamesHaveIndexes(t, intRange(2, 9, 0, 9), placeResultsToNames(r.place)) 1124 } 1125 1126 // Tests the reconciler cancels an old deployment when the job is being stopped 1127 func TestReconciler_CancelDeployment_JobStop(t *testing.T) { 1128 job := mock.Job() 1129 job.Stop = true 1130 1131 running := structs.NewDeployment(job) 1132 failed := structs.NewDeployment(job) 1133 failed.Status = structs.DeploymentStatusFailed 1134 1135 cases := []struct { 1136 name string 1137 job *structs.Job 1138 jobID, taskGroup string 1139 deployment *structs.Deployment 1140 cancel bool 1141 }{ 1142 { 1143 name: "stopped job, running deployment", 1144 job: job, 1145 jobID: job.ID, 1146 taskGroup: job.TaskGroups[0].Name, 1147 deployment: running, 1148 cancel: true, 1149 }, 1150 { 1151 name: "nil job, running deployment", 1152 job: nil, 1153 jobID: "foo", 1154 taskGroup: "bar", 1155 deployment: running, 1156 cancel: true, 1157 }, 1158 { 1159 name: "stopped job, failed deployment", 1160 job: job, 1161 jobID: job.ID, 1162 taskGroup: job.TaskGroups[0].Name, 1163 deployment: failed, 1164 cancel: false, 1165 }, 1166 { 1167 name: "nil job, failed deployment", 1168 job: nil, 1169 jobID: "foo", 1170 taskGroup: "bar", 1171 deployment: failed, 1172 cancel: false, 1173 }, 1174 } 1175 1176 for _, c := range cases { 1177 t.Run(c.name, func(t *testing.T) { 1178 // Create 10 allocations 1179 var allocs []*structs.Allocation 1180 for i := 0; i < 10; i++ { 1181 alloc := mock.Alloc() 1182 alloc.Job = c.job 1183 alloc.JobID = c.jobID 1184 alloc.NodeID = structs.GenerateUUID() 1185 alloc.Name = structs.AllocName(c.jobID, c.taskGroup, uint(i)) 1186 alloc.TaskGroup = c.taskGroup 1187 allocs = append(allocs, alloc) 1188 } 1189 1190 reconciler := NewAllocReconciler(testLogger(), allocUpdateFnIgnore, false, c.jobID, c.job, c.deployment, allocs, nil) 1191 r := reconciler.Compute() 1192 1193 var updates []*structs.DeploymentStatusUpdate 1194 if c.cancel { 1195 updates = []*structs.DeploymentStatusUpdate{ 1196 { 1197 DeploymentID: c.deployment.ID, 1198 Status: structs.DeploymentStatusCancelled, 1199 StatusDescription: structs.DeploymentStatusDescriptionStoppedJob, 1200 }, 1201 } 1202 } 1203 1204 // Assert the correct results 1205 assertResults(t, r, &resultExpectation{ 1206 createDeployment: nil, 1207 deploymentUpdates: updates, 1208 place: 0, 1209 inplace: 0, 1210 stop: 10, 1211 desiredTGUpdates: map[string]*structs.DesiredUpdates{ 1212 c.taskGroup: { 1213 Stop: 10, 1214 }, 1215 }, 1216 }) 1217 1218 assertNamesHaveIndexes(t, intRange(0, 9), stopResultsToNames(r.stop)) 1219 }) 1220 } 1221 } 1222 1223 // Tests the reconciler cancels an old deployment when the job is updated 1224 func TestReconciler_CancelDeployment_JobUpdate(t *testing.T) { 1225 // Create a base job 1226 job := mock.Job() 1227 1228 // Create two deployments 1229 running := structs.NewDeployment(job) 1230 failed := structs.NewDeployment(job) 1231 failed.Status = structs.DeploymentStatusFailed 1232 1233 // Make the job newer than the deployment 1234 job.Version += 10 1235 1236 cases := []struct { 1237 name string 1238 deployment *structs.Deployment 1239 cancel bool 1240 }{ 1241 { 1242 name: "running deployment", 1243 deployment: running, 1244 cancel: true, 1245 }, 1246 { 1247 name: "failed deployment", 1248 deployment: failed, 1249 cancel: false, 1250 }, 1251 } 1252 1253 for _, c := range cases { 1254 t.Run(c.name, func(t *testing.T) { 1255 // Create 10 allocations 1256 var allocs []*structs.Allocation 1257 for i := 0; i < 10; i++ { 1258 alloc := mock.Alloc() 1259 alloc.Job = job 1260 alloc.JobID = job.ID 1261 alloc.NodeID = structs.GenerateUUID() 1262 alloc.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i)) 1263 alloc.TaskGroup = job.TaskGroups[0].Name 1264 allocs = append(allocs, alloc) 1265 } 1266 1267 reconciler := NewAllocReconciler(testLogger(), allocUpdateFnIgnore, false, job.ID, job, c.deployment, allocs, nil) 1268 r := reconciler.Compute() 1269 1270 var updates []*structs.DeploymentStatusUpdate 1271 if c.cancel { 1272 updates = []*structs.DeploymentStatusUpdate{ 1273 { 1274 DeploymentID: c.deployment.ID, 1275 Status: structs.DeploymentStatusCancelled, 1276 StatusDescription: structs.DeploymentStatusDescriptionNewerJob, 1277 }, 1278 } 1279 } 1280 1281 // Assert the correct results 1282 assertResults(t, r, &resultExpectation{ 1283 createDeployment: nil, 1284 deploymentUpdates: updates, 1285 place: 0, 1286 inplace: 0, 1287 stop: 0, 1288 desiredTGUpdates: map[string]*structs.DesiredUpdates{ 1289 job.TaskGroups[0].Name: { 1290 Ignore: 10, 1291 }, 1292 }, 1293 }) 1294 }) 1295 } 1296 } 1297 1298 // Tests the reconciler creates a deployment and does a rolling upgrade with 1299 // destructive changes 1300 func TestReconciler_CreateDeployment_RollingUpgrade_Destructive(t *testing.T) { 1301 job := mock.Job() 1302 job.TaskGroups[0].Update = noCanaryUpdate 1303 1304 // Create 10 allocations from the old job 1305 var allocs []*structs.Allocation 1306 for i := 0; i < 10; i++ { 1307 alloc := mock.Alloc() 1308 alloc.Job = job 1309 alloc.JobID = job.ID 1310 alloc.NodeID = structs.GenerateUUID() 1311 alloc.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i)) 1312 alloc.TaskGroup = job.TaskGroups[0].Name 1313 allocs = append(allocs, alloc) 1314 } 1315 1316 reconciler := NewAllocReconciler(testLogger(), allocUpdateFnDestructive, false, job.ID, job, nil, allocs, nil) 1317 r := reconciler.Compute() 1318 1319 d := structs.NewDeployment(job) 1320 d.TaskGroups[job.TaskGroups[0].Name] = &structs.DeploymentState{ 1321 DesiredTotal: 10, 1322 } 1323 1324 // Assert the correct results 1325 assertResults(t, r, &resultExpectation{ 1326 createDeployment: d, 1327 deploymentUpdates: nil, 1328 destructive: 4, 1329 desiredTGUpdates: map[string]*structs.DesiredUpdates{ 1330 job.TaskGroups[0].Name: { 1331 DestructiveUpdate: 4, 1332 Ignore: 6, 1333 }, 1334 }, 1335 }) 1336 1337 assertNamesHaveIndexes(t, intRange(0, 3), destructiveResultsToNames(r.destructiveUpdate)) 1338 } 1339 1340 // Tests the reconciler creates a deployment for inplace updates 1341 func TestReconciler_CreateDeployment_RollingUpgrade_Inplace(t *testing.T) { 1342 job := mock.Job() 1343 job.TaskGroups[0].Update = noCanaryUpdate 1344 1345 // Create 10 allocations from the old job 1346 var allocs []*structs.Allocation 1347 for i := 0; i < 10; i++ { 1348 alloc := mock.Alloc() 1349 alloc.Job = job 1350 alloc.JobID = job.ID 1351 alloc.NodeID = structs.GenerateUUID() 1352 alloc.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i)) 1353 alloc.TaskGroup = job.TaskGroups[0].Name 1354 allocs = append(allocs, alloc) 1355 } 1356 1357 reconciler := NewAllocReconciler(testLogger(), allocUpdateFnInplace, false, job.ID, job, nil, allocs, nil) 1358 r := reconciler.Compute() 1359 1360 d := structs.NewDeployment(job) 1361 d.TaskGroups[job.TaskGroups[0].Name] = &structs.DeploymentState{ 1362 DesiredTotal: 10, 1363 } 1364 1365 // Assert the correct results 1366 assertResults(t, r, &resultExpectation{ 1367 createDeployment: d, 1368 deploymentUpdates: nil, 1369 place: 0, 1370 inplace: 10, 1371 stop: 0, 1372 desiredTGUpdates: map[string]*structs.DesiredUpdates{ 1373 job.TaskGroups[0].Name: { 1374 InPlaceUpdate: 10, 1375 }, 1376 }, 1377 }) 1378 } 1379 1380 // Tests the reconciler doesn't creates a deployment if there are no changes 1381 func TestReconciler_DontCreateDeployment_NoChanges(t *testing.T) { 1382 job := mock.Job() 1383 job.TaskGroups[0].Update = noCanaryUpdate 1384 1385 // Create 10 allocations from the job 1386 var allocs []*structs.Allocation 1387 for i := 0; i < 10; i++ { 1388 alloc := mock.Alloc() 1389 alloc.Job = job 1390 alloc.JobID = job.ID 1391 alloc.NodeID = structs.GenerateUUID() 1392 alloc.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i)) 1393 alloc.TaskGroup = job.TaskGroups[0].Name 1394 allocs = append(allocs, alloc) 1395 } 1396 1397 reconciler := NewAllocReconciler(testLogger(), allocUpdateFnIgnore, false, job.ID, job, nil, allocs, nil) 1398 r := reconciler.Compute() 1399 1400 // Assert the correct results 1401 assertResults(t, r, &resultExpectation{ 1402 createDeployment: nil, 1403 deploymentUpdates: nil, 1404 place: 0, 1405 inplace: 0, 1406 stop: 0, 1407 desiredTGUpdates: map[string]*structs.DesiredUpdates{ 1408 job.TaskGroups[0].Name: { 1409 DestructiveUpdate: 0, 1410 Ignore: 10, 1411 }, 1412 }, 1413 }) 1414 } 1415 1416 // Tests the reconciler doesn't place any more canaries when the deployment is 1417 // paused or failed 1418 func TestReconciler_PausedOrFailedDeployment_NoMoreCanaries(t *testing.T) { 1419 job := mock.Job() 1420 job.TaskGroups[0].Update = canaryUpdate 1421 1422 cases := []struct { 1423 name string 1424 deploymentStatus string 1425 stop uint64 1426 }{ 1427 { 1428 name: "paused deployment", 1429 deploymentStatus: structs.DeploymentStatusPaused, 1430 stop: 0, 1431 }, 1432 { 1433 name: "failed deployment", 1434 deploymentStatus: structs.DeploymentStatusFailed, 1435 stop: 1, 1436 }, 1437 } 1438 1439 for _, c := range cases { 1440 t.Run(c.name, func(t *testing.T) { 1441 // Create a deployment that is paused/failed and has placed some canaries 1442 d := structs.NewDeployment(job) 1443 d.Status = c.deploymentStatus 1444 d.TaskGroups[job.TaskGroups[0].Name] = &structs.DeploymentState{ 1445 Promoted: false, 1446 DesiredCanaries: 2, 1447 DesiredTotal: 10, 1448 PlacedAllocs: 1, 1449 } 1450 1451 // Create 10 allocations for the original job 1452 var allocs []*structs.Allocation 1453 for i := 0; i < 10; i++ { 1454 alloc := mock.Alloc() 1455 alloc.Job = job 1456 alloc.JobID = job.ID 1457 alloc.NodeID = structs.GenerateUUID() 1458 alloc.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i)) 1459 alloc.TaskGroup = job.TaskGroups[0].Name 1460 allocs = append(allocs, alloc) 1461 } 1462 1463 // Create one canary 1464 canary := mock.Alloc() 1465 canary.Job = job 1466 canary.JobID = job.ID 1467 canary.NodeID = structs.GenerateUUID() 1468 canary.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, 0) 1469 canary.TaskGroup = job.TaskGroups[0].Name 1470 canary.DeploymentID = d.ID 1471 allocs = append(allocs, canary) 1472 d.TaskGroups[canary.TaskGroup].PlacedCanaries = []string{canary.ID} 1473 1474 mockUpdateFn := allocUpdateFnMock(map[string]allocUpdateType{canary.ID: allocUpdateFnIgnore}, allocUpdateFnDestructive) 1475 reconciler := NewAllocReconciler(testLogger(), mockUpdateFn, false, job.ID, job, d, allocs, nil) 1476 r := reconciler.Compute() 1477 1478 // Assert the correct results 1479 assertResults(t, r, &resultExpectation{ 1480 createDeployment: nil, 1481 deploymentUpdates: nil, 1482 place: 0, 1483 inplace: 0, 1484 stop: int(c.stop), 1485 desiredTGUpdates: map[string]*structs.DesiredUpdates{ 1486 job.TaskGroups[0].Name: { 1487 Ignore: 11 - c.stop, 1488 Stop: c.stop, 1489 }, 1490 }, 1491 }) 1492 }) 1493 } 1494 } 1495 1496 // Tests the reconciler doesn't place any more allocs when the deployment is 1497 // paused or failed 1498 func TestReconciler_PausedOrFailedDeployment_NoMorePlacements(t *testing.T) { 1499 job := mock.Job() 1500 job.TaskGroups[0].Update = noCanaryUpdate 1501 job.TaskGroups[0].Count = 15 1502 1503 cases := []struct { 1504 name string 1505 deploymentStatus string 1506 }{ 1507 { 1508 name: "paused deployment", 1509 deploymentStatus: structs.DeploymentStatusPaused, 1510 }, 1511 { 1512 name: "failed deployment", 1513 deploymentStatus: structs.DeploymentStatusFailed, 1514 }, 1515 } 1516 1517 for _, c := range cases { 1518 t.Run(c.name, func(t *testing.T) { 1519 // Create a deployment that is paused and has placed some canaries 1520 d := structs.NewDeployment(job) 1521 d.Status = c.deploymentStatus 1522 d.TaskGroups[job.TaskGroups[0].Name] = &structs.DeploymentState{ 1523 Promoted: false, 1524 DesiredTotal: 15, 1525 PlacedAllocs: 10, 1526 } 1527 1528 // Create 10 allocations for the new job 1529 var allocs []*structs.Allocation 1530 for i := 0; i < 10; i++ { 1531 alloc := mock.Alloc() 1532 alloc.Job = job 1533 alloc.JobID = job.ID 1534 alloc.NodeID = structs.GenerateUUID() 1535 alloc.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i)) 1536 alloc.TaskGroup = job.TaskGroups[0].Name 1537 allocs = append(allocs, alloc) 1538 } 1539 1540 reconciler := NewAllocReconciler(testLogger(), allocUpdateFnIgnore, false, job.ID, job, d, allocs, nil) 1541 r := reconciler.Compute() 1542 1543 // Assert the correct results 1544 assertResults(t, r, &resultExpectation{ 1545 createDeployment: nil, 1546 deploymentUpdates: nil, 1547 place: 0, 1548 inplace: 0, 1549 stop: 0, 1550 desiredTGUpdates: map[string]*structs.DesiredUpdates{ 1551 job.TaskGroups[0].Name: { 1552 Ignore: 10, 1553 }, 1554 }, 1555 }) 1556 }) 1557 } 1558 } 1559 1560 // Tests the reconciler doesn't do any more destructive updates when the 1561 // deployment is paused or failed 1562 func TestReconciler_PausedOrFailedDeployment_NoMoreDestructiveUpdates(t *testing.T) { 1563 job := mock.Job() 1564 job.TaskGroups[0].Update = noCanaryUpdate 1565 1566 cases := []struct { 1567 name string 1568 deploymentStatus string 1569 }{ 1570 { 1571 name: "paused deployment", 1572 deploymentStatus: structs.DeploymentStatusPaused, 1573 }, 1574 { 1575 name: "failed deployment", 1576 deploymentStatus: structs.DeploymentStatusFailed, 1577 }, 1578 } 1579 1580 for _, c := range cases { 1581 t.Run(c.name, func(t *testing.T) { 1582 // Create a deployment that is paused and has placed some canaries 1583 d := structs.NewDeployment(job) 1584 d.Status = c.deploymentStatus 1585 d.TaskGroups[job.TaskGroups[0].Name] = &structs.DeploymentState{ 1586 Promoted: false, 1587 DesiredTotal: 10, 1588 PlacedAllocs: 1, 1589 } 1590 1591 // Create 9 allocations for the original job 1592 var allocs []*structs.Allocation 1593 for i := 1; i < 10; i++ { 1594 alloc := mock.Alloc() 1595 alloc.Job = job 1596 alloc.JobID = job.ID 1597 alloc.NodeID = structs.GenerateUUID() 1598 alloc.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i)) 1599 alloc.TaskGroup = job.TaskGroups[0].Name 1600 allocs = append(allocs, alloc) 1601 } 1602 1603 // Create one for the new job 1604 newAlloc := mock.Alloc() 1605 newAlloc.Job = job 1606 newAlloc.JobID = job.ID 1607 newAlloc.NodeID = structs.GenerateUUID() 1608 newAlloc.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, 0) 1609 newAlloc.TaskGroup = job.TaskGroups[0].Name 1610 newAlloc.DeploymentID = d.ID 1611 allocs = append(allocs, newAlloc) 1612 1613 mockUpdateFn := allocUpdateFnMock(map[string]allocUpdateType{newAlloc.ID: allocUpdateFnIgnore}, allocUpdateFnDestructive) 1614 reconciler := NewAllocReconciler(testLogger(), mockUpdateFn, false, job.ID, job, d, allocs, nil) 1615 r := reconciler.Compute() 1616 1617 // Assert the correct results 1618 assertResults(t, r, &resultExpectation{ 1619 createDeployment: nil, 1620 deploymentUpdates: nil, 1621 place: 0, 1622 inplace: 0, 1623 stop: 0, 1624 desiredTGUpdates: map[string]*structs.DesiredUpdates{ 1625 job.TaskGroups[0].Name: { 1626 Ignore: 10, 1627 }, 1628 }, 1629 }) 1630 }) 1631 } 1632 } 1633 1634 // Tests the reconciler handles migrations correctly when a deployment is paused 1635 // or failed 1636 func TestReconciler_PausedOrFailedDeployment_Migrations(t *testing.T) { 1637 job := mock.Job() 1638 job.TaskGroups[0].Update = noCanaryUpdate 1639 1640 cases := []struct { 1641 name string 1642 deploymentStatus string 1643 place int 1644 stop int 1645 ignoreAnnotation uint64 1646 migrateAnnotation uint64 1647 stopAnnotation uint64 1648 }{ 1649 { 1650 name: "paused deployment", 1651 deploymentStatus: structs.DeploymentStatusPaused, 1652 place: 0, 1653 stop: 3, 1654 ignoreAnnotation: 5, 1655 stopAnnotation: 3, 1656 }, 1657 { 1658 name: "failed deployment", 1659 deploymentStatus: structs.DeploymentStatusFailed, 1660 place: 0, 1661 stop: 3, 1662 ignoreAnnotation: 5, 1663 migrateAnnotation: 0, 1664 stopAnnotation: 3, 1665 }, 1666 } 1667 1668 for _, c := range cases { 1669 t.Run(c.name, func(t *testing.T) { 1670 // Create a deployment that is paused and has placed some canaries 1671 d := structs.NewDeployment(job) 1672 d.Status = c.deploymentStatus 1673 d.TaskGroups[job.TaskGroups[0].Name] = &structs.DeploymentState{ 1674 Promoted: false, 1675 DesiredTotal: 10, 1676 PlacedAllocs: 8, 1677 } 1678 1679 // Create 8 allocations in the deployment 1680 var allocs []*structs.Allocation 1681 for i := 0; i < 8; i++ { 1682 alloc := mock.Alloc() 1683 alloc.Job = job 1684 alloc.JobID = job.ID 1685 alloc.NodeID = structs.GenerateUUID() 1686 alloc.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i)) 1687 alloc.TaskGroup = job.TaskGroups[0].Name 1688 alloc.DeploymentID = d.ID 1689 allocs = append(allocs, alloc) 1690 } 1691 1692 // Build a map of tainted nodes 1693 tainted := make(map[string]*structs.Node, 3) 1694 for i := 0; i < 3; i++ { 1695 n := mock.Node() 1696 n.ID = allocs[i].NodeID 1697 n.Drain = true 1698 tainted[n.ID] = n 1699 } 1700 1701 reconciler := NewAllocReconciler(testLogger(), allocUpdateFnIgnore, false, job.ID, job, d, allocs, tainted) 1702 r := reconciler.Compute() 1703 1704 // Assert the correct results 1705 assertResults(t, r, &resultExpectation{ 1706 createDeployment: nil, 1707 deploymentUpdates: nil, 1708 place: c.place, 1709 inplace: 0, 1710 stop: c.stop, 1711 desiredTGUpdates: map[string]*structs.DesiredUpdates{ 1712 job.TaskGroups[0].Name: { 1713 Migrate: c.migrateAnnotation, 1714 Ignore: c.ignoreAnnotation, 1715 Stop: c.stopAnnotation, 1716 }, 1717 }, 1718 }) 1719 }) 1720 } 1721 } 1722 1723 // Tests the reconciler handles migrating a canary correctly on a draining node 1724 func TestReconciler_DrainNode_Canary(t *testing.T) { 1725 job := mock.Job() 1726 job.TaskGroups[0].Update = canaryUpdate 1727 1728 // Create a deployment that is paused and has placed some canaries 1729 d := structs.NewDeployment(job) 1730 s := &structs.DeploymentState{ 1731 Promoted: false, 1732 DesiredTotal: 10, 1733 DesiredCanaries: 2, 1734 PlacedAllocs: 2, 1735 } 1736 d.TaskGroups[job.TaskGroups[0].Name] = s 1737 1738 // Create 10 allocations from the old job 1739 var allocs []*structs.Allocation 1740 for i := 0; i < 10; i++ { 1741 alloc := mock.Alloc() 1742 alloc.Job = job 1743 alloc.JobID = job.ID 1744 alloc.NodeID = structs.GenerateUUID() 1745 alloc.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i)) 1746 alloc.TaskGroup = job.TaskGroups[0].Name 1747 allocs = append(allocs, alloc) 1748 } 1749 1750 // Create two canaries for the new job 1751 handled := make(map[string]allocUpdateType) 1752 for i := 0; i < 2; i++ { 1753 // Create one canary 1754 canary := mock.Alloc() 1755 canary.Job = job 1756 canary.JobID = job.ID 1757 canary.NodeID = structs.GenerateUUID() 1758 canary.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i)) 1759 canary.TaskGroup = job.TaskGroups[0].Name 1760 canary.DeploymentID = d.ID 1761 s.PlacedCanaries = append(s.PlacedCanaries, canary.ID) 1762 allocs = append(allocs, canary) 1763 handled[canary.ID] = allocUpdateFnIgnore 1764 } 1765 1766 // Build a map of tainted nodes that contains the last canary 1767 tainted := make(map[string]*structs.Node, 1) 1768 n := mock.Node() 1769 n.ID = allocs[11].NodeID 1770 n.Drain = true 1771 tainted[n.ID] = n 1772 1773 mockUpdateFn := allocUpdateFnMock(handled, allocUpdateFnDestructive) 1774 reconciler := NewAllocReconciler(testLogger(), mockUpdateFn, false, job.ID, job, d, allocs, tainted) 1775 r := reconciler.Compute() 1776 1777 // Assert the correct results 1778 assertResults(t, r, &resultExpectation{ 1779 createDeployment: nil, 1780 deploymentUpdates: nil, 1781 place: 1, 1782 inplace: 0, 1783 stop: 1, 1784 desiredTGUpdates: map[string]*structs.DesiredUpdates{ 1785 job.TaskGroups[0].Name: { 1786 Canary: 1, 1787 Ignore: 11, 1788 }, 1789 }, 1790 }) 1791 assertNamesHaveIndexes(t, intRange(1, 1), stopResultsToNames(r.stop)) 1792 assertNamesHaveIndexes(t, intRange(1, 1), placeResultsToNames(r.place)) 1793 } 1794 1795 // Tests the reconciler handles migrating a canary correctly on a lost node 1796 func TestReconciler_LostNode_Canary(t *testing.T) { 1797 job := mock.Job() 1798 job.TaskGroups[0].Update = canaryUpdate 1799 1800 // Create a deployment that is paused and has placed some canaries 1801 d := structs.NewDeployment(job) 1802 s := &structs.DeploymentState{ 1803 Promoted: false, 1804 DesiredTotal: 10, 1805 DesiredCanaries: 2, 1806 PlacedAllocs: 2, 1807 } 1808 d.TaskGroups[job.TaskGroups[0].Name] = s 1809 1810 // Create 10 allocations from the old job 1811 var allocs []*structs.Allocation 1812 for i := 0; i < 10; i++ { 1813 alloc := mock.Alloc() 1814 alloc.Job = job 1815 alloc.JobID = job.ID 1816 alloc.NodeID = structs.GenerateUUID() 1817 alloc.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i)) 1818 alloc.TaskGroup = job.TaskGroups[0].Name 1819 allocs = append(allocs, alloc) 1820 } 1821 1822 // Create two canaries for the new job 1823 handled := make(map[string]allocUpdateType) 1824 for i := 0; i < 2; i++ { 1825 // Create one canary 1826 canary := mock.Alloc() 1827 canary.Job = job 1828 canary.JobID = job.ID 1829 canary.NodeID = structs.GenerateUUID() 1830 canary.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i)) 1831 canary.TaskGroup = job.TaskGroups[0].Name 1832 s.PlacedCanaries = append(s.PlacedCanaries, canary.ID) 1833 canary.DeploymentID = d.ID 1834 allocs = append(allocs, canary) 1835 handled[canary.ID] = allocUpdateFnIgnore 1836 } 1837 1838 // Build a map of tainted nodes that contains the last canary 1839 tainted := make(map[string]*structs.Node, 1) 1840 n := mock.Node() 1841 n.ID = allocs[11].NodeID 1842 n.Status = structs.NodeStatusDown 1843 tainted[n.ID] = n 1844 1845 mockUpdateFn := allocUpdateFnMock(handled, allocUpdateFnDestructive) 1846 reconciler := NewAllocReconciler(testLogger(), mockUpdateFn, false, job.ID, job, d, allocs, tainted) 1847 r := reconciler.Compute() 1848 1849 // Assert the correct results 1850 assertResults(t, r, &resultExpectation{ 1851 createDeployment: nil, 1852 deploymentUpdates: nil, 1853 place: 1, 1854 inplace: 0, 1855 stop: 1, 1856 desiredTGUpdates: map[string]*structs.DesiredUpdates{ 1857 job.TaskGroups[0].Name: { 1858 Canary: 1, 1859 Ignore: 11, 1860 }, 1861 }, 1862 }) 1863 1864 assertNamesHaveIndexes(t, intRange(1, 1), stopResultsToNames(r.stop)) 1865 assertNamesHaveIndexes(t, intRange(1, 1), placeResultsToNames(r.place)) 1866 } 1867 1868 // Tests the reconciler handles stopping canaries from older deployments 1869 func TestReconciler_StopOldCanaries(t *testing.T) { 1870 job := mock.Job() 1871 job.TaskGroups[0].Update = canaryUpdate 1872 1873 // Create an old deployment that has placed some canaries 1874 d := structs.NewDeployment(job) 1875 s := &structs.DeploymentState{ 1876 Promoted: false, 1877 DesiredTotal: 10, 1878 DesiredCanaries: 2, 1879 PlacedAllocs: 2, 1880 } 1881 d.TaskGroups[job.TaskGroups[0].Name] = s 1882 1883 // Update the job 1884 job.Version += 10 1885 1886 // Create 10 allocations from the old job 1887 var allocs []*structs.Allocation 1888 for i := 0; i < 10; i++ { 1889 alloc := mock.Alloc() 1890 alloc.Job = job 1891 alloc.JobID = job.ID 1892 alloc.NodeID = structs.GenerateUUID() 1893 alloc.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i)) 1894 alloc.TaskGroup = job.TaskGroups[0].Name 1895 allocs = append(allocs, alloc) 1896 } 1897 1898 // Create canaries 1899 for i := 0; i < 2; i++ { 1900 // Create one canary 1901 canary := mock.Alloc() 1902 canary.Job = job 1903 canary.JobID = job.ID 1904 canary.NodeID = structs.GenerateUUID() 1905 canary.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i)) 1906 canary.TaskGroup = job.TaskGroups[0].Name 1907 s.PlacedCanaries = append(s.PlacedCanaries, canary.ID) 1908 canary.DeploymentID = d.ID 1909 allocs = append(allocs, canary) 1910 } 1911 1912 reconciler := NewAllocReconciler(testLogger(), allocUpdateFnDestructive, false, job.ID, job, d, allocs, nil) 1913 r := reconciler.Compute() 1914 1915 newD := structs.NewDeployment(job) 1916 newD.StatusDescription = structs.DeploymentStatusDescriptionRunningNeedsPromotion 1917 newD.TaskGroups[job.TaskGroups[0].Name] = &structs.DeploymentState{ 1918 DesiredCanaries: 2, 1919 DesiredTotal: 10, 1920 } 1921 1922 // Assert the correct results 1923 assertResults(t, r, &resultExpectation{ 1924 createDeployment: newD, 1925 deploymentUpdates: []*structs.DeploymentStatusUpdate{ 1926 { 1927 DeploymentID: d.ID, 1928 Status: structs.DeploymentStatusCancelled, 1929 StatusDescription: structs.DeploymentStatusDescriptionNewerJob, 1930 }, 1931 }, 1932 place: 2, 1933 inplace: 0, 1934 stop: 2, 1935 desiredTGUpdates: map[string]*structs.DesiredUpdates{ 1936 job.TaskGroups[0].Name: { 1937 Canary: 2, 1938 Stop: 2, 1939 Ignore: 10, 1940 }, 1941 }, 1942 }) 1943 1944 assertNamesHaveIndexes(t, intRange(0, 1), stopResultsToNames(r.stop)) 1945 assertNamesHaveIndexes(t, intRange(0, 1), placeResultsToNames(r.place)) 1946 } 1947 1948 // Tests the reconciler creates new canaries when the job changes 1949 func TestReconciler_NewCanaries(t *testing.T) { 1950 job := mock.Job() 1951 job.TaskGroups[0].Update = canaryUpdate 1952 1953 // Create 10 allocations from the old job 1954 var allocs []*structs.Allocation 1955 for i := 0; i < 10; i++ { 1956 alloc := mock.Alloc() 1957 alloc.Job = job 1958 alloc.JobID = job.ID 1959 alloc.NodeID = structs.GenerateUUID() 1960 alloc.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i)) 1961 alloc.TaskGroup = job.TaskGroups[0].Name 1962 allocs = append(allocs, alloc) 1963 } 1964 1965 reconciler := NewAllocReconciler(testLogger(), allocUpdateFnDestructive, false, job.ID, job, nil, allocs, nil) 1966 r := reconciler.Compute() 1967 1968 newD := structs.NewDeployment(job) 1969 newD.StatusDescription = structs.DeploymentStatusDescriptionRunningNeedsPromotion 1970 newD.TaskGroups[job.TaskGroups[0].Name] = &structs.DeploymentState{ 1971 DesiredCanaries: 2, 1972 DesiredTotal: 10, 1973 } 1974 1975 // Assert the correct results 1976 assertResults(t, r, &resultExpectation{ 1977 createDeployment: newD, 1978 deploymentUpdates: nil, 1979 place: 2, 1980 inplace: 0, 1981 stop: 0, 1982 desiredTGUpdates: map[string]*structs.DesiredUpdates{ 1983 job.TaskGroups[0].Name: { 1984 Canary: 2, 1985 Ignore: 10, 1986 }, 1987 }, 1988 }) 1989 1990 assertNamesHaveIndexes(t, intRange(0, 1), placeResultsToNames(r.place)) 1991 } 1992 1993 // Tests the reconciler creates new canaries when the job changes for multiple 1994 // task groups 1995 func TestReconciler_NewCanaries_MultiTG(t *testing.T) { 1996 job := mock.Job() 1997 job.TaskGroups[0].Update = canaryUpdate 1998 job.TaskGroups = append(job.TaskGroups, job.TaskGroups[0].Copy()) 1999 job.TaskGroups[0].Name = "tg2" 2000 2001 // Create 10 allocations from the old job for each tg 2002 var allocs []*structs.Allocation 2003 for j := 0; j < 2; j++ { 2004 for i := 0; i < 10; i++ { 2005 alloc := mock.Alloc() 2006 alloc.Job = job 2007 alloc.JobID = job.ID 2008 alloc.NodeID = structs.GenerateUUID() 2009 alloc.Name = structs.AllocName(job.ID, job.TaskGroups[j].Name, uint(i)) 2010 alloc.TaskGroup = job.TaskGroups[j].Name 2011 allocs = append(allocs, alloc) 2012 } 2013 } 2014 2015 reconciler := NewAllocReconciler(testLogger(), allocUpdateFnDestructive, false, job.ID, job, nil, allocs, nil) 2016 r := reconciler.Compute() 2017 2018 newD := structs.NewDeployment(job) 2019 newD.StatusDescription = structs.DeploymentStatusDescriptionRunningNeedsPromotion 2020 state := &structs.DeploymentState{ 2021 DesiredCanaries: 2, 2022 DesiredTotal: 10, 2023 } 2024 newD.TaskGroups[job.TaskGroups[0].Name] = state 2025 newD.TaskGroups[job.TaskGroups[1].Name] = state.Copy() 2026 2027 // Assert the correct results 2028 assertResults(t, r, &resultExpectation{ 2029 createDeployment: newD, 2030 deploymentUpdates: nil, 2031 place: 4, 2032 inplace: 0, 2033 stop: 0, 2034 desiredTGUpdates: map[string]*structs.DesiredUpdates{ 2035 job.TaskGroups[0].Name: { 2036 Canary: 2, 2037 Ignore: 10, 2038 }, 2039 job.TaskGroups[1].Name: { 2040 Canary: 2, 2041 Ignore: 10, 2042 }, 2043 }, 2044 }) 2045 2046 assertNamesHaveIndexes(t, intRange(0, 1, 0, 1), placeResultsToNames(r.place)) 2047 } 2048 2049 // Tests the reconciler creates new canaries when the job changes and scales up 2050 func TestReconciler_NewCanaries_ScaleUp(t *testing.T) { 2051 // Scale the job up to 15 2052 job := mock.Job() 2053 job.TaskGroups[0].Update = canaryUpdate 2054 job.TaskGroups[0].Count = 15 2055 2056 // Create 10 allocations from the old job 2057 var allocs []*structs.Allocation 2058 for i := 0; i < 10; i++ { 2059 alloc := mock.Alloc() 2060 alloc.Job = job 2061 alloc.JobID = job.ID 2062 alloc.NodeID = structs.GenerateUUID() 2063 alloc.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i)) 2064 alloc.TaskGroup = job.TaskGroups[0].Name 2065 allocs = append(allocs, alloc) 2066 } 2067 2068 reconciler := NewAllocReconciler(testLogger(), allocUpdateFnDestructive, false, job.ID, job, nil, allocs, nil) 2069 r := reconciler.Compute() 2070 2071 newD := structs.NewDeployment(job) 2072 newD.StatusDescription = structs.DeploymentStatusDescriptionRunningNeedsPromotion 2073 newD.TaskGroups[job.TaskGroups[0].Name] = &structs.DeploymentState{ 2074 DesiredCanaries: 2, 2075 DesiredTotal: 15, 2076 } 2077 2078 // Assert the correct results 2079 assertResults(t, r, &resultExpectation{ 2080 createDeployment: newD, 2081 deploymentUpdates: nil, 2082 place: 2, 2083 inplace: 0, 2084 stop: 0, 2085 desiredTGUpdates: map[string]*structs.DesiredUpdates{ 2086 job.TaskGroups[0].Name: { 2087 Canary: 2, 2088 Ignore: 10, 2089 }, 2090 }, 2091 }) 2092 2093 assertNamesHaveIndexes(t, intRange(0, 1), placeResultsToNames(r.place)) 2094 } 2095 2096 // Tests the reconciler creates new canaries when the job changes and scales 2097 // down 2098 func TestReconciler_NewCanaries_ScaleDown(t *testing.T) { 2099 // Scale the job down to 5 2100 job := mock.Job() 2101 job.TaskGroups[0].Update = canaryUpdate 2102 job.TaskGroups[0].Count = 5 2103 2104 // Create 10 allocations from the old job 2105 var allocs []*structs.Allocation 2106 for i := 0; i < 10; i++ { 2107 alloc := mock.Alloc() 2108 alloc.Job = job 2109 alloc.JobID = job.ID 2110 alloc.NodeID = structs.GenerateUUID() 2111 alloc.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i)) 2112 alloc.TaskGroup = job.TaskGroups[0].Name 2113 allocs = append(allocs, alloc) 2114 } 2115 2116 reconciler := NewAllocReconciler(testLogger(), allocUpdateFnDestructive, false, job.ID, job, nil, allocs, nil) 2117 r := reconciler.Compute() 2118 2119 newD := structs.NewDeployment(job) 2120 newD.StatusDescription = structs.DeploymentStatusDescriptionRunningNeedsPromotion 2121 newD.TaskGroups[job.TaskGroups[0].Name] = &structs.DeploymentState{ 2122 DesiredCanaries: 2, 2123 DesiredTotal: 5, 2124 } 2125 2126 // Assert the correct results 2127 assertResults(t, r, &resultExpectation{ 2128 createDeployment: newD, 2129 deploymentUpdates: nil, 2130 place: 2, 2131 inplace: 0, 2132 stop: 5, 2133 desiredTGUpdates: map[string]*structs.DesiredUpdates{ 2134 job.TaskGroups[0].Name: { 2135 Canary: 2, 2136 Stop: 5, 2137 Ignore: 5, 2138 }, 2139 }, 2140 }) 2141 2142 assertNamesHaveIndexes(t, intRange(0, 1), placeResultsToNames(r.place)) 2143 assertNamesHaveIndexes(t, intRange(5, 9), stopResultsToNames(r.stop)) 2144 } 2145 2146 // Tests the reconciler handles filling the names of partially placed canaries 2147 func TestReconciler_NewCanaries_FillNames(t *testing.T) { 2148 job := mock.Job() 2149 job.TaskGroups[0].Update = &structs.UpdateStrategy{ 2150 Canary: 4, 2151 MaxParallel: 2, 2152 HealthCheck: structs.UpdateStrategyHealthCheck_Checks, 2153 MinHealthyTime: 10 * time.Second, 2154 HealthyDeadline: 10 * time.Minute, 2155 } 2156 2157 // Create an existing deployment that has placed some canaries 2158 d := structs.NewDeployment(job) 2159 s := &structs.DeploymentState{ 2160 Promoted: false, 2161 DesiredTotal: 10, 2162 DesiredCanaries: 4, 2163 PlacedAllocs: 2, 2164 } 2165 d.TaskGroups[job.TaskGroups[0].Name] = s 2166 2167 // Create 10 allocations from the old job 2168 var allocs []*structs.Allocation 2169 for i := 0; i < 10; i++ { 2170 alloc := mock.Alloc() 2171 alloc.Job = job 2172 alloc.JobID = job.ID 2173 alloc.NodeID = structs.GenerateUUID() 2174 alloc.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i)) 2175 alloc.TaskGroup = job.TaskGroups[0].Name 2176 allocs = append(allocs, alloc) 2177 } 2178 2179 // Create canaries but pick names at the ends 2180 for i := 0; i < 4; i += 3 { 2181 // Create one canary 2182 canary := mock.Alloc() 2183 canary.Job = job 2184 canary.JobID = job.ID 2185 canary.NodeID = structs.GenerateUUID() 2186 canary.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i)) 2187 canary.TaskGroup = job.TaskGroups[0].Name 2188 s.PlacedCanaries = append(s.PlacedCanaries, canary.ID) 2189 canary.DeploymentID = d.ID 2190 allocs = append(allocs, canary) 2191 } 2192 2193 reconciler := NewAllocReconciler(testLogger(), allocUpdateFnDestructive, false, job.ID, job, d, allocs, nil) 2194 r := reconciler.Compute() 2195 2196 // Assert the correct results 2197 assertResults(t, r, &resultExpectation{ 2198 createDeployment: nil, 2199 deploymentUpdates: nil, 2200 place: 2, 2201 inplace: 0, 2202 stop: 0, 2203 desiredTGUpdates: map[string]*structs.DesiredUpdates{ 2204 job.TaskGroups[0].Name: { 2205 Canary: 2, 2206 Ignore: 12, 2207 }, 2208 }, 2209 }) 2210 2211 assertNamesHaveIndexes(t, intRange(1, 2), placeResultsToNames(r.place)) 2212 } 2213 2214 // Tests the reconciler handles canary promotion by unblocking max_parallel 2215 func TestReconciler_PromoteCanaries_Unblock(t *testing.T) { 2216 job := mock.Job() 2217 job.TaskGroups[0].Update = canaryUpdate 2218 2219 // Create an existing deployment that has placed some canaries and mark them 2220 // promoted 2221 d := structs.NewDeployment(job) 2222 s := &structs.DeploymentState{ 2223 Promoted: true, 2224 DesiredTotal: 10, 2225 DesiredCanaries: 2, 2226 PlacedAllocs: 2, 2227 } 2228 d.TaskGroups[job.TaskGroups[0].Name] = s 2229 2230 // Create 10 allocations from the old job 2231 var allocs []*structs.Allocation 2232 for i := 0; i < 10; i++ { 2233 alloc := mock.Alloc() 2234 alloc.Job = job 2235 alloc.JobID = job.ID 2236 alloc.NodeID = structs.GenerateUUID() 2237 alloc.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i)) 2238 alloc.TaskGroup = job.TaskGroups[0].Name 2239 allocs = append(allocs, alloc) 2240 } 2241 2242 // Create the canaries 2243 handled := make(map[string]allocUpdateType) 2244 for i := 0; i < 2; i++ { 2245 // Create one canary 2246 canary := mock.Alloc() 2247 canary.Job = job 2248 canary.JobID = job.ID 2249 canary.NodeID = structs.GenerateUUID() 2250 canary.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i)) 2251 canary.TaskGroup = job.TaskGroups[0].Name 2252 s.PlacedCanaries = append(s.PlacedCanaries, canary.ID) 2253 canary.DeploymentID = d.ID 2254 canary.DeploymentStatus = &structs.AllocDeploymentStatus{ 2255 Healthy: helper.BoolToPtr(true), 2256 } 2257 allocs = append(allocs, canary) 2258 handled[canary.ID] = allocUpdateFnIgnore 2259 } 2260 2261 mockUpdateFn := allocUpdateFnMock(handled, allocUpdateFnDestructive) 2262 reconciler := NewAllocReconciler(testLogger(), mockUpdateFn, false, job.ID, job, d, allocs, nil) 2263 r := reconciler.Compute() 2264 2265 // Assert the correct results 2266 assertResults(t, r, &resultExpectation{ 2267 createDeployment: nil, 2268 deploymentUpdates: nil, 2269 destructive: 2, 2270 stop: 2, 2271 desiredTGUpdates: map[string]*structs.DesiredUpdates{ 2272 job.TaskGroups[0].Name: { 2273 Stop: 2, 2274 DestructiveUpdate: 2, 2275 Ignore: 8, 2276 }, 2277 }, 2278 }) 2279 2280 assertNoCanariesStopped(t, d, r.stop) 2281 assertNamesHaveIndexes(t, intRange(2, 3), destructiveResultsToNames(r.destructiveUpdate)) 2282 assertNamesHaveIndexes(t, intRange(0, 1), stopResultsToNames(r.stop)) 2283 } 2284 2285 // Tests the reconciler handles canary promotion when the canary count equals 2286 // the total correctly 2287 func TestReconciler_PromoteCanaries_CanariesEqualCount(t *testing.T) { 2288 job := mock.Job() 2289 job.TaskGroups[0].Update = canaryUpdate 2290 job.TaskGroups[0].Count = 2 2291 2292 // Create an existing deployment that has placed some canaries and mark them 2293 // promoted 2294 d := structs.NewDeployment(job) 2295 s := &structs.DeploymentState{ 2296 Promoted: true, 2297 DesiredTotal: 2, 2298 DesiredCanaries: 2, 2299 PlacedAllocs: 2, 2300 } 2301 d.TaskGroups[job.TaskGroups[0].Name] = s 2302 2303 // Create 2 allocations from the old job 2304 var allocs []*structs.Allocation 2305 for i := 0; i < 2; i++ { 2306 alloc := mock.Alloc() 2307 alloc.Job = job 2308 alloc.JobID = job.ID 2309 alloc.NodeID = structs.GenerateUUID() 2310 alloc.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i)) 2311 alloc.TaskGroup = job.TaskGroups[0].Name 2312 allocs = append(allocs, alloc) 2313 } 2314 2315 // Create the canaries 2316 handled := make(map[string]allocUpdateType) 2317 for i := 0; i < 2; i++ { 2318 // Create one canary 2319 canary := mock.Alloc() 2320 canary.Job = job 2321 canary.JobID = job.ID 2322 canary.NodeID = structs.GenerateUUID() 2323 canary.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i)) 2324 canary.TaskGroup = job.TaskGroups[0].Name 2325 s.PlacedCanaries = append(s.PlacedCanaries, canary.ID) 2326 canary.DeploymentID = d.ID 2327 canary.DeploymentStatus = &structs.AllocDeploymentStatus{ 2328 Healthy: helper.BoolToPtr(true), 2329 } 2330 allocs = append(allocs, canary) 2331 handled[canary.ID] = allocUpdateFnIgnore 2332 } 2333 2334 mockUpdateFn := allocUpdateFnMock(handled, allocUpdateFnDestructive) 2335 reconciler := NewAllocReconciler(testLogger(), mockUpdateFn, false, job.ID, job, d, allocs, nil) 2336 r := reconciler.Compute() 2337 2338 updates := []*structs.DeploymentStatusUpdate{ 2339 { 2340 DeploymentID: d.ID, 2341 Status: structs.DeploymentStatusSuccessful, 2342 StatusDescription: structs.DeploymentStatusDescriptionSuccessful, 2343 }, 2344 } 2345 2346 // Assert the correct results 2347 assertResults(t, r, &resultExpectation{ 2348 createDeployment: nil, 2349 deploymentUpdates: updates, 2350 place: 0, 2351 inplace: 0, 2352 stop: 2, 2353 desiredTGUpdates: map[string]*structs.DesiredUpdates{ 2354 job.TaskGroups[0].Name: { 2355 Stop: 2, 2356 Ignore: 2, 2357 }, 2358 }, 2359 }) 2360 2361 assertNoCanariesStopped(t, d, r.stop) 2362 assertNamesHaveIndexes(t, intRange(0, 1), stopResultsToNames(r.stop)) 2363 } 2364 2365 // Tests the reconciler checks the health of placed allocs to determine the 2366 // limit 2367 func TestReconciler_DeploymentLimit_HealthAccounting(t *testing.T) { 2368 job := mock.Job() 2369 job.TaskGroups[0].Update = noCanaryUpdate 2370 2371 cases := []struct { 2372 healthy int 2373 }{ 2374 { 2375 healthy: 0, 2376 }, 2377 { 2378 healthy: 1, 2379 }, 2380 { 2381 healthy: 2, 2382 }, 2383 { 2384 healthy: 3, 2385 }, 2386 { 2387 healthy: 4, 2388 }, 2389 } 2390 2391 for _, c := range cases { 2392 t.Run(fmt.Sprintf("%d healthy", c.healthy), func(t *testing.T) { 2393 // Create an existing deployment that has placed some canaries and mark them 2394 // promoted 2395 d := structs.NewDeployment(job) 2396 d.TaskGroups[job.TaskGroups[0].Name] = &structs.DeploymentState{ 2397 Promoted: true, 2398 DesiredTotal: 10, 2399 PlacedAllocs: 4, 2400 } 2401 2402 // Create 6 allocations from the old job 2403 var allocs []*structs.Allocation 2404 for i := 4; i < 10; i++ { 2405 alloc := mock.Alloc() 2406 alloc.Job = job 2407 alloc.JobID = job.ID 2408 alloc.NodeID = structs.GenerateUUID() 2409 alloc.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i)) 2410 alloc.TaskGroup = job.TaskGroups[0].Name 2411 allocs = append(allocs, alloc) 2412 } 2413 2414 // Create the new allocs 2415 handled := make(map[string]allocUpdateType) 2416 for i := 0; i < 4; i++ { 2417 new := mock.Alloc() 2418 new.Job = job 2419 new.JobID = job.ID 2420 new.NodeID = structs.GenerateUUID() 2421 new.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i)) 2422 new.TaskGroup = job.TaskGroups[0].Name 2423 new.DeploymentID = d.ID 2424 if i < c.healthy { 2425 new.DeploymentStatus = &structs.AllocDeploymentStatus{ 2426 Healthy: helper.BoolToPtr(true), 2427 } 2428 } 2429 allocs = append(allocs, new) 2430 handled[new.ID] = allocUpdateFnIgnore 2431 } 2432 2433 mockUpdateFn := allocUpdateFnMock(handled, allocUpdateFnDestructive) 2434 reconciler := NewAllocReconciler(testLogger(), mockUpdateFn, false, job.ID, job, d, allocs, nil) 2435 r := reconciler.Compute() 2436 2437 // Assert the correct results 2438 assertResults(t, r, &resultExpectation{ 2439 createDeployment: nil, 2440 deploymentUpdates: nil, 2441 destructive: c.healthy, 2442 desiredTGUpdates: map[string]*structs.DesiredUpdates{ 2443 job.TaskGroups[0].Name: { 2444 DestructiveUpdate: uint64(c.healthy), 2445 Ignore: uint64(10 - c.healthy), 2446 }, 2447 }, 2448 }) 2449 2450 if c.healthy != 0 { 2451 assertNamesHaveIndexes(t, intRange(4, 3+c.healthy), destructiveResultsToNames(r.destructiveUpdate)) 2452 } 2453 }) 2454 } 2455 } 2456 2457 // Tests the reconciler handles an alloc on a tainted node during a rolling 2458 // update 2459 func TestReconciler_TaintedNode_RollingUpgrade(t *testing.T) { 2460 job := mock.Job() 2461 job.TaskGroups[0].Update = noCanaryUpdate 2462 2463 // Create an existing deployment that has some placed allocs 2464 d := structs.NewDeployment(job) 2465 d.TaskGroups[job.TaskGroups[0].Name] = &structs.DeploymentState{ 2466 Promoted: true, 2467 DesiredTotal: 10, 2468 PlacedAllocs: 7, 2469 } 2470 2471 // Create 3 allocations from the old job 2472 var allocs []*structs.Allocation 2473 for i := 7; i < 10; i++ { 2474 alloc := mock.Alloc() 2475 alloc.Job = job 2476 alloc.JobID = job.ID 2477 alloc.NodeID = structs.GenerateUUID() 2478 alloc.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i)) 2479 alloc.TaskGroup = job.TaskGroups[0].Name 2480 allocs = append(allocs, alloc) 2481 } 2482 2483 // Create the healthy replacements 2484 handled := make(map[string]allocUpdateType) 2485 for i := 0; i < 7; i++ { 2486 new := mock.Alloc() 2487 new.Job = job 2488 new.JobID = job.ID 2489 new.NodeID = structs.GenerateUUID() 2490 new.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i)) 2491 new.TaskGroup = job.TaskGroups[0].Name 2492 new.DeploymentID = d.ID 2493 new.DeploymentStatus = &structs.AllocDeploymentStatus{ 2494 Healthy: helper.BoolToPtr(true), 2495 } 2496 allocs = append(allocs, new) 2497 handled[new.ID] = allocUpdateFnIgnore 2498 } 2499 2500 // Build a map of tainted nodes 2501 tainted := make(map[string]*structs.Node, 3) 2502 for i := 0; i < 3; i++ { 2503 n := mock.Node() 2504 n.ID = allocs[3+i].NodeID 2505 if i == 0 { 2506 n.Status = structs.NodeStatusDown 2507 } else { 2508 n.Drain = true 2509 } 2510 tainted[n.ID] = n 2511 } 2512 2513 mockUpdateFn := allocUpdateFnMock(handled, allocUpdateFnDestructive) 2514 reconciler := NewAllocReconciler(testLogger(), mockUpdateFn, false, job.ID, job, d, allocs, tainted) 2515 r := reconciler.Compute() 2516 2517 // Assert the correct results 2518 assertResults(t, r, &resultExpectation{ 2519 createDeployment: nil, 2520 deploymentUpdates: nil, 2521 place: 2, 2522 destructive: 3, 2523 stop: 2, 2524 followupEvalWait: 31 * time.Second, 2525 desiredTGUpdates: map[string]*structs.DesiredUpdates{ 2526 job.TaskGroups[0].Name: { 2527 Place: 1, // Place the lost 2528 Stop: 1, // Stop the lost 2529 Migrate: 1, // Migrate the tainted 2530 DestructiveUpdate: 3, 2531 Ignore: 5, 2532 }, 2533 }, 2534 }) 2535 2536 assertNamesHaveIndexes(t, intRange(7, 9), destructiveResultsToNames(r.destructiveUpdate)) 2537 assertNamesHaveIndexes(t, intRange(0, 1), placeResultsToNames(r.place)) 2538 assertNamesHaveIndexes(t, intRange(0, 1), stopResultsToNames(r.stop)) 2539 } 2540 2541 // Tests the reconciler handles a failed deployment and does no placements 2542 func TestReconciler_FailedDeployment_NoPlacements(t *testing.T) { 2543 job := mock.Job() 2544 job.TaskGroups[0].Update = noCanaryUpdate 2545 2546 // Create an existing failed deployment that has some placed allocs 2547 d := structs.NewDeployment(job) 2548 d.Status = structs.DeploymentStatusFailed 2549 d.TaskGroups[job.TaskGroups[0].Name] = &structs.DeploymentState{ 2550 Promoted: true, 2551 DesiredTotal: 10, 2552 PlacedAllocs: 4, 2553 } 2554 2555 // Create 6 allocations from the old job 2556 var allocs []*structs.Allocation 2557 for i := 4; i < 10; i++ { 2558 alloc := mock.Alloc() 2559 alloc.Job = job 2560 alloc.JobID = job.ID 2561 alloc.NodeID = structs.GenerateUUID() 2562 alloc.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i)) 2563 alloc.TaskGroup = job.TaskGroups[0].Name 2564 allocs = append(allocs, alloc) 2565 } 2566 2567 // Create the healthy replacements 2568 handled := make(map[string]allocUpdateType) 2569 for i := 0; i < 4; i++ { 2570 new := mock.Alloc() 2571 new.Job = job 2572 new.JobID = job.ID 2573 new.NodeID = structs.GenerateUUID() 2574 new.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i)) 2575 new.TaskGroup = job.TaskGroups[0].Name 2576 new.DeploymentID = d.ID 2577 new.DeploymentStatus = &structs.AllocDeploymentStatus{ 2578 Healthy: helper.BoolToPtr(true), 2579 } 2580 allocs = append(allocs, new) 2581 handled[new.ID] = allocUpdateFnIgnore 2582 } 2583 2584 // Build a map of tainted nodes 2585 tainted := make(map[string]*structs.Node, 2) 2586 for i := 0; i < 2; i++ { 2587 n := mock.Node() 2588 n.ID = allocs[6+i].NodeID 2589 if i == 0 { 2590 n.Status = structs.NodeStatusDown 2591 } else { 2592 n.Drain = true 2593 } 2594 tainted[n.ID] = n 2595 } 2596 2597 mockUpdateFn := allocUpdateFnMock(handled, allocUpdateFnDestructive) 2598 reconciler := NewAllocReconciler(testLogger(), mockUpdateFn, false, job.ID, job, d, allocs, tainted) 2599 r := reconciler.Compute() 2600 2601 // Assert the correct results 2602 assertResults(t, r, &resultExpectation{ 2603 createDeployment: nil, 2604 deploymentUpdates: nil, 2605 place: 0, 2606 inplace: 0, 2607 stop: 2, 2608 followupEvalWait: 0, // Since the deployment is failed, there should be no followup 2609 desiredTGUpdates: map[string]*structs.DesiredUpdates{ 2610 job.TaskGroups[0].Name: { 2611 Stop: 2, 2612 Ignore: 8, 2613 }, 2614 }, 2615 }) 2616 2617 assertNamesHaveIndexes(t, intRange(0, 1), stopResultsToNames(r.stop)) 2618 } 2619 2620 // Tests the reconciler handles a run after a deployment is complete 2621 // successfully. 2622 func TestReconciler_CompleteDeployment(t *testing.T) { 2623 job := mock.Job() 2624 job.TaskGroups[0].Update = canaryUpdate 2625 2626 d := structs.NewDeployment(job) 2627 d.Status = structs.DeploymentStatusSuccessful 2628 d.TaskGroups[job.TaskGroups[0].Name] = &structs.DeploymentState{ 2629 Promoted: true, 2630 DesiredTotal: 10, 2631 DesiredCanaries: 2, 2632 PlacedAllocs: 10, 2633 HealthyAllocs: 10, 2634 } 2635 2636 // Create allocations from the old job 2637 var allocs []*structs.Allocation 2638 for i := 0; i < 10; i++ { 2639 alloc := mock.Alloc() 2640 alloc.Job = job 2641 alloc.JobID = job.ID 2642 alloc.NodeID = structs.GenerateUUID() 2643 alloc.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i)) 2644 alloc.TaskGroup = job.TaskGroups[0].Name 2645 alloc.DeploymentID = d.ID 2646 alloc.DeploymentStatus = &structs.AllocDeploymentStatus{ 2647 Healthy: helper.BoolToPtr(true), 2648 } 2649 allocs = append(allocs, alloc) 2650 } 2651 2652 reconciler := NewAllocReconciler(testLogger(), allocUpdateFnIgnore, false, job.ID, job, d, allocs, nil) 2653 r := reconciler.Compute() 2654 2655 // Assert the correct results 2656 assertResults(t, r, &resultExpectation{ 2657 createDeployment: nil, 2658 deploymentUpdates: nil, 2659 place: 0, 2660 inplace: 0, 2661 stop: 0, 2662 desiredTGUpdates: map[string]*structs.DesiredUpdates{ 2663 job.TaskGroups[0].Name: { 2664 Ignore: 10, 2665 }, 2666 }, 2667 }) 2668 } 2669 2670 // Test that a failed deployment cancels non-promoted canaries 2671 func TestReconciler_FailedDeployment_CancelCanaries(t *testing.T) { 2672 // Create a job with two task groups 2673 job := mock.Job() 2674 job.TaskGroups[0].Update = canaryUpdate 2675 job.TaskGroups = append(job.TaskGroups, job.TaskGroups[0].Copy()) 2676 job.TaskGroups[1].Name = "two" 2677 2678 // Create an existing failed deployment that has promoted one task group 2679 d := structs.NewDeployment(job) 2680 d.Status = structs.DeploymentStatusFailed 2681 s0 := &structs.DeploymentState{ 2682 Promoted: true, 2683 DesiredTotal: 10, 2684 DesiredCanaries: 2, 2685 PlacedAllocs: 4, 2686 } 2687 s1 := &structs.DeploymentState{ 2688 Promoted: false, 2689 DesiredTotal: 10, 2690 DesiredCanaries: 2, 2691 PlacedAllocs: 2, 2692 } 2693 d.TaskGroups[job.TaskGroups[0].Name] = s0 2694 d.TaskGroups[job.TaskGroups[1].Name] = s1 2695 2696 // Create 6 allocations from the old job 2697 var allocs []*structs.Allocation 2698 handled := make(map[string]allocUpdateType) 2699 for _, group := range []int{0, 1} { 2700 replacements := 4 2701 state := s0 2702 if group == 1 { 2703 replacements = 2 2704 state = s1 2705 } 2706 2707 // Create the healthy replacements 2708 for i := 0; i < replacements; i++ { 2709 new := mock.Alloc() 2710 new.Job = job 2711 new.JobID = job.ID 2712 new.NodeID = structs.GenerateUUID() 2713 new.Name = structs.AllocName(job.ID, job.TaskGroups[group].Name, uint(i)) 2714 new.TaskGroup = job.TaskGroups[group].Name 2715 new.DeploymentID = d.ID 2716 new.DeploymentStatus = &structs.AllocDeploymentStatus{ 2717 Healthy: helper.BoolToPtr(true), 2718 } 2719 allocs = append(allocs, new) 2720 handled[new.ID] = allocUpdateFnIgnore 2721 2722 // Add the alloc to the canary list 2723 if i < 2 { 2724 state.PlacedCanaries = append(state.PlacedCanaries, new.ID) 2725 } 2726 } 2727 for i := replacements; i < 10; i++ { 2728 alloc := mock.Alloc() 2729 alloc.Job = job 2730 alloc.JobID = job.ID 2731 alloc.NodeID = structs.GenerateUUID() 2732 alloc.Name = structs.AllocName(job.ID, job.TaskGroups[group].Name, uint(i)) 2733 alloc.TaskGroup = job.TaskGroups[group].Name 2734 allocs = append(allocs, alloc) 2735 } 2736 } 2737 2738 mockUpdateFn := allocUpdateFnMock(handled, allocUpdateFnDestructive) 2739 reconciler := NewAllocReconciler(testLogger(), mockUpdateFn, false, job.ID, job, d, allocs, nil) 2740 r := reconciler.Compute() 2741 2742 // Assert the correct results 2743 assertResults(t, r, &resultExpectation{ 2744 createDeployment: nil, 2745 deploymentUpdates: nil, 2746 place: 0, 2747 inplace: 0, 2748 stop: 2, 2749 desiredTGUpdates: map[string]*structs.DesiredUpdates{ 2750 job.TaskGroups[0].Name: { 2751 Ignore: 10, 2752 }, 2753 job.TaskGroups[1].Name: { 2754 Stop: 2, 2755 Ignore: 8, 2756 }, 2757 }, 2758 }) 2759 2760 assertNamesHaveIndexes(t, intRange(0, 1), stopResultsToNames(r.stop)) 2761 } 2762 2763 // Test that a failed deployment and updated job works 2764 func TestReconciler_FailedDeployment_NewJob(t *testing.T) { 2765 job := mock.Job() 2766 job.TaskGroups[0].Update = noCanaryUpdate 2767 2768 // Create an existing failed deployment that has some placed allocs 2769 d := structs.NewDeployment(job) 2770 d.Status = structs.DeploymentStatusFailed 2771 d.TaskGroups[job.TaskGroups[0].Name] = &structs.DeploymentState{ 2772 Promoted: true, 2773 DesiredTotal: 10, 2774 PlacedAllocs: 4, 2775 } 2776 2777 // Create 6 allocations from the old job 2778 var allocs []*structs.Allocation 2779 for i := 4; i < 10; i++ { 2780 alloc := mock.Alloc() 2781 alloc.Job = job 2782 alloc.JobID = job.ID 2783 alloc.NodeID = structs.GenerateUUID() 2784 alloc.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i)) 2785 alloc.TaskGroup = job.TaskGroups[0].Name 2786 allocs = append(allocs, alloc) 2787 } 2788 2789 // Create the healthy replacements 2790 for i := 0; i < 4; i++ { 2791 new := mock.Alloc() 2792 new.Job = job 2793 new.JobID = job.ID 2794 new.NodeID = structs.GenerateUUID() 2795 new.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i)) 2796 new.TaskGroup = job.TaskGroups[0].Name 2797 new.DeploymentID = d.ID 2798 new.DeploymentStatus = &structs.AllocDeploymentStatus{ 2799 Healthy: helper.BoolToPtr(true), 2800 } 2801 allocs = append(allocs, new) 2802 } 2803 2804 // Up the job version 2805 jobNew := job.Copy() 2806 jobNew.Version += 100 2807 2808 reconciler := NewAllocReconciler(testLogger(), allocUpdateFnDestructive, false, job.ID, jobNew, d, allocs, nil) 2809 r := reconciler.Compute() 2810 2811 dnew := structs.NewDeployment(jobNew) 2812 dnew.TaskGroups[job.TaskGroups[0].Name] = &structs.DeploymentState{ 2813 DesiredTotal: 10, 2814 } 2815 2816 // Assert the correct results 2817 assertResults(t, r, &resultExpectation{ 2818 createDeployment: dnew, 2819 deploymentUpdates: nil, 2820 destructive: 4, 2821 desiredTGUpdates: map[string]*structs.DesiredUpdates{ 2822 job.TaskGroups[0].Name: { 2823 DestructiveUpdate: 4, 2824 Ignore: 6, 2825 }, 2826 }, 2827 }) 2828 2829 assertNamesHaveIndexes(t, intRange(0, 3), destructiveResultsToNames(r.destructiveUpdate)) 2830 } 2831 2832 // Tests the reconciler marks a deployment as complete 2833 func TestReconciler_MarkDeploymentComplete(t *testing.T) { 2834 job := mock.Job() 2835 job.TaskGroups[0].Update = noCanaryUpdate 2836 2837 d := structs.NewDeployment(job) 2838 d.TaskGroups[job.TaskGroups[0].Name] = &structs.DeploymentState{ 2839 Promoted: true, 2840 DesiredTotal: 10, 2841 PlacedAllocs: 10, 2842 HealthyAllocs: 10, 2843 } 2844 2845 // Create allocations from the old job 2846 var allocs []*structs.Allocation 2847 for i := 0; i < 10; i++ { 2848 alloc := mock.Alloc() 2849 alloc.Job = job 2850 alloc.JobID = job.ID 2851 alloc.NodeID = structs.GenerateUUID() 2852 alloc.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i)) 2853 alloc.TaskGroup = job.TaskGroups[0].Name 2854 alloc.DeploymentID = d.ID 2855 alloc.DeploymentStatus = &structs.AllocDeploymentStatus{ 2856 Healthy: helper.BoolToPtr(true), 2857 } 2858 allocs = append(allocs, alloc) 2859 } 2860 2861 reconciler := NewAllocReconciler(testLogger(), allocUpdateFnIgnore, false, job.ID, job, d, allocs, nil) 2862 r := reconciler.Compute() 2863 2864 updates := []*structs.DeploymentStatusUpdate{ 2865 { 2866 DeploymentID: d.ID, 2867 Status: structs.DeploymentStatusSuccessful, 2868 StatusDescription: structs.DeploymentStatusDescriptionSuccessful, 2869 }, 2870 } 2871 2872 // Assert the correct results 2873 assertResults(t, r, &resultExpectation{ 2874 createDeployment: nil, 2875 deploymentUpdates: updates, 2876 place: 0, 2877 inplace: 0, 2878 stop: 0, 2879 desiredTGUpdates: map[string]*structs.DesiredUpdates{ 2880 job.TaskGroups[0].Name: { 2881 Ignore: 10, 2882 }, 2883 }, 2884 }) 2885 } 2886 2887 // Tests the reconciler picks the maximum of the staggers when multiple task 2888 // groups are under going node drains. 2889 func TestReconciler_TaintedNode_MultiGroups(t *testing.T) { 2890 // Create a job with two task groups 2891 job := mock.Job() 2892 job.TaskGroups[0].Update = noCanaryUpdate 2893 job.TaskGroups = append(job.TaskGroups, job.TaskGroups[0].Copy()) 2894 job.TaskGroups[1].Name = "two" 2895 job.TaskGroups[1].Update.Stagger = 100 * time.Second 2896 2897 // Create the allocations 2898 var allocs []*structs.Allocation 2899 for j := 0; j < 2; j++ { 2900 for i := 0; i < 10; i++ { 2901 alloc := mock.Alloc() 2902 alloc.Job = job 2903 alloc.JobID = job.ID 2904 alloc.NodeID = structs.GenerateUUID() 2905 alloc.Name = structs.AllocName(job.ID, job.TaskGroups[j].Name, uint(i)) 2906 alloc.TaskGroup = job.TaskGroups[j].Name 2907 allocs = append(allocs, alloc) 2908 } 2909 } 2910 2911 // Build a map of tainted nodes 2912 tainted := make(map[string]*structs.Node, 15) 2913 for i := 0; i < 15; i++ { 2914 n := mock.Node() 2915 n.ID = allocs[i].NodeID 2916 n.Drain = true 2917 tainted[n.ID] = n 2918 } 2919 2920 reconciler := NewAllocReconciler(testLogger(), allocUpdateFnIgnore, false, job.ID, job, nil, allocs, tainted) 2921 r := reconciler.Compute() 2922 2923 // Assert the correct results 2924 assertResults(t, r, &resultExpectation{ 2925 createDeployment: nil, 2926 deploymentUpdates: nil, 2927 place: 8, 2928 inplace: 0, 2929 stop: 8, 2930 followupEvalWait: 100 * time.Second, 2931 desiredTGUpdates: map[string]*structs.DesiredUpdates{ 2932 job.TaskGroups[0].Name: { 2933 Place: 0, 2934 Stop: 0, 2935 Migrate: 4, 2936 DestructiveUpdate: 0, 2937 Ignore: 6, 2938 }, 2939 job.TaskGroups[1].Name: { 2940 Place: 0, 2941 Stop: 0, 2942 Migrate: 4, 2943 DestructiveUpdate: 0, 2944 Ignore: 6, 2945 }, 2946 }, 2947 }) 2948 2949 assertNamesHaveIndexes(t, intRange(0, 3, 0, 3), placeResultsToNames(r.place)) 2950 assertNamesHaveIndexes(t, intRange(0, 3, 0, 3), stopResultsToNames(r.stop)) 2951 } 2952 2953 // Tests the reconciler handles changing a job such that a deployment is created 2954 // while doing a scale up but as the second eval. 2955 func TestReconciler_JobChange_ScaleUp_SecondEval(t *testing.T) { 2956 // Scale the job up to 15 2957 job := mock.Job() 2958 job.TaskGroups[0].Update = noCanaryUpdate 2959 job.TaskGroups[0].Count = 30 2960 2961 // Create a deployment that is paused and has placed some canaries 2962 d := structs.NewDeployment(job) 2963 d.TaskGroups[job.TaskGroups[0].Name] = &structs.DeploymentState{ 2964 Promoted: false, 2965 DesiredTotal: 30, 2966 PlacedAllocs: 20, 2967 } 2968 2969 // Create 10 allocations from the old job 2970 var allocs []*structs.Allocation 2971 for i := 0; i < 10; i++ { 2972 alloc := mock.Alloc() 2973 alloc.Job = job 2974 alloc.JobID = job.ID 2975 alloc.NodeID = structs.GenerateUUID() 2976 alloc.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i)) 2977 alloc.TaskGroup = job.TaskGroups[0].Name 2978 allocs = append(allocs, alloc) 2979 } 2980 2981 // Create 20 from new job 2982 handled := make(map[string]allocUpdateType) 2983 for i := 10; i < 30; i++ { 2984 alloc := mock.Alloc() 2985 alloc.Job = job 2986 alloc.JobID = job.ID 2987 alloc.DeploymentID = d.ID 2988 alloc.NodeID = structs.GenerateUUID() 2989 alloc.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i)) 2990 alloc.TaskGroup = job.TaskGroups[0].Name 2991 allocs = append(allocs, alloc) 2992 handled[alloc.ID] = allocUpdateFnIgnore 2993 } 2994 2995 mockUpdateFn := allocUpdateFnMock(handled, allocUpdateFnDestructive) 2996 reconciler := NewAllocReconciler(testLogger(), mockUpdateFn, false, job.ID, job, d, allocs, nil) 2997 r := reconciler.Compute() 2998 2999 // Assert the correct results 3000 assertResults(t, r, &resultExpectation{ 3001 createDeployment: nil, 3002 deploymentUpdates: nil, 3003 desiredTGUpdates: map[string]*structs.DesiredUpdates{ 3004 job.TaskGroups[0].Name: { 3005 // All should be ignored becasue nothing has been marked as 3006 // healthy. 3007 Ignore: 30, 3008 }, 3009 }, 3010 }) 3011 }