github.com/hhrutter/nomad@v0.6.0-rc2.0.20170723054333-80c4b03f0705/scheduler/reconcile.go (about) 1 package scheduler 2 3 import ( 4 "fmt" 5 "log" 6 "time" 7 8 "github.com/hashicorp/nomad/helper" 9 "github.com/hashicorp/nomad/nomad/structs" 10 ) 11 12 // allocUpdateType takes an existing allocation and a new job definition and 13 // returns whether the allocation can ignore the change, requires a destructive 14 // update, or can be inplace updated. If it can be inplace updated, an updated 15 // allocation that has the new resources and alloc metrics attached will be 16 // returned. 17 type allocUpdateType func(existing *structs.Allocation, newJob *structs.Job, 18 newTG *structs.TaskGroup) (ignore, destructive bool, updated *structs.Allocation) 19 20 // allocReconciler is used to determine the set of allocations that require 21 // placement, inplace updating or stopping given the job specification and 22 // existing cluster state. The reconciler should only be used for batch and 23 // service jobs. 24 type allocReconciler struct { 25 // logger is used to log debug information. Logging should be kept at a 26 // minimal here 27 logger *log.Logger 28 29 // canInplace is used to check if the allocation can be inplace upgraded 30 allocUpdateFn allocUpdateType 31 32 // batch marks whether the job is a batch job 33 batch bool 34 35 // job is the job being operated on, it may be nil if the job is being 36 // stopped via a purge 37 job *structs.Job 38 39 // jobID is the ID of the job being operated on. The job may be nil if it is 40 // being stopped so we require this seperately. 41 jobID string 42 43 // oldDeployment is the last deployment for the job 44 oldDeployment *structs.Deployment 45 46 // deployment is the current deployment for the job 47 deployment *structs.Deployment 48 49 // deploymentPaused marks whether the deployment is paused 50 deploymentPaused bool 51 52 // deploymentFailed marks whether the deployment is failed 53 deploymentFailed bool 54 55 // taintedNodes contains a map of nodes that are tainted 56 taintedNodes map[string]*structs.Node 57 58 // existingAllocs is non-terminal existing allocations 59 existingAllocs []*structs.Allocation 60 61 // result is the results of the reconcile. During computation it can be 62 // used to store intermediate state 63 result *reconcileResults 64 } 65 66 // reconcileResults contains the results of the reconciliation and should be 67 // applied by the scheduler. 68 type reconcileResults struct { 69 // deployment is the deployment that should be created or updated as a 70 // result of scheduling 71 deployment *structs.Deployment 72 73 // deploymentUpdates contains a set of deployment updates that should be 74 // applied as a result of scheduling 75 deploymentUpdates []*structs.DeploymentStatusUpdate 76 77 // place is the set of allocations to place by the scheduler 78 place []allocPlaceResult 79 80 // destructiveUpdate is the set of allocations to apply a destructive update to 81 destructiveUpdate []allocDestructiveResult 82 83 // inplaceUpdate is the set of allocations to apply an inplace update to 84 inplaceUpdate []*structs.Allocation 85 86 // stop is the set of allocations to stop 87 stop []allocStopResult 88 89 // desiredTGUpdates captures the desired set of changes to make for each 90 // task group. 91 desiredTGUpdates map[string]*structs.DesiredUpdates 92 93 // followupEvalWait is set if there should be a followup eval run after the 94 // given duration 95 followupEvalWait time.Duration 96 } 97 98 func (r *reconcileResults) GoString() string { 99 base := fmt.Sprintf("Total changes: (place %d) (destructive %d) (inplace %d) (stop %d)", 100 len(r.place), len(r.destructiveUpdate), len(r.inplaceUpdate), len(r.stop)) 101 102 if r.deployment != nil { 103 base += fmt.Sprintf("\nCreated Deployment: %q", r.deployment.ID) 104 } 105 for _, u := range r.deploymentUpdates { 106 base += fmt.Sprintf("\nDeployment Update for ID %q: Status %q; Description %q", 107 u.DeploymentID, u.Status, u.StatusDescription) 108 } 109 if r.followupEvalWait != 0 { 110 base += fmt.Sprintf("\nFollowup Eval in %v", r.followupEvalWait) 111 } 112 for tg, u := range r.desiredTGUpdates { 113 base += fmt.Sprintf("\nDesired Changes for %q: %#v", tg, u) 114 } 115 return base 116 } 117 118 // Changes returns the number of total changes 119 func (r *reconcileResults) Changes() int { 120 return len(r.place) + len(r.inplaceUpdate) + len(r.stop) 121 } 122 123 // NewAllocReconciler creates a new reconciler that should be used to determine 124 // the changes required to bring the cluster state inline with the declared jobspec 125 func NewAllocReconciler(logger *log.Logger, allocUpdateFn allocUpdateType, batch bool, 126 jobID string, job *structs.Job, deployment *structs.Deployment, 127 existingAllocs []*structs.Allocation, taintedNodes map[string]*structs.Node) *allocReconciler { 128 129 return &allocReconciler{ 130 logger: logger, 131 allocUpdateFn: allocUpdateFn, 132 batch: batch, 133 jobID: jobID, 134 job: job, 135 deployment: deployment.Copy(), 136 existingAllocs: existingAllocs, 137 taintedNodes: taintedNodes, 138 result: &reconcileResults{ 139 desiredTGUpdates: make(map[string]*structs.DesiredUpdates), 140 }, 141 } 142 } 143 144 // Compute reconciles the existing cluster state and returns the set of changes 145 // required to converge the job spec and state 146 func (a *allocReconciler) Compute() *reconcileResults { 147 // Create the allocation matrix 148 m := newAllocMatrix(a.job, a.existingAllocs) 149 150 // Handle stopping unneeded deployments 151 a.cancelDeployments() 152 153 // If we are just stopping a job we do not need to do anything more than 154 // stopping all running allocs 155 if a.job.Stopped() { 156 a.handleStop(m) 157 return a.result 158 } 159 160 // Detect if the deployment is paused 161 if a.deployment != nil { 162 a.deploymentPaused = a.deployment.Status == structs.DeploymentStatusPaused 163 a.deploymentFailed = a.deployment.Status == structs.DeploymentStatusFailed 164 } 165 166 // Reconcile each group 167 complete := true 168 for group, as := range m { 169 groupComplete := a.computeGroup(group, as) 170 complete = complete && groupComplete 171 } 172 173 // Mark the deployment as complete if possible 174 if a.deployment != nil && complete { 175 a.result.deploymentUpdates = append(a.result.deploymentUpdates, &structs.DeploymentStatusUpdate{ 176 DeploymentID: a.deployment.ID, 177 Status: structs.DeploymentStatusSuccessful, 178 StatusDescription: structs.DeploymentStatusDescriptionSuccessful, 179 }) 180 } 181 182 // Set the description of a created deployment 183 if d := a.result.deployment; d != nil { 184 if d.RequiresPromotion() { 185 d.StatusDescription = structs.DeploymentStatusDescriptionRunningNeedsPromotion 186 } 187 } 188 189 return a.result 190 } 191 192 // cancelDeployments cancels any deployment that is not needed 193 func (a *allocReconciler) cancelDeployments() { 194 // If the job is stopped and there is a non-terminal deployment, cancel it 195 if a.job.Stopped() { 196 if a.deployment != nil && a.deployment.Active() { 197 a.result.deploymentUpdates = append(a.result.deploymentUpdates, &structs.DeploymentStatusUpdate{ 198 DeploymentID: a.deployment.ID, 199 Status: structs.DeploymentStatusCancelled, 200 StatusDescription: structs.DeploymentStatusDescriptionStoppedJob, 201 }) 202 } 203 204 // Nothing else to do 205 a.oldDeployment = a.deployment 206 a.deployment = nil 207 return 208 } 209 210 d := a.deployment 211 if d == nil { 212 return 213 } 214 215 // Check if the deployment is active and referencing an older job and cancel it 216 if d.JobCreateIndex != a.job.CreateIndex || d.JobVersion != a.job.Version { 217 if d.Active() { 218 a.result.deploymentUpdates = append(a.result.deploymentUpdates, &structs.DeploymentStatusUpdate{ 219 DeploymentID: a.deployment.ID, 220 Status: structs.DeploymentStatusCancelled, 221 StatusDescription: structs.DeploymentStatusDescriptionNewerJob, 222 }) 223 } 224 225 a.oldDeployment = d 226 a.deployment = nil 227 } 228 229 // Clear it as the current deployment if it is successful 230 if d.Status == structs.DeploymentStatusSuccessful { 231 a.oldDeployment = d 232 a.deployment = nil 233 } 234 } 235 236 // handleStop marks all allocations to be stopped, handling the lost case 237 func (a *allocReconciler) handleStop(m allocMatrix) { 238 for group, as := range m { 239 untainted, migrate, lost := as.filterByTainted(a.taintedNodes) 240 a.markStop(untainted, "", allocNotNeeded) 241 a.markStop(migrate, "", allocNotNeeded) 242 a.markStop(lost, structs.AllocClientStatusLost, allocLost) 243 desiredChanges := new(structs.DesiredUpdates) 244 desiredChanges.Stop = uint64(len(as)) 245 a.result.desiredTGUpdates[group] = desiredChanges 246 } 247 } 248 249 // markStop is a helper for marking a set of allocation for stop with a 250 // particular client status and description. 251 func (a *allocReconciler) markStop(allocs allocSet, clientStatus, statusDescription string) { 252 for _, alloc := range allocs { 253 a.result.stop = append(a.result.stop, allocStopResult{ 254 alloc: alloc, 255 clientStatus: clientStatus, 256 statusDescription: statusDescription, 257 }) 258 } 259 } 260 261 // computeGroup reconciles state for a particular task group. It returns whether 262 // the deployment it is for is complete with regards to the task group. 263 func (a *allocReconciler) computeGroup(group string, all allocSet) bool { 264 // Create the desired update object for the group 265 desiredChanges := new(structs.DesiredUpdates) 266 a.result.desiredTGUpdates[group] = desiredChanges 267 268 // Get the task group. The task group may be nil if the job was updates such 269 // that the task group no longer exists 270 tg := a.job.LookupTaskGroup(group) 271 272 // If the task group is nil, then the task group has been removed so all we 273 // need to do is stop everything 274 if tg == nil { 275 untainted, migrate, lost := all.filterByTainted(a.taintedNodes) 276 a.markStop(untainted, "", allocNotNeeded) 277 a.markStop(migrate, "", allocNotNeeded) 278 a.markStop(lost, structs.AllocClientStatusLost, allocLost) 279 desiredChanges.Stop = uint64(len(untainted) + len(migrate) + len(lost)) 280 return true 281 } 282 283 // Get the deployment state for the group 284 var dstate *structs.DeploymentState 285 existingDeployment := false 286 if a.deployment != nil { 287 dstate, existingDeployment = a.deployment.TaskGroups[group] 288 } 289 if !existingDeployment { 290 autorevert := false 291 if tg.Update != nil && tg.Update.AutoRevert { 292 autorevert = true 293 } 294 dstate = &structs.DeploymentState{ 295 AutoRevert: autorevert, 296 } 297 } 298 299 canaries, all := a.handleGroupCanaries(all, desiredChanges) 300 301 // Determine what set of allocations are on tainted nodes 302 untainted, migrate, lost := all.filterByTainted(a.taintedNodes) 303 304 // Create a structure for choosing names. Seed with the taken names which is 305 // the union of untainted and migrating nodes (includes canaries) 306 nameIndex := newAllocNameIndex(a.jobID, group, tg.Count, untainted.union(migrate)) 307 308 // Stop any unneeded allocations and update the untainted set to not 309 // included stopped allocations. 310 canaryState := dstate != nil && dstate.DesiredCanaries != 0 && !dstate.Promoted 311 stop := a.computeStop(tg, nameIndex, untainted, migrate, lost, canaries, canaryState) 312 desiredChanges.Stop += uint64(len(stop)) 313 untainted = untainted.difference(stop) 314 315 // Having stopped un-needed allocations, append the canaries to the existing 316 // set of untainted because they are promoted. This will cause them to be 317 // treated like non-canaries 318 if !canaryState { 319 untainted = untainted.union(canaries) 320 nameIndex.Set(canaries) 321 } 322 323 // Do inplace upgrades where possible and capture the set of upgrades that 324 // need to be done destructively. 325 ignore, inplace, destructive := a.computeUpdates(tg, untainted) 326 desiredChanges.Ignore += uint64(len(ignore)) 327 desiredChanges.InPlaceUpdate += uint64(len(inplace)) 328 if !existingDeployment { 329 dstate.DesiredTotal += len(destructive) + len(inplace) 330 } 331 332 // The fact that we have destructive updates and have less canaries than is 333 // desired means we need to create canaries 334 numDestructive := len(destructive) 335 strategy := tg.Update 336 canariesPromoted := dstate != nil && dstate.Promoted 337 requireCanary := numDestructive != 0 && strategy != nil && len(canaries) < strategy.Canary && !canariesPromoted 338 if requireCanary && !a.deploymentPaused && !a.deploymentFailed { 339 number := strategy.Canary - len(canaries) 340 number = helper.IntMin(numDestructive, number) 341 desiredChanges.Canary += uint64(number) 342 if !existingDeployment { 343 dstate.DesiredCanaries = strategy.Canary 344 } 345 346 for _, name := range nameIndex.NextCanaries(uint(number), canaries, destructive) { 347 a.result.place = append(a.result.place, allocPlaceResult{ 348 name: name, 349 canary: true, 350 taskGroup: tg, 351 }) 352 } 353 } 354 355 // Determine how many we can place 356 canaryState = dstate != nil && dstate.DesiredCanaries != 0 && !dstate.Promoted 357 limit := a.computeLimit(tg, untainted, destructive, migrate, canaryState) 358 359 // Place if: 360 // * The deployment is not paused or failed 361 // * Not placing any canaries 362 // * If there are any canaries that they have been promoted 363 place := a.computePlacements(tg, nameIndex, untainted, migrate) 364 if !existingDeployment { 365 dstate.DesiredTotal += len(place) 366 } 367 368 if !a.deploymentPaused && !a.deploymentFailed && !canaryState { 369 // Place all new allocations 370 desiredChanges.Place += uint64(len(place)) 371 for _, p := range place { 372 a.result.place = append(a.result.place, p) 373 } 374 375 // Do all destructive updates 376 min := helper.IntMin(len(destructive), limit) 377 limit -= min 378 desiredChanges.DestructiveUpdate += uint64(min) 379 desiredChanges.Ignore += uint64(len(destructive) - min) 380 for _, alloc := range destructive.nameOrder()[:min] { 381 a.result.destructiveUpdate = append(a.result.destructiveUpdate, allocDestructiveResult{ 382 placeName: alloc.Name, 383 placeTaskGroup: tg, 384 stopAlloc: alloc, 385 stopStatusDescription: allocUpdating, 386 }) 387 } 388 } else { 389 desiredChanges.Ignore += uint64(len(destructive)) 390 } 391 392 // Calculate the allowed number of changes and set the desired changes 393 // accordingly. 394 min := helper.IntMin(len(migrate), limit) 395 if !a.deploymentFailed && !a.deploymentPaused { 396 desiredChanges.Migrate += uint64(min) 397 desiredChanges.Ignore += uint64(len(migrate) - min) 398 } else { 399 desiredChanges.Stop += uint64(len(migrate)) 400 } 401 402 followup := false 403 migrated := 0 404 for _, alloc := range migrate.nameOrder() { 405 // If the deployment is failed or paused, don't replace it, just mark as stop. 406 if a.deploymentFailed || a.deploymentPaused { 407 a.result.stop = append(a.result.stop, allocStopResult{ 408 alloc: alloc, 409 statusDescription: allocNodeTainted, 410 }) 411 continue 412 } 413 414 if migrated >= limit { 415 followup = true 416 break 417 } 418 419 migrated++ 420 a.result.stop = append(a.result.stop, allocStopResult{ 421 alloc: alloc, 422 statusDescription: allocMigrating, 423 }) 424 a.result.place = append(a.result.place, allocPlaceResult{ 425 name: alloc.Name, 426 canary: false, 427 taskGroup: tg, 428 previousAlloc: alloc, 429 }) 430 } 431 432 // We need to create a followup evaluation. 433 if followup && strategy != nil && a.result.followupEvalWait < strategy.Stagger { 434 a.result.followupEvalWait = strategy.Stagger 435 } 436 437 // Create a new deployment if necessary 438 if a.deployment == nil && strategy != nil && dstate.DesiredTotal != 0 { 439 a.deployment = structs.NewDeployment(a.job) 440 a.result.deployment = a.deployment 441 a.deployment.TaskGroups[group] = dstate 442 } 443 444 // deploymentComplete is whether the deployment is complete which largely 445 // means that no placements were made or desired to be made 446 deploymentComplete := len(destructive)+len(inplace)+len(place)+len(migrate) == 0 && !requireCanary 447 448 // Final check to see if the deployment is complete is to ensure everything 449 // is healthy 450 if deploymentComplete && a.deployment != nil { 451 partOf, _ := untainted.filterByDeployment(a.deployment.ID) 452 for _, alloc := range partOf { 453 if !alloc.DeploymentStatus.IsHealthy() { 454 deploymentComplete = false 455 break 456 } 457 } 458 } 459 460 return deploymentComplete 461 } 462 463 // handleGroupCanaries handles the canaries for the group by stopping the 464 // unneeded ones and returning the current set of canaries and the updated total 465 // set of allocs for the group 466 func (a *allocReconciler) handleGroupCanaries(all allocSet, desiredChanges *structs.DesiredUpdates) (canaries, newAll allocSet) { 467 // Stop any canary from an older deployment or from a failed one 468 var stop []string 469 470 // Cancel any non-promoted canaries from the older deployment 471 if a.oldDeployment != nil { 472 for _, s := range a.oldDeployment.TaskGroups { 473 if !s.Promoted { 474 stop = append(stop, s.PlacedCanaries...) 475 } 476 } 477 } 478 479 // Cancel any non-promoted canaries from a failed deployment 480 if a.deployment != nil && a.deployment.Status == structs.DeploymentStatusFailed { 481 for _, s := range a.deployment.TaskGroups { 482 if !s.Promoted { 483 stop = append(stop, s.PlacedCanaries...) 484 } 485 } 486 } 487 488 // stopSet is the allocSet that contains the canaries we desire to stop from 489 // above. 490 stopSet := all.fromKeys(stop) 491 a.markStop(stopSet, "", allocNotNeeded) 492 desiredChanges.Stop += uint64(len(stopSet)) 493 all = all.difference(stopSet) 494 495 // Capture our current set of canaries and handle any migrations that are 496 // needed by just stopping them. 497 if a.deployment != nil { 498 var canaryIDs []string 499 for _, s := range a.deployment.TaskGroups { 500 canaryIDs = append(canaryIDs, s.PlacedCanaries...) 501 } 502 503 canaries = all.fromKeys(canaryIDs) 504 untainted, migrate, lost := canaries.filterByTainted(a.taintedNodes) 505 a.markStop(migrate, "", allocMigrating) 506 a.markStop(lost, structs.AllocClientStatusLost, allocLost) 507 508 canaries = untainted 509 all = all.difference(migrate, lost) 510 } 511 512 return canaries, all 513 } 514 515 // computeLimit returns the placement limit for a particular group. The inputs 516 // are the group definition, the untainted, destructive, and migrate allocation 517 // set and whether we are in a canary state. 518 func (a *allocReconciler) computeLimit(group *structs.TaskGroup, untainted, destructive, migrate allocSet, canaryState bool) int { 519 // If there is no update stategy or deployment for the group we can deploy 520 // as many as the group has 521 if group.Update == nil || len(destructive)+len(migrate) == 0 { 522 return group.Count 523 } else if a.deploymentPaused || a.deploymentFailed { 524 // If the deployment is paused or failed, do not create anything else 525 return 0 526 } 527 528 // If we have canaries and they have not been promoted the limit is 0 529 if canaryState { 530 return 0 531 } 532 533 // If we have been promoted or there are no canaries, the limit is the 534 // configured MaxParallel minus any outstanding non-healthy alloc for the 535 // deployment 536 limit := group.Update.MaxParallel 537 if a.deployment != nil { 538 partOf, _ := untainted.filterByDeployment(a.deployment.ID) 539 for _, alloc := range partOf { 540 // An unhealthy allocation means nothing else should be happen. 541 if alloc.DeploymentStatus.IsUnhealthy() { 542 return 0 543 } 544 545 if !alloc.DeploymentStatus.IsHealthy() { 546 limit-- 547 } 548 } 549 } 550 551 // The limit can be less than zero in the case that the job was changed such 552 // that it required destructive changes and the count was scaled up. 553 if limit < 0 { 554 return 0 555 } 556 557 return limit 558 } 559 560 // computePlacement returns the set of allocations to place given the group 561 // definiton, the set of untainted and migrating allocations for the group. 562 func (a *allocReconciler) computePlacements(group *structs.TaskGroup, 563 nameIndex *allocNameIndex, untainted, migrate allocSet) []allocPlaceResult { 564 565 // Hot path the nothing to do case 566 existing := len(untainted) + len(migrate) 567 if existing >= group.Count { 568 return nil 569 } 570 571 var place []allocPlaceResult 572 for _, name := range nameIndex.Next(uint(group.Count - existing)) { 573 place = append(place, allocPlaceResult{ 574 name: name, 575 taskGroup: group, 576 }) 577 } 578 579 return place 580 } 581 582 // computeStop returns the set of allocations that are marked for stopping given 583 // the group definiton, the set of allocations in various states and whether we 584 // are canarying. 585 func (a *allocReconciler) computeStop(group *structs.TaskGroup, nameIndex *allocNameIndex, 586 untainted, migrate, lost, canaries allocSet, canaryState bool) allocSet { 587 588 // Mark all lost allocations for stop. Previous allocation doesn't matter 589 // here since it is on a lost node 590 var stop allocSet 591 stop = stop.union(lost) 592 a.markStop(lost, structs.AllocClientStatusLost, allocLost) 593 594 // If we are still deploying or creating canaries, don't stop them 595 if canaryState { 596 untainted = untainted.difference(canaries) 597 } 598 599 // Hot path the nothing to do case 600 remove := len(untainted) + len(migrate) - group.Count 601 if remove <= 0 { 602 return stop 603 } 604 605 // Prefer stopping any alloc that has the same name as the canaries if we 606 // are promoted 607 if !canaryState && len(canaries) != 0 { 608 canaryNames := canaries.nameSet() 609 for id, alloc := range untainted.difference(canaries) { 610 if _, match := canaryNames[alloc.Name]; match { 611 stop[id] = alloc 612 a.result.stop = append(a.result.stop, allocStopResult{ 613 alloc: alloc, 614 statusDescription: allocNotNeeded, 615 }) 616 delete(untainted, id) 617 618 remove-- 619 if remove == 0 { 620 return stop 621 } 622 } 623 } 624 } 625 626 // Prefer selecting from the migrating set before stopping existing allocs 627 if len(migrate) != 0 { 628 mNames := newAllocNameIndex(a.jobID, group.Name, group.Count, migrate) 629 removeNames := mNames.Highest(uint(remove)) 630 for id, alloc := range migrate { 631 if _, match := removeNames[alloc.Name]; !match { 632 continue 633 } 634 a.result.stop = append(a.result.stop, allocStopResult{ 635 alloc: alloc, 636 statusDescription: allocNotNeeded, 637 }) 638 delete(migrate, id) 639 stop[id] = alloc 640 nameIndex.UnsetIndex(alloc.Index()) 641 642 remove-- 643 if remove == 0 { 644 return stop 645 } 646 } 647 } 648 649 // Select the allocs with the highest count to remove 650 removeNames := nameIndex.Highest(uint(remove)) 651 for id, alloc := range untainted { 652 if _, remove := removeNames[alloc.Name]; remove { 653 stop[id] = alloc 654 a.result.stop = append(a.result.stop, allocStopResult{ 655 alloc: alloc, 656 statusDescription: allocNotNeeded, 657 }) 658 } 659 } 660 661 return stop 662 } 663 664 // computeUpdates determines which allocations for the passed group require 665 // updates. Three groups are returned: 666 // 1. Those that require no upgrades 667 // 2. Those that can be upgraded in-place. These are added to the results 668 // automatically since the function contains the correct state to do so, 669 // 3. Those that require destructive updates 670 func (a *allocReconciler) computeUpdates(group *structs.TaskGroup, untainted allocSet) (ignore, inplace, destructive allocSet) { 671 // Determine the set of allocations that need to be updated 672 ignore = make(map[string]*structs.Allocation) 673 inplace = make(map[string]*structs.Allocation) 674 destructive = make(map[string]*structs.Allocation) 675 676 for _, alloc := range untainted { 677 ignoreChange, destructiveChange, inplaceAlloc := a.allocUpdateFn(alloc, a.job, group) 678 if ignoreChange { 679 ignore[alloc.ID] = alloc 680 } else if destructiveChange { 681 destructive[alloc.ID] = alloc 682 } else { 683 // Attach the deployment ID and and clear the health if the 684 // deployment has changed 685 inplace[alloc.ID] = alloc 686 a.result.inplaceUpdate = append(a.result.inplaceUpdate, inplaceAlloc) 687 } 688 } 689 690 return 691 }