github.com/Ilhicas/nomad@v1.0.4-0.20210304152020-e86851182bc3/nomad/deploymentwatcher/deployment_watcher.go (about) 1 package deploymentwatcher 2 3 import ( 4 "context" 5 "fmt" 6 "sync" 7 "time" 8 9 log "github.com/hashicorp/go-hclog" 10 memdb "github.com/hashicorp/go-memdb" 11 "github.com/hashicorp/nomad/helper" 12 "github.com/hashicorp/nomad/helper/uuid" 13 "github.com/hashicorp/nomad/nomad/state" 14 "github.com/hashicorp/nomad/nomad/structs" 15 "golang.org/x/time/rate" 16 ) 17 18 const ( 19 // perJobEvalBatchPeriod is the batching length before creating an evaluation to 20 // trigger the scheduler when allocations are marked as healthy. 21 perJobEvalBatchPeriod = 1 * time.Second 22 ) 23 24 var ( 25 // allowRescheduleTransition is the transition that allows failed 26 // allocations part of a deployment to be rescheduled. We create a one off 27 // variable to avoid creating a new object for every request. 28 allowRescheduleTransition = &structs.DesiredTransition{ 29 Reschedule: helper.BoolToPtr(true), 30 } 31 ) 32 33 // deploymentTriggers are the set of functions required to trigger changes on 34 // behalf of a deployment 35 type deploymentTriggers interface { 36 // createUpdate is used to create allocation desired transition updates and 37 // an evaluation. 38 createUpdate(allocs map[string]*structs.DesiredTransition, eval *structs.Evaluation) (uint64, error) 39 40 // upsertJob is used to roll back a job when autoreverting for a deployment 41 upsertJob(job *structs.Job) (uint64, error) 42 43 // upsertDeploymentStatusUpdate is used to upsert a deployment status update 44 // and an optional evaluation and job to upsert 45 upsertDeploymentStatusUpdate(u *structs.DeploymentStatusUpdate, eval *structs.Evaluation, job *structs.Job) (uint64, error) 46 47 // upsertDeploymentPromotion is used to promote canaries in a deployment 48 upsertDeploymentPromotion(req *structs.ApplyDeploymentPromoteRequest) (uint64, error) 49 50 // upsertDeploymentAllocHealth is used to set the health of allocations in a 51 // deployment 52 upsertDeploymentAllocHealth(req *structs.ApplyDeploymentAllocHealthRequest) (uint64, error) 53 } 54 55 // deploymentWatcher is used to watch a single deployment and trigger the 56 // scheduler when allocation health transitions. 57 type deploymentWatcher struct { 58 // queryLimiter is used to limit the rate of blocking queries 59 queryLimiter *rate.Limiter 60 61 // deploymentTriggers holds the methods required to trigger changes on behalf of the 62 // deployment 63 deploymentTriggers 64 65 // DeploymentRPC holds methods for interacting with peer regions 66 // in enterprise edition 67 DeploymentRPC 68 69 // JobRPC holds methods for interacting with peer regions 70 // in enterprise edition 71 JobRPC 72 73 // state is the state that is watched for state changes. 74 state *state.StateStore 75 76 // deploymentID is the deployment's ID being watched 77 deploymentID string 78 79 // deploymentUpdateCh is triggered when there is an updated deployment 80 deploymentUpdateCh chan struct{} 81 82 // d is the deployment being watched 83 d *structs.Deployment 84 85 // j is the job the deployment is for 86 j *structs.Job 87 88 // outstandingBatch marks whether an outstanding function exists to create 89 // the evaluation. Access should be done through the lock. 90 outstandingBatch bool 91 92 // outstandingAllowReplacements is the map of allocations that will be 93 // marked as allowing a replacement. Access should be done through the lock. 94 outstandingAllowReplacements map[string]*structs.DesiredTransition 95 96 // latestEval is the latest eval for the job. It is updated by the watch 97 // loop and any time an evaluation is created. The field should be accessed 98 // by holding the lock or using the setter and getter methods. 99 latestEval uint64 100 101 logger log.Logger 102 ctx context.Context 103 exitFn context.CancelFunc 104 l sync.RWMutex 105 } 106 107 // newDeploymentWatcher returns a deployment watcher that is used to watch 108 // deployments and trigger the scheduler as needed. 109 func newDeploymentWatcher(parent context.Context, queryLimiter *rate.Limiter, 110 logger log.Logger, state *state.StateStore, d *structs.Deployment, 111 j *structs.Job, triggers deploymentTriggers, 112 deploymentRPC DeploymentRPC, jobRPC JobRPC) *deploymentWatcher { 113 114 ctx, exitFn := context.WithCancel(parent) 115 w := &deploymentWatcher{ 116 queryLimiter: queryLimiter, 117 deploymentID: d.ID, 118 deploymentUpdateCh: make(chan struct{}, 1), 119 d: d, 120 j: j, 121 state: state, 122 deploymentTriggers: triggers, 123 DeploymentRPC: deploymentRPC, 124 JobRPC: jobRPC, 125 logger: logger.With("deployment_id", d.ID, "job", j.NamespacedID()), 126 ctx: ctx, 127 exitFn: exitFn, 128 } 129 130 // Start the long lived watcher that scans for allocation updates 131 go w.watch() 132 133 return w 134 } 135 136 // updateDeployment is used to update the tracked deployment. 137 func (w *deploymentWatcher) updateDeployment(d *structs.Deployment) { 138 w.l.Lock() 139 defer w.l.Unlock() 140 141 // Update and trigger 142 w.d = d 143 select { 144 case w.deploymentUpdateCh <- struct{}{}: 145 default: 146 } 147 } 148 149 // getDeployment returns the tracked deployment. 150 func (w *deploymentWatcher) getDeployment() *structs.Deployment { 151 w.l.RLock() 152 defer w.l.RUnlock() 153 return w.d 154 } 155 156 func (w *deploymentWatcher) SetAllocHealth( 157 req *structs.DeploymentAllocHealthRequest, 158 resp *structs.DeploymentUpdateResponse) error { 159 160 // If we are failing the deployment, update the status and potentially 161 // rollback 162 var j *structs.Job 163 var u *structs.DeploymentStatusUpdate 164 165 // If there are unhealthy allocations we need to mark the deployment as 166 // failed and check if we should roll back to a stable job. 167 if l := len(req.UnhealthyAllocationIDs); l != 0 { 168 unhealthy := make(map[string]struct{}, l) 169 for _, alloc := range req.UnhealthyAllocationIDs { 170 unhealthy[alloc] = struct{}{} 171 } 172 173 // Get the allocations for the deployment 174 snap, err := w.state.Snapshot() 175 if err != nil { 176 return err 177 } 178 179 allocs, err := snap.AllocsByDeployment(nil, req.DeploymentID) 180 if err != nil { 181 return err 182 } 183 184 // Determine if we should autorevert to an older job 185 desc := structs.DeploymentStatusDescriptionFailedAllocations 186 for _, alloc := range allocs { 187 // Check that the alloc has been marked unhealthy 188 if _, ok := unhealthy[alloc.ID]; !ok { 189 continue 190 } 191 192 // Check if the group has autorevert set 193 dstate, ok := w.getDeployment().TaskGroups[alloc.TaskGroup] 194 if !ok || !dstate.AutoRevert { 195 continue 196 } 197 198 var err error 199 j, err = w.latestStableJob() 200 if err != nil { 201 return err 202 } 203 204 if j != nil { 205 j, desc = w.handleRollbackValidity(j, desc) 206 } 207 break 208 } 209 210 u = w.getDeploymentStatusUpdate(structs.DeploymentStatusFailed, desc) 211 } 212 213 // Canonicalize the job in case it doesn't have namespace set 214 j.Canonicalize() 215 216 // Create the request 217 areq := &structs.ApplyDeploymentAllocHealthRequest{ 218 DeploymentAllocHealthRequest: *req, 219 Timestamp: time.Now(), 220 Eval: w.getEval(), 221 DeploymentUpdate: u, 222 Job: j, 223 } 224 225 index, err := w.upsertDeploymentAllocHealth(areq) 226 if err != nil { 227 return err 228 } 229 230 // Build the response 231 resp.EvalID = areq.Eval.ID 232 resp.EvalCreateIndex = index 233 resp.DeploymentModifyIndex = index 234 resp.Index = index 235 if j != nil { 236 resp.RevertedJobVersion = helper.Uint64ToPtr(j.Version) 237 } 238 return nil 239 } 240 241 // handleRollbackValidity checks if the job being rolled back to has the same spec as the existing job 242 // Returns a modified description and job accordingly. 243 func (w *deploymentWatcher) handleRollbackValidity(rollbackJob *structs.Job, desc string) (*structs.Job, string) { 244 // Only rollback if job being changed has a different spec. 245 // This prevents an infinite revert cycle when a previously stable version of the job fails to start up during a rollback 246 // If the job we are trying to rollback to is identical to the current job, we stop because the rollback will not succeed. 247 if w.j.SpecChanged(rollbackJob) { 248 desc = structs.DeploymentStatusDescriptionRollback(desc, rollbackJob.Version) 249 } else { 250 desc = structs.DeploymentStatusDescriptionRollbackNoop(desc, rollbackJob.Version) 251 rollbackJob = nil 252 } 253 return rollbackJob, desc 254 } 255 256 func (w *deploymentWatcher) PromoteDeployment( 257 req *structs.DeploymentPromoteRequest, 258 resp *structs.DeploymentUpdateResponse) error { 259 260 // Create the request 261 areq := &structs.ApplyDeploymentPromoteRequest{ 262 DeploymentPromoteRequest: *req, 263 Eval: w.getEval(), 264 } 265 266 index, err := w.upsertDeploymentPromotion(areq) 267 if err != nil { 268 return err 269 } 270 271 // Build the response 272 resp.EvalID = areq.Eval.ID 273 resp.EvalCreateIndex = index 274 resp.DeploymentModifyIndex = index 275 resp.Index = index 276 return nil 277 } 278 279 // autoPromoteDeployment creates a synthetic promotion request, and upserts it for processing 280 func (w *deploymentWatcher) autoPromoteDeployment(allocs []*structs.AllocListStub) error { 281 d := w.getDeployment() 282 if !d.HasPlacedCanaries() || !d.RequiresPromotion() { 283 return nil 284 } 285 286 // AutoPromote iff every task group is marked auto_promote and is healthy. The whole 287 // job version has been incremented, so we promote together. See also AutoRevert 288 for _, dstate := range d.TaskGroups { 289 if !dstate.AutoPromote || dstate.DesiredCanaries != len(dstate.PlacedCanaries) { 290 return nil 291 } 292 293 // Find the health status of each canary 294 for _, c := range dstate.PlacedCanaries { 295 for _, a := range allocs { 296 if c == a.ID && !a.DeploymentStatus.IsHealthy() { 297 return nil 298 } 299 } 300 } 301 } 302 303 // Send the request 304 _, err := w.upsertDeploymentPromotion(&structs.ApplyDeploymentPromoteRequest{ 305 DeploymentPromoteRequest: structs.DeploymentPromoteRequest{DeploymentID: d.GetID(), All: true}, 306 Eval: w.getEval(), 307 }) 308 return err 309 } 310 311 func (w *deploymentWatcher) PauseDeployment( 312 req *structs.DeploymentPauseRequest, 313 resp *structs.DeploymentUpdateResponse) error { 314 // Determine the status we should transition to and if we need to create an 315 // evaluation 316 status, desc := structs.DeploymentStatusPaused, structs.DeploymentStatusDescriptionPaused 317 var eval *structs.Evaluation 318 evalID := "" 319 if !req.Pause { 320 status, desc = structs.DeploymentStatusRunning, structs.DeploymentStatusDescriptionRunning 321 eval = w.getEval() 322 evalID = eval.ID 323 } 324 update := w.getDeploymentStatusUpdate(status, desc) 325 326 // Commit the change 327 i, err := w.upsertDeploymentStatusUpdate(update, eval, nil) 328 if err != nil { 329 return err 330 } 331 332 // Build the response 333 if evalID != "" { 334 resp.EvalID = evalID 335 resp.EvalCreateIndex = i 336 } 337 resp.DeploymentModifyIndex = i 338 resp.Index = i 339 return nil 340 } 341 342 func (w *deploymentWatcher) FailDeployment( 343 req *structs.DeploymentFailRequest, 344 resp *structs.DeploymentUpdateResponse) error { 345 346 status, desc := structs.DeploymentStatusFailed, structs.DeploymentStatusDescriptionFailedByUser 347 348 // Determine if we should rollback 349 rollback := false 350 for _, dstate := range w.getDeployment().TaskGroups { 351 if dstate.AutoRevert { 352 rollback = true 353 break 354 } 355 } 356 357 var rollbackJob *structs.Job 358 if rollback { 359 var err error 360 rollbackJob, err = w.latestStableJob() 361 if err != nil { 362 return err 363 } 364 365 if rollbackJob != nil { 366 rollbackJob, desc = w.handleRollbackValidity(rollbackJob, desc) 367 } else { 368 desc = structs.DeploymentStatusDescriptionNoRollbackTarget(desc) 369 } 370 } 371 372 // Commit the change 373 update := w.getDeploymentStatusUpdate(status, desc) 374 eval := w.getEval() 375 i, err := w.upsertDeploymentStatusUpdate(update, eval, rollbackJob) 376 if err != nil { 377 return err 378 } 379 380 // Build the response 381 resp.EvalID = eval.ID 382 resp.EvalCreateIndex = i 383 resp.DeploymentModifyIndex = i 384 resp.Index = i 385 if rollbackJob != nil { 386 resp.RevertedJobVersion = helper.Uint64ToPtr(rollbackJob.Version) 387 } 388 return nil 389 } 390 391 // StopWatch stops watching the deployment. This should be called whenever a 392 // deployment is completed or the watcher is no longer needed. 393 func (w *deploymentWatcher) StopWatch() { 394 w.exitFn() 395 } 396 397 // watch is the long running watcher that watches for both allocation and 398 // deployment changes. Its function is to create evaluations to trigger the 399 // scheduler when more progress can be made, to fail the deployment if it has 400 // failed and potentially rolling back the job. Progress can be made when an 401 // allocation transitions to healthy, so we create an eval. 402 func (w *deploymentWatcher) watch() { 403 // Get the deadline. This is likely a zero time to begin with but we need to 404 // handle the case that the deployment has already progressed and we are now 405 // just starting to watch it. This must likely would occur if there was a 406 // leader transition and we are now starting our watcher. 407 currentDeadline := w.getDeploymentProgressCutoff(w.getDeployment()) 408 var deadlineTimer *time.Timer 409 if currentDeadline.IsZero() { 410 deadlineTimer = time.NewTimer(0) 411 if !deadlineTimer.Stop() { 412 <-deadlineTimer.C 413 } 414 } else { 415 deadlineTimer = time.NewTimer(time.Until(currentDeadline)) 416 } 417 418 allocIndex := uint64(1) 419 var updates *allocUpdates 420 421 rollback, deadlineHit := false, false 422 423 FAIL: 424 for { 425 select { 426 case <-w.ctx.Done(): 427 // This is the successful case, and we stop the loop 428 return 429 case <-deadlineTimer.C: 430 // We have hit the progress deadline, so fail the deployment 431 // unless we're waiting for manual promotion. We need to determine 432 // whether we should roll back the job by inspecting which allocs 433 // as part of the deployment are healthy and which aren't. The 434 // deadlineHit flag is never reset, so even in the case of a 435 // manual promotion, we'll describe any failure as a progress 436 // deadline failure at this point. 437 deadlineHit = true 438 fail, rback, err := w.shouldFail() 439 if err != nil { 440 w.logger.Error("failed to determine whether to rollback job", "error", err) 441 } 442 if !fail { 443 w.logger.Debug("skipping deadline") 444 continue 445 } 446 447 w.logger.Debug("deadline hit", "rollback", rback) 448 rollback = rback 449 err = w.nextRegion(structs.DeploymentStatusFailed) 450 if err != nil { 451 w.logger.Error("multiregion deployment error", "error", err) 452 } 453 break FAIL 454 case <-w.deploymentUpdateCh: 455 // Get the updated deployment and check if we should change the 456 // deadline timer 457 next := w.getDeploymentProgressCutoff(w.getDeployment()) 458 if !next.Equal(currentDeadline) { 459 prevDeadlineZero := currentDeadline.IsZero() 460 currentDeadline = next 461 // The most recent deadline can be zero if no allocs were created for this deployment. 462 // The deadline timer would have already been stopped once in that case. To prevent 463 // deadlocking on the already stopped deadline timer, we only drain the channel if 464 // the previous deadline was not zero. 465 if !prevDeadlineZero && !deadlineTimer.Stop() { 466 select { 467 case <-deadlineTimer.C: 468 default: 469 } 470 } 471 472 // If the next deadline is zero, we should not reset the timer 473 // as we aren't tracking towards a progress deadline yet. This 474 // can happen if you have multiple task groups with progress 475 // deadlines and one of the task groups hasn't made any 476 // placements. As soon as the other task group finishes its 477 // rollout, the next progress deadline becomes zero, so we want 478 // to avoid resetting, causing a deployment failure. 479 if !next.IsZero() { 480 deadlineTimer.Reset(time.Until(next)) 481 w.logger.Trace("resetting deadline") 482 } 483 } 484 485 err := w.nextRegion(w.getStatus()) 486 if err != nil { 487 break FAIL 488 } 489 490 case updates = <-w.getAllocsCh(allocIndex): 491 if err := updates.err; err != nil { 492 if err == context.Canceled || w.ctx.Err() == context.Canceled { 493 return 494 } 495 496 w.logger.Error("failed to retrieve allocations", "error", err) 497 return 498 } 499 allocIndex = updates.index 500 501 // We have allocation changes for this deployment so determine the 502 // steps to take. 503 res, err := w.handleAllocUpdate(updates.allocs) 504 if err != nil { 505 if err == context.Canceled || w.ctx.Err() == context.Canceled { 506 return 507 } 508 509 w.logger.Error("failed handling allocation updates", "error", err) 510 return 511 } 512 513 // The deployment has failed, so break out of the watch loop and 514 // handle the failure 515 if res.failDeployment { 516 rollback = res.rollback 517 err := w.nextRegion(structs.DeploymentStatusFailed) 518 if err != nil { 519 w.logger.Error("multiregion deployment error", "error", err) 520 } 521 break FAIL 522 } 523 524 // If permitted, automatically promote this canary deployment 525 err = w.autoPromoteDeployment(updates.allocs) 526 if err != nil { 527 w.logger.Error("failed to auto promote deployment", "error", err) 528 } 529 530 // Create an eval to push the deployment along 531 if res.createEval || len(res.allowReplacements) != 0 { 532 w.createBatchedUpdate(res.allowReplacements, allocIndex) 533 } 534 } 535 } 536 537 // Change the deployments status to failed 538 desc := structs.DeploymentStatusDescriptionFailedAllocations 539 if deadlineHit { 540 desc = structs.DeploymentStatusDescriptionProgressDeadline 541 } 542 543 // Rollback to the old job if necessary 544 var j *structs.Job 545 if rollback { 546 var err error 547 j, err = w.latestStableJob() 548 if err != nil { 549 w.logger.Error("failed to lookup latest stable job", "error", err) 550 } 551 552 // Description should include that the job is being rolled back to 553 // version N 554 if j != nil { 555 j, desc = w.handleRollbackValidity(j, desc) 556 } else { 557 desc = structs.DeploymentStatusDescriptionNoRollbackTarget(desc) 558 } 559 } 560 561 // Update the status of the deployment to failed and create an evaluation. 562 e := w.getEval() 563 u := w.getDeploymentStatusUpdate(structs.DeploymentStatusFailed, desc) 564 if _, err := w.upsertDeploymentStatusUpdate(u, e, j); err != nil { 565 w.logger.Error("failed to update deployment status", "error", err) 566 } 567 } 568 569 // allocUpdateResult is used to return the desired actions given the newest set 570 // of allocations for the deployment. 571 type allocUpdateResult struct { 572 createEval bool 573 failDeployment bool 574 rollback bool 575 allowReplacements []string 576 } 577 578 // handleAllocUpdate is used to compute the set of actions to take based on the 579 // updated allocations for the deployment. 580 func (w *deploymentWatcher) handleAllocUpdate(allocs []*structs.AllocListStub) (allocUpdateResult, error) { 581 var res allocUpdateResult 582 583 // Get the latest evaluation index 584 latestEval, err := w.jobEvalStatus() 585 if err != nil { 586 if err == context.Canceled || w.ctx.Err() == context.Canceled { 587 return res, err 588 } 589 590 return res, fmt.Errorf("failed to determine last evaluation index for job %q: %v", w.j.ID, err) 591 } 592 593 deployment := w.getDeployment() 594 for _, alloc := range allocs { 595 dstate, ok := deployment.TaskGroups[alloc.TaskGroup] 596 if !ok { 597 continue 598 } 599 600 // Determine if the update stanza for this group is progress based 601 progressBased := dstate.ProgressDeadline != 0 602 603 // Check if the allocation has failed and we need to mark it for allow 604 // replacements 605 if progressBased && alloc.DeploymentStatus.IsUnhealthy() && 606 deployment.Active() && !alloc.DesiredTransition.ShouldReschedule() { 607 res.allowReplacements = append(res.allowReplacements, alloc.ID) 608 continue 609 } 610 611 // We need to create an eval so the job can progress. 612 if alloc.DeploymentStatus.IsHealthy() && alloc.DeploymentStatus.ModifyIndex > latestEval { 613 res.createEval = true 614 } 615 616 // If the group is using a progress deadline, we don't have to do anything. 617 if progressBased { 618 continue 619 } 620 621 // Fail on the first bad allocation 622 if alloc.DeploymentStatus.IsUnhealthy() { 623 // Check if the group has autorevert set 624 if dstate.AutoRevert { 625 res.rollback = true 626 } 627 628 // Since we have an unhealthy allocation, fail the deployment 629 res.failDeployment = true 630 } 631 632 // All conditions have been hit so we can break 633 if res.createEval && res.failDeployment && res.rollback { 634 break 635 } 636 } 637 638 return res, nil 639 } 640 641 // shouldFail returns whether the job should be failed and whether it should 642 // rolled back to an earlier stable version by examining the allocations in the 643 // deployment. 644 func (w *deploymentWatcher) shouldFail() (fail, rollback bool, err error) { 645 snap, err := w.state.Snapshot() 646 if err != nil { 647 return false, false, err 648 } 649 650 d, err := snap.DeploymentByID(nil, w.deploymentID) 651 if err != nil { 652 return false, false, err 653 } 654 if d == nil { 655 // The deployment wasn't in the state store, possibly due to a system gc 656 return false, false, fmt.Errorf("deployment id not found: %q", w.deploymentID) 657 } 658 659 fail = false 660 for tg, dstate := range d.TaskGroups { 661 // If we are in a canary state we fail if there aren't enough healthy 662 // allocs to satisfy DesiredCanaries 663 if dstate.DesiredCanaries > 0 && !dstate.Promoted { 664 if dstate.HealthyAllocs >= dstate.DesiredCanaries { 665 continue 666 } 667 } else if dstate.HealthyAllocs >= dstate.DesiredTotal { 668 continue 669 } 670 671 // We have failed this TG 672 fail = true 673 674 // We don't need to autorevert this group 675 upd := w.j.LookupTaskGroup(tg).Update 676 if upd == nil || !upd.AutoRevert { 677 continue 678 } 679 680 // Unhealthy allocs and we need to autorevert 681 return fail, true, nil 682 } 683 684 return fail, false, nil 685 } 686 687 // getDeploymentProgressCutoff returns the progress cutoff for the given 688 // deployment 689 func (w *deploymentWatcher) getDeploymentProgressCutoff(d *structs.Deployment) time.Time { 690 var next time.Time 691 doneTGs := w.doneGroups(d) 692 for name, dstate := range d.TaskGroups { 693 // This task group is done so we don't have to concern ourselves with 694 // its progress deadline. 695 if done, ok := doneTGs[name]; ok && done { 696 continue 697 } 698 699 if dstate.RequireProgressBy.IsZero() { 700 continue 701 } 702 703 if next.IsZero() || dstate.RequireProgressBy.Before(next) { 704 next = dstate.RequireProgressBy 705 } 706 } 707 return next 708 } 709 710 // doneGroups returns a map of task group to whether the deployment appears to 711 // be done for the group. A true value doesn't mean no more action will be taken 712 // in the life time of the deployment because there could always be node 713 // failures, or rescheduling events. 714 func (w *deploymentWatcher) doneGroups(d *structs.Deployment) map[string]bool { 715 if d == nil { 716 return nil 717 } 718 719 // Collect the allocations by the task group 720 snap, err := w.state.Snapshot() 721 if err != nil { 722 return nil 723 } 724 725 allocs, err := snap.AllocsByDeployment(nil, d.ID) 726 if err != nil { 727 return nil 728 } 729 730 // Go through the allocs and count up how many healthy allocs we have 731 healthy := make(map[string]int, len(d.TaskGroups)) 732 for _, a := range allocs { 733 if a.TerminalStatus() || !a.DeploymentStatus.IsHealthy() { 734 continue 735 } 736 healthy[a.TaskGroup]++ 737 } 738 739 // Go through each group and check if it done 740 groups := make(map[string]bool, len(d.TaskGroups)) 741 for name, dstate := range d.TaskGroups { 742 // Requires promotion 743 if dstate.DesiredCanaries != 0 && !dstate.Promoted { 744 groups[name] = false 745 continue 746 } 747 748 // Check we have enough healthy currently running allocations 749 groups[name] = healthy[name] >= dstate.DesiredTotal 750 } 751 752 return groups 753 } 754 755 // latestStableJob returns the latest stable job. It may be nil if none exist 756 func (w *deploymentWatcher) latestStableJob() (*structs.Job, error) { 757 snap, err := w.state.Snapshot() 758 if err != nil { 759 return nil, err 760 } 761 762 versions, err := snap.JobVersionsByID(nil, w.j.Namespace, w.j.ID) 763 if err != nil { 764 return nil, err 765 } 766 767 var stable *structs.Job 768 for _, job := range versions { 769 if job.Stable { 770 stable = job 771 break 772 } 773 } 774 775 return stable, nil 776 } 777 778 // createBatchedUpdate creates an eval for the given index as well as updating 779 // the given allocations to allow them to reschedule. 780 func (w *deploymentWatcher) createBatchedUpdate(allowReplacements []string, forIndex uint64) { 781 w.l.Lock() 782 defer w.l.Unlock() 783 784 // Store the allocations that can be replaced 785 for _, allocID := range allowReplacements { 786 if w.outstandingAllowReplacements == nil { 787 w.outstandingAllowReplacements = make(map[string]*structs.DesiredTransition, len(allowReplacements)) 788 } 789 w.outstandingAllowReplacements[allocID] = allowRescheduleTransition 790 } 791 792 if w.outstandingBatch || (forIndex < w.latestEval && len(allowReplacements) == 0) { 793 return 794 } 795 796 w.outstandingBatch = true 797 798 time.AfterFunc(perJobEvalBatchPeriod, func() { 799 // If the timer has been created and then we shutdown, we need to no-op 800 // the evaluation creation. 801 select { 802 case <-w.ctx.Done(): 803 return 804 default: 805 } 806 807 w.l.Lock() 808 replacements := w.outstandingAllowReplacements 809 w.outstandingAllowReplacements = nil 810 w.outstandingBatch = false 811 w.l.Unlock() 812 813 // Create the eval 814 if _, err := w.createUpdate(replacements, w.getEval()); err != nil { 815 w.logger.Error("failed to create evaluation for deployment", "deployment_id", w.deploymentID, "error", err) 816 } 817 }) 818 } 819 820 // getEval returns an evaluation suitable for the deployment 821 func (w *deploymentWatcher) getEval() *structs.Evaluation { 822 now := time.Now().UTC().UnixNano() 823 return &structs.Evaluation{ 824 ID: uuid.Generate(), 825 Namespace: w.j.Namespace, 826 Priority: w.j.Priority, 827 Type: w.j.Type, 828 TriggeredBy: structs.EvalTriggerDeploymentWatcher, 829 JobID: w.j.ID, 830 DeploymentID: w.deploymentID, 831 Status: structs.EvalStatusPending, 832 CreateTime: now, 833 ModifyTime: now, 834 } 835 } 836 837 // getDeploymentStatusUpdate returns a deployment status update 838 func (w *deploymentWatcher) getDeploymentStatusUpdate(status, desc string) *structs.DeploymentStatusUpdate { 839 return &structs.DeploymentStatusUpdate{ 840 DeploymentID: w.deploymentID, 841 Status: status, 842 StatusDescription: desc, 843 } 844 } 845 846 // getStatus returns the current status of the deployment 847 func (w *deploymentWatcher) getStatus() string { 848 w.l.RLock() 849 defer w.l.RUnlock() 850 return w.d.Status 851 } 852 853 type allocUpdates struct { 854 allocs []*structs.AllocListStub 855 index uint64 856 err error 857 } 858 859 // getAllocsCh creates a channel and starts a goroutine that 860 // 1. parks a blocking query for allocations on the state 861 // 2. reads those and drops them on the channel 862 // This query runs once here, but watch calls it in a loop 863 func (w *deploymentWatcher) getAllocsCh(index uint64) <-chan *allocUpdates { 864 out := make(chan *allocUpdates, 1) 865 go func() { 866 allocs, index, err := w.getAllocs(index) 867 out <- &allocUpdates{ 868 allocs: allocs, 869 index: index, 870 err: err, 871 } 872 }() 873 874 return out 875 } 876 877 // getAllocs retrieves the allocations that are part of the deployment blocking 878 // at the given index. 879 func (w *deploymentWatcher) getAllocs(index uint64) ([]*structs.AllocListStub, uint64, error) { 880 resp, index, err := w.state.BlockingQuery(w.getAllocsImpl, index, w.ctx) 881 if err != nil { 882 return nil, 0, err 883 } 884 if err := w.ctx.Err(); err != nil { 885 return nil, 0, err 886 } 887 888 return resp.([]*structs.AllocListStub), index, nil 889 } 890 891 // getDeploysImpl retrieves all deployments from the passed state store. 892 func (w *deploymentWatcher) getAllocsImpl(ws memdb.WatchSet, state *state.StateStore) (interface{}, uint64, error) { 893 if err := w.queryLimiter.Wait(w.ctx); err != nil { 894 return nil, 0, err 895 } 896 897 // Capture all the allocations 898 allocs, err := state.AllocsByDeployment(ws, w.deploymentID) 899 if err != nil { 900 return nil, 0, err 901 } 902 903 maxIndex := uint64(0) 904 stubs := make([]*structs.AllocListStub, 0, len(allocs)) 905 for _, alloc := range allocs { 906 stubs = append(stubs, alloc.Stub(nil)) 907 908 if maxIndex < alloc.ModifyIndex { 909 maxIndex = alloc.ModifyIndex 910 } 911 } 912 913 // Use the last index that affected the allocs table 914 if len(stubs) == 0 { 915 index, err := state.Index("allocs") 916 if err != nil { 917 return nil, index, err 918 } 919 maxIndex = index 920 } 921 922 return stubs, maxIndex, nil 923 } 924 925 // jobEvalStatus returns the latest eval index for a job. The index is used to 926 // determine if an allocation update requires an evaluation to be triggered. 927 func (w *deploymentWatcher) jobEvalStatus() (latestIndex uint64, err error) { 928 if err := w.queryLimiter.Wait(w.ctx); err != nil { 929 return 0, err 930 } 931 932 snap, err := w.state.Snapshot() 933 if err != nil { 934 return 0, err 935 } 936 937 evals, err := snap.EvalsByJob(nil, w.j.Namespace, w.j.ID) 938 if err != nil { 939 return 0, err 940 } 941 942 // If there are no evals for the job, return zero, since we want any 943 // allocation change to trigger an evaluation. 944 if len(evals) == 0 { 945 return 0, nil 946 } 947 948 var max uint64 949 for _, eval := range evals { 950 // A cancelled eval never impacts what the scheduler has saw, so do not 951 // use it's indexes. 952 if eval.Status == structs.EvalStatusCancelled { 953 continue 954 } 955 956 // Prefer using the snapshot index. Otherwise use the create index 957 if eval.SnapshotIndex != 0 && max < eval.SnapshotIndex { 958 max = eval.SnapshotIndex 959 } else if max < eval.CreateIndex { 960 max = eval.CreateIndex 961 } 962 } 963 964 return max, nil 965 }