github.com/djenriquez/nomad-1@v0.8.1/nomad/periodic.go (about) 1 package nomad 2 3 import ( 4 "container/heap" 5 "context" 6 "fmt" 7 "log" 8 "strconv" 9 "strings" 10 "sync" 11 "time" 12 13 memdb "github.com/hashicorp/go-memdb" 14 "github.com/hashicorp/nomad/helper/uuid" 15 "github.com/hashicorp/nomad/nomad/structs" 16 ) 17 18 // PeriodicDispatch is used to track and launch periodic jobs. It maintains the 19 // set of periodic jobs and creates derived jobs and evaluations per 20 // instantiation which is determined by the periodic spec. 21 type PeriodicDispatch struct { 22 dispatcher JobEvalDispatcher 23 enabled bool 24 25 tracked map[structs.NamespacedID]*structs.Job 26 heap *periodicHeap 27 28 updateCh chan struct{} 29 stopFn context.CancelFunc 30 logger *log.Logger 31 l sync.RWMutex 32 } 33 34 // JobEvalDispatcher is an interface to submit jobs and have evaluations created 35 // for them. 36 type JobEvalDispatcher interface { 37 // DispatchJob takes a job a new, untracked job and creates an evaluation 38 // for it and returns the eval. 39 DispatchJob(job *structs.Job) (*structs.Evaluation, error) 40 41 // RunningChildren returns whether the passed job has any running children. 42 RunningChildren(job *structs.Job) (bool, error) 43 } 44 45 // DispatchJob creates an evaluation for the passed job and commits both the 46 // evaluation and the job to the raft log. It returns the eval. 47 func (s *Server) DispatchJob(job *structs.Job) (*structs.Evaluation, error) { 48 // Commit this update via Raft 49 job.SetSubmitTime() 50 req := structs.JobRegisterRequest{ 51 Job: job, 52 WriteRequest: structs.WriteRequest{ 53 Namespace: job.Namespace, 54 }, 55 } 56 fsmErr, index, err := s.raftApply(structs.JobRegisterRequestType, req) 57 if err, ok := fsmErr.(error); ok && err != nil { 58 return nil, err 59 } 60 if err != nil { 61 return nil, err 62 } 63 64 // Create a new evaluation 65 eval := &structs.Evaluation{ 66 ID: uuid.Generate(), 67 Namespace: job.Namespace, 68 Priority: job.Priority, 69 Type: job.Type, 70 TriggeredBy: structs.EvalTriggerPeriodicJob, 71 JobID: job.ID, 72 JobModifyIndex: index, 73 Status: structs.EvalStatusPending, 74 } 75 update := &structs.EvalUpdateRequest{ 76 Evals: []*structs.Evaluation{eval}, 77 } 78 79 // Commit this evaluation via Raft 80 // XXX: There is a risk of partial failure where the JobRegister succeeds 81 // but that the EvalUpdate does not. 82 _, evalIndex, err := s.raftApply(structs.EvalUpdateRequestType, update) 83 if err != nil { 84 return nil, err 85 } 86 87 // Update its indexes. 88 eval.CreateIndex = evalIndex 89 eval.ModifyIndex = evalIndex 90 return eval, nil 91 } 92 93 // RunningChildren checks whether the passed job has any running children. 94 func (s *Server) RunningChildren(job *structs.Job) (bool, error) { 95 state, err := s.fsm.State().Snapshot() 96 if err != nil { 97 return false, err 98 } 99 100 ws := memdb.NewWatchSet() 101 prefix := fmt.Sprintf("%s%s", job.ID, structs.PeriodicLaunchSuffix) 102 iter, err := state.JobsByIDPrefix(ws, job.Namespace, prefix) 103 if err != nil { 104 return false, err 105 } 106 107 var child *structs.Job 108 for i := iter.Next(); i != nil; i = iter.Next() { 109 child = i.(*structs.Job) 110 111 // Ensure the job is actually a child. 112 if child.ParentID != job.ID { 113 continue 114 } 115 116 // Get the childs evaluations. 117 evals, err := state.EvalsByJob(ws, child.Namespace, child.ID) 118 if err != nil { 119 return false, err 120 } 121 122 // Check if any of the evals are active or have running allocations. 123 for _, eval := range evals { 124 if !eval.TerminalStatus() { 125 return true, nil 126 } 127 128 allocs, err := state.AllocsByEval(ws, eval.ID) 129 if err != nil { 130 return false, err 131 } 132 133 for _, alloc := range allocs { 134 if !alloc.TerminalStatus() { 135 return true, nil 136 } 137 } 138 } 139 } 140 141 // There are no evals or allocations that aren't terminal. 142 return false, nil 143 } 144 145 // NewPeriodicDispatch returns a periodic dispatcher that is used to track and 146 // launch periodic jobs. 147 func NewPeriodicDispatch(logger *log.Logger, dispatcher JobEvalDispatcher) *PeriodicDispatch { 148 return &PeriodicDispatch{ 149 dispatcher: dispatcher, 150 tracked: make(map[structs.NamespacedID]*structs.Job), 151 heap: NewPeriodicHeap(), 152 updateCh: make(chan struct{}, 1), 153 logger: logger, 154 } 155 } 156 157 // SetEnabled is used to control if the periodic dispatcher is enabled. It 158 // should only be enabled on the active leader. Disabling an active dispatcher 159 // will stop any launched go routine and flush the dispatcher. 160 func (p *PeriodicDispatch) SetEnabled(enabled bool) { 161 p.l.Lock() 162 defer p.l.Unlock() 163 wasRunning := p.enabled 164 p.enabled = enabled 165 166 // If we are transitioning from enabled to disabled, stop the daemon and 167 // flush. 168 if !enabled && wasRunning { 169 p.stopFn() 170 p.flush() 171 } else if enabled && !wasRunning { 172 // If we are transitioning from disabled to enabled, run the daemon. 173 ctx, cancel := context.WithCancel(context.Background()) 174 p.stopFn = cancel 175 go p.run(ctx) 176 } 177 } 178 179 // Tracked returns the set of tracked job IDs. 180 func (p *PeriodicDispatch) Tracked() []*structs.Job { 181 p.l.RLock() 182 defer p.l.RUnlock() 183 tracked := make([]*structs.Job, len(p.tracked)) 184 i := 0 185 for _, job := range p.tracked { 186 tracked[i] = job 187 i++ 188 } 189 return tracked 190 } 191 192 // Add begins tracking of a periodic job. If it is already tracked, it acts as 193 // an update to the jobs periodic spec. The method returns whether the job was 194 // added and any error that may have occurred. 195 func (p *PeriodicDispatch) Add(job *structs.Job) error { 196 p.l.Lock() 197 defer p.l.Unlock() 198 199 // Do nothing if not enabled 200 if !p.enabled { 201 return nil 202 } 203 204 // If we were tracking a job and it has been disabled, made non-periodic, 205 // stopped or is parameterized, remove it 206 disabled := !job.IsPeriodicActive() 207 208 tuple := structs.NamespacedID{ 209 ID: job.ID, 210 Namespace: job.Namespace, 211 } 212 _, tracked := p.tracked[tuple] 213 if disabled { 214 if tracked { 215 p.removeLocked(tuple) 216 } 217 218 // If the job is disabled and we aren't tracking it, do nothing. 219 return nil 220 } 221 222 // Add or update the job. 223 p.tracked[tuple] = job 224 next := job.Periodic.Next(time.Now().In(job.Periodic.GetLocation())) 225 if tracked { 226 if err := p.heap.Update(job, next); err != nil { 227 return fmt.Errorf("failed to update job %q (%s) launch time: %v", job.ID, job.Namespace, err) 228 } 229 p.logger.Printf("[DEBUG] nomad.periodic: updated periodic job %q (%s)", job.ID, job.Namespace) 230 } else { 231 if err := p.heap.Push(job, next); err != nil { 232 return fmt.Errorf("failed to add job %v: %v", job.ID, err) 233 } 234 p.logger.Printf("[DEBUG] nomad.periodic: registered periodic job %q (%s)", job.ID, job.Namespace) 235 } 236 237 // Signal an update. 238 select { 239 case p.updateCh <- struct{}{}: 240 default: 241 } 242 243 return nil 244 } 245 246 // Remove stops tracking the passed job. If the job is not tracked, it is a 247 // no-op. 248 func (p *PeriodicDispatch) Remove(namespace, jobID string) error { 249 p.l.Lock() 250 defer p.l.Unlock() 251 return p.removeLocked(structs.NamespacedID{ 252 ID: jobID, 253 Namespace: namespace, 254 }) 255 } 256 257 // Remove stops tracking the passed job. If the job is not tracked, it is a 258 // no-op. It assumes this is called while a lock is held. 259 func (p *PeriodicDispatch) removeLocked(jobID structs.NamespacedID) error { 260 // Do nothing if not enabled 261 if !p.enabled { 262 return nil 263 } 264 265 job, tracked := p.tracked[jobID] 266 if !tracked { 267 return nil 268 } 269 270 delete(p.tracked, jobID) 271 if err := p.heap.Remove(job); err != nil { 272 return fmt.Errorf("failed to remove tracked job %q (%s): %v", jobID.ID, jobID.Namespace, err) 273 } 274 275 // Signal an update. 276 select { 277 case p.updateCh <- struct{}{}: 278 default: 279 } 280 281 p.logger.Printf("[DEBUG] nomad.periodic: deregistered periodic job %q (%s)", jobID.ID, jobID.Namespace) 282 return nil 283 } 284 285 // ForceRun causes the periodic job to be evaluated immediately and returns the 286 // subsequent eval. 287 func (p *PeriodicDispatch) ForceRun(namespace, jobID string) (*structs.Evaluation, error) { 288 p.l.Lock() 289 290 // Do nothing if not enabled 291 if !p.enabled { 292 p.l.Unlock() 293 return nil, fmt.Errorf("periodic dispatch disabled") 294 } 295 296 tuple := structs.NamespacedID{ 297 ID: jobID, 298 Namespace: namespace, 299 } 300 job, tracked := p.tracked[tuple] 301 if !tracked { 302 p.l.Unlock() 303 return nil, fmt.Errorf("can't force run non-tracked job %q (%s)", jobID, namespace) 304 } 305 306 p.l.Unlock() 307 return p.createEval(job, time.Now().In(job.Periodic.GetLocation())) 308 } 309 310 // shouldRun returns whether the long lived run function should run. 311 func (p *PeriodicDispatch) shouldRun() bool { 312 p.l.RLock() 313 defer p.l.RUnlock() 314 return p.enabled 315 } 316 317 // run is a long-lived function that waits till a job's periodic spec is met and 318 // then creates an evaluation to run the job. 319 func (p *PeriodicDispatch) run(ctx context.Context) { 320 var launchCh <-chan time.Time 321 for p.shouldRun() { 322 job, launch := p.nextLaunch() 323 if launch.IsZero() { 324 launchCh = nil 325 } else { 326 launchDur := launch.Sub(time.Now().In(job.Periodic.GetLocation())) 327 launchCh = time.After(launchDur) 328 p.logger.Printf("[DEBUG] nomad.periodic: launching job %q (%s) in %s", job.ID, job.Namespace, launchDur) 329 } 330 331 select { 332 case <-ctx.Done(): 333 return 334 case <-p.updateCh: 335 continue 336 case <-launchCh: 337 p.dispatch(job, launch) 338 } 339 } 340 } 341 342 // dispatch creates an evaluation for the job and updates its next launchtime 343 // based on the passed launch time. 344 func (p *PeriodicDispatch) dispatch(job *structs.Job, launchTime time.Time) { 345 p.l.Lock() 346 347 nextLaunch := job.Periodic.Next(launchTime) 348 if err := p.heap.Update(job, nextLaunch); err != nil { 349 p.logger.Printf("[ERR] nomad.periodic: failed to update next launch of periodic job %q (%s): %v", job.ID, job.Namespace, err) 350 } 351 352 // If the job prohibits overlapping and there are running children, we skip 353 // the launch. 354 if job.Periodic.ProhibitOverlap { 355 running, err := p.dispatcher.RunningChildren(job) 356 if err != nil { 357 msg := fmt.Sprintf("[ERR] nomad.periodic: failed to determine if"+ 358 " periodic job %q (%s) has running children: %v", job.ID, job.Namespace, err) 359 p.logger.Println(msg) 360 p.l.Unlock() 361 return 362 } 363 364 if running { 365 msg := fmt.Sprintf("[DEBUG] nomad.periodic: skipping launch of"+ 366 " periodic job %q (%s) because job prohibits overlap", job.ID, job.Namespace) 367 p.logger.Println(msg) 368 p.l.Unlock() 369 return 370 } 371 } 372 373 p.logger.Printf("[DEBUG] nomad.periodic: launching job %q (%v) at %v", job.ID, job.Namespace, launchTime) 374 p.l.Unlock() 375 p.createEval(job, launchTime) 376 } 377 378 // nextLaunch returns the next job to launch and when it should be launched. If 379 // the next job can't be determined, an error is returned. If the dispatcher is 380 // stopped, a nil job will be returned. 381 func (p *PeriodicDispatch) nextLaunch() (*structs.Job, time.Time) { 382 // If there is nothing wait for an update. 383 p.l.RLock() 384 defer p.l.RUnlock() 385 if p.heap.Length() == 0 { 386 return nil, time.Time{} 387 } 388 389 nextJob := p.heap.Peek() 390 if nextJob == nil { 391 return nil, time.Time{} 392 } 393 394 return nextJob.job, nextJob.next 395 } 396 397 // createEval instantiates a job based on the passed periodic job and submits an 398 // evaluation for it. This should not be called with the lock held. 399 func (p *PeriodicDispatch) createEval(periodicJob *structs.Job, time time.Time) (*structs.Evaluation, error) { 400 derived, err := p.deriveJob(periodicJob, time) 401 if err != nil { 402 return nil, err 403 } 404 405 eval, err := p.dispatcher.DispatchJob(derived) 406 if err != nil { 407 p.logger.Printf("[ERR] nomad.periodic: failed to dispatch job %q (%s): %v", 408 periodicJob.ID, periodicJob.Namespace, err) 409 return nil, err 410 } 411 412 return eval, nil 413 } 414 415 // deriveJob instantiates a new job based on the passed periodic job and the 416 // launch time. 417 func (p *PeriodicDispatch) deriveJob(periodicJob *structs.Job, time time.Time) ( 418 derived *structs.Job, err error) { 419 420 // Have to recover in case the job copy panics. 421 defer func() { 422 if r := recover(); r != nil { 423 p.logger.Printf("[ERR] nomad.periodic: deriving job from"+ 424 " periodic job %q (%s) failed; deregistering from periodic runner: %v", 425 periodicJob.ID, periodicJob.Namespace, r) 426 427 p.Remove(periodicJob.Namespace, periodicJob.ID) 428 derived = nil 429 err = fmt.Errorf("Failed to create a copy of the periodic job %q (%s): %v", 430 periodicJob.ID, periodicJob.Namespace, r) 431 } 432 }() 433 434 // Create a copy of the periodic job, give it a derived ID/Name and make it 435 // non-periodic. 436 derived = periodicJob.Copy() 437 derived.ParentID = periodicJob.ID 438 derived.ID = p.derivedJobID(periodicJob, time) 439 derived.Name = derived.ID 440 derived.Periodic = nil 441 return 442 } 443 444 // deriveJobID returns a job ID based on the parent periodic job and the launch 445 // time. 446 func (p *PeriodicDispatch) derivedJobID(periodicJob *structs.Job, time time.Time) string { 447 return fmt.Sprintf("%s%s%d", periodicJob.ID, structs.PeriodicLaunchSuffix, time.Unix()) 448 } 449 450 // LaunchTime returns the launch time of the job. This is only valid for 451 // jobs created by PeriodicDispatch and will otherwise return an error. 452 func (p *PeriodicDispatch) LaunchTime(jobID string) (time.Time, error) { 453 index := strings.LastIndex(jobID, structs.PeriodicLaunchSuffix) 454 if index == -1 { 455 return time.Time{}, fmt.Errorf("couldn't parse launch time from eval: %v", jobID) 456 } 457 458 launch, err := strconv.Atoi(jobID[index+len(structs.PeriodicLaunchSuffix):]) 459 if err != nil { 460 return time.Time{}, fmt.Errorf("couldn't parse launch time from eval: %v", jobID) 461 } 462 463 return time.Unix(int64(launch), 0), nil 464 } 465 466 // flush clears the state of the PeriodicDispatcher 467 func (p *PeriodicDispatch) flush() { 468 p.updateCh = make(chan struct{}, 1) 469 p.tracked = make(map[structs.NamespacedID]*structs.Job) 470 p.heap = NewPeriodicHeap() 471 p.stopFn = nil 472 } 473 474 // periodicHeap wraps a heap and gives operations other than Push/Pop. 475 type periodicHeap struct { 476 index map[structs.NamespacedID]*periodicJob 477 heap periodicHeapImp 478 } 479 480 type periodicJob struct { 481 job *structs.Job 482 next time.Time 483 index int 484 } 485 486 func NewPeriodicHeap() *periodicHeap { 487 return &periodicHeap{ 488 index: make(map[structs.NamespacedID]*periodicJob), 489 heap: make(periodicHeapImp, 0), 490 } 491 } 492 493 func (p *periodicHeap) Push(job *structs.Job, next time.Time) error { 494 tuple := structs.NamespacedID{ 495 ID: job.ID, 496 Namespace: job.Namespace, 497 } 498 if _, ok := p.index[tuple]; ok { 499 return fmt.Errorf("job %q (%s) already exists", job.ID, job.Namespace) 500 } 501 502 pJob := &periodicJob{job, next, 0} 503 p.index[tuple] = pJob 504 heap.Push(&p.heap, pJob) 505 return nil 506 } 507 508 func (p *periodicHeap) Pop() *periodicJob { 509 if len(p.heap) == 0 { 510 return nil 511 } 512 513 pJob := heap.Pop(&p.heap).(*periodicJob) 514 tuple := structs.NamespacedID{ 515 ID: pJob.job.ID, 516 Namespace: pJob.job.Namespace, 517 } 518 delete(p.index, tuple) 519 return pJob 520 } 521 522 func (p *periodicHeap) Peek() *periodicJob { 523 if len(p.heap) == 0 { 524 return nil 525 } 526 527 return p.heap[0] 528 } 529 530 func (p *periodicHeap) Contains(job *structs.Job) bool { 531 tuple := structs.NamespacedID{ 532 ID: job.ID, 533 Namespace: job.Namespace, 534 } 535 _, ok := p.index[tuple] 536 return ok 537 } 538 539 func (p *periodicHeap) Update(job *structs.Job, next time.Time) error { 540 tuple := structs.NamespacedID{ 541 ID: job.ID, 542 Namespace: job.Namespace, 543 } 544 if pJob, ok := p.index[tuple]; ok { 545 // Need to update the job as well because its spec can change. 546 pJob.job = job 547 pJob.next = next 548 heap.Fix(&p.heap, pJob.index) 549 return nil 550 } 551 552 return fmt.Errorf("heap doesn't contain job %q (%s)", job.ID, job.Namespace) 553 } 554 555 func (p *periodicHeap) Remove(job *structs.Job) error { 556 tuple := structs.NamespacedID{ 557 ID: job.ID, 558 Namespace: job.Namespace, 559 } 560 if pJob, ok := p.index[tuple]; ok { 561 heap.Remove(&p.heap, pJob.index) 562 delete(p.index, tuple) 563 return nil 564 } 565 566 return fmt.Errorf("heap doesn't contain job %q (%s)", job.ID, job.Namespace) 567 } 568 569 func (p *periodicHeap) Length() int { 570 return len(p.heap) 571 } 572 573 type periodicHeapImp []*periodicJob 574 575 func (h periodicHeapImp) Len() int { return len(h) } 576 577 func (h periodicHeapImp) Less(i, j int) bool { 578 // Two zero times should return false. 579 // Otherwise, zero is "greater" than any other time. 580 // (To sort it at the end of the list.) 581 // Sort such that zero times are at the end of the list. 582 iZero, jZero := h[i].next.IsZero(), h[j].next.IsZero() 583 if iZero && jZero { 584 return false 585 } else if iZero { 586 return false 587 } else if jZero { 588 return true 589 } 590 591 return h[i].next.Before(h[j].next) 592 } 593 594 func (h periodicHeapImp) Swap(i, j int) { 595 h[i], h[j] = h[j], h[i] 596 h[i].index = i 597 h[j].index = j 598 } 599 600 func (h *periodicHeapImp) Push(x interface{}) { 601 n := len(*h) 602 job := x.(*periodicJob) 603 job.index = n 604 *h = append(*h, job) 605 } 606 607 func (h *periodicHeapImp) Pop() interface{} { 608 old := *h 609 n := len(old) 610 job := old[n-1] 611 job.index = -1 // for safety 612 *h = old[0 : n-1] 613 return job 614 }