github.com/mattyr/nomad@v0.3.3-0.20160919021406-3485a065154a/nomad/periodic_test.go (about) 1 package nomad 2 3 import ( 4 "fmt" 5 "log" 6 "math/rand" 7 "os" 8 "reflect" 9 "sort" 10 "strconv" 11 "strings" 12 "sync" 13 "testing" 14 "time" 15 16 "github.com/hashicorp/nomad/nomad/mock" 17 "github.com/hashicorp/nomad/nomad/structs" 18 "github.com/hashicorp/nomad/testutil" 19 ) 20 21 type MockJobEvalDispatcher struct { 22 Jobs map[string]*structs.Job 23 lock sync.Mutex 24 } 25 26 func NewMockJobEvalDispatcher() *MockJobEvalDispatcher { 27 return &MockJobEvalDispatcher{Jobs: make(map[string]*structs.Job)} 28 } 29 30 func (m *MockJobEvalDispatcher) DispatchJob(job *structs.Job) (*structs.Evaluation, error) { 31 m.lock.Lock() 32 defer m.lock.Unlock() 33 m.Jobs[job.ID] = job 34 return nil, nil 35 } 36 37 func (m *MockJobEvalDispatcher) RunningChildren(parent *structs.Job) (bool, error) { 38 m.lock.Lock() 39 defer m.lock.Unlock() 40 for _, job := range m.Jobs { 41 if job.ParentID == parent.ID { 42 return true, nil 43 } 44 } 45 return false, nil 46 } 47 48 // LaunchTimes returns the launch times of child jobs in sorted order. 49 func (m *MockJobEvalDispatcher) LaunchTimes(p *PeriodicDispatch, parentID string) ([]time.Time, error) { 50 m.lock.Lock() 51 defer m.lock.Unlock() 52 var launches []time.Time 53 for _, job := range m.Jobs { 54 if job.ParentID != parentID { 55 continue 56 } 57 58 t, err := p.LaunchTime(job.ID) 59 if err != nil { 60 return nil, err 61 } 62 launches = append(launches, t) 63 } 64 sort.Sort(times(launches)) 65 return launches, nil 66 } 67 68 type times []time.Time 69 70 func (t times) Len() int { return len(t) } 71 func (t times) Swap(i, j int) { t[i], t[j] = t[j], t[i] } 72 func (t times) Less(i, j int) bool { return t[i].Before(t[j]) } 73 74 // testPeriodicDispatcher returns an enabled PeriodicDispatcher which uses the 75 // MockJobEvalDispatcher. 76 func testPeriodicDispatcher() (*PeriodicDispatch, *MockJobEvalDispatcher) { 77 logger := log.New(os.Stderr, "", log.LstdFlags) 78 m := NewMockJobEvalDispatcher() 79 d := NewPeriodicDispatch(logger, m) 80 d.SetEnabled(true) 81 d.Start() 82 return d, m 83 } 84 85 // testPeriodicJob is a helper that creates a periodic job that launches at the 86 // passed times. 87 func testPeriodicJob(times ...time.Time) *structs.Job { 88 job := mock.PeriodicJob() 89 job.Periodic.SpecType = structs.PeriodicSpecTest 90 91 l := make([]string, len(times)) 92 for i, t := range times { 93 l[i] = strconv.Itoa(int(t.Round(1 * time.Second).Unix())) 94 } 95 96 job.Periodic.Spec = strings.Join(l, ",") 97 return job 98 } 99 100 func TestPeriodicDispatch_Add_NonPeriodic(t *testing.T) { 101 p, _ := testPeriodicDispatcher() 102 job := mock.Job() 103 if err := p.Add(job); err != nil { 104 t.Fatalf("Add of non-periodic job failed: %v; expect no-op", err) 105 } 106 107 tracked := p.Tracked() 108 if len(tracked) != 0 { 109 t.Fatalf("Add of non-periodic job should be no-op: %v", tracked) 110 } 111 } 112 113 func TestPeriodicDispatch_Add_UpdateJob(t *testing.T) { 114 p, _ := testPeriodicDispatcher() 115 job := mock.PeriodicJob() 116 if err := p.Add(job); err != nil { 117 t.Fatalf("Add failed %v", err) 118 } 119 120 tracked := p.Tracked() 121 if len(tracked) != 1 { 122 t.Fatalf("Add didn't track the job: %v", tracked) 123 } 124 125 // Update the job and add it again. 126 job.Periodic.Spec = "foo" 127 if err := p.Add(job); err != nil { 128 t.Fatalf("Add failed %v", err) 129 } 130 131 tracked = p.Tracked() 132 if len(tracked) != 1 { 133 t.Fatalf("Add didn't update: %v", tracked) 134 } 135 136 if !reflect.DeepEqual(job, tracked[0]) { 137 t.Fatalf("Add didn't properly update: got %v; want %v", tracked[0], job) 138 } 139 } 140 141 func TestPeriodicDispatch_Add_RemoveJob(t *testing.T) { 142 p, _ := testPeriodicDispatcher() 143 job := mock.PeriodicJob() 144 if err := p.Add(job); err != nil { 145 t.Fatalf("Add failed %v", err) 146 } 147 148 tracked := p.Tracked() 149 if len(tracked) != 1 { 150 t.Fatalf("Add didn't track the job: %v", tracked) 151 } 152 153 // Update the job to be non-periodic and add it again. 154 job.Periodic = nil 155 if err := p.Add(job); err != nil { 156 t.Fatalf("Add failed %v", err) 157 } 158 159 tracked = p.Tracked() 160 if len(tracked) != 0 { 161 t.Fatalf("Add didn't remove: %v", tracked) 162 } 163 } 164 165 func TestPeriodicDispatch_Add_TriggersUpdate(t *testing.T) { 166 p, m := testPeriodicDispatcher() 167 168 // Create a job that won't be evalauted for a while. 169 job := testPeriodicJob(time.Now().Add(10 * time.Second)) 170 171 // Add it. 172 if err := p.Add(job); err != nil { 173 t.Fatalf("Add failed %v", err) 174 } 175 176 // Update it to be sooner and re-add. 177 expected := time.Now().Round(1 * time.Second).Add(1 * time.Second) 178 job.Periodic.Spec = fmt.Sprintf("%d", expected.Unix()) 179 if err := p.Add(job); err != nil { 180 t.Fatalf("Add failed %v", err) 181 } 182 183 // Check that nothing is created. 184 if _, ok := m.Jobs[job.ID]; ok { 185 t.Fatalf("periodic dispatcher created eval at the wrong time") 186 } 187 188 time.Sleep(2 * time.Second) 189 190 // Check that job was launched correctly. 191 times, err := m.LaunchTimes(p, job.ID) 192 if err != nil { 193 t.Fatalf("failed to get launch times for job %q", job.ID) 194 } 195 if len(times) != 1 { 196 t.Fatalf("incorrect number of launch times for job %q", job.ID) 197 } 198 if times[0] != expected { 199 t.Fatalf("periodic dispatcher created eval for time %v; want %v", times[0], expected) 200 } 201 } 202 203 func TestPeriodicDispatch_Remove_Untracked(t *testing.T) { 204 p, _ := testPeriodicDispatcher() 205 if err := p.Remove("foo"); err != nil { 206 t.Fatalf("Remove failed %v; expected a no-op", err) 207 } 208 } 209 210 func TestPeriodicDispatch_Remove_Tracked(t *testing.T) { 211 p, _ := testPeriodicDispatcher() 212 213 job := mock.PeriodicJob() 214 if err := p.Add(job); err != nil { 215 t.Fatalf("Add failed %v", err) 216 } 217 218 tracked := p.Tracked() 219 if len(tracked) != 1 { 220 t.Fatalf("Add didn't track the job: %v", tracked) 221 } 222 223 if err := p.Remove(job.ID); err != nil { 224 t.Fatalf("Remove failed %v", err) 225 } 226 227 tracked = p.Tracked() 228 if len(tracked) != 0 { 229 t.Fatalf("Remove didn't untrack the job: %v", tracked) 230 } 231 } 232 233 func TestPeriodicDispatch_Remove_TriggersUpdate(t *testing.T) { 234 p, _ := testPeriodicDispatcher() 235 236 // Create a job that will be evaluated soon. 237 job := testPeriodicJob(time.Now().Add(1 * time.Second)) 238 239 // Add it. 240 if err := p.Add(job); err != nil { 241 t.Fatalf("Add failed %v", err) 242 } 243 244 // Remove the job. 245 if err := p.Remove(job.ID); err != nil { 246 t.Fatalf("Add failed %v", err) 247 } 248 249 time.Sleep(2 * time.Second) 250 251 // Check that an eval wasn't created. 252 d := p.dispatcher.(*MockJobEvalDispatcher) 253 if _, ok := d.Jobs[job.ID]; ok { 254 t.Fatalf("Remove didn't cancel creation of an eval") 255 } 256 } 257 258 func TestPeriodicDispatch_ForceRun_Untracked(t *testing.T) { 259 p, _ := testPeriodicDispatcher() 260 261 if _, err := p.ForceRun("foo"); err == nil { 262 t.Fatal("ForceRun of untracked job should fail") 263 } 264 } 265 266 func TestPeriodicDispatch_ForceRun_Tracked(t *testing.T) { 267 p, m := testPeriodicDispatcher() 268 269 // Create a job that won't be evalauted for a while. 270 job := testPeriodicJob(time.Now().Add(10 * time.Second)) 271 272 // Add it. 273 if err := p.Add(job); err != nil { 274 t.Fatalf("Add failed %v", err) 275 } 276 277 // ForceRun the job 278 if _, err := p.ForceRun(job.ID); err != nil { 279 t.Fatalf("ForceRun failed %v", err) 280 } 281 282 // Check that job was launched correctly. 283 launches, err := m.LaunchTimes(p, job.ID) 284 if err != nil { 285 t.Fatalf("failed to get launch times for job %q: %v", job.ID, err) 286 } 287 l := len(launches) 288 if l != 1 { 289 t.Fatalf("restorePeriodicDispatcher() created an unexpected"+ 290 " number of evals; got %d; want 1", l) 291 } 292 } 293 294 func TestPeriodicDispatch_Run_DisallowOverlaps(t *testing.T) { 295 p, m := testPeriodicDispatcher() 296 297 // Create a job that will trigger two launches but disallows overlapping. 298 launch1 := time.Now().Round(1 * time.Second).Add(1 * time.Second) 299 launch2 := time.Now().Round(1 * time.Second).Add(2 * time.Second) 300 job := testPeriodicJob(launch1, launch2) 301 job.Periodic.ProhibitOverlap = true 302 303 // Add it. 304 if err := p.Add(job); err != nil { 305 t.Fatalf("Add failed %v", err) 306 } 307 308 time.Sleep(3 * time.Second) 309 310 // Check that only one job was launched. 311 times, err := m.LaunchTimes(p, job.ID) 312 if err != nil { 313 t.Fatalf("failed to get launch times for job %q", job.ID) 314 } 315 if len(times) != 1 { 316 t.Fatalf("incorrect number of launch times for job %q; got %v", job.ID, times) 317 } 318 if times[0] != launch1 { 319 t.Fatalf("periodic dispatcher created eval for time %v; want %v", times[0], launch1) 320 } 321 } 322 323 func TestPeriodicDispatch_Run_Multiple(t *testing.T) { 324 p, m := testPeriodicDispatcher() 325 326 // Create a job that will be launched twice. 327 launch1 := time.Now().Round(1 * time.Second).Add(1 * time.Second) 328 launch2 := time.Now().Round(1 * time.Second).Add(2 * time.Second) 329 job := testPeriodicJob(launch1, launch2) 330 331 // Add it. 332 if err := p.Add(job); err != nil { 333 t.Fatalf("Add failed %v", err) 334 } 335 336 time.Sleep(3 * time.Second) 337 338 // Check that job was launched correctly. 339 times, err := m.LaunchTimes(p, job.ID) 340 if err != nil { 341 t.Fatalf("failed to get launch times for job %q", job.ID) 342 } 343 if len(times) != 2 { 344 t.Fatalf("incorrect number of launch times for job %q", job.ID) 345 } 346 if times[0] != launch1 { 347 t.Fatalf("periodic dispatcher created eval for time %v; want %v", times[0], launch1) 348 } 349 if times[1] != launch2 { 350 t.Fatalf("periodic dispatcher created eval for time %v; want %v", times[1], launch2) 351 } 352 } 353 354 func TestPeriodicDispatch_Run_SameTime(t *testing.T) { 355 p, m := testPeriodicDispatcher() 356 357 // Create two job that will be launched at the same time. 358 launch := time.Now().Round(1 * time.Second).Add(1 * time.Second) 359 job := testPeriodicJob(launch) 360 job2 := testPeriodicJob(launch) 361 362 // Add them. 363 if err := p.Add(job); err != nil { 364 t.Fatalf("Add failed %v", err) 365 } 366 if err := p.Add(job2); err != nil { 367 t.Fatalf("Add failed %v", err) 368 } 369 370 time.Sleep(2 * time.Second) 371 372 // Check that the jobs were launched correctly. 373 for _, job := range []*structs.Job{job, job2} { 374 times, err := m.LaunchTimes(p, job.ID) 375 if err != nil { 376 t.Fatalf("failed to get launch times for job %q", job.ID) 377 } 378 if len(times) != 1 { 379 t.Fatalf("incorrect number of launch times for job %q; got %d; want 1", job.ID, len(times)) 380 } 381 if times[0] != launch { 382 t.Fatalf("periodic dispatcher created eval for time %v; want %v", times[0], launch) 383 } 384 } 385 } 386 387 // This test adds and removes a bunch of jobs, some launching at the same time, 388 // some after each other and some invalid times, and ensures the correct 389 // behavior. 390 func TestPeriodicDispatch_Complex(t *testing.T) { 391 p, m := testPeriodicDispatcher() 392 393 // Create some jobs launching at different times. 394 now := time.Now().Round(1 * time.Second) 395 same := now.Add(1 * time.Second) 396 launch1 := same.Add(1 * time.Second) 397 launch2 := same.Add(2 * time.Second) 398 launch3 := same.Add(3 * time.Second) 399 invalid := now.Add(-200 * time.Second) 400 401 // Create two jobs launching at the same time. 402 job1 := testPeriodicJob(same) 403 job2 := testPeriodicJob(same) 404 405 // Create a job that will never launch. 406 job3 := testPeriodicJob(invalid) 407 408 // Create a job that launches twice. 409 job4 := testPeriodicJob(launch1, launch3) 410 411 // Create a job that launches once. 412 job5 := testPeriodicJob(launch2) 413 414 // Create 3 jobs we will delete. 415 job6 := testPeriodicJob(same) 416 job7 := testPeriodicJob(launch1, launch3) 417 job8 := testPeriodicJob(launch2) 418 419 // Create a map of expected eval job ids. 420 expected := map[string][]time.Time{ 421 job1.ID: []time.Time{same}, 422 job2.ID: []time.Time{same}, 423 job3.ID: nil, 424 job4.ID: []time.Time{launch1, launch3}, 425 job5.ID: []time.Time{launch2}, 426 job6.ID: nil, 427 job7.ID: nil, 428 job8.ID: nil, 429 } 430 431 // Shuffle the jobs so they can be added randomly 432 jobs := []*structs.Job{job1, job2, job3, job4, job5, job6, job7, job8} 433 toDelete := []*structs.Job{job6, job7, job8} 434 shuffle(jobs) 435 shuffle(toDelete) 436 437 for _, job := range jobs { 438 if err := p.Add(job); err != nil { 439 t.Fatalf("Add failed %v", err) 440 } 441 } 442 443 for _, job := range toDelete { 444 if err := p.Remove(job.ID); err != nil { 445 t.Fatalf("Remove failed %v", err) 446 } 447 } 448 449 time.Sleep(5 * time.Second) 450 actual := make(map[string][]time.Time, len(expected)) 451 for _, job := range jobs { 452 launches, err := m.LaunchTimes(p, job.ID) 453 if err != nil { 454 t.Fatalf("LaunchTimes(%v) failed %v", job.ID, err) 455 } 456 457 actual[job.ID] = launches 458 } 459 460 if !reflect.DeepEqual(actual, expected) { 461 t.Fatalf("Unexpected launches; got %#v; want %#v", actual, expected) 462 } 463 } 464 465 func shuffle(jobs []*structs.Job) { 466 rand.Seed(time.Now().Unix()) 467 for i := range jobs { 468 j := rand.Intn(len(jobs)) 469 jobs[i], jobs[j] = jobs[j], jobs[i] 470 } 471 } 472 473 func TestPeriodicHeap_Order(t *testing.T) { 474 h := NewPeriodicHeap() 475 j1 := mock.PeriodicJob() 476 j2 := mock.PeriodicJob() 477 j3 := mock.PeriodicJob() 478 479 lookup := map[*structs.Job]string{ 480 j1: "j1", 481 j2: "j2", 482 j3: "j3", 483 } 484 485 h.Push(j1, time.Time{}) 486 h.Push(j2, time.Unix(10, 0)) 487 h.Push(j3, time.Unix(11, 0)) 488 489 exp := []string{"j2", "j3", "j1"} 490 var act []string 491 for i := 0; i < 3; i++ { 492 pJob := h.Pop() 493 act = append(act, lookup[pJob.job]) 494 } 495 496 if !reflect.DeepEqual(act, exp) { 497 t.Fatalf("Wrong ordering; got %v; want %v", act, exp) 498 } 499 } 500 501 // deriveChildJob takes a parent periodic job and returns a job with fields set 502 // such that it appears spawned from the parent. 503 func deriveChildJob(parent *structs.Job) *structs.Job { 504 childjob := mock.Job() 505 childjob.ParentID = parent.ID 506 childjob.ID = fmt.Sprintf("%s%s%v", parent.ID, structs.PeriodicLaunchSuffix, time.Now().Unix()) 507 return childjob 508 } 509 510 func TestPeriodicDispatch_RunningChildren_NoEvals(t *testing.T) { 511 s1 := testServer(t, nil) 512 defer s1.Shutdown() 513 testutil.WaitForLeader(t, s1.RPC) 514 515 // Insert job. 516 state := s1.fsm.State() 517 job := mock.PeriodicJob() 518 if err := state.UpsertJob(1000, job); err != nil { 519 t.Fatalf("UpsertJob failed: %v", err) 520 } 521 522 running, err := s1.RunningChildren(job) 523 if err != nil { 524 t.Fatalf("RunningChildren failed: %v", err) 525 } 526 527 if running { 528 t.Fatalf("RunningChildren should return false") 529 } 530 } 531 532 func TestPeriodicDispatch_RunningChildren_ActiveEvals(t *testing.T) { 533 s1 := testServer(t, nil) 534 defer s1.Shutdown() 535 testutil.WaitForLeader(t, s1.RPC) 536 537 // Insert periodic job and child. 538 state := s1.fsm.State() 539 job := mock.PeriodicJob() 540 if err := state.UpsertJob(1000, job); err != nil { 541 t.Fatalf("UpsertJob failed: %v", err) 542 } 543 544 childjob := deriveChildJob(job) 545 if err := state.UpsertJob(1001, childjob); err != nil { 546 t.Fatalf("UpsertJob failed: %v", err) 547 } 548 549 // Insert non-terminal eval 550 eval := mock.Eval() 551 eval.JobID = childjob.ID 552 eval.Status = structs.EvalStatusPending 553 if err := state.UpsertEvals(1002, []*structs.Evaluation{eval}); err != nil { 554 t.Fatalf("UpsertEvals failed: %v", err) 555 } 556 557 running, err := s1.RunningChildren(job) 558 if err != nil { 559 t.Fatalf("RunningChildren failed: %v", err) 560 } 561 562 if !running { 563 t.Fatalf("RunningChildren should return true") 564 } 565 } 566 567 func TestPeriodicDispatch_RunningChildren_ActiveAllocs(t *testing.T) { 568 s1 := testServer(t, nil) 569 defer s1.Shutdown() 570 testutil.WaitForLeader(t, s1.RPC) 571 572 // Insert periodic job and child. 573 state := s1.fsm.State() 574 job := mock.PeriodicJob() 575 if err := state.UpsertJob(1000, job); err != nil { 576 t.Fatalf("UpsertJob failed: %v", err) 577 } 578 579 childjob := deriveChildJob(job) 580 if err := state.UpsertJob(1001, childjob); err != nil { 581 t.Fatalf("UpsertJob failed: %v", err) 582 } 583 584 // Insert terminal eval 585 eval := mock.Eval() 586 eval.JobID = childjob.ID 587 eval.Status = structs.EvalStatusPending 588 if err := state.UpsertEvals(1002, []*structs.Evaluation{eval}); err != nil { 589 t.Fatalf("UpsertEvals failed: %v", err) 590 } 591 592 // Insert active alloc 593 alloc := mock.Alloc() 594 alloc.JobID = childjob.ID 595 alloc.EvalID = eval.ID 596 alloc.DesiredStatus = structs.AllocDesiredStatusRun 597 if err := state.UpsertAllocs(1003, []*structs.Allocation{alloc}); err != nil { 598 t.Fatalf("UpsertAllocs failed: %v", err) 599 } 600 601 running, err := s1.RunningChildren(job) 602 if err != nil { 603 t.Fatalf("RunningChildren failed: %v", err) 604 } 605 606 if !running { 607 t.Fatalf("RunningChildren should return true") 608 } 609 }