github.com/kardianos/nomad@v0.1.3-0.20151022182107-b13df73ee850/scheduler/system_sched_test.go (about) 1 package scheduler 2 3 import ( 4 "testing" 5 "time" 6 7 "github.com/hashicorp/nomad/nomad/mock" 8 "github.com/hashicorp/nomad/nomad/structs" 9 ) 10 11 func TestSystemSched_JobRegister(t *testing.T) { 12 h := NewHarness(t) 13 14 // Create some nodes 15 for i := 0; i < 10; i++ { 16 node := mock.Node() 17 noErr(t, h.State.UpsertNode(h.NextIndex(), node)) 18 } 19 20 // Create a job 21 job := mock.SystemJob() 22 noErr(t, h.State.UpsertJob(h.NextIndex(), job)) 23 24 // Create a mock evaluation to deregister the job 25 eval := &structs.Evaluation{ 26 ID: structs.GenerateUUID(), 27 Priority: job.Priority, 28 TriggeredBy: structs.EvalTriggerJobRegister, 29 JobID: job.ID, 30 } 31 32 // Process the evaluation 33 err := h.Process(NewSystemScheduler, eval) 34 if err != nil { 35 t.Fatalf("err: %v", err) 36 } 37 38 // Ensure a single plan 39 if len(h.Plans) != 1 { 40 t.Fatalf("bad: %#v", h.Plans) 41 } 42 plan := h.Plans[0] 43 44 // Ensure the plan allocated 45 var planned []*structs.Allocation 46 for _, allocList := range plan.NodeAllocation { 47 planned = append(planned, allocList...) 48 } 49 if len(planned) != 10 { 50 t.Fatalf("bad: %#v", plan) 51 } 52 53 // Lookup the allocations by JobID 54 out, err := h.State.AllocsByJob(job.ID) 55 noErr(t, err) 56 57 // Ensure all allocations placed 58 if len(out) != 10 { 59 t.Fatalf("bad: %#v", out) 60 } 61 62 h.AssertEvalStatus(t, structs.EvalStatusComplete) 63 } 64 65 func TestSystemSched_JobRegister_AddNode(t *testing.T) { 66 h := NewHarness(t) 67 68 // Create some nodes 69 var nodes []*structs.Node 70 for i := 0; i < 10; i++ { 71 node := mock.Node() 72 nodes = append(nodes, node) 73 noErr(t, h.State.UpsertNode(h.NextIndex(), node)) 74 } 75 76 // Generate a fake job with allocations 77 job := mock.SystemJob() 78 noErr(t, h.State.UpsertJob(h.NextIndex(), job)) 79 80 var allocs []*structs.Allocation 81 for _, node := range nodes { 82 alloc := mock.Alloc() 83 alloc.Job = job 84 alloc.JobID = job.ID 85 alloc.NodeID = node.ID 86 alloc.Name = "my-job.web[0]" 87 allocs = append(allocs, alloc) 88 } 89 noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) 90 91 // Add a new node. 92 node := mock.Node() 93 noErr(t, h.State.UpsertNode(h.NextIndex(), node)) 94 95 // Create a mock evaluation to deal with the node update 96 eval := &structs.Evaluation{ 97 ID: structs.GenerateUUID(), 98 Priority: 50, 99 TriggeredBy: structs.EvalTriggerNodeUpdate, 100 JobID: job.ID, 101 } 102 103 // Process the evaluation 104 err := h.Process(NewSystemScheduler, eval) 105 if err != nil { 106 t.Fatalf("err: %v", err) 107 } 108 109 // Ensure a single plan 110 if len(h.Plans) != 1 { 111 t.Fatalf("bad: %#v", h.Plans) 112 } 113 plan := h.Plans[0] 114 115 // Ensure the plan had no node updates 116 var update []*structs.Allocation 117 for _, updateList := range plan.NodeUpdate { 118 update = append(update, updateList...) 119 } 120 if len(update) != 0 { 121 t.Log(len(update)) 122 t.Fatalf("bad: %#v", plan) 123 } 124 125 // Ensure the plan allocated on the new node 126 var planned []*structs.Allocation 127 for _, allocList := range plan.NodeAllocation { 128 planned = append(planned, allocList...) 129 } 130 if len(planned) != 1 { 131 t.Fatalf("bad: %#v", plan) 132 } 133 134 // Ensure it allocated on the right node 135 if _, ok := plan.NodeAllocation[node.ID]; !ok { 136 t.Fatalf("allocated on wrong node: %#v", plan) 137 } 138 139 // Lookup the allocations by JobID 140 out, err := h.State.AllocsByJob(job.ID) 141 noErr(t, err) 142 143 // Ensure all allocations placed 144 out = structs.FilterTerminalAllocs(out) 145 if len(out) != 11 { 146 t.Fatalf("bad: %#v", out) 147 } 148 149 h.AssertEvalStatus(t, structs.EvalStatusComplete) 150 } 151 152 func TestSystemSched_JobRegister_AllocFail(t *testing.T) { 153 h := NewHarness(t) 154 155 // Create NO nodes 156 // Create a job 157 job := mock.SystemJob() 158 noErr(t, h.State.UpsertJob(h.NextIndex(), job)) 159 160 // Create a mock evaluation to register the job 161 eval := &structs.Evaluation{ 162 ID: structs.GenerateUUID(), 163 Priority: job.Priority, 164 TriggeredBy: structs.EvalTriggerJobRegister, 165 JobID: job.ID, 166 } 167 168 // Process the evaluation 169 err := h.Process(NewSystemScheduler, eval) 170 if err != nil { 171 t.Fatalf("err: %v", err) 172 } 173 174 // Ensure no plan as this should be a no-op. 175 if len(h.Plans) != 0 { 176 t.Fatalf("bad: %#v", h.Plans) 177 } 178 179 h.AssertEvalStatus(t, structs.EvalStatusComplete) 180 } 181 182 func TestSystemSched_JobModify(t *testing.T) { 183 h := NewHarness(t) 184 185 // Create some nodes 186 var nodes []*structs.Node 187 for i := 0; i < 10; i++ { 188 node := mock.Node() 189 nodes = append(nodes, node) 190 noErr(t, h.State.UpsertNode(h.NextIndex(), node)) 191 } 192 193 // Generate a fake job with allocations 194 job := mock.SystemJob() 195 noErr(t, h.State.UpsertJob(h.NextIndex(), job)) 196 197 var allocs []*structs.Allocation 198 for _, node := range nodes { 199 alloc := mock.Alloc() 200 alloc.Job = job 201 alloc.JobID = job.ID 202 alloc.NodeID = node.ID 203 alloc.Name = "my-job.web[0]" 204 allocs = append(allocs, alloc) 205 } 206 noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) 207 208 // Add a few terminal status allocations, these should be ignored 209 var terminal []*structs.Allocation 210 for i := 0; i < 5; i++ { 211 alloc := mock.Alloc() 212 alloc.Job = job 213 alloc.JobID = job.ID 214 alloc.NodeID = nodes[i].ID 215 alloc.Name = "my-job.web[0]" 216 alloc.DesiredStatus = structs.AllocDesiredStatusFailed 217 terminal = append(terminal, alloc) 218 } 219 noErr(t, h.State.UpsertAllocs(h.NextIndex(), terminal)) 220 221 // Update the job 222 job2 := mock.SystemJob() 223 job2.ID = job.ID 224 225 // Update the task, such that it cannot be done in-place 226 job2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other" 227 noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) 228 229 // Create a mock evaluation to deal with drain 230 eval := &structs.Evaluation{ 231 ID: structs.GenerateUUID(), 232 Priority: 50, 233 TriggeredBy: structs.EvalTriggerJobRegister, 234 JobID: job.ID, 235 } 236 237 // Process the evaluation 238 err := h.Process(NewSystemScheduler, eval) 239 if err != nil { 240 t.Fatalf("err: %v", err) 241 } 242 243 // Ensure a single plan 244 if len(h.Plans) != 1 { 245 t.Fatalf("bad: %#v", h.Plans) 246 } 247 plan := h.Plans[0] 248 249 // Ensure the plan evicted all allocs 250 var update []*structs.Allocation 251 for _, updateList := range plan.NodeUpdate { 252 update = append(update, updateList...) 253 } 254 if len(update) != len(allocs) { 255 t.Fatalf("bad: %#v", plan) 256 } 257 258 // Ensure the plan allocated 259 var planned []*structs.Allocation 260 for _, allocList := range plan.NodeAllocation { 261 planned = append(planned, allocList...) 262 } 263 if len(planned) != 10 { 264 t.Fatalf("bad: %#v", plan) 265 } 266 267 // Lookup the allocations by JobID 268 out, err := h.State.AllocsByJob(job.ID) 269 noErr(t, err) 270 271 // Ensure all allocations placed 272 out = structs.FilterTerminalAllocs(out) 273 if len(out) != 10 { 274 t.Fatalf("bad: %#v", out) 275 } 276 277 h.AssertEvalStatus(t, structs.EvalStatusComplete) 278 } 279 280 func TestSystemSched_JobModify_Rolling(t *testing.T) { 281 h := NewHarness(t) 282 283 // Create some nodes 284 var nodes []*structs.Node 285 for i := 0; i < 10; i++ { 286 node := mock.Node() 287 nodes = append(nodes, node) 288 noErr(t, h.State.UpsertNode(h.NextIndex(), node)) 289 } 290 291 // Generate a fake job with allocations 292 job := mock.SystemJob() 293 noErr(t, h.State.UpsertJob(h.NextIndex(), job)) 294 295 var allocs []*structs.Allocation 296 for _, node := range nodes { 297 alloc := mock.Alloc() 298 alloc.Job = job 299 alloc.JobID = job.ID 300 alloc.NodeID = node.ID 301 alloc.Name = "my-job.web[0]" 302 allocs = append(allocs, alloc) 303 } 304 noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) 305 306 // Update the job 307 job2 := mock.SystemJob() 308 job2.ID = job.ID 309 job2.Update = structs.UpdateStrategy{ 310 Stagger: 30 * time.Second, 311 MaxParallel: 5, 312 } 313 314 // Update the task, such that it cannot be done in-place 315 job2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other" 316 noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) 317 318 // Create a mock evaluation to deal with drain 319 eval := &structs.Evaluation{ 320 ID: structs.GenerateUUID(), 321 Priority: 50, 322 TriggeredBy: structs.EvalTriggerJobRegister, 323 JobID: job.ID, 324 } 325 326 // Process the evaluation 327 err := h.Process(NewSystemScheduler, eval) 328 if err != nil { 329 t.Fatalf("err: %v", err) 330 } 331 332 // Ensure a single plan 333 if len(h.Plans) != 1 { 334 t.Fatalf("bad: %#v", h.Plans) 335 } 336 plan := h.Plans[0] 337 338 // Ensure the plan evicted only MaxParallel 339 var update []*structs.Allocation 340 for _, updateList := range plan.NodeUpdate { 341 update = append(update, updateList...) 342 } 343 if len(update) != job2.Update.MaxParallel { 344 t.Fatalf("bad: %#v", plan) 345 } 346 347 // Ensure the plan allocated 348 var planned []*structs.Allocation 349 for _, allocList := range plan.NodeAllocation { 350 planned = append(planned, allocList...) 351 } 352 if len(planned) != job2.Update.MaxParallel { 353 t.Fatalf("bad: %#v", plan) 354 } 355 356 h.AssertEvalStatus(t, structs.EvalStatusComplete) 357 358 // Ensure a follow up eval was created 359 eval = h.Evals[0] 360 if eval.NextEval == "" { 361 t.Fatalf("missing next eval") 362 } 363 364 // Check for create 365 if len(h.CreateEvals) == 0 { 366 t.Fatalf("missing created eval") 367 } 368 create := h.CreateEvals[0] 369 if eval.NextEval != create.ID { 370 t.Fatalf("ID mismatch") 371 } 372 if create.PreviousEval != eval.ID { 373 t.Fatalf("missing previous eval") 374 } 375 376 if create.TriggeredBy != structs.EvalTriggerRollingUpdate { 377 t.Fatalf("bad: %#v", create) 378 } 379 } 380 381 func TestSystemSched_JobModify_InPlace(t *testing.T) { 382 h := NewHarness(t) 383 384 // Create some nodes 385 var nodes []*structs.Node 386 for i := 0; i < 10; i++ { 387 node := mock.Node() 388 nodes = append(nodes, node) 389 noErr(t, h.State.UpsertNode(h.NextIndex(), node)) 390 } 391 392 // Generate a fake job with allocations 393 job := mock.SystemJob() 394 noErr(t, h.State.UpsertJob(h.NextIndex(), job)) 395 396 var allocs []*structs.Allocation 397 for _, node := range nodes { 398 alloc := mock.Alloc() 399 alloc.Job = job 400 alloc.JobID = job.ID 401 alloc.NodeID = node.ID 402 alloc.Name = "my-job.web[0]" 403 allocs = append(allocs, alloc) 404 } 405 noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) 406 407 // Update the job 408 job2 := mock.SystemJob() 409 job2.ID = job.ID 410 noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) 411 412 // Create a mock evaluation to deal with drain 413 eval := &structs.Evaluation{ 414 ID: structs.GenerateUUID(), 415 Priority: 50, 416 TriggeredBy: structs.EvalTriggerJobRegister, 417 JobID: job.ID, 418 } 419 420 // Process the evaluation 421 err := h.Process(NewSystemScheduler, eval) 422 if err != nil { 423 t.Fatalf("err: %v", err) 424 } 425 426 // Ensure a single plan 427 if len(h.Plans) != 1 { 428 t.Fatalf("bad: %#v", h.Plans) 429 } 430 plan := h.Plans[0] 431 432 // Ensure the plan did not evict any allocs 433 var update []*structs.Allocation 434 for _, updateList := range plan.NodeUpdate { 435 update = append(update, updateList...) 436 } 437 if len(update) != 0 { 438 t.Fatalf("bad: %#v", plan) 439 } 440 441 // Ensure the plan updated the existing allocs 442 var planned []*structs.Allocation 443 for _, allocList := range plan.NodeAllocation { 444 planned = append(planned, allocList...) 445 } 446 if len(planned) != 10 { 447 t.Fatalf("bad: %#v", plan) 448 } 449 for _, p := range planned { 450 if p.Job != job2 { 451 t.Fatalf("should update job") 452 } 453 } 454 455 // Lookup the allocations by JobID 456 out, err := h.State.AllocsByJob(job.ID) 457 noErr(t, err) 458 459 // Ensure all allocations placed 460 if len(out) != 10 { 461 t.Fatalf("bad: %#v", out) 462 } 463 h.AssertEvalStatus(t, structs.EvalStatusComplete) 464 465 // Verify the network did not change 466 for _, alloc := range out { 467 for _, resources := range alloc.TaskResources { 468 if resources.Networks[0].ReservedPorts[0] != 5000 { 469 t.Fatalf("bad: %#v", alloc) 470 } 471 } 472 } 473 } 474 475 func TestSystemSched_JobDeregister(t *testing.T) { 476 h := NewHarness(t) 477 478 // Create some nodes 479 var nodes []*structs.Node 480 for i := 0; i < 10; i++ { 481 node := mock.Node() 482 nodes = append(nodes, node) 483 noErr(t, h.State.UpsertNode(h.NextIndex(), node)) 484 } 485 486 // Generate a fake job with allocations 487 job := mock.SystemJob() 488 489 var allocs []*structs.Allocation 490 for _, node := range nodes { 491 alloc := mock.Alloc() 492 alloc.Job = job 493 alloc.JobID = job.ID 494 alloc.NodeID = node.ID 495 alloc.Name = "my-job.web[0]" 496 allocs = append(allocs, alloc) 497 } 498 noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) 499 500 // Create a mock evaluation to deregister the job 501 eval := &structs.Evaluation{ 502 ID: structs.GenerateUUID(), 503 Priority: 50, 504 TriggeredBy: structs.EvalTriggerJobDeregister, 505 JobID: job.ID, 506 } 507 508 // Process the evaluation 509 err := h.Process(NewSystemScheduler, eval) 510 if err != nil { 511 t.Fatalf("err: %v", err) 512 } 513 514 // Ensure a single plan 515 if len(h.Plans) != 1 { 516 t.Fatalf("bad: %#v", h.Plans) 517 } 518 plan := h.Plans[0] 519 520 // Ensure the plan evicted the job from all nodes. 521 for _, node := range nodes { 522 if len(plan.NodeUpdate[node.ID]) != 1 { 523 t.Fatalf("bad: %#v", plan) 524 } 525 } 526 527 // Lookup the allocations by JobID 528 out, err := h.State.AllocsByJob(job.ID) 529 noErr(t, err) 530 531 // Ensure no remaining allocations 532 out = structs.FilterTerminalAllocs(out) 533 if len(out) != 0 { 534 t.Fatalf("bad: %#v", out) 535 } 536 537 h.AssertEvalStatus(t, structs.EvalStatusComplete) 538 } 539 540 func TestSystemSched_NodeDrain(t *testing.T) { 541 h := NewHarness(t) 542 543 // Register a draining node 544 node := mock.Node() 545 node.Drain = true 546 noErr(t, h.State.UpsertNode(h.NextIndex(), node)) 547 548 // Generate a fake job allocated on that node. 549 job := mock.SystemJob() 550 noErr(t, h.State.UpsertJob(h.NextIndex(), job)) 551 552 alloc := mock.Alloc() 553 alloc.Job = job 554 alloc.JobID = job.ID 555 alloc.NodeID = node.ID 556 alloc.Name = "my-job.web[0]" 557 noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) 558 559 // Create a mock evaluation to deal with drain 560 eval := &structs.Evaluation{ 561 ID: structs.GenerateUUID(), 562 Priority: 50, 563 TriggeredBy: structs.EvalTriggerNodeUpdate, 564 JobID: job.ID, 565 NodeID: node.ID, 566 } 567 568 // Process the evaluation 569 err := h.Process(NewSystemScheduler, eval) 570 if err != nil { 571 t.Fatalf("err: %v", err) 572 } 573 574 // Ensure a single plan 575 if len(h.Plans) != 1 { 576 t.Fatalf("bad: %#v", h.Plans) 577 } 578 plan := h.Plans[0] 579 580 // Ensure the plan evicted all allocs 581 if len(plan.NodeUpdate[node.ID]) != 1 { 582 t.Fatalf("bad: %#v", plan) 583 } 584 585 // Ensure the plan updated the allocation. 586 var planned []*structs.Allocation 587 for _, allocList := range plan.NodeUpdate { 588 planned = append(planned, allocList...) 589 } 590 if len(planned) != 1 { 591 t.Log(len(planned)) 592 t.Fatalf("bad: %#v", plan) 593 } 594 595 // Lookup the allocations by JobID 596 out, err := h.State.AllocsByJob(job.ID) 597 noErr(t, err) 598 599 // Ensure the allocations is stopped 600 if planned[0].DesiredStatus != structs.AllocDesiredStatusStop { 601 t.Fatalf("bad: %#v", out) 602 } 603 604 h.AssertEvalStatus(t, structs.EvalStatusComplete) 605 } 606 607 func TestSystemSched_RetryLimit(t *testing.T) { 608 h := NewHarness(t) 609 h.Planner = &RejectPlan{h} 610 611 // Create some nodes 612 for i := 0; i < 10; i++ { 613 node := mock.Node() 614 noErr(t, h.State.UpsertNode(h.NextIndex(), node)) 615 } 616 617 // Create a job 618 job := mock.SystemJob() 619 noErr(t, h.State.UpsertJob(h.NextIndex(), job)) 620 621 // Create a mock evaluation to deregister the job 622 eval := &structs.Evaluation{ 623 ID: structs.GenerateUUID(), 624 Priority: job.Priority, 625 TriggeredBy: structs.EvalTriggerJobRegister, 626 JobID: job.ID, 627 } 628 629 // Process the evaluation 630 err := h.Process(NewSystemScheduler, eval) 631 if err != nil { 632 t.Fatalf("err: %v", err) 633 } 634 635 // Ensure multiple plans 636 if len(h.Plans) == 0 { 637 t.Fatalf("bad: %#v", h.Plans) 638 } 639 640 // Lookup the allocations by JobID 641 out, err := h.State.AllocsByJob(job.ID) 642 noErr(t, err) 643 644 // Ensure no allocations placed 645 if len(out) != 0 { 646 t.Fatalf("bad: %#v", out) 647 } 648 649 // Should hit the retry limit 650 h.AssertEvalStatus(t, structs.EvalStatusFailed) 651 }