github.com/kardianos/nomad@v0.1.3-0.20151022182107-b13df73ee850/scheduler/generic_sched_test.go (about) 1 package scheduler 2 3 import ( 4 "fmt" 5 "testing" 6 "time" 7 8 "github.com/hashicorp/nomad/nomad/mock" 9 "github.com/hashicorp/nomad/nomad/structs" 10 ) 11 12 func TestServiceSched_JobRegister(t *testing.T) { 13 h := NewHarness(t) 14 15 // Create some nodes 16 for i := 0; i < 10; i++ { 17 node := mock.Node() 18 noErr(t, h.State.UpsertNode(h.NextIndex(), node)) 19 } 20 21 // Create a job 22 job := mock.Job() 23 noErr(t, h.State.UpsertJob(h.NextIndex(), job)) 24 25 // Create a mock evaluation to register the job 26 eval := &structs.Evaluation{ 27 ID: structs.GenerateUUID(), 28 Priority: job.Priority, 29 TriggeredBy: structs.EvalTriggerJobRegister, 30 JobID: job.ID, 31 } 32 33 // Process the evaluation 34 err := h.Process(NewServiceScheduler, eval) 35 if err != nil { 36 t.Fatalf("err: %v", err) 37 } 38 39 // Ensure a single plan 40 if len(h.Plans) != 1 { 41 t.Fatalf("bad: %#v", h.Plans) 42 } 43 plan := h.Plans[0] 44 45 // Ensure the plan allocated 46 var planned []*structs.Allocation 47 for _, allocList := range plan.NodeAllocation { 48 planned = append(planned, allocList...) 49 } 50 if len(planned) != 10 { 51 t.Fatalf("bad: %#v", plan) 52 } 53 54 // Lookup the allocations by JobID 55 out, err := h.State.AllocsByJob(job.ID) 56 noErr(t, err) 57 58 // Ensure all allocations placed 59 if len(out) != 10 { 60 t.Fatalf("bad: %#v", out) 61 } 62 63 h.AssertEvalStatus(t, structs.EvalStatusComplete) 64 } 65 66 func TestServiceSched_JobRegister_AllocFail(t *testing.T) { 67 h := NewHarness(t) 68 69 // Create NO nodes 70 // Create a job 71 job := mock.Job() 72 noErr(t, h.State.UpsertJob(h.NextIndex(), job)) 73 74 // Create a mock evaluation to register the job 75 eval := &structs.Evaluation{ 76 ID: structs.GenerateUUID(), 77 Priority: job.Priority, 78 TriggeredBy: structs.EvalTriggerJobRegister, 79 JobID: job.ID, 80 } 81 82 // Process the evaluation 83 err := h.Process(NewServiceScheduler, eval) 84 if err != nil { 85 t.Fatalf("err: %v", err) 86 } 87 88 // Ensure a single plan 89 if len(h.Plans) != 1 { 90 t.Fatalf("bad: %#v", h.Plans) 91 } 92 plan := h.Plans[0] 93 94 // Ensure the plan failed to alloc 95 if len(plan.FailedAllocs) != 1 { 96 t.Fatalf("bad: %#v", plan) 97 } 98 99 // Lookup the allocations by JobID 100 out, err := h.State.AllocsByJob(job.ID) 101 noErr(t, err) 102 103 // Ensure all allocations placed 104 if len(out) != 1 { 105 t.Fatalf("bad: %#v", out) 106 } 107 108 // Check the coalesced failures 109 if out[0].Metrics.CoalescedFailures != 9 { 110 t.Fatalf("bad: %#v", out[0].Metrics) 111 } 112 113 h.AssertEvalStatus(t, structs.EvalStatusComplete) 114 } 115 116 func TestServiceSched_JobModify(t *testing.T) { 117 h := NewHarness(t) 118 119 // Create some nodes 120 var nodes []*structs.Node 121 for i := 0; i < 10; i++ { 122 node := mock.Node() 123 nodes = append(nodes, node) 124 noErr(t, h.State.UpsertNode(h.NextIndex(), node)) 125 } 126 127 // Generate a fake job with allocations 128 job := mock.Job() 129 noErr(t, h.State.UpsertJob(h.NextIndex(), job)) 130 131 var allocs []*structs.Allocation 132 for i := 0; i < 10; i++ { 133 alloc := mock.Alloc() 134 alloc.Job = job 135 alloc.JobID = job.ID 136 alloc.NodeID = nodes[i].ID 137 alloc.Name = fmt.Sprintf("my-job.web[%d]", i) 138 allocs = append(allocs, alloc) 139 } 140 noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) 141 142 // Add a few terminal status allocations, these should be ignored 143 var terminal []*structs.Allocation 144 for i := 0; i < 5; i++ { 145 alloc := mock.Alloc() 146 alloc.Job = job 147 alloc.JobID = job.ID 148 alloc.NodeID = nodes[i].ID 149 alloc.Name = fmt.Sprintf("my-job.web[%d]", i) 150 alloc.DesiredStatus = structs.AllocDesiredStatusFailed 151 terminal = append(terminal, alloc) 152 } 153 noErr(t, h.State.UpsertAllocs(h.NextIndex(), terminal)) 154 155 // Update the job 156 job2 := mock.Job() 157 job2.ID = job.ID 158 159 // Update the task, such that it cannot be done in-place 160 job2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other" 161 noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) 162 163 // Create a mock evaluation to deal with drain 164 eval := &structs.Evaluation{ 165 ID: structs.GenerateUUID(), 166 Priority: 50, 167 TriggeredBy: structs.EvalTriggerJobRegister, 168 JobID: job.ID, 169 } 170 171 // Process the evaluation 172 err := h.Process(NewServiceScheduler, eval) 173 if err != nil { 174 t.Fatalf("err: %v", err) 175 } 176 177 // Ensure a single plan 178 if len(h.Plans) != 1 { 179 t.Fatalf("bad: %#v", h.Plans) 180 } 181 plan := h.Plans[0] 182 183 // Ensure the plan evicted all allocs 184 var update []*structs.Allocation 185 for _, updateList := range plan.NodeUpdate { 186 update = append(update, updateList...) 187 } 188 if len(update) != len(allocs) { 189 t.Fatalf("bad: %#v", plan) 190 } 191 192 // Ensure the plan allocated 193 var planned []*structs.Allocation 194 for _, allocList := range plan.NodeAllocation { 195 planned = append(planned, allocList...) 196 } 197 if len(planned) != 10 { 198 t.Fatalf("bad: %#v", plan) 199 } 200 201 // Lookup the allocations by JobID 202 out, err := h.State.AllocsByJob(job.ID) 203 noErr(t, err) 204 205 // Ensure all allocations placed 206 out = structs.FilterTerminalAllocs(out) 207 if len(out) != 10 { 208 t.Fatalf("bad: %#v", out) 209 } 210 211 h.AssertEvalStatus(t, structs.EvalStatusComplete) 212 } 213 214 func TestServiceSched_JobModify_Rolling(t *testing.T) { 215 h := NewHarness(t) 216 217 // Create some nodes 218 var nodes []*structs.Node 219 for i := 0; i < 10; i++ { 220 node := mock.Node() 221 nodes = append(nodes, node) 222 noErr(t, h.State.UpsertNode(h.NextIndex(), node)) 223 } 224 225 // Generate a fake job with allocations 226 job := mock.Job() 227 noErr(t, h.State.UpsertJob(h.NextIndex(), job)) 228 229 var allocs []*structs.Allocation 230 for i := 0; i < 10; i++ { 231 alloc := mock.Alloc() 232 alloc.Job = job 233 alloc.JobID = job.ID 234 alloc.NodeID = nodes[i].ID 235 alloc.Name = fmt.Sprintf("my-job.web[%d]", i) 236 allocs = append(allocs, alloc) 237 } 238 noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) 239 240 // Update the job 241 job2 := mock.Job() 242 job2.ID = job.ID 243 job2.Update = structs.UpdateStrategy{ 244 Stagger: 30 * time.Second, 245 MaxParallel: 5, 246 } 247 248 // Update the task, such that it cannot be done in-place 249 job2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other" 250 noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) 251 252 // Create a mock evaluation to deal with drain 253 eval := &structs.Evaluation{ 254 ID: structs.GenerateUUID(), 255 Priority: 50, 256 TriggeredBy: structs.EvalTriggerJobRegister, 257 JobID: job.ID, 258 } 259 260 // Process the evaluation 261 err := h.Process(NewServiceScheduler, eval) 262 if err != nil { 263 t.Fatalf("err: %v", err) 264 } 265 266 // Ensure a single plan 267 if len(h.Plans) != 1 { 268 t.Fatalf("bad: %#v", h.Plans) 269 } 270 plan := h.Plans[0] 271 272 // Ensure the plan evicted only MaxParallel 273 var update []*structs.Allocation 274 for _, updateList := range plan.NodeUpdate { 275 update = append(update, updateList...) 276 } 277 if len(update) != job2.Update.MaxParallel { 278 t.Fatalf("bad: %#v", plan) 279 } 280 281 // Ensure the plan allocated 282 var planned []*structs.Allocation 283 for _, allocList := range plan.NodeAllocation { 284 planned = append(planned, allocList...) 285 } 286 if len(planned) != job2.Update.MaxParallel { 287 t.Fatalf("bad: %#v", plan) 288 } 289 290 h.AssertEvalStatus(t, structs.EvalStatusComplete) 291 292 // Ensure a follow up eval was created 293 eval = h.Evals[0] 294 if eval.NextEval == "" { 295 t.Fatalf("missing next eval") 296 } 297 298 // Check for create 299 if len(h.CreateEvals) == 0 { 300 t.Fatalf("missing created eval") 301 } 302 create := h.CreateEvals[0] 303 if eval.NextEval != create.ID { 304 t.Fatalf("ID mismatch") 305 } 306 if create.PreviousEval != eval.ID { 307 t.Fatalf("missing previous eval") 308 } 309 310 if create.TriggeredBy != structs.EvalTriggerRollingUpdate { 311 t.Fatalf("bad: %#v", create) 312 } 313 } 314 315 func TestServiceSched_JobModify_InPlace(t *testing.T) { 316 h := NewHarness(t) 317 318 // Create some nodes 319 var nodes []*structs.Node 320 for i := 0; i < 10; i++ { 321 node := mock.Node() 322 nodes = append(nodes, node) 323 noErr(t, h.State.UpsertNode(h.NextIndex(), node)) 324 } 325 326 // Generate a fake job with allocations 327 job := mock.Job() 328 noErr(t, h.State.UpsertJob(h.NextIndex(), job)) 329 330 var allocs []*structs.Allocation 331 for i := 0; i < 10; i++ { 332 alloc := mock.Alloc() 333 alloc.Job = job 334 alloc.JobID = job.ID 335 alloc.NodeID = nodes[i].ID 336 alloc.Name = fmt.Sprintf("my-job.web[%d]", i) 337 allocs = append(allocs, alloc) 338 } 339 noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) 340 341 // Update the job 342 job2 := mock.Job() 343 job2.ID = job.ID 344 noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) 345 346 // Create a mock evaluation to deal with drain 347 eval := &structs.Evaluation{ 348 ID: structs.GenerateUUID(), 349 Priority: 50, 350 TriggeredBy: structs.EvalTriggerJobRegister, 351 JobID: job.ID, 352 } 353 354 // Process the evaluation 355 err := h.Process(NewServiceScheduler, eval) 356 if err != nil { 357 t.Fatalf("err: %v", err) 358 } 359 360 // Ensure a single plan 361 if len(h.Plans) != 1 { 362 t.Fatalf("bad: %#v", h.Plans) 363 } 364 plan := h.Plans[0] 365 366 // Ensure the plan did not evict any allocs 367 var update []*structs.Allocation 368 for _, updateList := range plan.NodeUpdate { 369 update = append(update, updateList...) 370 } 371 if len(update) != 0 { 372 t.Fatalf("bad: %#v", plan) 373 } 374 375 // Ensure the plan updated the existing allocs 376 var planned []*structs.Allocation 377 for _, allocList := range plan.NodeAllocation { 378 planned = append(planned, allocList...) 379 } 380 if len(planned) != 10 { 381 t.Fatalf("bad: %#v", plan) 382 } 383 for _, p := range planned { 384 if p.Job != job2 { 385 t.Fatalf("should update job") 386 } 387 } 388 389 // Lookup the allocations by JobID 390 out, err := h.State.AllocsByJob(job.ID) 391 noErr(t, err) 392 393 // Ensure all allocations placed 394 if len(out) != 10 { 395 t.Fatalf("bad: %#v", out) 396 } 397 h.AssertEvalStatus(t, structs.EvalStatusComplete) 398 399 // Verify the network did not change 400 for _, alloc := range out { 401 for _, resources := range alloc.TaskResources { 402 if resources.Networks[0].ReservedPorts[0] != 5000 { 403 t.Fatalf("bad: %#v", alloc) 404 } 405 } 406 } 407 } 408 409 func TestServiceSched_JobDeregister(t *testing.T) { 410 h := NewHarness(t) 411 412 // Generate a fake job with allocations 413 job := mock.Job() 414 415 var allocs []*structs.Allocation 416 for i := 0; i < 10; i++ { 417 alloc := mock.Alloc() 418 alloc.Job = job 419 alloc.JobID = job.ID 420 allocs = append(allocs, alloc) 421 } 422 noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) 423 424 // Create a mock evaluation to deregister the job 425 eval := &structs.Evaluation{ 426 ID: structs.GenerateUUID(), 427 Priority: 50, 428 TriggeredBy: structs.EvalTriggerJobDeregister, 429 JobID: job.ID, 430 } 431 432 // Process the evaluation 433 err := h.Process(NewServiceScheduler, eval) 434 if err != nil { 435 t.Fatalf("err: %v", err) 436 } 437 438 // Ensure a single plan 439 if len(h.Plans) != 1 { 440 t.Fatalf("bad: %#v", h.Plans) 441 } 442 plan := h.Plans[0] 443 444 // Ensure the plan evicted all nodes 445 if len(plan.NodeUpdate["foo"]) != len(allocs) { 446 t.Fatalf("bad: %#v", plan) 447 } 448 449 // Lookup the allocations by JobID 450 out, err := h.State.AllocsByJob(job.ID) 451 noErr(t, err) 452 453 // Ensure no remaining allocations 454 out = structs.FilterTerminalAllocs(out) 455 if len(out) != 0 { 456 t.Fatalf("bad: %#v", out) 457 } 458 459 h.AssertEvalStatus(t, structs.EvalStatusComplete) 460 } 461 462 func TestServiceSched_NodeDrain(t *testing.T) { 463 h := NewHarness(t) 464 465 // Register a draining node 466 node := mock.Node() 467 node.Drain = true 468 noErr(t, h.State.UpsertNode(h.NextIndex(), node)) 469 470 // Create some nodes 471 for i := 0; i < 10; i++ { 472 node := mock.Node() 473 noErr(t, h.State.UpsertNode(h.NextIndex(), node)) 474 } 475 476 // Generate a fake job with allocations 477 job := mock.Job() 478 noErr(t, h.State.UpsertJob(h.NextIndex(), job)) 479 480 var allocs []*structs.Allocation 481 for i := 0; i < 10; i++ { 482 alloc := mock.Alloc() 483 alloc.Job = job 484 alloc.JobID = job.ID 485 alloc.NodeID = node.ID 486 alloc.Name = fmt.Sprintf("my-job.web[%d]", i) 487 allocs = append(allocs, alloc) 488 } 489 noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) 490 491 // Create a mock evaluation to deal with drain 492 eval := &structs.Evaluation{ 493 ID: structs.GenerateUUID(), 494 Priority: 50, 495 TriggeredBy: structs.EvalTriggerNodeUpdate, 496 JobID: job.ID, 497 NodeID: node.ID, 498 } 499 500 // Process the evaluation 501 err := h.Process(NewServiceScheduler, eval) 502 if err != nil { 503 t.Fatalf("err: %v", err) 504 } 505 506 // Ensure a single plan 507 if len(h.Plans) != 1 { 508 t.Fatalf("bad: %#v", h.Plans) 509 } 510 plan := h.Plans[0] 511 512 // Ensure the plan evicted all allocs 513 if len(plan.NodeUpdate[node.ID]) != len(allocs) { 514 t.Fatalf("bad: %#v", plan) 515 } 516 517 // Ensure the plan allocated 518 var planned []*structs.Allocation 519 for _, allocList := range plan.NodeAllocation { 520 planned = append(planned, allocList...) 521 } 522 if len(planned) != 10 { 523 t.Fatalf("bad: %#v", plan) 524 } 525 526 // Lookup the allocations by JobID 527 out, err := h.State.AllocsByJob(job.ID) 528 noErr(t, err) 529 530 // Ensure all allocations placed 531 out = structs.FilterTerminalAllocs(out) 532 if len(out) != 10 { 533 t.Fatalf("bad: %#v", out) 534 } 535 536 h.AssertEvalStatus(t, structs.EvalStatusComplete) 537 } 538 539 func TestServiceSched_RetryLimit(t *testing.T) { 540 h := NewHarness(t) 541 h.Planner = &RejectPlan{h} 542 543 // Create some nodes 544 for i := 0; i < 10; i++ { 545 node := mock.Node() 546 noErr(t, h.State.UpsertNode(h.NextIndex(), node)) 547 } 548 549 // Create a job 550 job := mock.Job() 551 noErr(t, h.State.UpsertJob(h.NextIndex(), job)) 552 553 // Create a mock evaluation to register the job 554 eval := &structs.Evaluation{ 555 ID: structs.GenerateUUID(), 556 Priority: job.Priority, 557 TriggeredBy: structs.EvalTriggerJobRegister, 558 JobID: job.ID, 559 } 560 561 // Process the evaluation 562 err := h.Process(NewServiceScheduler, eval) 563 if err != nil { 564 t.Fatalf("err: %v", err) 565 } 566 567 // Ensure multiple plans 568 if len(h.Plans) == 0 { 569 t.Fatalf("bad: %#v", h.Plans) 570 } 571 572 // Lookup the allocations by JobID 573 out, err := h.State.AllocsByJob(job.ID) 574 noErr(t, err) 575 576 // Ensure no allocations placed 577 if len(out) != 0 { 578 t.Fatalf("bad: %#v", out) 579 } 580 581 // Should hit the retry limit 582 h.AssertEvalStatus(t, structs.EvalStatusFailed) 583 }