github.com/Ilhicas/nomad@v1.0.4-0.20210304152020-e86851182bc3/nomad/worker_test.go (about) 1 package nomad 2 3 import ( 4 "fmt" 5 "reflect" 6 "sync" 7 "testing" 8 "time" 9 10 log "github.com/hashicorp/go-hclog" 11 "github.com/hashicorp/go-memdb" 12 "github.com/stretchr/testify/require" 13 14 "github.com/hashicorp/nomad/helper/uuid" 15 "github.com/hashicorp/nomad/nomad/mock" 16 "github.com/hashicorp/nomad/nomad/structs" 17 "github.com/hashicorp/nomad/scheduler" 18 "github.com/hashicorp/nomad/testutil" 19 "github.com/stretchr/testify/assert" 20 ) 21 22 type NoopScheduler struct { 23 state scheduler.State 24 planner scheduler.Planner 25 eval *structs.Evaluation 26 err error 27 } 28 29 func (n *NoopScheduler) Process(eval *structs.Evaluation) error { 30 if n.state == nil { 31 panic("missing state") 32 } 33 if n.planner == nil { 34 panic("missing planner") 35 } 36 n.eval = eval 37 return n.err 38 } 39 40 func init() { 41 scheduler.BuiltinSchedulers["noop"] = func(logger log.Logger, s scheduler.State, p scheduler.Planner) scheduler.Scheduler { 42 n := &NoopScheduler{ 43 state: s, 44 planner: p, 45 } 46 return n 47 } 48 } 49 50 func TestWorker_dequeueEvaluation(t *testing.T) { 51 t.Parallel() 52 53 s1, cleanupS1 := TestServer(t, func(c *Config) { 54 c.NumSchedulers = 0 55 c.EnabledSchedulers = []string{structs.JobTypeService} 56 }) 57 defer cleanupS1() 58 testutil.WaitForLeader(t, s1.RPC) 59 60 // Create the evaluation 61 eval1 := mock.Eval() 62 s1.evalBroker.Enqueue(eval1) 63 64 // Create a worker 65 w := &Worker{srv: s1, logger: s1.logger} 66 67 // Attempt dequeue 68 eval, token, waitIndex, shutdown := w.dequeueEvaluation(10 * time.Millisecond) 69 if shutdown { 70 t.Fatalf("should not shutdown") 71 } 72 if token == "" { 73 t.Fatalf("should get token") 74 } 75 if waitIndex != eval1.ModifyIndex { 76 t.Fatalf("bad wait index; got %d; want %d", waitIndex, eval1.ModifyIndex) 77 } 78 79 // Ensure we get a sane eval 80 if !reflect.DeepEqual(eval, eval1) { 81 t.Fatalf("bad: %#v %#v", eval, eval1) 82 } 83 } 84 85 // Test that the worker picks up the correct wait index when there are multiple 86 // evals for the same job. 87 func TestWorker_dequeueEvaluation_SerialJobs(t *testing.T) { 88 t.Parallel() 89 90 s1, cleanupS1 := TestServer(t, func(c *Config) { 91 c.NumSchedulers = 0 92 c.EnabledSchedulers = []string{structs.JobTypeService} 93 }) 94 defer cleanupS1() 95 testutil.WaitForLeader(t, s1.RPC) 96 97 // Create the evaluation 98 eval1 := mock.Eval() 99 eval2 := mock.Eval() 100 eval2.JobID = eval1.JobID 101 102 // Insert the evals into the state store 103 if err := s1.fsm.State().UpsertEvals(structs.MsgTypeTestSetup, 1000, []*structs.Evaluation{eval1, eval2}); err != nil { 104 t.Fatal(err) 105 } 106 107 s1.evalBroker.Enqueue(eval1) 108 s1.evalBroker.Enqueue(eval2) 109 110 // Create a worker 111 w := &Worker{srv: s1, logger: s1.logger} 112 113 // Attempt dequeue 114 eval, token, waitIndex, shutdown := w.dequeueEvaluation(10 * time.Millisecond) 115 if shutdown { 116 t.Fatalf("should not shutdown") 117 } 118 if token == "" { 119 t.Fatalf("should get token") 120 } 121 if waitIndex != eval1.ModifyIndex { 122 t.Fatalf("bad wait index; got %d; want %d", waitIndex, eval1.ModifyIndex) 123 } 124 125 // Ensure we get a sane eval 126 if !reflect.DeepEqual(eval, eval1) { 127 t.Fatalf("bad: %#v %#v", eval, eval1) 128 } 129 130 // Update the modify index of the first eval 131 if err := s1.fsm.State().UpsertEvals(structs.MsgTypeTestSetup, 2000, []*structs.Evaluation{eval1}); err != nil { 132 t.Fatal(err) 133 } 134 135 // Send the Ack 136 w.sendAck(eval1.ID, token, true) 137 138 // Attempt second dequeue 139 eval, token, waitIndex, shutdown = w.dequeueEvaluation(10 * time.Millisecond) 140 if shutdown { 141 t.Fatalf("should not shutdown") 142 } 143 if token == "" { 144 t.Fatalf("should get token") 145 } 146 if waitIndex != 2000 { 147 t.Fatalf("bad wait index; got %d; want 2000", eval2.ModifyIndex) 148 } 149 150 // Ensure we get a sane eval 151 if !reflect.DeepEqual(eval, eval2) { 152 t.Fatalf("bad: %#v %#v", eval, eval2) 153 } 154 } 155 156 func TestWorker_dequeueEvaluation_paused(t *testing.T) { 157 t.Parallel() 158 159 s1, cleanupS1 := TestServer(t, func(c *Config) { 160 c.NumSchedulers = 0 161 c.EnabledSchedulers = []string{structs.JobTypeService} 162 }) 163 defer cleanupS1() 164 testutil.WaitForLeader(t, s1.RPC) 165 166 // Create the evaluation 167 eval1 := mock.Eval() 168 s1.evalBroker.Enqueue(eval1) 169 170 // Create a worker 171 w := &Worker{srv: s1, logger: s1.logger} 172 w.pauseCond = sync.NewCond(&w.pauseLock) 173 174 // PAUSE the worker 175 w.SetPause(true) 176 177 go func() { 178 time.Sleep(100 * time.Millisecond) 179 w.SetPause(false) 180 }() 181 182 // Attempt dequeue 183 start := time.Now() 184 eval, token, waitIndex, shutdown := w.dequeueEvaluation(10 * time.Millisecond) 185 if diff := time.Since(start); diff < 100*time.Millisecond { 186 t.Fatalf("should have paused: %v", diff) 187 } 188 if shutdown { 189 t.Fatalf("should not shutdown") 190 } 191 if token == "" { 192 t.Fatalf("should get token") 193 } 194 if waitIndex != eval1.ModifyIndex { 195 t.Fatalf("bad wait index; got %d; want %d", waitIndex, eval1.ModifyIndex) 196 } 197 198 // Ensure we get a sane eval 199 if !reflect.DeepEqual(eval, eval1) { 200 t.Fatalf("bad: %#v %#v", eval, eval1) 201 } 202 } 203 204 func TestWorker_dequeueEvaluation_shutdown(t *testing.T) { 205 t.Parallel() 206 207 s1, cleanupS1 := TestServer(t, func(c *Config) { 208 c.NumSchedulers = 0 209 c.EnabledSchedulers = []string{structs.JobTypeService} 210 }) 211 defer cleanupS1() 212 testutil.WaitForLeader(t, s1.RPC) 213 214 // Create a worker 215 w := &Worker{srv: s1, logger: s1.logger} 216 217 go func() { 218 time.Sleep(10 * time.Millisecond) 219 s1.Shutdown() 220 }() 221 222 // Attempt dequeue 223 eval, _, _, shutdown := w.dequeueEvaluation(10 * time.Millisecond) 224 if !shutdown { 225 t.Fatalf("should not shutdown") 226 } 227 228 // Ensure we get a sane eval 229 if eval != nil { 230 t.Fatalf("bad: %#v", eval) 231 } 232 } 233 234 func TestWorker_sendAck(t *testing.T) { 235 t.Parallel() 236 237 s1, cleanupS1 := TestServer(t, func(c *Config) { 238 c.NumSchedulers = 0 239 c.EnabledSchedulers = []string{structs.JobTypeService} 240 }) 241 defer cleanupS1() 242 testutil.WaitForLeader(t, s1.RPC) 243 244 // Create the evaluation 245 eval1 := mock.Eval() 246 s1.evalBroker.Enqueue(eval1) 247 248 // Create a worker 249 w := &Worker{srv: s1, logger: s1.logger} 250 251 // Attempt dequeue 252 eval, token, _, _ := w.dequeueEvaluation(10 * time.Millisecond) 253 254 // Check the depth is 0, 1 unacked 255 stats := s1.evalBroker.Stats() 256 if stats.TotalReady != 0 && stats.TotalUnacked != 1 { 257 t.Fatalf("bad: %#v", stats) 258 } 259 260 // Send the Nack 261 w.sendAck(eval.ID, token, false) 262 263 // Check the depth is 1, nothing unacked 264 stats = s1.evalBroker.Stats() 265 if stats.TotalReady != 1 && stats.TotalUnacked != 0 { 266 t.Fatalf("bad: %#v", stats) 267 } 268 269 // Attempt dequeue 270 eval, token, _, _ = w.dequeueEvaluation(10 * time.Millisecond) 271 272 // Send the Ack 273 w.sendAck(eval.ID, token, true) 274 275 // Check the depth is 0 276 stats = s1.evalBroker.Stats() 277 if stats.TotalReady != 0 && stats.TotalUnacked != 0 { 278 t.Fatalf("bad: %#v", stats) 279 } 280 } 281 282 func TestWorker_waitForIndex(t *testing.T) { 283 t.Parallel() 284 285 s1, cleanupS1 := TestServer(t, func(c *Config) { 286 c.NumSchedulers = 0 287 c.EnabledSchedulers = []string{structs.JobTypeService} 288 }) 289 defer cleanupS1() 290 testutil.WaitForLeader(t, s1.RPC) 291 292 // Get the current index 293 index := s1.raft.AppliedIndex() 294 295 // Cause an increment 296 errCh := make(chan error, 1) 297 go func() { 298 time.Sleep(10 * time.Millisecond) 299 n := mock.Node() 300 errCh <- s1.fsm.state.UpsertNode(structs.MsgTypeTestSetup, index+1, n) 301 }() 302 303 // Wait for a future index 304 w := &Worker{srv: s1, logger: s1.logger} 305 snap, err := w.snapshotMinIndex(index+1, time.Second) 306 require.NoError(t, err) 307 require.NotNil(t, snap) 308 309 // No error from upserting 310 require.NoError(t, <-errCh) 311 312 // Cause a timeout 313 waitIndex := index + 100 314 timeout := 10 * time.Millisecond 315 snap, err = w.snapshotMinIndex(index+100, timeout) 316 require.Nil(t, snap) 317 require.EqualError(t, err, 318 fmt.Sprintf("timed out after %s waiting for index=%d", timeout, waitIndex)) 319 } 320 321 func TestWorker_invokeScheduler(t *testing.T) { 322 t.Parallel() 323 324 s1, cleanupS1 := TestServer(t, func(c *Config) { 325 c.NumSchedulers = 0 326 c.EnabledSchedulers = []string{structs.JobTypeService} 327 }) 328 defer cleanupS1() 329 330 w := &Worker{srv: s1, logger: s1.logger} 331 eval := mock.Eval() 332 eval.Type = "noop" 333 334 snap, err := s1.fsm.state.Snapshot() 335 require.NoError(t, err) 336 337 err = w.invokeScheduler(snap, eval, uuid.Generate()) 338 require.NoError(t, err) 339 } 340 341 func TestWorker_SubmitPlan(t *testing.T) { 342 t.Parallel() 343 344 s1, cleanupS1 := TestServer(t, func(c *Config) { 345 c.NumSchedulers = 0 346 c.EnabledSchedulers = []string{structs.JobTypeService} 347 }) 348 defer cleanupS1() 349 testutil.WaitForLeader(t, s1.RPC) 350 351 // Register node 352 node := mock.Node() 353 testRegisterNode(t, s1, node) 354 355 job := mock.Job() 356 eval1 := mock.Eval() 357 eval1.JobID = job.ID 358 s1.fsm.State().UpsertJob(structs.MsgTypeTestSetup, 1000, job) 359 s1.fsm.State().UpsertEvals(structs.MsgTypeTestSetup, 1000, []*structs.Evaluation{eval1}) 360 361 // Create the register request 362 s1.evalBroker.Enqueue(eval1) 363 364 evalOut, token, err := s1.evalBroker.Dequeue([]string{eval1.Type}, time.Second) 365 if err != nil { 366 t.Fatalf("err: %v", err) 367 } 368 if evalOut != eval1 { 369 t.Fatalf("Bad eval") 370 } 371 372 // Create an allocation plan 373 alloc := mock.Alloc() 374 plan := &structs.Plan{ 375 Job: job, 376 EvalID: eval1.ID, 377 NodeAllocation: map[string][]*structs.Allocation{ 378 node.ID: {alloc}, 379 }, 380 } 381 382 // Attempt to submit a plan 383 w := &Worker{srv: s1, logger: s1.logger, evalToken: token} 384 result, state, err := w.SubmitPlan(plan) 385 if err != nil { 386 t.Fatalf("err: %v", err) 387 } 388 389 // Should have no update 390 if state != nil { 391 t.Fatalf("unexpected state update") 392 } 393 394 // Result should have allocated 395 if result == nil { 396 t.Fatalf("missing result") 397 } 398 399 if result.AllocIndex == 0 { 400 t.Fatalf("Bad: %#v", result) 401 } 402 if len(result.NodeAllocation) != 1 { 403 t.Fatalf("Bad: %#v", result) 404 } 405 } 406 407 func TestWorker_SubmitPlanNormalizedAllocations(t *testing.T) { 408 t.Parallel() 409 410 s1, cleanupS1 := TestServer(t, func(c *Config) { 411 c.NumSchedulers = 0 412 c.EnabledSchedulers = []string{structs.JobTypeService} 413 c.Build = "0.9.2" 414 }) 415 defer cleanupS1() 416 testutil.WaitForLeader(t, s1.RPC) 417 418 // Register node 419 node := mock.Node() 420 testRegisterNode(t, s1, node) 421 422 job := mock.Job() 423 eval1 := mock.Eval() 424 eval1.JobID = job.ID 425 s1.fsm.State().UpsertJob(structs.MsgTypeTestSetup, 0, job) 426 s1.fsm.State().UpsertEvals(structs.MsgTypeTestSetup, 0, []*structs.Evaluation{eval1}) 427 428 stoppedAlloc := mock.Alloc() 429 preemptedAlloc := mock.Alloc() 430 s1.fsm.State().UpsertAllocs(structs.MsgTypeTestSetup, 5, []*structs.Allocation{stoppedAlloc, preemptedAlloc}) 431 432 // Create an allocation plan 433 plan := &structs.Plan{ 434 Job: job, 435 EvalID: eval1.ID, 436 NodeUpdate: make(map[string][]*structs.Allocation), 437 NodePreemptions: make(map[string][]*structs.Allocation), 438 } 439 desiredDescription := "desired desc" 440 plan.AppendStoppedAlloc(stoppedAlloc, desiredDescription, structs.AllocClientStatusLost, "") 441 preemptingAllocID := uuid.Generate() 442 plan.AppendPreemptedAlloc(preemptedAlloc, preemptingAllocID) 443 444 // Attempt to submit a plan 445 w := &Worker{srv: s1, logger: s1.logger} 446 w.SubmitPlan(plan) 447 448 assert.Equal(t, &structs.Allocation{ 449 ID: preemptedAlloc.ID, 450 PreemptedByAllocation: preemptingAllocID, 451 }, plan.NodePreemptions[preemptedAlloc.NodeID][0]) 452 assert.Equal(t, &structs.Allocation{ 453 ID: stoppedAlloc.ID, 454 DesiredDescription: desiredDescription, 455 ClientStatus: structs.AllocClientStatusLost, 456 }, plan.NodeUpdate[stoppedAlloc.NodeID][0]) 457 } 458 459 func TestWorker_SubmitPlan_MissingNodeRefresh(t *testing.T) { 460 t.Parallel() 461 462 s1, cleanupS1 := TestServer(t, func(c *Config) { 463 c.NumSchedulers = 0 464 c.EnabledSchedulers = []string{structs.JobTypeService} 465 }) 466 defer cleanupS1() 467 testutil.WaitForLeader(t, s1.RPC) 468 469 // Register node 470 node := mock.Node() 471 testRegisterNode(t, s1, node) 472 473 // Create the job 474 job := mock.Job() 475 s1.fsm.State().UpsertJob(structs.MsgTypeTestSetup, 1000, job) 476 477 // Create the register request 478 eval1 := mock.Eval() 479 eval1.JobID = job.ID 480 s1.evalBroker.Enqueue(eval1) 481 482 evalOut, token, err := s1.evalBroker.Dequeue([]string{eval1.Type}, time.Second) 483 if err != nil { 484 t.Fatalf("err: %v", err) 485 } 486 if evalOut != eval1 { 487 t.Fatalf("Bad eval") 488 } 489 490 // Create an allocation plan, with unregistered node 491 node2 := mock.Node() 492 alloc := mock.Alloc() 493 plan := &structs.Plan{ 494 Job: job, 495 EvalID: eval1.ID, 496 NodeAllocation: map[string][]*structs.Allocation{ 497 node2.ID: {alloc}, 498 }, 499 } 500 501 // Attempt to submit a plan 502 w := &Worker{srv: s1, logger: s1.logger, evalToken: token} 503 result, state, err := w.SubmitPlan(plan) 504 if err != nil { 505 t.Fatalf("err: %v", err) 506 } 507 508 // Result should have allocated 509 if result == nil { 510 t.Fatalf("missing result") 511 } 512 513 // Expect no allocation and forced refresh 514 if result.AllocIndex != 0 { 515 t.Fatalf("Bad: %#v", result) 516 } 517 if result.RefreshIndex == 0 { 518 t.Fatalf("Bad: %#v", result) 519 } 520 if len(result.NodeAllocation) != 0 { 521 t.Fatalf("Bad: %#v", result) 522 } 523 524 // Should have an update 525 if state == nil { 526 t.Fatalf("expected state update") 527 } 528 } 529 530 func TestWorker_UpdateEval(t *testing.T) { 531 t.Parallel() 532 533 s1, cleanupS1 := TestServer(t, func(c *Config) { 534 c.NumSchedulers = 0 535 c.EnabledSchedulers = []string{structs.JobTypeService} 536 }) 537 defer cleanupS1() 538 testutil.WaitForLeader(t, s1.RPC) 539 540 // Register node 541 node := mock.Node() 542 testRegisterNode(t, s1, node) 543 544 // Create the register request 545 eval1 := mock.Eval() 546 s1.evalBroker.Enqueue(eval1) 547 evalOut, token, err := s1.evalBroker.Dequeue([]string{eval1.Type}, time.Second) 548 if err != nil { 549 t.Fatalf("err: %v", err) 550 } 551 if evalOut != eval1 { 552 t.Fatalf("Bad eval") 553 } 554 555 eval2 := evalOut.Copy() 556 eval2.Status = structs.EvalStatusComplete 557 558 // Attempt to update eval 559 w := &Worker{srv: s1, logger: s1.logger, evalToken: token} 560 err = w.UpdateEval(eval2) 561 if err != nil { 562 t.Fatalf("err: %v", err) 563 } 564 565 ws := memdb.NewWatchSet() 566 out, err := s1.fsm.State().EvalByID(ws, eval2.ID) 567 if err != nil { 568 t.Fatalf("err: %v", err) 569 } 570 if out.Status != structs.EvalStatusComplete { 571 t.Fatalf("bad: %v", out) 572 } 573 if out.SnapshotIndex != w.snapshotIndex { 574 t.Fatalf("bad: %v", out) 575 } 576 } 577 578 func TestWorker_CreateEval(t *testing.T) { 579 t.Parallel() 580 581 s1, cleanupS1 := TestServer(t, func(c *Config) { 582 c.NumSchedulers = 0 583 c.EnabledSchedulers = []string{structs.JobTypeService} 584 }) 585 defer cleanupS1() 586 testutil.WaitForLeader(t, s1.RPC) 587 588 // Register node 589 node := mock.Node() 590 testRegisterNode(t, s1, node) 591 592 // Create the register request 593 eval1 := mock.Eval() 594 s1.evalBroker.Enqueue(eval1) 595 596 evalOut, token, err := s1.evalBroker.Dequeue([]string{eval1.Type}, time.Second) 597 if err != nil { 598 t.Fatalf("err: %v", err) 599 } 600 if evalOut != eval1 { 601 t.Fatalf("Bad eval") 602 } 603 604 eval2 := mock.Eval() 605 eval2.PreviousEval = eval1.ID 606 607 // Attempt to create eval 608 w := &Worker{srv: s1, logger: s1.logger, evalToken: token} 609 err = w.CreateEval(eval2) 610 if err != nil { 611 t.Fatalf("err: %v", err) 612 } 613 614 ws := memdb.NewWatchSet() 615 out, err := s1.fsm.State().EvalByID(ws, eval2.ID) 616 if err != nil { 617 t.Fatalf("err: %v", err) 618 } 619 if out.PreviousEval != eval1.ID { 620 t.Fatalf("bad: %v", out) 621 } 622 if out.SnapshotIndex != w.snapshotIndex { 623 t.Fatalf("bad: %v", out) 624 } 625 } 626 627 func TestWorker_ReblockEval(t *testing.T) { 628 t.Parallel() 629 630 s1, cleanupS1 := TestServer(t, func(c *Config) { 631 c.NumSchedulers = 0 632 c.EnabledSchedulers = []string{structs.JobTypeService} 633 }) 634 defer cleanupS1() 635 testutil.WaitForLeader(t, s1.RPC) 636 637 // Create the blocked eval 638 eval1 := mock.Eval() 639 eval1.Status = structs.EvalStatusBlocked 640 eval1.QueuedAllocations = map[string]int{"cache": 100} 641 642 // Insert it into the state store 643 if err := s1.fsm.State().UpsertEvals(structs.MsgTypeTestSetup, 1000, []*structs.Evaluation{eval1}); err != nil { 644 t.Fatal(err) 645 } 646 647 // Create the job summary 648 js := mock.JobSummary(eval1.JobID) 649 tg := js.Summary["web"] 650 tg.Queued = 100 651 js.Summary["web"] = tg 652 if err := s1.fsm.State().UpsertJobSummary(1001, js); err != nil { 653 t.Fatal(err) 654 } 655 656 // Enqueue the eval and then dequeue 657 s1.evalBroker.Enqueue(eval1) 658 evalOut, token, err := s1.evalBroker.Dequeue([]string{eval1.Type}, time.Second) 659 if err != nil { 660 t.Fatalf("err: %v", err) 661 } 662 if evalOut != eval1 { 663 t.Fatalf("Bad eval") 664 } 665 666 eval2 := evalOut.Copy() 667 eval2.QueuedAllocations = map[string]int{"web": 50} 668 669 // Attempt to reblock eval 670 w := &Worker{srv: s1, logger: s1.logger, evalToken: token} 671 err = w.ReblockEval(eval2) 672 if err != nil { 673 t.Fatalf("err: %v", err) 674 } 675 676 // Ack the eval 677 w.sendAck(evalOut.ID, token, true) 678 679 // Check that it is blocked 680 bStats := s1.blockedEvals.Stats() 681 if bStats.TotalBlocked+bStats.TotalEscaped != 1 { 682 t.Fatalf("ReblockEval didn't insert eval into the blocked eval tracker: %#v", bStats) 683 } 684 685 // Check that the eval was updated 686 ws := memdb.NewWatchSet() 687 eval, err := s1.fsm.State().EvalByID(ws, eval2.ID) 688 if err != nil { 689 t.Fatal(err) 690 } 691 if !reflect.DeepEqual(eval.QueuedAllocations, eval2.QueuedAllocations) { 692 t.Fatalf("expected: %#v, actual: %#v", eval2.QueuedAllocations, eval.QueuedAllocations) 693 } 694 695 // Check that the snapshot index was set properly by unblocking the eval and 696 // then dequeuing. 697 s1.blockedEvals.Unblock("foobar", 1000) 698 699 reblockedEval, _, err := s1.evalBroker.Dequeue([]string{eval1.Type}, 1*time.Second) 700 if err != nil { 701 t.Fatalf("err: %v", err) 702 } 703 if reblockedEval == nil { 704 t.Fatalf("Nil eval") 705 } 706 if reblockedEval.ID != eval1.ID { 707 t.Fatalf("Bad eval") 708 } 709 710 // Check that the SnapshotIndex is set 711 if reblockedEval.SnapshotIndex != w.snapshotIndex { 712 t.Fatalf("incorrect snapshot index; got %d; want %d", 713 reblockedEval.SnapshotIndex, w.snapshotIndex) 714 } 715 }