github.com/blixtra/nomad@v0.7.2-0.20171221000451-da9a1d7bb050/nomad/eval_broker_test.go (about) 1 package nomad 2 3 import ( 4 "fmt" 5 "testing" 6 "time" 7 8 "github.com/hashicorp/nomad/nomad/mock" 9 "github.com/hashicorp/nomad/nomad/structs" 10 "github.com/hashicorp/nomad/testutil" 11 ) 12 13 var ( 14 defaultSched = []string{ 15 structs.JobTypeService, 16 structs.JobTypeBatch, 17 } 18 ) 19 20 func testBrokerConfig() *Config { 21 config := DefaultConfig() 22 23 // Tune the Nack timeout 24 config.EvalNackTimeout = 5 * time.Second 25 26 // Tune the Nack delay 27 config.EvalNackInitialReenqueueDelay = 5 * time.Millisecond 28 config.EvalNackSubsequentReenqueueDelay = 50 * time.Millisecond 29 return config 30 } 31 32 func testBroker(t *testing.T, timeout time.Duration) *EvalBroker { 33 config := testBrokerConfig() 34 35 if timeout != 0 { 36 config.EvalNackTimeout = timeout 37 } 38 39 return testBrokerFromConfig(t, config) 40 } 41 42 func testBrokerFromConfig(t *testing.T, c *Config) *EvalBroker { 43 b, err := NewEvalBroker(c.EvalNackTimeout, c.EvalNackInitialReenqueueDelay, c.EvalNackSubsequentReenqueueDelay, 3) 44 if err != nil { 45 t.Fatalf("err: %v", err) 46 } 47 48 return b 49 } 50 51 func TestEvalBroker_Enqueue_Dequeue_Nack_Ack(t *testing.T) { 52 t.Parallel() 53 b := testBroker(t, 0) 54 55 // Enqueue, but broker is disabled! 56 eval := mock.Eval() 57 b.Enqueue(eval) 58 59 // Verify nothing was done 60 stats := b.Stats() 61 if stats.TotalReady != 0 { 62 t.Fatalf("bad: %#v", stats) 63 } 64 65 if b.Enabled() { 66 t.Fatalf("should not be enabled") 67 } 68 69 // Enable the broker, and enqueue 70 b.SetEnabled(true) 71 b.Enqueue(eval) 72 73 // Double enqueue is a no-op 74 b.Enqueue(eval) 75 76 if !b.Enabled() { 77 t.Fatalf("should be enabled") 78 } 79 80 // Verify enqueue is done 81 stats = b.Stats() 82 if stats.TotalReady != 1 { 83 t.Fatalf("bad: %#v", stats) 84 } 85 if stats.ByScheduler[eval.Type].Ready != 1 { 86 t.Fatalf("bad: %#v", stats) 87 } 88 89 // Dequeue should work 90 out, token, err := b.Dequeue(defaultSched, time.Second) 91 if err != nil { 92 t.Fatalf("err: %v", err) 93 } 94 if out != eval { 95 t.Fatalf("bad : %#v", out) 96 } 97 98 tokenOut, ok := b.Outstanding(out.ID) 99 if !ok { 100 t.Fatalf("should be outstanding") 101 } 102 if tokenOut != token { 103 t.Fatalf("Bad: %#v %#v", token, tokenOut) 104 } 105 106 // OutstandingReset should verify the token 107 err = b.OutstandingReset("nope", "foo") 108 if err != ErrNotOutstanding { 109 t.Fatalf("err: %v", err) 110 } 111 err = b.OutstandingReset(out.ID, "foo") 112 if err != ErrTokenMismatch { 113 t.Fatalf("err: %v", err) 114 } 115 err = b.OutstandingReset(out.ID, tokenOut) 116 if err != nil { 117 t.Fatalf("err: %v", err) 118 } 119 120 // Check the stats 121 stats = b.Stats() 122 if stats.TotalReady != 0 { 123 t.Fatalf("bad: %#v", stats) 124 } 125 if stats.TotalUnacked != 1 { 126 t.Fatalf("bad: %#v", stats) 127 } 128 if stats.ByScheduler[eval.Type].Ready != 0 { 129 t.Fatalf("bad: %#v", stats) 130 } 131 if stats.ByScheduler[eval.Type].Unacked != 1 { 132 t.Fatalf("bad: %#v", stats) 133 } 134 135 // Nack with wrong token should fail 136 err = b.Nack(eval.ID, "foobarbaz") 137 if err == nil { 138 t.Fatalf("should fail to nack") 139 } 140 141 // Nack back into the queue 142 err = b.Nack(eval.ID, token) 143 if err != nil { 144 t.Fatalf("err: %v", err) 145 } 146 147 if _, ok := b.Outstanding(out.ID); ok { 148 t.Fatalf("should not be outstanding") 149 } 150 151 // Check the stats 152 testutil.WaitForResult(func() (bool, error) { 153 stats = b.Stats() 154 if stats.TotalReady != 1 { 155 return false, fmt.Errorf("bad: %#v", stats) 156 } 157 if stats.TotalUnacked != 0 { 158 return false, fmt.Errorf("bad: %#v", stats) 159 } 160 if stats.TotalWaiting != 0 { 161 return false, fmt.Errorf("bad: %#v", stats) 162 } 163 if stats.ByScheduler[eval.Type].Ready != 1 { 164 return false, fmt.Errorf("bad: %#v", stats) 165 } 166 if stats.ByScheduler[eval.Type].Unacked != 0 { 167 return false, fmt.Errorf("bad: %#v", stats) 168 } 169 170 return true, nil 171 }, func(e error) { 172 t.Fatal(e) 173 }) 174 175 // Dequeue should work again 176 out2, token2, err := b.Dequeue(defaultSched, time.Second) 177 if err != nil { 178 t.Fatalf("err: %v", err) 179 } 180 if out2 != eval { 181 t.Fatalf("bad : %#v", out2) 182 } 183 if token2 == token { 184 t.Fatalf("should get a new token") 185 } 186 187 tokenOut2, ok := b.Outstanding(out.ID) 188 if !ok { 189 t.Fatalf("should be outstanding") 190 } 191 if tokenOut2 != token2 { 192 t.Fatalf("Bad: %#v %#v", token2, tokenOut2) 193 } 194 195 // Ack with wrong token 196 err = b.Ack(eval.ID, "zip") 197 if err == nil { 198 t.Fatalf("should fail to ack") 199 } 200 201 // Ack finally 202 err = b.Ack(eval.ID, token2) 203 if err != nil { 204 t.Fatalf("err: %v", err) 205 } 206 207 if _, ok := b.Outstanding(out.ID); ok { 208 t.Fatalf("should not be outstanding") 209 } 210 211 // Check the stats 212 stats = b.Stats() 213 if stats.TotalReady != 0 { 214 t.Fatalf("bad: %#v", stats) 215 } 216 if stats.TotalUnacked != 0 { 217 t.Fatalf("bad: %#v", stats) 218 } 219 if stats.ByScheduler[eval.Type].Ready != 0 { 220 t.Fatalf("bad: %#v", stats) 221 } 222 if stats.ByScheduler[eval.Type].Unacked != 0 { 223 t.Fatalf("bad: %#v", stats) 224 } 225 } 226 227 func TestEvalBroker_Nack_Delay(t *testing.T) { 228 t.Parallel() 229 b := testBroker(t, 0) 230 231 // Enqueue, but broker is disabled! 232 b.SetEnabled(true) 233 eval := mock.Eval() 234 b.Enqueue(eval) 235 236 // Dequeue should work 237 out, token, err := b.Dequeue(defaultSched, time.Second) 238 if err != nil { 239 t.Fatalf("err: %v", err) 240 } 241 if out != eval { 242 t.Fatalf("bad : %#v", out) 243 } 244 245 // Nack back into the queue 246 err = b.Nack(eval.ID, token) 247 if err != nil { 248 t.Fatalf("err: %v", err) 249 } 250 251 if _, ok := b.Outstanding(out.ID); ok { 252 t.Fatalf("should not be outstanding") 253 } 254 255 // Check the stats to ensure that it is waiting 256 stats := b.Stats() 257 if stats.TotalReady != 0 { 258 t.Fatalf("bad: %#v", stats) 259 } 260 if stats.TotalUnacked != 0 { 261 t.Fatalf("bad: %#v", stats) 262 } 263 if stats.TotalWaiting != 1 { 264 t.Fatalf("bad: %#v", stats) 265 } 266 if stats.ByScheduler[eval.Type].Ready != 0 { 267 t.Fatalf("bad: %#v", stats) 268 } 269 if stats.ByScheduler[eval.Type].Unacked != 0 { 270 t.Fatalf("bad: %#v", stats) 271 } 272 273 // Now wait for it to be re-enqueued 274 testutil.WaitForResult(func() (bool, error) { 275 stats = b.Stats() 276 if stats.TotalReady != 1 { 277 return false, fmt.Errorf("bad: %#v", stats) 278 } 279 if stats.TotalUnacked != 0 { 280 return false, fmt.Errorf("bad: %#v", stats) 281 } 282 if stats.TotalWaiting != 0 { 283 return false, fmt.Errorf("bad: %#v", stats) 284 } 285 if stats.ByScheduler[eval.Type].Ready != 1 { 286 return false, fmt.Errorf("bad: %#v", stats) 287 } 288 if stats.ByScheduler[eval.Type].Unacked != 0 { 289 return false, fmt.Errorf("bad: %#v", stats) 290 } 291 292 return true, nil 293 }, func(e error) { 294 t.Fatal(e) 295 }) 296 297 // Dequeue should work again 298 out2, token2, err := b.Dequeue(defaultSched, time.Second) 299 if err != nil { 300 t.Fatalf("err: %v", err) 301 } 302 if out2 != eval { 303 t.Fatalf("bad : %#v", out2) 304 } 305 if token2 == token { 306 t.Fatalf("should get a new token") 307 } 308 309 // Capture the time 310 start := time.Now() 311 312 // Nack back into the queue 313 err = b.Nack(eval.ID, token2) 314 if err != nil { 315 t.Fatalf("err: %v", err) 316 } 317 318 // Now wait for it to be re-enqueued 319 testutil.WaitForResult(func() (bool, error) { 320 stats = b.Stats() 321 if stats.TotalReady != 1 { 322 return false, fmt.Errorf("bad: %#v", stats) 323 } 324 if stats.TotalUnacked != 0 { 325 return false, fmt.Errorf("bad: %#v", stats) 326 } 327 if stats.TotalWaiting != 0 { 328 return false, fmt.Errorf("bad: %#v", stats) 329 } 330 if stats.ByScheduler[eval.Type].Ready != 1 { 331 return false, fmt.Errorf("bad: %#v", stats) 332 } 333 if stats.ByScheduler[eval.Type].Unacked != 0 { 334 return false, fmt.Errorf("bad: %#v", stats) 335 } 336 337 return true, nil 338 }, func(e error) { 339 t.Fatal(e) 340 }) 341 342 delay := time.Now().Sub(start) 343 if delay < b.subsequentNackDelay { 344 t.Fatalf("bad: delay was %v; want at least %v", delay, b.subsequentNackDelay) 345 } 346 347 // Dequeue should work again 348 out3, token3, err := b.Dequeue(defaultSched, time.Second) 349 if err != nil { 350 t.Fatalf("err: %v", err) 351 } 352 if out3 != eval { 353 t.Fatalf("bad : %#v", out3) 354 } 355 if token3 == token || token3 == token2 { 356 t.Fatalf("should get a new token") 357 } 358 359 // Ack finally 360 err = b.Ack(eval.ID, token3) 361 if err != nil { 362 t.Fatalf("err: %v", err) 363 } 364 365 if _, ok := b.Outstanding(out.ID); ok { 366 t.Fatalf("should not be outstanding") 367 } 368 369 // Check the stats 370 stats = b.Stats() 371 if stats.TotalReady != 0 { 372 t.Fatalf("bad: %#v", stats) 373 } 374 if stats.TotalUnacked != 0 { 375 t.Fatalf("bad: %#v", stats) 376 } 377 if stats.ByScheduler[eval.Type].Ready != 0 { 378 t.Fatalf("bad: %#v", stats) 379 } 380 if stats.ByScheduler[eval.Type].Unacked != 0 { 381 t.Fatalf("bad: %#v", stats) 382 } 383 } 384 385 func TestEvalBroker_Serialize_DuplicateJobID(t *testing.T) { 386 t.Parallel() 387 b := testBroker(t, 0) 388 b.SetEnabled(true) 389 390 ns1 := "namespace-one" 391 ns2 := "namespace-two" 392 eval := mock.Eval() 393 eval.Namespace = ns1 394 b.Enqueue(eval) 395 396 eval2 := mock.Eval() 397 eval2.JobID = eval.JobID 398 eval2.Namespace = ns1 399 eval2.CreateIndex = eval.CreateIndex + 1 400 b.Enqueue(eval2) 401 402 eval3 := mock.Eval() 403 eval3.JobID = eval.JobID 404 eval3.Namespace = ns1 405 eval3.CreateIndex = eval.CreateIndex + 2 406 b.Enqueue(eval3) 407 408 eval4 := mock.Eval() 409 eval4.JobID = eval.JobID 410 eval4.Namespace = ns2 411 eval4.CreateIndex = eval.CreateIndex + 3 412 b.Enqueue(eval4) 413 414 eval5 := mock.Eval() 415 eval5.JobID = eval.JobID 416 eval5.Namespace = ns2 417 eval5.CreateIndex = eval.CreateIndex + 4 418 b.Enqueue(eval5) 419 420 stats := b.Stats() 421 if stats.TotalReady != 2 { 422 t.Fatalf("bad: %#v", stats) 423 } 424 if stats.TotalBlocked != 3 { 425 t.Fatalf("bad: %#v", stats) 426 } 427 428 // Dequeue should work 429 out, token, err := b.Dequeue(defaultSched, time.Second) 430 if err != nil { 431 t.Fatalf("err: %v", err) 432 } 433 if out != eval { 434 t.Fatalf("bad : %#v", out) 435 } 436 437 // Check the stats 438 stats = b.Stats() 439 if stats.TotalReady != 1 { 440 t.Fatalf("bad: %#v", stats) 441 } 442 if stats.TotalUnacked != 1 { 443 t.Fatalf("bad: %#v", stats) 444 } 445 if stats.TotalBlocked != 3 { 446 t.Fatalf("bad: %#v", stats) 447 } 448 449 // Ack out 450 err = b.Ack(eval.ID, token) 451 if err != nil { 452 t.Fatalf("err: %v", err) 453 } 454 455 // Check the stats 456 stats = b.Stats() 457 if stats.TotalReady != 2 { 458 t.Fatalf("bad: %#v", stats) 459 } 460 if stats.TotalUnacked != 0 { 461 t.Fatalf("bad: %#v", stats) 462 } 463 if stats.TotalBlocked != 2 { 464 t.Fatalf("bad: %#v", stats) 465 } 466 467 // Dequeue should work 468 out, token, err = b.Dequeue(defaultSched, time.Second) 469 if err != nil { 470 t.Fatalf("err: %v", err) 471 } 472 if out != eval2 { 473 t.Fatalf("bad : %#v", out) 474 } 475 476 // Check the stats 477 stats = b.Stats() 478 if stats.TotalReady != 1 { 479 t.Fatalf("bad: %#v", stats) 480 } 481 if stats.TotalUnacked != 1 { 482 t.Fatalf("bad: %#v", stats) 483 } 484 if stats.TotalBlocked != 2 { 485 t.Fatalf("bad: %#v", stats) 486 } 487 488 // Ack out 489 err = b.Ack(eval2.ID, token) 490 if err != nil { 491 t.Fatalf("err: %v", err) 492 } 493 494 // Check the stats 495 stats = b.Stats() 496 if stats.TotalReady != 2 { 497 t.Fatalf("bad: %#v", stats) 498 } 499 if stats.TotalUnacked != 0 { 500 t.Fatalf("bad: %#v", stats) 501 } 502 if stats.TotalBlocked != 1 { 503 t.Fatalf("bad: %#v", stats) 504 } 505 506 // Dequeue should work 507 out, token, err = b.Dequeue(defaultSched, time.Second) 508 if err != nil { 509 t.Fatalf("err: %v", err) 510 } 511 if out != eval3 { 512 t.Fatalf("bad : %#v", out) 513 } 514 515 // Check the stats 516 stats = b.Stats() 517 if stats.TotalReady != 1 { 518 t.Fatalf("bad: %#v", stats) 519 } 520 if stats.TotalUnacked != 1 { 521 t.Fatalf("bad: %#v", stats) 522 } 523 if stats.TotalBlocked != 1 { 524 t.Fatalf("bad: %#v", stats) 525 } 526 527 // Ack out 528 err = b.Ack(eval3.ID, token) 529 if err != nil { 530 t.Fatalf("err: %v", err) 531 } 532 533 // Check the stats 534 stats = b.Stats() 535 if stats.TotalReady != 1 { 536 t.Fatalf("bad: %#v", stats) 537 } 538 if stats.TotalUnacked != 0 { 539 t.Fatalf("bad: %#v", stats) 540 } 541 if stats.TotalBlocked != 1 { 542 t.Fatalf("bad: %#v", stats) 543 } 544 545 // Dequeue should work 546 out, token, err = b.Dequeue(defaultSched, time.Second) 547 if err != nil { 548 t.Fatalf("err: %v", err) 549 } 550 if out != eval4 { 551 t.Fatalf("bad : %#v", out) 552 } 553 554 // Check the stats 555 stats = b.Stats() 556 if stats.TotalReady != 0 { 557 t.Fatalf("bad: %#v", stats) 558 } 559 if stats.TotalUnacked != 1 { 560 t.Fatalf("bad: %#v", stats) 561 } 562 if stats.TotalBlocked != 1 { 563 t.Fatalf("bad: %#v", stats) 564 } 565 566 // Ack out 567 err = b.Ack(eval4.ID, token) 568 if err != nil { 569 t.Fatalf("err: %v", err) 570 } 571 572 // Check the stats 573 stats = b.Stats() 574 if stats.TotalReady != 1 { 575 t.Fatalf("bad: %#v", stats) 576 } 577 if stats.TotalUnacked != 0 { 578 t.Fatalf("bad: %#v", stats) 579 } 580 if stats.TotalBlocked != 0 { 581 t.Fatalf("bad: %#v", stats) 582 } 583 584 // Dequeue should work 585 out, token, err = b.Dequeue(defaultSched, time.Second) 586 if err != nil { 587 t.Fatalf("err: %v", err) 588 } 589 if out != eval5 { 590 t.Fatalf("bad : %#v", out) 591 } 592 593 // Check the stats 594 stats = b.Stats() 595 if stats.TotalReady != 0 { 596 t.Fatalf("bad: %#v", stats) 597 } 598 if stats.TotalUnacked != 1 { 599 t.Fatalf("bad: %#v", stats) 600 } 601 if stats.TotalBlocked != 0 { 602 t.Fatalf("bad: %#v", stats) 603 } 604 605 // Ack out 606 err = b.Ack(eval5.ID, token) 607 if err != nil { 608 t.Fatalf("err: %v", err) 609 } 610 611 // Check the stats 612 stats = b.Stats() 613 if stats.TotalReady != 0 { 614 t.Fatalf("bad: %#v", stats) 615 } 616 if stats.TotalUnacked != 0 { 617 t.Fatalf("bad: %#v", stats) 618 } 619 if stats.TotalBlocked != 0 { 620 t.Fatalf("bad: %#v", stats) 621 } 622 } 623 624 func TestEvalBroker_Enqueue_Disable(t *testing.T) { 625 t.Parallel() 626 b := testBroker(t, 0) 627 628 // Enqueue 629 eval := mock.Eval() 630 b.SetEnabled(true) 631 b.Enqueue(eval) 632 633 // Flush via SetEnabled 634 b.SetEnabled(false) 635 636 // Check the stats 637 stats := b.Stats() 638 if stats.TotalReady != 0 { 639 t.Fatalf("bad: %#v", stats) 640 } 641 if stats.TotalUnacked != 0 { 642 t.Fatalf("bad: %#v", stats) 643 } 644 if _, ok := stats.ByScheduler[eval.Type]; ok { 645 t.Fatalf("bad: %#v", stats) 646 } 647 } 648 649 func TestEvalBroker_Dequeue_Timeout(t *testing.T) { 650 t.Parallel() 651 b := testBroker(t, 0) 652 b.SetEnabled(true) 653 654 start := time.Now() 655 out, _, err := b.Dequeue(defaultSched, 5*time.Millisecond) 656 end := time.Now() 657 658 if err != nil { 659 t.Fatalf("err: %v", err) 660 } 661 if out != nil { 662 t.Fatalf("unexpected: %#v", out) 663 } 664 665 if diff := end.Sub(start); diff < 5*time.Millisecond { 666 t.Fatalf("bad: %#v", diff) 667 } 668 } 669 670 func TestEvalBroker_Dequeue_Empty_Timeout(t *testing.T) { 671 t.Parallel() 672 b := testBroker(t, 0) 673 b.SetEnabled(true) 674 doneCh := make(chan struct{}, 1) 675 676 go func() { 677 out, _, err := b.Dequeue(defaultSched, 0) 678 if err != nil { 679 t.Fatalf("err: %v", err) 680 } 681 if out == nil { 682 t.Fatal("Expect an eval") 683 } 684 doneCh <- struct{}{} 685 }() 686 687 // Sleep for a little bit 688 select { 689 case <-time.After(5 * time.Millisecond): 690 case <-doneCh: 691 t.Fatalf("Dequeue(0) should block") 692 } 693 694 // Enqueue to unblock the dequeue. 695 eval := mock.Eval() 696 b.Enqueue(eval) 697 698 select { 699 case <-doneCh: 700 return 701 case <-time.After(5 * time.Millisecond): 702 t.Fatal("timeout: Dequeue(0) should return after enqueue") 703 } 704 } 705 706 // Ensure higher priority dequeued first 707 func TestEvalBroker_Dequeue_Priority(t *testing.T) { 708 t.Parallel() 709 b := testBroker(t, 0) 710 b.SetEnabled(true) 711 712 eval1 := mock.Eval() 713 eval1.Priority = 10 714 b.Enqueue(eval1) 715 716 eval2 := mock.Eval() 717 eval2.Priority = 30 718 b.Enqueue(eval2) 719 720 eval3 := mock.Eval() 721 eval3.Priority = 20 722 b.Enqueue(eval3) 723 724 out1, _, _ := b.Dequeue(defaultSched, time.Second) 725 if out1 != eval2 { 726 t.Fatalf("bad: %#v", out1) 727 } 728 729 out2, _, _ := b.Dequeue(defaultSched, time.Second) 730 if out2 != eval3 { 731 t.Fatalf("bad: %#v", out2) 732 } 733 734 out3, _, _ := b.Dequeue(defaultSched, time.Second) 735 if out3 != eval1 { 736 t.Fatalf("bad: %#v", out3) 737 } 738 } 739 740 // Ensure FIFO at fixed priority 741 func TestEvalBroker_Dequeue_FIFO(t *testing.T) { 742 t.Parallel() 743 b := testBroker(t, 0) 744 b.SetEnabled(true) 745 NUM := 100 746 747 for i := 0; i < NUM; i++ { 748 eval1 := mock.Eval() 749 eval1.CreateIndex = uint64(i) 750 eval1.ModifyIndex = uint64(i) 751 b.Enqueue(eval1) 752 } 753 754 for i := 0; i < NUM; i++ { 755 out1, _, _ := b.Dequeue(defaultSched, time.Second) 756 if out1.CreateIndex != uint64(i) { 757 t.Fatalf("bad: %d %#v", i, out1) 758 } 759 } 760 } 761 762 // Ensure fairness between schedulers 763 func TestEvalBroker_Dequeue_Fairness(t *testing.T) { 764 t.Parallel() 765 b := testBroker(t, 0) 766 b.SetEnabled(true) 767 NUM := 1000 768 769 for i := 0; i < NUM; i++ { 770 eval1 := mock.Eval() 771 if i < (NUM / 2) { 772 eval1.Type = structs.JobTypeService 773 } else { 774 eval1.Type = structs.JobTypeBatch 775 } 776 b.Enqueue(eval1) 777 } 778 779 counter := 0 780 for i := 0; i < NUM; i++ { 781 out1, _, _ := b.Dequeue(defaultSched, time.Second) 782 783 switch out1.Type { 784 case structs.JobTypeService: 785 if counter < 0 { 786 counter = 0 787 } 788 counter += 1 789 case structs.JobTypeBatch: 790 if counter > 0 { 791 counter = 0 792 } 793 counter -= 1 794 } 795 796 // This will fail randomly at times. It is very hard to 797 // test deterministically that its acting randomly. 798 if counter >= 250 || counter <= -250 { 799 t.Fatalf("unlikely sequence: %d", counter) 800 } 801 } 802 } 803 804 // Ensure we get unblocked 805 func TestEvalBroker_Dequeue_Blocked(t *testing.T) { 806 t.Parallel() 807 b := testBroker(t, 0) 808 b.SetEnabled(true) 809 810 // Start with a blocked dequeue 811 outCh := make(chan *structs.Evaluation, 1) 812 go func() { 813 start := time.Now() 814 out, _, err := b.Dequeue(defaultSched, time.Second) 815 end := time.Now() 816 outCh <- out 817 if err != nil { 818 t.Fatalf("err: %v", err) 819 } 820 if d := end.Sub(start); d < 5*time.Millisecond { 821 t.Fatalf("bad: %v", d) 822 } 823 }() 824 825 // Wait for a bit 826 time.Sleep(5 * time.Millisecond) 827 828 // Enqueue 829 eval := mock.Eval() 830 b.Enqueue(eval) 831 832 // Ensure dequeue 833 select { 834 case out := <-outCh: 835 if out != eval { 836 t.Fatalf("bad: %v", out) 837 } 838 case <-time.After(time.Second): 839 t.Fatalf("timeout") 840 } 841 } 842 843 // Ensure we nack in a timely manner 844 func TestEvalBroker_Nack_Timeout(t *testing.T) { 845 t.Parallel() 846 b := testBroker(t, 5*time.Millisecond) 847 b.SetEnabled(true) 848 849 // Enqueue 850 eval := mock.Eval() 851 b.Enqueue(eval) 852 853 // Dequeue 854 out, _, err := b.Dequeue(defaultSched, time.Second) 855 start := time.Now() 856 if err != nil { 857 t.Fatalf("err: %v", err) 858 } 859 if out != eval { 860 t.Fatalf("bad: %v", out) 861 } 862 863 // Dequeue, should block on Nack timer 864 out, _, err = b.Dequeue(defaultSched, time.Second) 865 end := time.Now() 866 if err != nil { 867 t.Fatalf("err: %v", err) 868 } 869 if out != eval { 870 t.Fatalf("bad: %v", out) 871 } 872 873 // Check the nack timer 874 if diff := end.Sub(start); diff < 5*time.Millisecond { 875 t.Fatalf("bad: %#v", diff) 876 } 877 } 878 879 // Ensure we nack in a timely manner 880 func TestEvalBroker_Nack_TimeoutReset(t *testing.T) { 881 t.Parallel() 882 b := testBroker(t, 50*time.Millisecond) 883 b.SetEnabled(true) 884 885 // Enqueue 886 eval := mock.Eval() 887 b.Enqueue(eval) 888 889 // Dequeue 890 out, token, err := b.Dequeue(defaultSched, time.Second) 891 start := time.Now() 892 if err != nil { 893 t.Fatalf("err: %v", err) 894 } 895 if out != eval { 896 t.Fatalf("bad: %v", out) 897 } 898 899 // Reset in 20 milliseconds 900 time.Sleep(20 * time.Millisecond) 901 if err := b.OutstandingReset(out.ID, token); err != nil { 902 t.Fatalf("err: %v", err) 903 } 904 905 // Dequeue, should block on Nack timer 906 out, _, err = b.Dequeue(defaultSched, time.Second) 907 end := time.Now() 908 if err != nil { 909 t.Fatalf("err: %v", err) 910 } 911 if out != eval { 912 t.Fatalf("bad: %v", out) 913 } 914 915 // Check the nack timer 916 if diff := end.Sub(start); diff < 75*time.Millisecond { 917 t.Fatalf("bad: %#v", diff) 918 } 919 } 920 921 func TestEvalBroker_PauseResumeNackTimeout(t *testing.T) { 922 t.Parallel() 923 b := testBroker(t, 50*time.Millisecond) 924 b.SetEnabled(true) 925 926 // Enqueue 927 eval := mock.Eval() 928 b.Enqueue(eval) 929 930 // Dequeue 931 out, token, err := b.Dequeue(defaultSched, time.Second) 932 start := time.Now() 933 if err != nil { 934 t.Fatalf("err: %v", err) 935 } 936 if out != eval { 937 t.Fatalf("bad: %v", out) 938 } 939 940 // Pause in 20 milliseconds 941 time.Sleep(20 * time.Millisecond) 942 if err := b.PauseNackTimeout(out.ID, token); err != nil { 943 t.Fatalf("err: %v", err) 944 } 945 946 go func() { 947 time.Sleep(20 * time.Millisecond) 948 if err := b.ResumeNackTimeout(out.ID, token); err != nil { 949 t.Fatalf("err: %v", err) 950 } 951 }() 952 953 // Dequeue, should block until the timer is resumed 954 out, _, err = b.Dequeue(defaultSched, time.Second) 955 end := time.Now() 956 if err != nil { 957 t.Fatalf("err: %v", err) 958 } 959 if out != eval { 960 t.Fatalf("bad: %v", out) 961 } 962 963 // Check the nack timer 964 if diff := end.Sub(start); diff < 95*time.Millisecond { 965 t.Fatalf("bad: %#v", diff) 966 } 967 } 968 969 func TestEvalBroker_DeliveryLimit(t *testing.T) { 970 t.Parallel() 971 b := testBroker(t, 0) 972 b.SetEnabled(true) 973 974 eval := mock.Eval() 975 b.Enqueue(eval) 976 977 for i := 0; i < 3; i++ { 978 // Dequeue should work 979 out, token, err := b.Dequeue(defaultSched, time.Second) 980 if err != nil { 981 t.Fatalf("err: %v", err) 982 } 983 if out != eval { 984 t.Fatalf("bad : %#v", out) 985 } 986 987 // Nack with wrong token should fail 988 err = b.Nack(eval.ID, token) 989 if err != nil { 990 t.Fatalf("err: %v", err) 991 } 992 } 993 994 // Check the stats 995 stats := b.Stats() 996 if stats.TotalReady != 1 { 997 t.Fatalf("bad: %#v", stats) 998 } 999 if stats.TotalUnacked != 0 { 1000 t.Fatalf("bad: %#v", stats) 1001 } 1002 if stats.ByScheduler[failedQueue].Ready != 1 { 1003 t.Fatalf("bad: %#v", stats) 1004 } 1005 if stats.ByScheduler[failedQueue].Unacked != 0 { 1006 t.Fatalf("bad: %#v", stats) 1007 } 1008 1009 // Dequeue from failed queue 1010 out, token, err := b.Dequeue([]string{failedQueue}, time.Second) 1011 if err != nil { 1012 t.Fatalf("err: %v", err) 1013 } 1014 if out != eval { 1015 t.Fatalf("bad : %#v", out) 1016 } 1017 1018 // Check the stats 1019 stats = b.Stats() 1020 if stats.TotalReady != 0 { 1021 t.Fatalf("bad: %#v", stats) 1022 } 1023 if stats.TotalUnacked != 1 { 1024 t.Fatalf("bad: %#v", stats) 1025 } 1026 if stats.ByScheduler[failedQueue].Ready != 0 { 1027 t.Fatalf("bad: %#v", stats) 1028 } 1029 if stats.ByScheduler[failedQueue].Unacked != 1 { 1030 t.Fatalf("bad: %#v", stats) 1031 } 1032 1033 // Ack finally 1034 err = b.Ack(out.ID, token) 1035 if err != nil { 1036 t.Fatalf("err: %v", err) 1037 } 1038 1039 if _, ok := b.Outstanding(out.ID); ok { 1040 t.Fatalf("should not be outstanding") 1041 } 1042 1043 // Check the stats 1044 stats = b.Stats() 1045 if stats.TotalReady != 0 { 1046 t.Fatalf("bad: %#v", stats) 1047 } 1048 if stats.TotalUnacked != 0 { 1049 t.Fatalf("bad: %#v", stats) 1050 } 1051 if stats.ByScheduler[failedQueue].Ready != 0 { 1052 t.Fatalf("bad: %#v", stats.ByScheduler[failedQueue]) 1053 } 1054 if stats.ByScheduler[failedQueue].Unacked != 0 { 1055 t.Fatalf("bad: %#v", stats.ByScheduler[failedQueue]) 1056 } 1057 } 1058 1059 func TestEvalBroker_AckAtDeliveryLimit(t *testing.T) { 1060 t.Parallel() 1061 b := testBroker(t, 0) 1062 b.SetEnabled(true) 1063 1064 eval := mock.Eval() 1065 b.Enqueue(eval) 1066 1067 for i := 0; i < 3; i++ { 1068 // Dequeue should work 1069 out, token, err := b.Dequeue(defaultSched, time.Second) 1070 if err != nil { 1071 t.Fatalf("err: %v", err) 1072 } 1073 if out != eval { 1074 t.Fatalf("bad : %#v", out) 1075 } 1076 1077 if i == 2 { 1078 b.Ack(eval.ID, token) 1079 } else { 1080 // Nack with wrong token should fail 1081 err = b.Nack(eval.ID, token) 1082 if err != nil { 1083 t.Fatalf("err: %v", err) 1084 } 1085 } 1086 } 1087 1088 // Check the stats 1089 stats := b.Stats() 1090 if stats.TotalReady != 0 { 1091 t.Fatalf("bad: %#v", stats) 1092 } 1093 if stats.TotalUnacked != 0 { 1094 t.Fatalf("bad: %#v", stats) 1095 } 1096 if _, ok := stats.ByScheduler[failedQueue]; ok { 1097 t.Fatalf("bad: %#v", stats) 1098 } 1099 } 1100 1101 // Ensure fairness between schedulers 1102 func TestEvalBroker_Wait(t *testing.T) { 1103 t.Parallel() 1104 b := testBroker(t, 0) 1105 b.SetEnabled(true) 1106 1107 // Create an eval that should wait 1108 eval := mock.Eval() 1109 eval.Wait = 10 * time.Millisecond 1110 b.Enqueue(eval) 1111 1112 // Verify waiting 1113 stats := b.Stats() 1114 if stats.TotalReady != 0 { 1115 t.Fatalf("bad: %#v", stats) 1116 } 1117 if stats.TotalWaiting != 1 { 1118 t.Fatalf("bad: %#v", stats) 1119 } 1120 1121 // Let the wait elapse 1122 time.Sleep(20 * time.Millisecond) 1123 1124 // Verify ready 1125 stats = b.Stats() 1126 if stats.TotalReady != 1 { 1127 t.Fatalf("bad: %#v", stats) 1128 } 1129 if stats.TotalWaiting != 0 { 1130 t.Fatalf("bad: %#v", stats) 1131 } 1132 1133 // Dequeue should work 1134 out, _, err := b.Dequeue(defaultSched, time.Second) 1135 if err != nil { 1136 t.Fatalf("err: %v", err) 1137 } 1138 if out != eval { 1139 t.Fatalf("bad : %#v", out) 1140 } 1141 } 1142 1143 // Ensure that priority is taken into account when enqueueing many evaluations. 1144 func TestEvalBroker_EnqueueAll_Dequeue_Fair(t *testing.T) { 1145 t.Parallel() 1146 b := testBroker(t, 0) 1147 b.SetEnabled(true) 1148 1149 // Start with a blocked dequeue 1150 outCh := make(chan *structs.Evaluation, 1) 1151 go func() { 1152 start := time.Now() 1153 out, _, err := b.Dequeue(defaultSched, time.Second) 1154 end := time.Now() 1155 outCh <- out 1156 if err != nil { 1157 t.Fatalf("err: %v", err) 1158 } 1159 if d := end.Sub(start); d < 5*time.Millisecond { 1160 t.Fatalf("bad: %v", d) 1161 } 1162 }() 1163 1164 // Wait for a bit 1165 time.Sleep(5 * time.Millisecond) 1166 1167 // Enqueue 1168 evals := make(map[*structs.Evaluation]string, 8) 1169 expectedPriority := 90 1170 for i := 10; i <= expectedPriority; i += 10 { 1171 eval := mock.Eval() 1172 eval.Priority = i 1173 evals[eval] = "" 1174 1175 } 1176 b.EnqueueAll(evals) 1177 1178 // Ensure dequeue 1179 select { 1180 case out := <-outCh: 1181 if out.Priority != expectedPriority { 1182 t.Fatalf("bad: %v", out) 1183 } 1184 case <-time.After(time.Second): 1185 t.Fatalf("timeout") 1186 } 1187 } 1188 1189 func TestEvalBroker_EnqueueAll_Requeue_Ack(t *testing.T) { 1190 t.Parallel() 1191 b := testBroker(t, 0) 1192 b.SetEnabled(true) 1193 1194 // Create the evaluation, enqueue and dequeue 1195 eval := mock.Eval() 1196 b.Enqueue(eval) 1197 1198 out, token, err := b.Dequeue(defaultSched, time.Second) 1199 if err != nil { 1200 t.Fatalf("err: %v", err) 1201 } 1202 if out != eval { 1203 t.Fatalf("bad : %#v", out) 1204 } 1205 1206 // Requeue the same evaluation. 1207 b.EnqueueAll(map[*structs.Evaluation]string{eval: token}) 1208 1209 // The stats should show one unacked 1210 stats := b.Stats() 1211 if stats.TotalReady != 0 { 1212 t.Fatalf("bad: %#v", stats) 1213 } 1214 if stats.TotalUnacked != 1 { 1215 t.Fatalf("bad: %#v", stats) 1216 } 1217 1218 // Ack the evaluation. 1219 if err := b.Ack(eval.ID, token); err != nil { 1220 t.Fatalf("err: %v", err) 1221 } 1222 1223 // Check stats again as this should cause the re-enqueued one to transition 1224 // into the ready state 1225 stats = b.Stats() 1226 if stats.TotalReady != 1 { 1227 t.Fatalf("bad: %#v", stats) 1228 } 1229 if stats.TotalUnacked != 0 { 1230 t.Fatalf("bad: %#v", stats) 1231 } 1232 1233 // Another dequeue should be successful 1234 out2, token2, err := b.Dequeue(defaultSched, time.Second) 1235 if err != nil { 1236 t.Fatalf("err: %v", err) 1237 } 1238 if out2 != eval { 1239 t.Fatalf("bad : %#v", out) 1240 } 1241 if token == token2 { 1242 t.Fatalf("bad : %s and %s", token, token2) 1243 } 1244 } 1245 1246 func TestEvalBroker_EnqueueAll_Requeue_Nack(t *testing.T) { 1247 t.Parallel() 1248 b := testBroker(t, 0) 1249 b.SetEnabled(true) 1250 1251 // Create the evaluation, enqueue and dequeue 1252 eval := mock.Eval() 1253 b.Enqueue(eval) 1254 1255 out, token, err := b.Dequeue(defaultSched, time.Second) 1256 if err != nil { 1257 t.Fatalf("err: %v", err) 1258 } 1259 if out != eval { 1260 t.Fatalf("bad : %#v", out) 1261 } 1262 1263 // Requeue the same evaluation. 1264 b.EnqueueAll(map[*structs.Evaluation]string{eval: token}) 1265 1266 // The stats should show one unacked 1267 stats := b.Stats() 1268 if stats.TotalReady != 0 { 1269 t.Fatalf("bad: %#v", stats) 1270 } 1271 if stats.TotalUnacked != 1 { 1272 t.Fatalf("bad: %#v", stats) 1273 } 1274 1275 // Nack the evaluation. 1276 if err := b.Nack(eval.ID, token); err != nil { 1277 t.Fatalf("err: %v", err) 1278 } 1279 1280 // Check stats again as this should cause the re-enqueued one to be dropped 1281 testutil.WaitForResult(func() (bool, error) { 1282 stats = b.Stats() 1283 if stats.TotalReady != 1 { 1284 return false, fmt.Errorf("bad: %#v", stats) 1285 } 1286 if stats.TotalUnacked != 0 { 1287 return false, fmt.Errorf("bad: %#v", stats) 1288 } 1289 if len(b.requeue) != 0 { 1290 return false, fmt.Errorf("bad: %#v", b.requeue) 1291 } 1292 1293 return true, nil 1294 }, func(e error) { 1295 t.Fatal(e) 1296 }) 1297 }