github.com/manicqin/nomad@v0.9.5/nomad/eval_broker_test.go (about) 1 package nomad 2 3 import ( 4 "encoding/json" 5 "errors" 6 "fmt" 7 "testing" 8 "time" 9 10 "github.com/hashicorp/nomad/nomad/mock" 11 "github.com/hashicorp/nomad/nomad/structs" 12 "github.com/hashicorp/nomad/testutil" 13 "github.com/stretchr/testify/require" 14 ) 15 16 var ( 17 defaultSched = []string{ 18 structs.JobTypeService, 19 structs.JobTypeBatch, 20 } 21 ) 22 23 func testBrokerConfig() *Config { 24 config := DefaultConfig() 25 26 // Tune the Nack timeout 27 config.EvalNackTimeout = 5 * time.Second 28 29 // Tune the Nack delay 30 config.EvalNackInitialReenqueueDelay = 5 * time.Millisecond 31 config.EvalNackSubsequentReenqueueDelay = 50 * time.Millisecond 32 return config 33 } 34 35 func testBroker(t *testing.T, timeout time.Duration) *EvalBroker { 36 config := testBrokerConfig() 37 38 if timeout != 0 { 39 config.EvalNackTimeout = timeout 40 } 41 42 return testBrokerFromConfig(t, config) 43 } 44 45 func testBrokerFromConfig(t *testing.T, c *Config) *EvalBroker { 46 b, err := NewEvalBroker(c.EvalNackTimeout, c.EvalNackInitialReenqueueDelay, c.EvalNackSubsequentReenqueueDelay, 3) 47 if err != nil { 48 t.Fatalf("err: %v", err) 49 } 50 51 return b 52 } 53 54 func TestEvalBroker_Enqueue_Dequeue_Nack_Ack(t *testing.T) { 55 t.Parallel() 56 b := testBroker(t, 0) 57 58 // Enqueue, but broker is disabled! 59 eval := mock.Eval() 60 b.Enqueue(eval) 61 62 // Verify nothing was done 63 stats := b.Stats() 64 if stats.TotalReady != 0 { 65 t.Fatalf("bad: %#v", stats) 66 } 67 68 if b.Enabled() { 69 t.Fatalf("should not be enabled") 70 } 71 72 // Enable the broker, and enqueue 73 b.SetEnabled(true) 74 b.Enqueue(eval) 75 76 // Double enqueue is a no-op 77 b.Enqueue(eval) 78 79 if !b.Enabled() { 80 t.Fatalf("should be enabled") 81 } 82 83 // Verify enqueue is done 84 stats = b.Stats() 85 if stats.TotalReady != 1 { 86 t.Fatalf("bad: %#v", stats) 87 } 88 if stats.ByScheduler[eval.Type].Ready != 1 { 89 t.Fatalf("bad: %#v", stats) 90 } 91 92 // Dequeue should work 93 out, token, err := b.Dequeue(defaultSched, time.Second) 94 if err != nil { 95 t.Fatalf("err: %v", err) 96 } 97 if out != eval { 98 t.Fatalf("bad : %#v", out) 99 } 100 101 tokenOut, ok := b.Outstanding(out.ID) 102 if !ok { 103 t.Fatalf("should be outstanding") 104 } 105 if tokenOut != token { 106 t.Fatalf("Bad: %#v %#v", token, tokenOut) 107 } 108 109 // OutstandingReset should verify the token 110 err = b.OutstandingReset("nope", "foo") 111 if err != ErrNotOutstanding { 112 t.Fatalf("err: %v", err) 113 } 114 err = b.OutstandingReset(out.ID, "foo") 115 if err != ErrTokenMismatch { 116 t.Fatalf("err: %v", err) 117 } 118 err = b.OutstandingReset(out.ID, tokenOut) 119 if err != nil { 120 t.Fatalf("err: %v", err) 121 } 122 123 // Check the stats 124 stats = b.Stats() 125 if stats.TotalReady != 0 { 126 t.Fatalf("bad: %#v", stats) 127 } 128 if stats.TotalUnacked != 1 { 129 t.Fatalf("bad: %#v", stats) 130 } 131 if stats.ByScheduler[eval.Type].Ready != 0 { 132 t.Fatalf("bad: %#v", stats) 133 } 134 if stats.ByScheduler[eval.Type].Unacked != 1 { 135 t.Fatalf("bad: %#v", stats) 136 } 137 138 // Nack with wrong token should fail 139 err = b.Nack(eval.ID, "foobarbaz") 140 if err == nil { 141 t.Fatalf("should fail to nack") 142 } 143 144 // Nack back into the queue 145 err = b.Nack(eval.ID, token) 146 if err != nil { 147 t.Fatalf("err: %v", err) 148 } 149 150 if _, ok := b.Outstanding(out.ID); ok { 151 t.Fatalf("should not be outstanding") 152 } 153 154 // Check the stats 155 testutil.WaitForResult(func() (bool, error) { 156 stats = b.Stats() 157 if stats.TotalReady != 1 { 158 return false, fmt.Errorf("bad: %#v", stats) 159 } 160 if stats.TotalUnacked != 0 { 161 return false, fmt.Errorf("bad: %#v", stats) 162 } 163 if stats.TotalWaiting != 0 { 164 return false, fmt.Errorf("bad: %#v", stats) 165 } 166 if stats.ByScheduler[eval.Type].Ready != 1 { 167 return false, fmt.Errorf("bad: %#v", stats) 168 } 169 if stats.ByScheduler[eval.Type].Unacked != 0 { 170 return false, fmt.Errorf("bad: %#v", stats) 171 } 172 173 return true, nil 174 }, func(e error) { 175 t.Fatal(e) 176 }) 177 178 // Dequeue should work again 179 out2, token2, err := b.Dequeue(defaultSched, time.Second) 180 if err != nil { 181 t.Fatalf("err: %v", err) 182 } 183 if out2 != eval { 184 t.Fatalf("bad : %#v", out2) 185 } 186 if token2 == token { 187 t.Fatalf("should get a new token") 188 } 189 190 tokenOut2, ok := b.Outstanding(out.ID) 191 if !ok { 192 t.Fatalf("should be outstanding") 193 } 194 if tokenOut2 != token2 { 195 t.Fatalf("Bad: %#v %#v", token2, tokenOut2) 196 } 197 198 // Ack with wrong token 199 err = b.Ack(eval.ID, "zip") 200 if err == nil { 201 t.Fatalf("should fail to ack") 202 } 203 204 // Ack finally 205 err = b.Ack(eval.ID, token2) 206 if err != nil { 207 t.Fatalf("err: %v", err) 208 } 209 210 if _, ok := b.Outstanding(out.ID); ok { 211 t.Fatalf("should not be outstanding") 212 } 213 214 // Check the stats 215 stats = b.Stats() 216 if stats.TotalReady != 0 { 217 t.Fatalf("bad: %#v", stats) 218 } 219 if stats.TotalUnacked != 0 { 220 t.Fatalf("bad: %#v", stats) 221 } 222 if stats.ByScheduler[eval.Type].Ready != 0 { 223 t.Fatalf("bad: %#v", stats) 224 } 225 if stats.ByScheduler[eval.Type].Unacked != 0 { 226 t.Fatalf("bad: %#v", stats) 227 } 228 } 229 230 func TestEvalBroker_Nack_Delay(t *testing.T) { 231 t.Parallel() 232 b := testBroker(t, 0) 233 234 // Enqueue, but broker is disabled! 235 b.SetEnabled(true) 236 eval := mock.Eval() 237 b.Enqueue(eval) 238 239 // Dequeue should work 240 out, token, err := b.Dequeue(defaultSched, time.Second) 241 if err != nil { 242 t.Fatalf("err: %v", err) 243 } 244 if out != eval { 245 t.Fatalf("bad : %#v", out) 246 } 247 248 // Nack back into the queue 249 err = b.Nack(eval.ID, token) 250 if err != nil { 251 t.Fatalf("err: %v", err) 252 } 253 254 if _, ok := b.Outstanding(out.ID); ok { 255 t.Fatalf("should not be outstanding") 256 } 257 258 // Check the stats to ensure that it is waiting 259 stats := b.Stats() 260 if stats.TotalReady != 0 { 261 t.Fatalf("bad: %#v", stats) 262 } 263 if stats.TotalUnacked != 0 { 264 t.Fatalf("bad: %#v", stats) 265 } 266 if stats.TotalWaiting != 1 { 267 t.Fatalf("bad: %#v", stats) 268 } 269 if stats.ByScheduler[eval.Type].Ready != 0 { 270 t.Fatalf("bad: %#v", stats) 271 } 272 if stats.ByScheduler[eval.Type].Unacked != 0 { 273 t.Fatalf("bad: %#v", stats) 274 } 275 276 // Now wait for it to be re-enqueued 277 testutil.WaitForResult(func() (bool, error) { 278 stats = b.Stats() 279 if stats.TotalReady != 1 { 280 return false, fmt.Errorf("bad: %#v", stats) 281 } 282 if stats.TotalUnacked != 0 { 283 return false, fmt.Errorf("bad: %#v", stats) 284 } 285 if stats.TotalWaiting != 0 { 286 return false, fmt.Errorf("bad: %#v", stats) 287 } 288 if stats.ByScheduler[eval.Type].Ready != 1 { 289 return false, fmt.Errorf("bad: %#v", stats) 290 } 291 if stats.ByScheduler[eval.Type].Unacked != 0 { 292 return false, fmt.Errorf("bad: %#v", stats) 293 } 294 295 return true, nil 296 }, func(e error) { 297 t.Fatal(e) 298 }) 299 300 // Dequeue should work again 301 out2, token2, err := b.Dequeue(defaultSched, time.Second) 302 if err != nil { 303 t.Fatalf("err: %v", err) 304 } 305 if out2 != eval { 306 t.Fatalf("bad : %#v", out2) 307 } 308 if token2 == token { 309 t.Fatalf("should get a new token") 310 } 311 312 // Capture the time 313 start := time.Now() 314 315 // Nack back into the queue 316 err = b.Nack(eval.ID, token2) 317 if err != nil { 318 t.Fatalf("err: %v", err) 319 } 320 321 // Now wait for it to be re-enqueued 322 testutil.WaitForResult(func() (bool, error) { 323 stats = b.Stats() 324 if stats.TotalReady != 1 { 325 return false, fmt.Errorf("bad: %#v", stats) 326 } 327 if stats.TotalUnacked != 0 { 328 return false, fmt.Errorf("bad: %#v", stats) 329 } 330 if stats.TotalWaiting != 0 { 331 return false, fmt.Errorf("bad: %#v", stats) 332 } 333 if stats.ByScheduler[eval.Type].Ready != 1 { 334 return false, fmt.Errorf("bad: %#v", stats) 335 } 336 if stats.ByScheduler[eval.Type].Unacked != 0 { 337 return false, fmt.Errorf("bad: %#v", stats) 338 } 339 340 return true, nil 341 }, func(e error) { 342 t.Fatal(e) 343 }) 344 345 delay := time.Now().Sub(start) 346 if delay < b.subsequentNackDelay { 347 t.Fatalf("bad: delay was %v; want at least %v", delay, b.subsequentNackDelay) 348 } 349 350 // Dequeue should work again 351 out3, token3, err := b.Dequeue(defaultSched, time.Second) 352 if err != nil { 353 t.Fatalf("err: %v", err) 354 } 355 if out3 != eval { 356 t.Fatalf("bad : %#v", out3) 357 } 358 if token3 == token || token3 == token2 { 359 t.Fatalf("should get a new token") 360 } 361 362 // Ack finally 363 err = b.Ack(eval.ID, token3) 364 if err != nil { 365 t.Fatalf("err: %v", err) 366 } 367 368 if _, ok := b.Outstanding(out.ID); ok { 369 t.Fatalf("should not be outstanding") 370 } 371 372 // Check the stats 373 stats = b.Stats() 374 if stats.TotalReady != 0 { 375 t.Fatalf("bad: %#v", stats) 376 } 377 if stats.TotalUnacked != 0 { 378 t.Fatalf("bad: %#v", stats) 379 } 380 if stats.ByScheduler[eval.Type].Ready != 0 { 381 t.Fatalf("bad: %#v", stats) 382 } 383 if stats.ByScheduler[eval.Type].Unacked != 0 { 384 t.Fatalf("bad: %#v", stats) 385 } 386 } 387 388 func TestEvalBroker_Serialize_DuplicateJobID(t *testing.T) { 389 t.Parallel() 390 b := testBroker(t, 0) 391 b.SetEnabled(true) 392 393 ns1 := "namespace-one" 394 ns2 := "namespace-two" 395 eval := mock.Eval() 396 eval.Namespace = ns1 397 b.Enqueue(eval) 398 399 eval2 := mock.Eval() 400 eval2.JobID = eval.JobID 401 eval2.Namespace = ns1 402 eval2.CreateIndex = eval.CreateIndex + 1 403 b.Enqueue(eval2) 404 405 eval3 := mock.Eval() 406 eval3.JobID = eval.JobID 407 eval3.Namespace = ns1 408 eval3.CreateIndex = eval.CreateIndex + 2 409 b.Enqueue(eval3) 410 411 eval4 := mock.Eval() 412 eval4.JobID = eval.JobID 413 eval4.Namespace = ns2 414 eval4.CreateIndex = eval.CreateIndex + 3 415 b.Enqueue(eval4) 416 417 eval5 := mock.Eval() 418 eval5.JobID = eval.JobID 419 eval5.Namespace = ns2 420 eval5.CreateIndex = eval.CreateIndex + 4 421 b.Enqueue(eval5) 422 423 stats := b.Stats() 424 if stats.TotalReady != 2 { 425 t.Fatalf("bad: %#v", stats) 426 } 427 if stats.TotalBlocked != 3 { 428 t.Fatalf("bad: %#v", stats) 429 } 430 431 // Dequeue should work 432 out, token, err := b.Dequeue(defaultSched, time.Second) 433 if err != nil { 434 t.Fatalf("err: %v", err) 435 } 436 if out != eval { 437 t.Fatalf("bad : %#v", out) 438 } 439 440 // Check the stats 441 stats = b.Stats() 442 if stats.TotalReady != 1 { 443 t.Fatalf("bad: %#v", stats) 444 } 445 if stats.TotalUnacked != 1 { 446 t.Fatalf("bad: %#v", stats) 447 } 448 if stats.TotalBlocked != 3 { 449 t.Fatalf("bad: %#v", stats) 450 } 451 452 // Ack out 453 err = b.Ack(eval.ID, token) 454 if err != nil { 455 t.Fatalf("err: %v", err) 456 } 457 458 // Check the stats 459 stats = b.Stats() 460 if stats.TotalReady != 2 { 461 t.Fatalf("bad: %#v", stats) 462 } 463 if stats.TotalUnacked != 0 { 464 t.Fatalf("bad: %#v", stats) 465 } 466 if stats.TotalBlocked != 2 { 467 t.Fatalf("bad: %#v", stats) 468 } 469 470 // Dequeue should work 471 out, token, err = b.Dequeue(defaultSched, time.Second) 472 if err != nil { 473 t.Fatalf("err: %v", err) 474 } 475 if out != eval2 { 476 t.Fatalf("bad : %#v", out) 477 } 478 479 // Check the stats 480 stats = b.Stats() 481 if stats.TotalReady != 1 { 482 t.Fatalf("bad: %#v", stats) 483 } 484 if stats.TotalUnacked != 1 { 485 t.Fatalf("bad: %#v", stats) 486 } 487 if stats.TotalBlocked != 2 { 488 t.Fatalf("bad: %#v", stats) 489 } 490 491 // Ack out 492 err = b.Ack(eval2.ID, token) 493 if err != nil { 494 t.Fatalf("err: %v", err) 495 } 496 497 // Check the stats 498 stats = b.Stats() 499 if stats.TotalReady != 2 { 500 t.Fatalf("bad: %#v", stats) 501 } 502 if stats.TotalUnacked != 0 { 503 t.Fatalf("bad: %#v", stats) 504 } 505 if stats.TotalBlocked != 1 { 506 t.Fatalf("bad: %#v", stats) 507 } 508 509 // Dequeue should work 510 out, token, err = b.Dequeue(defaultSched, time.Second) 511 if err != nil { 512 t.Fatalf("err: %v", err) 513 } 514 if out != eval3 { 515 t.Fatalf("bad : %#v", out) 516 } 517 518 // Check the stats 519 stats = b.Stats() 520 if stats.TotalReady != 1 { 521 t.Fatalf("bad: %#v", stats) 522 } 523 if stats.TotalUnacked != 1 { 524 t.Fatalf("bad: %#v", stats) 525 } 526 if stats.TotalBlocked != 1 { 527 t.Fatalf("bad: %#v", stats) 528 } 529 530 // Ack out 531 err = b.Ack(eval3.ID, token) 532 if err != nil { 533 t.Fatalf("err: %v", err) 534 } 535 536 // Check the stats 537 stats = b.Stats() 538 if stats.TotalReady != 1 { 539 t.Fatalf("bad: %#v", stats) 540 } 541 if stats.TotalUnacked != 0 { 542 t.Fatalf("bad: %#v", stats) 543 } 544 if stats.TotalBlocked != 1 { 545 t.Fatalf("bad: %#v", stats) 546 } 547 548 // Dequeue should work 549 out, token, err = b.Dequeue(defaultSched, time.Second) 550 if err != nil { 551 t.Fatalf("err: %v", err) 552 } 553 if out != eval4 { 554 t.Fatalf("bad : %#v", out) 555 } 556 557 // Check the stats 558 stats = b.Stats() 559 if stats.TotalReady != 0 { 560 t.Fatalf("bad: %#v", stats) 561 } 562 if stats.TotalUnacked != 1 { 563 t.Fatalf("bad: %#v", stats) 564 } 565 if stats.TotalBlocked != 1 { 566 t.Fatalf("bad: %#v", stats) 567 } 568 569 // Ack out 570 err = b.Ack(eval4.ID, token) 571 if err != nil { 572 t.Fatalf("err: %v", err) 573 } 574 575 // Check the stats 576 stats = b.Stats() 577 if stats.TotalReady != 1 { 578 t.Fatalf("bad: %#v", stats) 579 } 580 if stats.TotalUnacked != 0 { 581 t.Fatalf("bad: %#v", stats) 582 } 583 if stats.TotalBlocked != 0 { 584 t.Fatalf("bad: %#v", stats) 585 } 586 587 // Dequeue should work 588 out, token, err = b.Dequeue(defaultSched, time.Second) 589 if err != nil { 590 t.Fatalf("err: %v", err) 591 } 592 if out != eval5 { 593 t.Fatalf("bad : %#v", out) 594 } 595 596 // Check the stats 597 stats = b.Stats() 598 if stats.TotalReady != 0 { 599 t.Fatalf("bad: %#v", stats) 600 } 601 if stats.TotalUnacked != 1 { 602 t.Fatalf("bad: %#v", stats) 603 } 604 if stats.TotalBlocked != 0 { 605 t.Fatalf("bad: %#v", stats) 606 } 607 608 // Ack out 609 err = b.Ack(eval5.ID, token) 610 if err != nil { 611 t.Fatalf("err: %v", err) 612 } 613 614 // Check the stats 615 stats = b.Stats() 616 if stats.TotalReady != 0 { 617 t.Fatalf("bad: %#v", stats) 618 } 619 if stats.TotalUnacked != 0 { 620 t.Fatalf("bad: %#v", stats) 621 } 622 if stats.TotalBlocked != 0 { 623 t.Fatalf("bad: %#v", stats) 624 } 625 } 626 627 func TestEvalBroker_Enqueue_Disable(t *testing.T) { 628 t.Parallel() 629 b := testBroker(t, 0) 630 631 // Enqueue 632 eval := mock.Eval() 633 b.SetEnabled(true) 634 b.Enqueue(eval) 635 636 // Flush via SetEnabled 637 b.SetEnabled(false) 638 639 // Check the stats 640 stats := b.Stats() 641 if stats.TotalReady != 0 { 642 t.Fatalf("bad: %#v", stats) 643 } 644 if stats.TotalUnacked != 0 { 645 t.Fatalf("bad: %#v", stats) 646 } 647 if _, ok := stats.ByScheduler[eval.Type]; ok { 648 t.Fatalf("bad: %#v", stats) 649 } 650 } 651 652 func TestEvalBroker_Enqueue_Disable_Delay(t *testing.T) { 653 t.Parallel() 654 b := testBroker(t, 0) 655 baseEval := mock.Eval() 656 b.SetEnabled(true) 657 658 { 659 // Enqueue 660 b.Enqueue(baseEval.Copy()) 661 662 delayedEval := baseEval.Copy() 663 delayedEval.Wait = 30 664 b.Enqueue(delayedEval) 665 666 waitEval := baseEval.Copy() 667 waitEval.WaitUntil = time.Now().Add(30 * time.Second) 668 b.Enqueue(waitEval) 669 } 670 671 // Flush via SetEnabled 672 b.SetEnabled(false) 673 674 { 675 // Check the stats 676 stats := b.Stats() 677 require.Equal(t, 0, stats.TotalReady, "Expected ready to be flushed") 678 require.Equal(t, 0, stats.TotalWaiting, "Expected waiting to be flushed") 679 require.Equal(t, 0, stats.TotalBlocked, "Expected blocked to be flushed") 680 require.Equal(t, 0, stats.TotalUnacked, "Expected unacked to be flushed") 681 _, ok := stats.ByScheduler[baseEval.Type] 682 require.False(t, ok, "Expected scheduler to have no stats") 683 } 684 685 { 686 // Enqueue again now we're disabled 687 b.Enqueue(baseEval.Copy()) 688 689 delayedEval := baseEval.Copy() 690 delayedEval.Wait = 30 * time.Second 691 b.Enqueue(delayedEval) 692 693 waitEval := baseEval.Copy() 694 waitEval.WaitUntil = time.Now().Add(30 * time.Second) 695 b.Enqueue(waitEval) 696 } 697 698 { 699 // Check the stats again 700 stats := b.Stats() 701 require.Equal(t, 0, stats.TotalReady, "Expected ready to be flushed") 702 require.Equal(t, 0, stats.TotalWaiting, "Expected waiting to be flushed") 703 require.Equal(t, 0, stats.TotalBlocked, "Expected blocked to be flushed") 704 require.Equal(t, 0, stats.TotalUnacked, "Expected unacked to be flushed") 705 _, ok := stats.ByScheduler[baseEval.Type] 706 require.False(t, ok, "Expected scheduler to have no stats") 707 } 708 } 709 710 func TestEvalBroker_Dequeue_Timeout(t *testing.T) { 711 t.Parallel() 712 b := testBroker(t, 0) 713 b.SetEnabled(true) 714 715 start := time.Now() 716 out, _, err := b.Dequeue(defaultSched, 5*time.Millisecond) 717 end := time.Now() 718 719 if err != nil { 720 t.Fatalf("err: %v", err) 721 } 722 if out != nil { 723 t.Fatalf("unexpected: %#v", out) 724 } 725 726 if diff := end.Sub(start); diff < 5*time.Millisecond { 727 t.Fatalf("bad: %#v", diff) 728 } 729 } 730 731 func TestEvalBroker_Dequeue_Empty_Timeout(t *testing.T) { 732 t.Parallel() 733 b := testBroker(t, 0) 734 b.SetEnabled(true) 735 736 errCh := make(chan error) 737 go func() { 738 defer close(errCh) 739 out, _, err := b.Dequeue(defaultSched, 0) 740 if err != nil { 741 errCh <- err 742 return 743 } 744 if out == nil { 745 errCh <- errors.New("expected a non-nil value") 746 return 747 } 748 }() 749 750 // Sleep for a little bit 751 select { 752 case <-time.After(5 * time.Millisecond): 753 case err := <-errCh: 754 if err != nil { 755 t.Fatalf("error from dequeue goroutine: %s", err) 756 } 757 t.Fatalf("Dequeue(0) should block, not finish") 758 } 759 760 // Enqueue to unblock the dequeue. 761 eval := mock.Eval() 762 b.Enqueue(eval) 763 764 select { 765 case err := <-errCh: 766 if err != nil { 767 t.Fatalf("error from dequeue goroutine: %s", err) 768 } 769 case <-time.After(5 * time.Millisecond): 770 t.Fatal("timeout: Dequeue(0) should return after enqueue") 771 } 772 } 773 774 // Ensure higher priority dequeued first 775 func TestEvalBroker_Dequeue_Priority(t *testing.T) { 776 t.Parallel() 777 b := testBroker(t, 0) 778 b.SetEnabled(true) 779 780 eval1 := mock.Eval() 781 eval1.Priority = 10 782 b.Enqueue(eval1) 783 784 eval2 := mock.Eval() 785 eval2.Priority = 30 786 b.Enqueue(eval2) 787 788 eval3 := mock.Eval() 789 eval3.Priority = 20 790 b.Enqueue(eval3) 791 792 out1, _, _ := b.Dequeue(defaultSched, time.Second) 793 if out1 != eval2 { 794 t.Fatalf("bad: %#v", out1) 795 } 796 797 out2, _, _ := b.Dequeue(defaultSched, time.Second) 798 if out2 != eval3 { 799 t.Fatalf("bad: %#v", out2) 800 } 801 802 out3, _, _ := b.Dequeue(defaultSched, time.Second) 803 if out3 != eval1 { 804 t.Fatalf("bad: %#v", out3) 805 } 806 } 807 808 // Ensure FIFO at fixed priority 809 func TestEvalBroker_Dequeue_FIFO(t *testing.T) { 810 t.Parallel() 811 b := testBroker(t, 0) 812 b.SetEnabled(true) 813 NUM := 100 814 815 for i := 0; i < NUM; i++ { 816 eval1 := mock.Eval() 817 eval1.CreateIndex = uint64(i) 818 eval1.ModifyIndex = uint64(i) 819 b.Enqueue(eval1) 820 } 821 822 for i := 0; i < NUM; i++ { 823 out1, _, _ := b.Dequeue(defaultSched, time.Second) 824 if out1.CreateIndex != uint64(i) { 825 t.Fatalf("bad: %d %#v", i, out1) 826 } 827 } 828 } 829 830 // Ensure fairness between schedulers 831 func TestEvalBroker_Dequeue_Fairness(t *testing.T) { 832 t.Parallel() 833 b := testBroker(t, 0) 834 b.SetEnabled(true) 835 NUM := 1000 836 837 for i := 0; i < NUM; i++ { 838 eval1 := mock.Eval() 839 if i < (NUM / 2) { 840 eval1.Type = structs.JobTypeService 841 } else { 842 eval1.Type = structs.JobTypeBatch 843 } 844 b.Enqueue(eval1) 845 } 846 847 counter := 0 848 for i := 0; i < NUM; i++ { 849 out1, _, _ := b.Dequeue(defaultSched, time.Second) 850 851 switch out1.Type { 852 case structs.JobTypeService: 853 if counter < 0 { 854 counter = 0 855 } 856 counter += 1 857 case structs.JobTypeBatch: 858 if counter > 0 { 859 counter = 0 860 } 861 counter -= 1 862 } 863 864 // This will fail randomly at times. It is very hard to 865 // test deterministically that its acting randomly. 866 if counter >= 250 || counter <= -250 { 867 t.Fatalf("unlikely sequence: %d", counter) 868 } 869 } 870 } 871 872 // Ensure we get unblocked 873 func TestEvalBroker_Dequeue_Blocked(t *testing.T) { 874 t.Parallel() 875 b := testBroker(t, 0) 876 b.SetEnabled(true) 877 878 // Start with a blocked dequeue 879 outCh := make(chan *structs.Evaluation) 880 errCh := make(chan error) 881 go func() { 882 defer close(errCh) 883 defer close(outCh) 884 start := time.Now() 885 out, _, err := b.Dequeue(defaultSched, time.Second) 886 if err != nil { 887 errCh <- err 888 return 889 } 890 end := time.Now() 891 if d := end.Sub(start); d < 5*time.Millisecond { 892 errCh <- fmt.Errorf("test broker dequeue duration too fast: %v", d) 893 return 894 } 895 outCh <- out 896 }() 897 898 // Wait for a bit, or t.Fatal if an error has already happened in 899 // the goroutine 900 select { 901 case <-time.After(5 * time.Millisecond): 902 // no errors yet, soldier on 903 case err := <-errCh: 904 if err != nil { 905 t.Fatalf("error from anonymous goroutine before enqueue: %v", err) 906 } 907 } 908 909 // Enqueue 910 eval := mock.Eval() 911 b.Enqueue(eval) 912 913 // Ensure dequeue 914 select { 915 case out := <-outCh: 916 if out != eval { 917 prettyExp, _ := json.MarshalIndent(eval, "", "\t") 918 prettyGot, _ := json.MarshalIndent(out, "", "\t") 919 t.Fatalf("dequeue result expected:\n%s\ngot:\n%s", 920 string(prettyExp), string(prettyGot)) 921 } 922 case err := <-errCh: 923 t.Fatalf("error from anonymous goroutine after enqueue: %v", err) 924 case <-time.After(time.Second): 925 t.Fatalf("timeout waiting for dequeue result") 926 } 927 } 928 929 // Ensure we nack in a timely manner 930 func TestEvalBroker_Nack_Timeout(t *testing.T) { 931 t.Parallel() 932 b := testBroker(t, 5*time.Millisecond) 933 b.SetEnabled(true) 934 935 // Enqueue 936 eval := mock.Eval() 937 b.Enqueue(eval) 938 939 // Dequeue 940 out, _, err := b.Dequeue(defaultSched, time.Second) 941 start := time.Now() 942 if err != nil { 943 t.Fatalf("err: %v", err) 944 } 945 if out != eval { 946 t.Fatalf("bad: %v", out) 947 } 948 949 // Dequeue, should block on Nack timer 950 out, _, err = b.Dequeue(defaultSched, time.Second) 951 end := time.Now() 952 if err != nil { 953 t.Fatalf("err: %v", err) 954 } 955 if out != eval { 956 t.Fatalf("bad: %v", out) 957 } 958 959 // Check the nack timer 960 if diff := end.Sub(start); diff < 5*time.Millisecond { 961 t.Fatalf("bad: %#v", diff) 962 } 963 } 964 965 // Ensure we nack in a timely manner 966 func TestEvalBroker_Nack_TimeoutReset(t *testing.T) { 967 t.Parallel() 968 b := testBroker(t, 50*time.Millisecond) 969 b.SetEnabled(true) 970 971 // Enqueue 972 eval := mock.Eval() 973 b.Enqueue(eval) 974 975 // Dequeue 976 out, token, err := b.Dequeue(defaultSched, time.Second) 977 start := time.Now() 978 if err != nil { 979 t.Fatalf("err: %v", err) 980 } 981 if out != eval { 982 t.Fatalf("bad: %v", out) 983 } 984 985 // Reset in 20 milliseconds 986 time.Sleep(20 * time.Millisecond) 987 if err := b.OutstandingReset(out.ID, token); err != nil { 988 t.Fatalf("err: %v", err) 989 } 990 991 // Dequeue, should block on Nack timer 992 out, _, err = b.Dequeue(defaultSched, time.Second) 993 end := time.Now() 994 if err != nil { 995 t.Fatalf("err: %v", err) 996 } 997 if out != eval { 998 t.Fatalf("bad: %v", out) 999 } 1000 1001 // Check the nack timer 1002 if diff := end.Sub(start); diff < 75*time.Millisecond { 1003 t.Fatalf("bad: %#v", diff) 1004 } 1005 } 1006 1007 func TestEvalBroker_PauseResumeNackTimeout(t *testing.T) { 1008 t.Parallel() 1009 b := testBroker(t, 50*time.Millisecond) 1010 b.SetEnabled(true) 1011 1012 // Enqueue 1013 eval := mock.Eval() 1014 b.Enqueue(eval) 1015 1016 // Dequeue 1017 out, token, err := b.Dequeue(defaultSched, time.Second) 1018 start := time.Now() 1019 if err != nil { 1020 t.Fatalf("err: %v", err) 1021 } 1022 if out != eval { 1023 t.Fatalf("bad: %v", out) 1024 } 1025 1026 // Pause in 20 milliseconds 1027 time.Sleep(20 * time.Millisecond) 1028 if err := b.PauseNackTimeout(out.ID, token); err != nil { 1029 t.Fatalf("pause nack timeout error: %v", err) 1030 } 1031 1032 errCh := make(chan error) 1033 go func() { 1034 defer close(errCh) 1035 time.Sleep(20 * time.Millisecond) 1036 if err := b.ResumeNackTimeout(out.ID, token); err != nil { 1037 errCh <- err 1038 return 1039 } 1040 }() 1041 1042 // Dequeue, should block until the timer is resumed 1043 out, _, err = b.Dequeue(defaultSched, time.Second) 1044 end := time.Now() 1045 if err != nil { 1046 t.Fatalf("dequeue error: %v", err) 1047 } 1048 if out != eval { 1049 prettyExp, _ := json.MarshalIndent(eval, "", "\t") 1050 prettyGot, _ := json.MarshalIndent(out, "", "\t") 1051 t.Fatalf("dequeue result expected:\n%s\ngot:\n%s", 1052 string(prettyExp), string(prettyGot)) 1053 } 1054 1055 // Check the nack timer 1056 if diff := end.Sub(start); diff < 95*time.Millisecond { 1057 t.Fatalf("deqeue happened too fast: %#v", diff) 1058 } 1059 1060 // check the result of ResumeNackTimeout 1061 err = <-errCh 1062 if err != nil { 1063 t.Fatalf("resume nack timeout error:%s", err) 1064 } 1065 } 1066 1067 func TestEvalBroker_DeliveryLimit(t *testing.T) { 1068 t.Parallel() 1069 b := testBroker(t, 0) 1070 b.SetEnabled(true) 1071 1072 eval := mock.Eval() 1073 b.Enqueue(eval) 1074 1075 for i := 0; i < 3; i++ { 1076 // Dequeue should work 1077 out, token, err := b.Dequeue(defaultSched, time.Second) 1078 if err != nil { 1079 t.Fatalf("err: %v", err) 1080 } 1081 if out != eval { 1082 t.Fatalf("bad : %#v", out) 1083 } 1084 1085 // Nack with wrong token should fail 1086 err = b.Nack(eval.ID, token) 1087 if err != nil { 1088 t.Fatalf("err: %v", err) 1089 } 1090 } 1091 1092 // Check the stats 1093 stats := b.Stats() 1094 if stats.TotalReady != 1 { 1095 t.Fatalf("bad: %#v", stats) 1096 } 1097 if stats.TotalUnacked != 0 { 1098 t.Fatalf("bad: %#v", stats) 1099 } 1100 if stats.ByScheduler[failedQueue].Ready != 1 { 1101 t.Fatalf("bad: %#v", stats) 1102 } 1103 if stats.ByScheduler[failedQueue].Unacked != 0 { 1104 t.Fatalf("bad: %#v", stats) 1105 } 1106 1107 // Dequeue from failed queue 1108 out, token, err := b.Dequeue([]string{failedQueue}, time.Second) 1109 if err != nil { 1110 t.Fatalf("err: %v", err) 1111 } 1112 if out != eval { 1113 t.Fatalf("bad : %#v", out) 1114 } 1115 1116 // Check the stats 1117 stats = b.Stats() 1118 if stats.TotalReady != 0 { 1119 t.Fatalf("bad: %#v", stats) 1120 } 1121 if stats.TotalUnacked != 1 { 1122 t.Fatalf("bad: %#v", stats) 1123 } 1124 if stats.ByScheduler[failedQueue].Ready != 0 { 1125 t.Fatalf("bad: %#v", stats) 1126 } 1127 if stats.ByScheduler[failedQueue].Unacked != 1 { 1128 t.Fatalf("bad: %#v", stats) 1129 } 1130 1131 // Ack finally 1132 err = b.Ack(out.ID, token) 1133 if err != nil { 1134 t.Fatalf("err: %v", err) 1135 } 1136 1137 if _, ok := b.Outstanding(out.ID); ok { 1138 t.Fatalf("should not be outstanding") 1139 } 1140 1141 // Check the stats 1142 stats = b.Stats() 1143 if stats.TotalReady != 0 { 1144 t.Fatalf("bad: %#v", stats) 1145 } 1146 if stats.TotalUnacked != 0 { 1147 t.Fatalf("bad: %#v", stats) 1148 } 1149 if stats.ByScheduler[failedQueue].Ready != 0 { 1150 t.Fatalf("bad: %#v", stats.ByScheduler[failedQueue]) 1151 } 1152 if stats.ByScheduler[failedQueue].Unacked != 0 { 1153 t.Fatalf("bad: %#v", stats.ByScheduler[failedQueue]) 1154 } 1155 } 1156 1157 func TestEvalBroker_AckAtDeliveryLimit(t *testing.T) { 1158 t.Parallel() 1159 b := testBroker(t, 0) 1160 b.SetEnabled(true) 1161 1162 eval := mock.Eval() 1163 b.Enqueue(eval) 1164 1165 for i := 0; i < 3; i++ { 1166 // Dequeue should work 1167 out, token, err := b.Dequeue(defaultSched, time.Second) 1168 if err != nil { 1169 t.Fatalf("err: %v", err) 1170 } 1171 if out != eval { 1172 t.Fatalf("bad : %#v", out) 1173 } 1174 1175 if i == 2 { 1176 b.Ack(eval.ID, token) 1177 } else { 1178 // Nack with wrong token should fail 1179 err = b.Nack(eval.ID, token) 1180 if err != nil { 1181 t.Fatalf("err: %v", err) 1182 } 1183 } 1184 } 1185 1186 // Check the stats 1187 stats := b.Stats() 1188 if stats.TotalReady != 0 { 1189 t.Fatalf("bad: %#v", stats) 1190 } 1191 if stats.TotalUnacked != 0 { 1192 t.Fatalf("bad: %#v", stats) 1193 } 1194 if _, ok := stats.ByScheduler[failedQueue]; ok { 1195 t.Fatalf("bad: %#v", stats) 1196 } 1197 } 1198 1199 // Ensure fairness between schedulers 1200 func TestEvalBroker_Wait(t *testing.T) { 1201 t.Parallel() 1202 b := testBroker(t, 0) 1203 b.SetEnabled(true) 1204 1205 // Create an eval that should wait 1206 eval := mock.Eval() 1207 eval.Wait = 10 * time.Millisecond 1208 b.Enqueue(eval) 1209 1210 // Verify waiting 1211 stats := b.Stats() 1212 if stats.TotalReady != 0 { 1213 t.Fatalf("bad: %#v", stats) 1214 } 1215 if stats.TotalWaiting != 1 { 1216 t.Fatalf("bad: %#v", stats) 1217 } 1218 1219 // Let the wait elapse 1220 time.Sleep(20 * time.Millisecond) 1221 1222 // Verify ready 1223 stats = b.Stats() 1224 if stats.TotalReady != 1 { 1225 t.Fatalf("bad: %#v", stats) 1226 } 1227 if stats.TotalWaiting != 0 { 1228 t.Fatalf("bad: %#v", stats) 1229 } 1230 1231 // Dequeue should work 1232 out, _, err := b.Dequeue(defaultSched, time.Second) 1233 if err != nil { 1234 t.Fatalf("err: %v", err) 1235 } 1236 if out != eval { 1237 t.Fatalf("bad : %#v", out) 1238 } 1239 } 1240 1241 // Ensure that delayed evaluations work as expected 1242 func TestEvalBroker_WaitUntil(t *testing.T) { 1243 t.Parallel() 1244 require := require.New(t) 1245 b := testBroker(t, 0) 1246 b.SetEnabled(true) 1247 1248 now := time.Now() 1249 // Create a few of evals with WaitUntil set 1250 eval1 := mock.Eval() 1251 eval1.WaitUntil = now.Add(1 * time.Second) 1252 eval1.CreateIndex = 1 1253 b.Enqueue(eval1) 1254 1255 eval2 := mock.Eval() 1256 eval2.WaitUntil = now.Add(100 * time.Millisecond) 1257 // set CreateIndex to use as a tie breaker when eval2 1258 // and eval3 are both in the pending evals heap 1259 eval2.CreateIndex = 2 1260 b.Enqueue(eval2) 1261 1262 eval3 := mock.Eval() 1263 eval3.WaitUntil = now.Add(20 * time.Millisecond) 1264 eval3.CreateIndex = 1 1265 b.Enqueue(eval3) 1266 require.Equal(3, b.stats.TotalWaiting) 1267 // sleep enough for two evals to be ready 1268 time.Sleep(200 * time.Millisecond) 1269 1270 // first dequeue should return eval3 1271 out, _, err := b.Dequeue(defaultSched, time.Second) 1272 require.Nil(err) 1273 require.Equal(eval3, out) 1274 1275 // second dequeue should return eval2 1276 out, _, err = b.Dequeue(defaultSched, time.Second) 1277 require.Nil(err) 1278 require.Equal(eval2, out) 1279 1280 // third dequeue should return eval1 1281 out, _, err = b.Dequeue(defaultSched, 2*time.Second) 1282 require.Nil(err) 1283 require.Equal(eval1, out) 1284 require.Equal(0, b.stats.TotalWaiting) 1285 } 1286 1287 // Ensure that priority is taken into account when enqueueing many evaluations. 1288 func TestEvalBroker_EnqueueAll_Dequeue_Fair(t *testing.T) { 1289 t.Parallel() 1290 b := testBroker(t, 0) 1291 b.SetEnabled(true) 1292 1293 // Start with a blocked dequeue 1294 outCh := make(chan *structs.Evaluation) 1295 errCh := make(chan error) 1296 go func() { 1297 defer close(errCh) 1298 defer close(outCh) 1299 start := time.Now() 1300 out, _, err := b.Dequeue(defaultSched, time.Second) 1301 if err != nil { 1302 errCh <- err 1303 return 1304 } 1305 end := time.Now() 1306 if d := end.Sub(start); d < 5*time.Millisecond { 1307 errCh <- fmt.Errorf("test broker dequeue duration too fast: %v", d) 1308 return 1309 } 1310 outCh <- out 1311 }() 1312 1313 // Wait for a bit, or t.Fatal if an error has already happened in 1314 // the goroutine 1315 select { 1316 case <-time.After(5 * time.Millisecond): 1317 // no errors yet, soldier on 1318 case err := <-errCh: 1319 if err != nil { 1320 t.Fatalf("error from anonymous goroutine before enqueue: %v", err) 1321 } 1322 } 1323 1324 // Enqueue 1325 evals := make(map[*structs.Evaluation]string, 8) 1326 expectedPriority := 90 1327 for i := 10; i <= expectedPriority; i += 10 { 1328 eval := mock.Eval() 1329 eval.Priority = i 1330 evals[eval] = "" 1331 1332 } 1333 b.EnqueueAll(evals) 1334 1335 // Ensure dequeue 1336 select { 1337 case out := <-outCh: 1338 if out.Priority != expectedPriority { 1339 pretty, _ := json.MarshalIndent(out, "", "\t") 1340 t.Logf("bad priority on *structs.Evaluation: %s", string(pretty)) 1341 t.Fatalf("priority wanted:%d, priority got:%d", expectedPriority, out.Priority) 1342 } 1343 case err := <-errCh: 1344 t.Fatalf("error from anonymous goroutine after enqueue: %v", err) 1345 case <-time.After(time.Second): 1346 t.Fatalf("timeout waiting for dequeue result") 1347 } 1348 } 1349 1350 func TestEvalBroker_EnqueueAll_Requeue_Ack(t *testing.T) { 1351 t.Parallel() 1352 b := testBroker(t, 0) 1353 b.SetEnabled(true) 1354 1355 // Create the evaluation, enqueue and dequeue 1356 eval := mock.Eval() 1357 b.Enqueue(eval) 1358 1359 out, token, err := b.Dequeue(defaultSched, time.Second) 1360 if err != nil { 1361 t.Fatalf("err: %v", err) 1362 } 1363 if out != eval { 1364 t.Fatalf("bad : %#v", out) 1365 } 1366 1367 // Requeue the same evaluation. 1368 b.EnqueueAll(map[*structs.Evaluation]string{eval: token}) 1369 1370 // The stats should show one unacked 1371 stats := b.Stats() 1372 if stats.TotalReady != 0 { 1373 t.Fatalf("bad: %#v", stats) 1374 } 1375 if stats.TotalUnacked != 1 { 1376 t.Fatalf("bad: %#v", stats) 1377 } 1378 1379 // Ack the evaluation. 1380 if err := b.Ack(eval.ID, token); err != nil { 1381 t.Fatalf("err: %v", err) 1382 } 1383 1384 // Check stats again as this should cause the re-enqueued one to transition 1385 // into the ready state 1386 stats = b.Stats() 1387 if stats.TotalReady != 1 { 1388 t.Fatalf("bad: %#v", stats) 1389 } 1390 if stats.TotalUnacked != 0 { 1391 t.Fatalf("bad: %#v", stats) 1392 } 1393 1394 // Another dequeue should be successful 1395 out2, token2, err := b.Dequeue(defaultSched, time.Second) 1396 if err != nil { 1397 t.Fatalf("err: %v", err) 1398 } 1399 if out2 != eval { 1400 t.Fatalf("bad : %#v", out) 1401 } 1402 if token == token2 { 1403 t.Fatalf("bad : %s and %s", token, token2) 1404 } 1405 } 1406 1407 func TestEvalBroker_EnqueueAll_Requeue_Nack(t *testing.T) { 1408 t.Parallel() 1409 b := testBroker(t, 0) 1410 b.SetEnabled(true) 1411 1412 // Create the evaluation, enqueue and dequeue 1413 eval := mock.Eval() 1414 b.Enqueue(eval) 1415 1416 out, token, err := b.Dequeue(defaultSched, time.Second) 1417 if err != nil { 1418 t.Fatalf("err: %v", err) 1419 } 1420 if out != eval { 1421 t.Fatalf("bad : %#v", out) 1422 } 1423 1424 // Requeue the same evaluation. 1425 b.EnqueueAll(map[*structs.Evaluation]string{eval: token}) 1426 1427 // The stats should show one unacked 1428 stats := b.Stats() 1429 if stats.TotalReady != 0 { 1430 t.Fatalf("bad: %#v", stats) 1431 } 1432 if stats.TotalUnacked != 1 { 1433 t.Fatalf("bad: %#v", stats) 1434 } 1435 1436 // Nack the evaluation. 1437 if err := b.Nack(eval.ID, token); err != nil { 1438 t.Fatalf("err: %v", err) 1439 } 1440 1441 // Check stats again as this should cause the re-enqueued one to be dropped 1442 testutil.WaitForResult(func() (bool, error) { 1443 stats = b.Stats() 1444 if stats.TotalReady != 1 { 1445 return false, fmt.Errorf("bad: %#v", stats) 1446 } 1447 if stats.TotalUnacked != 0 { 1448 return false, fmt.Errorf("bad: %#v", stats) 1449 } 1450 if len(b.requeue) != 0 { 1451 return false, fmt.Errorf("bad: %#v", b.requeue) 1452 } 1453 1454 return true, nil 1455 }, func(e error) { 1456 t.Fatal(e) 1457 }) 1458 } 1459 1460 func TestEvalBroker_NamespacedJobs(t *testing.T) { 1461 t.Parallel() 1462 b := testBroker(t, 0) 1463 b.SetEnabled(true) 1464 1465 // Create evals with the same jobid and different namespace 1466 jobId := "test-jobID" 1467 1468 eval1 := mock.Eval() 1469 eval1.JobID = jobId 1470 eval1.Namespace = "n1" 1471 b.Enqueue(eval1) 1472 1473 // This eval should not block 1474 eval2 := mock.Eval() 1475 eval2.JobID = jobId 1476 eval2.Namespace = "default" 1477 b.Enqueue(eval2) 1478 1479 // This eval should block 1480 eval3 := mock.Eval() 1481 eval3.JobID = jobId 1482 eval3.Namespace = "default" 1483 b.Enqueue(eval3) 1484 1485 require := require.New(t) 1486 out1, _, err := b.Dequeue(defaultSched, 5*time.Millisecond) 1487 require.Nil(err) 1488 require.Equal(eval1.ID, out1.ID) 1489 1490 out2, _, err := b.Dequeue(defaultSched, 5*time.Millisecond) 1491 require.Nil(err) 1492 require.Equal(eval2.ID, out2.ID) 1493 1494 out3, _, err := b.Dequeue(defaultSched, 5*time.Millisecond) 1495 require.Nil(err) 1496 require.Nil(out3) 1497 1498 require.Equal(1, len(b.blocked)) 1499 1500 }