github.com/hernad/nomad@v1.6.112/nomad/eval_broker_test.go (about) 1 // Copyright (c) HashiCorp, Inc. 2 // SPDX-License-Identifier: MPL-2.0 3 4 package nomad 5 6 import ( 7 "container/heap" 8 "encoding/json" 9 "errors" 10 "fmt" 11 "testing" 12 "time" 13 14 msgpackrpc "github.com/hashicorp/net-rpc-msgpackrpc" 15 "github.com/shoenig/test" 16 "github.com/shoenig/test/must" 17 "github.com/shoenig/test/wait" 18 "github.com/stretchr/testify/require" 19 20 "github.com/hernad/nomad/ci" 21 "github.com/hernad/nomad/nomad/mock" 22 "github.com/hernad/nomad/nomad/state" 23 "github.com/hernad/nomad/nomad/structs" 24 "github.com/hernad/nomad/testutil" 25 ) 26 27 var ( 28 defaultSched = []string{ 29 structs.JobTypeService, 30 structs.JobTypeBatch, 31 } 32 ) 33 34 func testBrokerConfig() *Config { 35 config := DefaultConfig() 36 37 // Tune the Nack timeout 38 config.EvalNackTimeout = 5 * time.Second 39 40 // Tune the Nack delay 41 config.EvalNackInitialReenqueueDelay = 5 * time.Millisecond 42 config.EvalNackSubsequentReenqueueDelay = 50 * time.Millisecond 43 return config 44 } 45 46 func testBroker(t *testing.T, timeout time.Duration) *EvalBroker { 47 config := testBrokerConfig() 48 49 if timeout != 0 { 50 config.EvalNackTimeout = timeout 51 } 52 53 return testBrokerFromConfig(t, config) 54 } 55 56 func testBrokerFromConfig(t *testing.T, c *Config) *EvalBroker { 57 b, err := NewEvalBroker(c.EvalNackTimeout, c.EvalNackInitialReenqueueDelay, c.EvalNackSubsequentReenqueueDelay, 3) 58 if err != nil { 59 t.Fatalf("err: %v", err) 60 } 61 62 return b 63 } 64 65 func TestEvalBroker_Enqueue_Dequeue_Nack_Ack(t *testing.T) { 66 ci.Parallel(t) 67 b := testBroker(t, 0) 68 69 // Enqueue, but broker is disabled! 70 eval := mock.Eval() 71 b.Enqueue(eval) 72 73 // Verify nothing was done 74 stats := b.Stats() 75 if stats.TotalReady != 0 { 76 t.Fatalf("bad: %#v", stats) 77 } 78 79 if b.Enabled() { 80 t.Fatalf("should not be enabled") 81 } 82 83 // Enable the broker, and enqueue 84 b.SetEnabled(true) 85 b.Enqueue(eval) 86 87 // Double enqueue is a no-op 88 b.Enqueue(eval) 89 90 if !b.Enabled() { 91 t.Fatalf("should be enabled") 92 } 93 94 // Verify enqueue is done 95 stats = b.Stats() 96 if stats.TotalReady != 1 { 97 t.Fatalf("bad: %#v", stats) 98 } 99 if stats.ByScheduler[eval.Type].Ready != 1 { 100 t.Fatalf("bad: %#v", stats) 101 } 102 103 // Dequeue should work 104 out, token, err := b.Dequeue(defaultSched, time.Second) 105 if err != nil { 106 t.Fatalf("err: %v", err) 107 } 108 if out != eval { 109 t.Fatalf("bad : %#v", out) 110 } 111 112 tokenOut, ok := b.Outstanding(out.ID) 113 if !ok { 114 t.Fatalf("should be outstanding") 115 } 116 if tokenOut != token { 117 t.Fatalf("Bad: %#v %#v", token, tokenOut) 118 } 119 120 // OutstandingReset should verify the token 121 err = b.OutstandingReset("nope", "foo") 122 if err != ErrNotOutstanding { 123 t.Fatalf("err: %v", err) 124 } 125 err = b.OutstandingReset(out.ID, "foo") 126 if err != ErrTokenMismatch { 127 t.Fatalf("err: %v", err) 128 } 129 err = b.OutstandingReset(out.ID, tokenOut) 130 if err != nil { 131 t.Fatalf("err: %v", err) 132 } 133 134 // Check the stats 135 stats = b.Stats() 136 if stats.TotalReady != 0 { 137 t.Fatalf("bad: %#v", stats) 138 } 139 if stats.TotalUnacked != 1 { 140 t.Fatalf("bad: %#v", stats) 141 } 142 if stats.ByScheduler[eval.Type].Ready != 0 { 143 t.Fatalf("bad: %#v", stats) 144 } 145 if stats.ByScheduler[eval.Type].Unacked != 1 { 146 t.Fatalf("bad: %#v", stats) 147 } 148 149 // Nack with wrong token should fail 150 err = b.Nack(eval.ID, "foobarbaz") 151 if err == nil { 152 t.Fatalf("should fail to nack") 153 } 154 155 // Nack back into the queue 156 err = b.Nack(eval.ID, token) 157 if err != nil { 158 t.Fatalf("err: %v", err) 159 } 160 161 if _, ok := b.Outstanding(out.ID); ok { 162 t.Fatalf("should not be outstanding") 163 } 164 165 // Check the stats 166 testutil.WaitForResult(func() (bool, error) { 167 stats = b.Stats() 168 if stats.TotalReady != 1 { 169 return false, fmt.Errorf("bad: %#v", stats) 170 } 171 if stats.TotalUnacked != 0 { 172 return false, fmt.Errorf("bad: %#v", stats) 173 } 174 if stats.TotalWaiting != 0 { 175 return false, fmt.Errorf("bad: %#v", stats) 176 } 177 if stats.ByScheduler[eval.Type].Ready != 1 { 178 return false, fmt.Errorf("bad: %#v", stats) 179 } 180 if stats.ByScheduler[eval.Type].Unacked != 0 { 181 return false, fmt.Errorf("bad: %#v", stats) 182 } 183 184 return true, nil 185 }, func(e error) { 186 t.Fatal(e) 187 }) 188 189 // Dequeue should work again 190 out2, token2, err := b.Dequeue(defaultSched, time.Second) 191 if err != nil { 192 t.Fatalf("err: %v", err) 193 } 194 if out2 != eval { 195 t.Fatalf("bad : %#v", out2) 196 } 197 if token2 == token { 198 t.Fatalf("should get a new token") 199 } 200 201 tokenOut2, ok := b.Outstanding(out.ID) 202 if !ok { 203 t.Fatalf("should be outstanding") 204 } 205 if tokenOut2 != token2 { 206 t.Fatalf("Bad: %#v %#v", token2, tokenOut2) 207 } 208 209 // Ack with wrong token 210 err = b.Ack(eval.ID, "zip") 211 if err == nil { 212 t.Fatalf("should fail to ack") 213 } 214 215 // Ack finally 216 err = b.Ack(eval.ID, token2) 217 if err != nil { 218 t.Fatalf("err: %v", err) 219 } 220 221 if _, ok := b.Outstanding(out.ID); ok { 222 t.Fatalf("should not be outstanding") 223 } 224 225 // Check the stats 226 stats = b.Stats() 227 if stats.TotalReady != 0 { 228 t.Fatalf("bad: %#v", stats) 229 } 230 if stats.TotalUnacked != 0 { 231 t.Fatalf("bad: %#v", stats) 232 } 233 if stats.ByScheduler[eval.Type].Ready != 0 { 234 t.Fatalf("bad: %#v", stats) 235 } 236 if stats.ByScheduler[eval.Type].Unacked != 0 { 237 t.Fatalf("bad: %#v", stats) 238 } 239 } 240 241 func TestEvalBroker_Nack_Delay(t *testing.T) { 242 ci.Parallel(t) 243 b := testBroker(t, 0) 244 245 // Enqueue, but broker is disabled! 246 b.SetEnabled(true) 247 eval := mock.Eval() 248 b.Enqueue(eval) 249 250 // Dequeue should work 251 out, token, err := b.Dequeue(defaultSched, time.Second) 252 if err != nil { 253 t.Fatalf("err: %v", err) 254 } 255 if out != eval { 256 t.Fatalf("bad : %#v", out) 257 } 258 259 // Nack back into the queue 260 err = b.Nack(eval.ID, token) 261 if err != nil { 262 t.Fatalf("err: %v", err) 263 } 264 265 if _, ok := b.Outstanding(out.ID); ok { 266 t.Fatalf("should not be outstanding") 267 } 268 269 // Check the stats to ensure that it is waiting 270 stats := b.Stats() 271 if stats.TotalReady != 0 { 272 t.Fatalf("bad: %#v", stats) 273 } 274 if stats.TotalUnacked != 0 { 275 t.Fatalf("bad: %#v", stats) 276 } 277 if stats.TotalWaiting != 1 { 278 t.Fatalf("bad: %#v", stats) 279 } 280 if stats.ByScheduler[eval.Type].Ready != 0 { 281 t.Fatalf("bad: %#v", stats) 282 } 283 if stats.ByScheduler[eval.Type].Unacked != 0 { 284 t.Fatalf("bad: %#v", stats) 285 } 286 287 // Now wait for it to be re-enqueued 288 testutil.WaitForResult(func() (bool, error) { 289 stats = b.Stats() 290 if stats.TotalReady != 1 { 291 return false, fmt.Errorf("bad: %#v", stats) 292 } 293 if stats.TotalUnacked != 0 { 294 return false, fmt.Errorf("bad: %#v", stats) 295 } 296 if stats.TotalWaiting != 0 { 297 return false, fmt.Errorf("bad: %#v", stats) 298 } 299 if stats.ByScheduler[eval.Type].Ready != 1 { 300 return false, fmt.Errorf("bad: %#v", stats) 301 } 302 if stats.ByScheduler[eval.Type].Unacked != 0 { 303 return false, fmt.Errorf("bad: %#v", stats) 304 } 305 306 return true, nil 307 }, func(e error) { 308 t.Fatal(e) 309 }) 310 311 // Dequeue should work again 312 out2, token2, err := b.Dequeue(defaultSched, time.Second) 313 if err != nil { 314 t.Fatalf("err: %v", err) 315 } 316 if out2 != eval { 317 t.Fatalf("bad : %#v", out2) 318 } 319 if token2 == token { 320 t.Fatalf("should get a new token") 321 } 322 323 // Capture the time 324 start := time.Now() 325 326 // Nack back into the queue 327 err = b.Nack(eval.ID, token2) 328 if err != nil { 329 t.Fatalf("err: %v", err) 330 } 331 332 // Now wait for it to be re-enqueued 333 testutil.WaitForResult(func() (bool, error) { 334 stats = b.Stats() 335 if stats.TotalReady != 1 { 336 return false, fmt.Errorf("bad: %#v", stats) 337 } 338 if stats.TotalUnacked != 0 { 339 return false, fmt.Errorf("bad: %#v", stats) 340 } 341 if stats.TotalWaiting != 0 { 342 return false, fmt.Errorf("bad: %#v", stats) 343 } 344 if stats.ByScheduler[eval.Type].Ready != 1 { 345 return false, fmt.Errorf("bad: %#v", stats) 346 } 347 if stats.ByScheduler[eval.Type].Unacked != 0 { 348 return false, fmt.Errorf("bad: %#v", stats) 349 } 350 351 return true, nil 352 }, func(e error) { 353 t.Fatal(e) 354 }) 355 356 delay := time.Now().Sub(start) 357 if delay < b.subsequentNackDelay { 358 t.Fatalf("bad: delay was %v; want at least %v", delay, b.subsequentNackDelay) 359 } 360 361 // Dequeue should work again 362 out3, token3, err := b.Dequeue(defaultSched, time.Second) 363 if err != nil { 364 t.Fatalf("err: %v", err) 365 } 366 if out3 != eval { 367 t.Fatalf("bad : %#v", out3) 368 } 369 if token3 == token || token3 == token2 { 370 t.Fatalf("should get a new token") 371 } 372 373 // Ack finally 374 err = b.Ack(eval.ID, token3) 375 if err != nil { 376 t.Fatalf("err: %v", err) 377 } 378 379 if _, ok := b.Outstanding(out.ID); ok { 380 t.Fatalf("should not be outstanding") 381 } 382 383 // Check the stats 384 stats = b.Stats() 385 if stats.TotalReady != 0 { 386 t.Fatalf("bad: %#v", stats) 387 } 388 if stats.TotalUnacked != 0 { 389 t.Fatalf("bad: %#v", stats) 390 } 391 if stats.ByScheduler[eval.Type].Ready != 0 { 392 t.Fatalf("bad: %#v", stats) 393 } 394 if stats.ByScheduler[eval.Type].Unacked != 0 { 395 t.Fatalf("bad: %#v", stats) 396 } 397 } 398 399 func TestEvalBroker_Serialize_DuplicateJobID(t *testing.T) { 400 ci.Parallel(t) 401 b := testBroker(t, 0) 402 b.SetEnabled(true) 403 404 ns1 := "namespace-one" 405 ns2 := "namespace-two" 406 jobID := "example" 407 408 newEval := func(idx uint64, ns string) *structs.Evaluation { 409 eval := mock.Eval() 410 eval.ID = fmt.Sprintf("eval:%d", idx) 411 eval.JobID = jobID 412 eval.Namespace = ns 413 eval.CreateIndex = idx 414 eval.ModifyIndex = idx 415 b.Enqueue(eval) 416 return eval 417 } 418 419 // first job 420 eval1 := newEval(1, ns1) 421 newEval(2, ns1) 422 newEval(3, ns1) 423 eval4 := newEval(4, ns1) 424 425 // second job 426 eval5 := newEval(5, ns2) 427 newEval(6, ns2) 428 eval7 := newEval(7, ns2) 429 430 // retreive the stats from the broker, less some stats that aren't 431 // interesting for this test and make the test much more verbose 432 // to include 433 getStats := func() BrokerStats { 434 t.Helper() 435 stats := b.Stats() 436 stats.DelayedEvals = nil 437 stats.ByScheduler = nil 438 return *stats 439 } 440 441 must.Eq(t, BrokerStats{TotalReady: 2, TotalUnacked: 0, 442 TotalPending: 5, TotalCancelable: 0}, getStats()) 443 444 // Dequeue should get 1st eval 445 out, token, err := b.Dequeue(defaultSched, time.Second) 446 must.NoError(t, err) 447 must.Eq(t, out, eval1, must.Sprint("expected 1st eval")) 448 449 must.Eq(t, BrokerStats{TotalReady: 1, TotalUnacked: 1, 450 TotalPending: 5, TotalCancelable: 0}, getStats()) 451 452 // Current wait index should be 4 but Ack to exercise behavior 453 // when worker's Eval.getWaitIndex gets a stale index 454 err = b.Ack(eval1.ID, token) 455 must.NoError(t, err) 456 457 must.Eq(t, BrokerStats{TotalReady: 2, TotalUnacked: 0, 458 TotalPending: 2, TotalCancelable: 2}, getStats()) 459 460 // eval4 and eval5 are ready 461 // eval6 and eval7 are pending 462 // Dequeue should get 4th eval 463 out, token, err = b.Dequeue(defaultSched, time.Second) 464 must.NoError(t, err) 465 must.Eq(t, out, eval4, must.Sprint("expected 4th eval")) 466 467 must.Eq(t, BrokerStats{TotalReady: 1, TotalUnacked: 1, 468 TotalPending: 2, TotalCancelable: 2}, getStats()) 469 470 // Ack should clear the rest of namespace-one pending but leave 471 // namespace-two untouched 472 err = b.Ack(eval4.ID, token) 473 must.NoError(t, err) 474 475 must.Eq(t, BrokerStats{TotalReady: 1, TotalUnacked: 0, 476 TotalPending: 2, TotalCancelable: 2}, getStats()) 477 478 // Dequeue should get 5th eval 479 out, token, err = b.Dequeue(defaultSched, time.Second) 480 must.NoError(t, err) 481 must.Eq(t, out, eval5, must.Sprint("expected 5th eval")) 482 483 must.Eq(t, BrokerStats{TotalReady: 0, TotalUnacked: 1, 484 TotalPending: 2, TotalCancelable: 2}, getStats()) 485 486 // Ack should clear remaining namespace-two pending evals 487 err = b.Ack(eval5.ID, token) 488 must.NoError(t, err) 489 490 must.Eq(t, BrokerStats{TotalReady: 1, TotalUnacked: 0, 491 TotalPending: 0, TotalCancelable: 3}, getStats()) 492 493 // Dequeue should get 7th eval because that's all that's left 494 out, token, err = b.Dequeue(defaultSched, time.Second) 495 must.NoError(t, err) 496 must.Eq(t, out, eval7, must.Sprint("expected 7th eval")) 497 498 must.Eq(t, BrokerStats{TotalReady: 0, TotalUnacked: 1, 499 TotalPending: 0, TotalCancelable: 3}, getStats()) 500 501 // Last ack should leave the broker empty except for cancels 502 err = b.Ack(eval7.ID, token) 503 must.NoError(t, err) 504 505 must.Eq(t, BrokerStats{TotalReady: 0, TotalUnacked: 0, 506 TotalPending: 0, TotalCancelable: 3}, getStats()) 507 } 508 509 func TestEvalBroker_Enqueue_Disable(t *testing.T) { 510 ci.Parallel(t) 511 b := testBroker(t, 0) 512 513 // Enqueue 514 eval := mock.Eval() 515 b.SetEnabled(true) 516 b.Enqueue(eval) 517 518 // Flush via SetEnabled 519 b.SetEnabled(false) 520 521 // Check the stats 522 stats := b.Stats() 523 if stats.TotalReady != 0 { 524 t.Fatalf("bad: %#v", stats) 525 } 526 if stats.TotalUnacked != 0 { 527 t.Fatalf("bad: %#v", stats) 528 } 529 if _, ok := stats.ByScheduler[eval.Type]; ok { 530 t.Fatalf("bad: %#v", stats) 531 } 532 } 533 534 func TestEvalBroker_Enqueue_Disable_Delay(t *testing.T) { 535 ci.Parallel(t) 536 b := testBroker(t, 0) 537 baseEval := mock.Eval() 538 b.SetEnabled(true) 539 540 { 541 // Enqueue 542 b.Enqueue(baseEval.Copy()) 543 544 delayedEval := baseEval.Copy() 545 delayedEval.Wait = 30 546 b.Enqueue(delayedEval) 547 548 waitEval := baseEval.Copy() 549 waitEval.WaitUntil = time.Now().Add(30 * time.Second) 550 b.Enqueue(waitEval) 551 } 552 553 // Flush via SetEnabled 554 b.SetEnabled(false) 555 556 { 557 // Check the stats 558 stats := b.Stats() 559 require.Equal(t, 0, stats.TotalReady, "Expected ready to be flushed") 560 require.Equal(t, 0, stats.TotalWaiting, "Expected waiting to be flushed") 561 require.Equal(t, 0, stats.TotalPending, "Expected pending to be flushed") 562 require.Equal(t, 0, stats.TotalUnacked, "Expected unacked to be flushed") 563 _, ok := stats.ByScheduler[baseEval.Type] 564 require.False(t, ok, "Expected scheduler to have no stats") 565 } 566 567 { 568 // Enqueue again now we're disabled 569 b.Enqueue(baseEval.Copy()) 570 571 delayedEval := baseEval.Copy() 572 delayedEval.Wait = 30 * time.Second 573 b.Enqueue(delayedEval) 574 575 waitEval := baseEval.Copy() 576 waitEval.WaitUntil = time.Now().Add(30 * time.Second) 577 b.Enqueue(waitEval) 578 } 579 580 { 581 // Check the stats again 582 stats := b.Stats() 583 require.Equal(t, 0, stats.TotalReady, "Expected ready to be flushed") 584 require.Equal(t, 0, stats.TotalWaiting, "Expected waiting to be flushed") 585 require.Equal(t, 0, stats.TotalPending, "Expected pending to be flushed") 586 require.Equal(t, 0, stats.TotalUnacked, "Expected unacked to be flushed") 587 _, ok := stats.ByScheduler[baseEval.Type] 588 require.False(t, ok, "Expected scheduler to have no stats") 589 } 590 } 591 592 func TestEvalBroker_Dequeue_Timeout(t *testing.T) { 593 ci.Parallel(t) 594 b := testBroker(t, 0) 595 b.SetEnabled(true) 596 597 start := time.Now() 598 out, _, err := b.Dequeue(defaultSched, 5*time.Millisecond) 599 end := time.Now() 600 601 if err != nil { 602 t.Fatalf("err: %v", err) 603 } 604 if out != nil { 605 t.Fatalf("unexpected: %#v", out) 606 } 607 608 if diff := end.Sub(start); diff < 5*time.Millisecond { 609 t.Fatalf("bad: %#v", diff) 610 } 611 } 612 613 func TestEvalBroker_Dequeue_Empty_Timeout(t *testing.T) { 614 ci.Parallel(t) 615 b := testBroker(t, 0) 616 b.SetEnabled(true) 617 618 errCh := make(chan error) 619 go func() { 620 defer close(errCh) 621 out, _, err := b.Dequeue(defaultSched, 0) 622 if err != nil { 623 errCh <- err 624 return 625 } 626 if out == nil { 627 errCh <- errors.New("expected a non-nil value") 628 return 629 } 630 }() 631 632 // Sleep for a little bit 633 select { 634 case <-time.After(5 * time.Millisecond): 635 case err := <-errCh: 636 if err != nil { 637 t.Fatalf("error from dequeue goroutine: %s", err) 638 } 639 t.Fatalf("Dequeue(0) should block, not finish") 640 } 641 642 // Enqueue to unblock the dequeue. 643 eval := mock.Eval() 644 b.Enqueue(eval) 645 646 select { 647 case err := <-errCh: 648 if err != nil { 649 t.Fatalf("error from dequeue goroutine: %s", err) 650 } 651 case <-time.After(5 * time.Millisecond): 652 t.Fatal("timeout: Dequeue(0) should return after enqueue") 653 } 654 } 655 656 // Ensure higher priority dequeued first 657 func TestEvalBroker_Dequeue_Priority(t *testing.T) { 658 ci.Parallel(t) 659 b := testBroker(t, 0) 660 b.SetEnabled(true) 661 662 eval1 := mock.Eval() 663 eval1.Priority = 10 664 b.Enqueue(eval1) 665 666 eval2 := mock.Eval() 667 eval2.Priority = 30 668 b.Enqueue(eval2) 669 670 eval3 := mock.Eval() 671 eval3.Priority = 20 672 b.Enqueue(eval3) 673 674 out1, _, _ := b.Dequeue(defaultSched, time.Second) 675 if out1 != eval2 { 676 t.Fatalf("bad: %#v", out1) 677 } 678 679 out2, _, _ := b.Dequeue(defaultSched, time.Second) 680 if out2 != eval3 { 681 t.Fatalf("bad: %#v", out2) 682 } 683 684 out3, _, _ := b.Dequeue(defaultSched, time.Second) 685 if out3 != eval1 { 686 t.Fatalf("bad: %#v", out3) 687 } 688 } 689 690 // Ensure FIFO at fixed priority 691 func TestEvalBroker_Dequeue_FIFO(t *testing.T) { 692 ci.Parallel(t) 693 b := testBroker(t, 0) 694 b.SetEnabled(true) 695 NUM := 100 696 697 for i := NUM; i > 0; i-- { 698 eval1 := mock.Eval() 699 eval1.CreateIndex = uint64(i) 700 eval1.ModifyIndex = uint64(i) 701 b.Enqueue(eval1) 702 } 703 704 for i := 1; i < NUM; i++ { 705 out1, _, _ := b.Dequeue(defaultSched, time.Second) 706 must.Eq(t, uint64(i), out1.CreateIndex, 707 must.Sprintf("eval was not FIFO by CreateIndex"), 708 ) 709 } 710 } 711 712 // Ensure fairness between schedulers 713 func TestEvalBroker_Dequeue_Fairness(t *testing.T) { 714 ci.Parallel(t) 715 b := testBroker(t, 0) 716 b.SetEnabled(true) 717 NUM := 1000 718 719 for i := 0; i < NUM; i++ { 720 eval1 := mock.Eval() 721 if i < (NUM / 2) { 722 eval1.Type = structs.JobTypeService 723 } else { 724 eval1.Type = structs.JobTypeBatch 725 } 726 b.Enqueue(eval1) 727 } 728 729 counter := 0 730 for i := 0; i < NUM; i++ { 731 out1, _, _ := b.Dequeue(defaultSched, time.Second) 732 733 switch out1.Type { 734 case structs.JobTypeService: 735 if counter < 0 { 736 counter = 0 737 } 738 counter += 1 739 case structs.JobTypeBatch: 740 if counter > 0 { 741 counter = 0 742 } 743 counter -= 1 744 } 745 746 // This will fail randomly at times. It is very hard to 747 // test deterministically that its acting randomly. 748 if counter >= 250 || counter <= -250 { 749 t.Fatalf("unlikely sequence: %d", counter) 750 } 751 } 752 } 753 754 // Ensure we get unblocked 755 func TestEvalBroker_Dequeue_Blocked(t *testing.T) { 756 ci.Parallel(t) 757 b := testBroker(t, 0) 758 b.SetEnabled(true) 759 760 // Start with a blocked dequeue 761 outCh := make(chan *structs.Evaluation) 762 errCh := make(chan error) 763 go func() { 764 defer close(errCh) 765 defer close(outCh) 766 start := time.Now() 767 out, _, err := b.Dequeue(defaultSched, time.Second) 768 if err != nil { 769 errCh <- err 770 return 771 } 772 end := time.Now() 773 if d := end.Sub(start); d < 5*time.Millisecond { 774 errCh <- fmt.Errorf("test broker dequeue duration too fast: %v", d) 775 return 776 } 777 outCh <- out 778 }() 779 780 // Wait for a bit, or t.Fatal if an error has already happened in 781 // the goroutine 782 select { 783 case <-time.After(5 * time.Millisecond): 784 // no errors yet, soldier on 785 case err := <-errCh: 786 if err != nil { 787 t.Fatalf("error from anonymous goroutine before enqueue: %v", err) 788 } 789 } 790 791 // Enqueue 792 eval := mock.Eval() 793 b.Enqueue(eval) 794 795 // Ensure dequeue 796 select { 797 case out := <-outCh: 798 if out != eval { 799 prettyExp, _ := json.MarshalIndent(eval, "", "\t") 800 prettyGot, _ := json.MarshalIndent(out, "", "\t") 801 t.Fatalf("dequeue result expected:\n%s\ngot:\n%s", 802 string(prettyExp), string(prettyGot)) 803 } 804 case err := <-errCh: 805 t.Fatalf("error from anonymous goroutine after enqueue: %v", err) 806 case <-time.After(time.Second): 807 t.Fatalf("timeout waiting for dequeue result") 808 } 809 } 810 811 // Ensure we nack in a timely manner 812 func TestEvalBroker_Nack_Timeout(t *testing.T) { 813 ci.Parallel(t) 814 b := testBroker(t, 5*time.Millisecond) 815 b.SetEnabled(true) 816 817 // Enqueue 818 eval := mock.Eval() 819 b.Enqueue(eval) 820 821 // Dequeue 822 out, _, err := b.Dequeue(defaultSched, time.Second) 823 start := time.Now() 824 if err != nil { 825 t.Fatalf("err: %v", err) 826 } 827 if out != eval { 828 t.Fatalf("bad: %v", out) 829 } 830 831 // Dequeue, should block on Nack timer 832 out, _, err = b.Dequeue(defaultSched, time.Second) 833 end := time.Now() 834 if err != nil { 835 t.Fatalf("err: %v", err) 836 } 837 if out != eval { 838 t.Fatalf("bad: %v", out) 839 } 840 841 // Check the nack timer 842 if diff := end.Sub(start); diff < 5*time.Millisecond { 843 t.Fatalf("bad: %#v", diff) 844 } 845 } 846 847 // Ensure we nack in a timely manner 848 func TestEvalBroker_Nack_TimeoutReset(t *testing.T) { 849 ci.Parallel(t) 850 b := testBroker(t, 50*time.Millisecond) 851 b.SetEnabled(true) 852 853 // Enqueue 854 eval := mock.Eval() 855 b.Enqueue(eval) 856 857 // Dequeue 858 out, token, err := b.Dequeue(defaultSched, time.Second) 859 start := time.Now() 860 if err != nil { 861 t.Fatalf("err: %v", err) 862 } 863 if out != eval { 864 t.Fatalf("bad: %v", out) 865 } 866 867 // Reset in 20 milliseconds 868 time.Sleep(20 * time.Millisecond) 869 if err := b.OutstandingReset(out.ID, token); err != nil { 870 t.Fatalf("err: %v", err) 871 } 872 873 // Dequeue, should block on Nack timer 874 out, _, err = b.Dequeue(defaultSched, time.Second) 875 end := time.Now() 876 if err != nil { 877 t.Fatalf("err: %v", err) 878 } 879 if out != eval { 880 t.Fatalf("bad: %v", out) 881 } 882 883 // Check the nack timer 884 if diff := end.Sub(start); diff < 75*time.Millisecond { 885 t.Fatalf("bad: %#v", diff) 886 } 887 } 888 889 func TestEvalBroker_PauseResumeNackTimeout(t *testing.T) { 890 ci.Parallel(t) 891 b := testBroker(t, 50*time.Millisecond) 892 b.SetEnabled(true) 893 894 // Enqueue 895 eval := mock.Eval() 896 b.Enqueue(eval) 897 898 // Dequeue 899 out, token, err := b.Dequeue(defaultSched, time.Second) 900 start := time.Now() 901 if err != nil { 902 t.Fatalf("err: %v", err) 903 } 904 if out != eval { 905 t.Fatalf("bad: %v", out) 906 } 907 908 // Pause in 20 milliseconds 909 time.Sleep(20 * time.Millisecond) 910 if err := b.PauseNackTimeout(out.ID, token); err != nil { 911 t.Fatalf("pause nack timeout error: %v", err) 912 } 913 914 errCh := make(chan error) 915 go func() { 916 defer close(errCh) 917 time.Sleep(20 * time.Millisecond) 918 if err := b.ResumeNackTimeout(out.ID, token); err != nil { 919 errCh <- err 920 return 921 } 922 }() 923 924 // Dequeue, should block until the timer is resumed 925 out, _, err = b.Dequeue(defaultSched, time.Second) 926 end := time.Now() 927 if err != nil { 928 t.Fatalf("dequeue error: %v", err) 929 } 930 if out != eval { 931 prettyExp, _ := json.MarshalIndent(eval, "", "\t") 932 prettyGot, _ := json.MarshalIndent(out, "", "\t") 933 t.Fatalf("dequeue result expected:\n%s\ngot:\n%s", 934 string(prettyExp), string(prettyGot)) 935 } 936 937 // Check the nack timer 938 if diff := end.Sub(start); diff < 95*time.Millisecond { 939 t.Fatalf("deqeue happened too fast: %#v", diff) 940 } 941 942 // check the result of ResumeNackTimeout 943 err = <-errCh 944 if err != nil { 945 t.Fatalf("resume nack timeout error:%s", err) 946 } 947 } 948 949 func TestEvalBroker_DeliveryLimit(t *testing.T) { 950 ci.Parallel(t) 951 b := testBroker(t, 0) 952 b.SetEnabled(true) 953 954 eval := mock.Eval() 955 b.Enqueue(eval) 956 957 for i := 0; i < 3; i++ { 958 // Dequeue should work 959 out, token, err := b.Dequeue(defaultSched, time.Second) 960 if err != nil { 961 t.Fatalf("err: %v", err) 962 } 963 if out != eval { 964 t.Fatalf("bad : %#v", out) 965 } 966 967 // Nack with wrong token should fail 968 err = b.Nack(eval.ID, token) 969 if err != nil { 970 t.Fatalf("err: %v", err) 971 } 972 } 973 974 // Check the stats 975 stats := b.Stats() 976 if stats.TotalReady != 1 { 977 t.Fatalf("bad: %#v", stats) 978 } 979 if stats.TotalUnacked != 0 { 980 t.Fatalf("bad: %#v", stats) 981 } 982 if stats.ByScheduler[failedQueue].Ready != 1 { 983 t.Fatalf("bad: %#v", stats) 984 } 985 if stats.ByScheduler[failedQueue].Unacked != 0 { 986 t.Fatalf("bad: %#v", stats) 987 } 988 989 // Dequeue from failed queue 990 out, token, err := b.Dequeue([]string{failedQueue}, time.Second) 991 if err != nil { 992 t.Fatalf("err: %v", err) 993 } 994 if out != eval { 995 t.Fatalf("bad : %#v", out) 996 } 997 998 // Check the stats 999 stats = b.Stats() 1000 if stats.TotalReady != 0 { 1001 t.Fatalf("bad: %#v", stats) 1002 } 1003 if stats.TotalUnacked != 1 { 1004 t.Fatalf("bad: %#v", stats) 1005 } 1006 if stats.ByScheduler[failedQueue].Ready != 0 { 1007 t.Fatalf("bad: %#v", stats) 1008 } 1009 if stats.ByScheduler[failedQueue].Unacked != 1 { 1010 t.Fatalf("bad: %#v", stats) 1011 } 1012 1013 // Ack finally 1014 err = b.Ack(out.ID, token) 1015 if err != nil { 1016 t.Fatalf("err: %v", err) 1017 } 1018 1019 if _, ok := b.Outstanding(out.ID); ok { 1020 t.Fatalf("should not be outstanding") 1021 } 1022 1023 // Check the stats 1024 stats = b.Stats() 1025 if stats.TotalReady != 0 { 1026 t.Fatalf("bad: %#v", stats) 1027 } 1028 if stats.TotalUnacked != 0 { 1029 t.Fatalf("bad: %#v", stats) 1030 } 1031 if stats.ByScheduler[failedQueue].Ready != 0 { 1032 t.Fatalf("bad: %#v", stats.ByScheduler[failedQueue]) 1033 } 1034 if stats.ByScheduler[failedQueue].Unacked != 0 { 1035 t.Fatalf("bad: %#v", stats.ByScheduler[failedQueue]) 1036 } 1037 } 1038 1039 func TestEvalBroker_AckAtDeliveryLimit(t *testing.T) { 1040 ci.Parallel(t) 1041 b := testBroker(t, 0) 1042 b.SetEnabled(true) 1043 1044 eval := mock.Eval() 1045 b.Enqueue(eval) 1046 1047 for i := 0; i < 3; i++ { 1048 // Dequeue should work 1049 out, token, err := b.Dequeue(defaultSched, time.Second) 1050 if err != nil { 1051 t.Fatalf("err: %v", err) 1052 } 1053 if out != eval { 1054 t.Fatalf("bad : %#v", out) 1055 } 1056 1057 if i == 2 { 1058 b.Ack(eval.ID, token) 1059 } else { 1060 // Nack with wrong token should fail 1061 err = b.Nack(eval.ID, token) 1062 if err != nil { 1063 t.Fatalf("err: %v", err) 1064 } 1065 } 1066 } 1067 1068 // Check the stats 1069 stats := b.Stats() 1070 if stats.TotalReady != 0 { 1071 t.Fatalf("bad: %#v", stats) 1072 } 1073 if stats.TotalUnacked != 0 { 1074 t.Fatalf("bad: %#v", stats) 1075 } 1076 if _, ok := stats.ByScheduler[failedQueue]; ok { 1077 t.Fatalf("bad: %#v", stats) 1078 } 1079 } 1080 1081 // TestEvalBroker_Wait asserts delayed evaluations cannot be dequeued until 1082 // their wait duration has elapsed. 1083 func TestEvalBroker_Wait(t *testing.T) { 1084 ci.Parallel(t) 1085 b := testBroker(t, 0) 1086 b.SetEnabled(true) 1087 1088 // Create an eval that should wait 1089 eval := mock.Eval() 1090 eval.Wait = 100 * time.Millisecond 1091 b.Enqueue(eval) 1092 1093 // Verify waiting 1094 stats := b.Stats() 1095 if stats.TotalReady != 0 { 1096 t.Fatalf("bad: %#v", stats) 1097 } 1098 if stats.TotalWaiting != 1 { 1099 t.Fatalf("bad: %#v", stats) 1100 } 1101 1102 // Dequeue should not return the eval until wait has elapsed 1103 out, token, err := b.Dequeue(defaultSched, 1) 1104 require.Nil(t, out) 1105 require.Empty(t, token) 1106 require.NoError(t, err) 1107 1108 // Let the wait elapse 1109 time.Sleep(200 * time.Millisecond) 1110 1111 // Verify ready 1112 stats = b.Stats() 1113 if stats.TotalReady != 1 { 1114 t.Fatalf("bad: %#v", stats) 1115 } 1116 if stats.TotalWaiting != 0 { 1117 t.Fatalf("bad: %#v", stats) 1118 } 1119 1120 // Dequeue should work 1121 out, _, err = b.Dequeue(defaultSched, time.Second) 1122 if err != nil { 1123 t.Fatalf("err: %v", err) 1124 } 1125 if out != eval { 1126 t.Fatalf("bad : %#v", out) 1127 } 1128 } 1129 1130 // Ensure that delayed evaluations work as expected 1131 func TestEvalBroker_WaitUntil(t *testing.T) { 1132 ci.Parallel(t) 1133 require := require.New(t) 1134 b := testBroker(t, 0) 1135 b.SetEnabled(true) 1136 1137 now := time.Now() 1138 // Create a few of evals with WaitUntil set 1139 eval1 := mock.Eval() 1140 eval1.WaitUntil = now.Add(1 * time.Second) 1141 eval1.CreateIndex = 1 1142 b.Enqueue(eval1) 1143 1144 eval2 := mock.Eval() 1145 eval2.WaitUntil = now.Add(100 * time.Millisecond) 1146 // set CreateIndex to use as a tie breaker when eval2 1147 // and eval3 are both in the pending evals heap 1148 eval2.CreateIndex = 2 1149 b.Enqueue(eval2) 1150 1151 eval3 := mock.Eval() 1152 eval3.WaitUntil = now.Add(20 * time.Millisecond) 1153 eval3.CreateIndex = 1 1154 b.Enqueue(eval3) 1155 require.Equal(3, b.stats.TotalWaiting) 1156 // sleep enough for two evals to be ready 1157 time.Sleep(200 * time.Millisecond) 1158 1159 // first dequeue should return eval3 1160 out, _, err := b.Dequeue(defaultSched, time.Second) 1161 require.Nil(err) 1162 require.Equal(eval3, out) 1163 1164 // second dequeue should return eval2 1165 out, _, err = b.Dequeue(defaultSched, time.Second) 1166 require.Nil(err) 1167 require.Equal(eval2, out) 1168 1169 // third dequeue should return eval1 1170 out, _, err = b.Dequeue(defaultSched, 2*time.Second) 1171 require.Nil(err) 1172 require.Equal(eval1, out) 1173 require.Equal(0, b.stats.TotalWaiting) 1174 } 1175 1176 // Ensure that priority is taken into account when enqueueing many evaluations. 1177 func TestEvalBroker_EnqueueAll_Dequeue_Fair(t *testing.T) { 1178 ci.Parallel(t) 1179 b := testBroker(t, 0) 1180 b.SetEnabled(true) 1181 1182 // Start with a blocked dequeue 1183 outCh := make(chan *structs.Evaluation) 1184 errCh := make(chan error) 1185 go func() { 1186 defer close(errCh) 1187 defer close(outCh) 1188 start := time.Now() 1189 out, _, err := b.Dequeue(defaultSched, time.Second) 1190 if err != nil { 1191 errCh <- err 1192 return 1193 } 1194 end := time.Now() 1195 if d := end.Sub(start); d < 5*time.Millisecond { 1196 errCh <- fmt.Errorf("test broker dequeue duration too fast: %v", d) 1197 return 1198 } 1199 outCh <- out 1200 }() 1201 1202 // Wait for a bit, or t.Fatal if an error has already happened in 1203 // the goroutine 1204 select { 1205 case <-time.After(5 * time.Millisecond): 1206 // no errors yet, soldier on 1207 case err := <-errCh: 1208 if err != nil { 1209 t.Fatalf("error from anonymous goroutine before enqueue: %v", err) 1210 } 1211 } 1212 1213 // Enqueue 1214 evals := make(map[*structs.Evaluation]string, 8) 1215 expectedPriority := 90 1216 for i := 10; i <= expectedPriority; i += 10 { 1217 eval := mock.Eval() 1218 eval.Priority = i 1219 evals[eval] = "" 1220 1221 } 1222 b.EnqueueAll(evals) 1223 1224 // Ensure dequeue 1225 select { 1226 case out := <-outCh: 1227 if out.Priority != expectedPriority { 1228 pretty, _ := json.MarshalIndent(out, "", "\t") 1229 t.Logf("bad priority on *structs.Evaluation: %s", string(pretty)) 1230 t.Fatalf("priority wanted:%d, priority got:%d", expectedPriority, out.Priority) 1231 } 1232 case err := <-errCh: 1233 t.Fatalf("error from anonymous goroutine after enqueue: %v", err) 1234 case <-time.After(time.Second): 1235 t.Fatalf("timeout waiting for dequeue result") 1236 } 1237 } 1238 1239 func TestEvalBroker_EnqueueAll_Requeue_Ack(t *testing.T) { 1240 ci.Parallel(t) 1241 b := testBroker(t, 0) 1242 b.SetEnabled(true) 1243 1244 // Create the evaluation, enqueue and dequeue 1245 eval := mock.Eval() 1246 b.Enqueue(eval) 1247 1248 out, token, err := b.Dequeue(defaultSched, time.Second) 1249 if err != nil { 1250 t.Fatalf("err: %v", err) 1251 } 1252 if out != eval { 1253 t.Fatalf("bad : %#v", out) 1254 } 1255 1256 // Requeue the same evaluation. 1257 b.EnqueueAll(map[*structs.Evaluation]string{eval: token}) 1258 1259 // The stats should show one unacked 1260 stats := b.Stats() 1261 if stats.TotalReady != 0 { 1262 t.Fatalf("bad: %#v", stats) 1263 } 1264 if stats.TotalUnacked != 1 { 1265 t.Fatalf("bad: %#v", stats) 1266 } 1267 1268 // Ack the evaluation. 1269 if err := b.Ack(eval.ID, token); err != nil { 1270 t.Fatalf("err: %v", err) 1271 } 1272 1273 // Check stats again as this should cause the re-enqueued one to transition 1274 // into the ready state 1275 stats = b.Stats() 1276 if stats.TotalReady != 1 { 1277 t.Fatalf("bad: %#v", stats) 1278 } 1279 if stats.TotalUnacked != 0 { 1280 t.Fatalf("bad: %#v", stats) 1281 } 1282 1283 // Another dequeue should be successful 1284 out2, token2, err := b.Dequeue(defaultSched, time.Second) 1285 if err != nil { 1286 t.Fatalf("err: %v", err) 1287 } 1288 if out2 != eval { 1289 t.Fatalf("bad : %#v", out) 1290 } 1291 if token == token2 { 1292 t.Fatalf("bad : %s and %s", token, token2) 1293 } 1294 } 1295 1296 func TestEvalBroker_EnqueueAll_Requeue_Nack(t *testing.T) { 1297 ci.Parallel(t) 1298 b := testBroker(t, 0) 1299 b.SetEnabled(true) 1300 1301 // Create the evaluation, enqueue and dequeue 1302 eval := mock.Eval() 1303 b.Enqueue(eval) 1304 1305 out, token, err := b.Dequeue(defaultSched, time.Second) 1306 if err != nil { 1307 t.Fatalf("err: %v", err) 1308 } 1309 if out != eval { 1310 t.Fatalf("bad : %#v", out) 1311 } 1312 1313 // Requeue the same evaluation. 1314 b.EnqueueAll(map[*structs.Evaluation]string{eval: token}) 1315 1316 // The stats should show one unacked 1317 stats := b.Stats() 1318 if stats.TotalReady != 0 { 1319 t.Fatalf("bad: %#v", stats) 1320 } 1321 if stats.TotalUnacked != 1 { 1322 t.Fatalf("bad: %#v", stats) 1323 } 1324 1325 // Nack the evaluation. 1326 if err := b.Nack(eval.ID, token); err != nil { 1327 t.Fatalf("err: %v", err) 1328 } 1329 1330 // Check stats again as this should cause the re-enqueued one to be dropped 1331 testutil.WaitForResult(func() (bool, error) { 1332 stats = b.Stats() 1333 if stats.TotalReady != 1 { 1334 return false, fmt.Errorf("bad: %#v", stats) 1335 } 1336 if stats.TotalUnacked != 0 { 1337 return false, fmt.Errorf("bad: %#v", stats) 1338 } 1339 if len(b.requeue) != 0 { 1340 return false, fmt.Errorf("bad: %#v", b.requeue) 1341 } 1342 1343 return true, nil 1344 }, func(e error) { 1345 t.Fatal(e) 1346 }) 1347 } 1348 1349 func TestEvalBroker_NamespacedJobs(t *testing.T) { 1350 ci.Parallel(t) 1351 b := testBroker(t, 0) 1352 b.SetEnabled(true) 1353 1354 // Create evals with the same jobid and different namespace 1355 jobId := "test-jobID" 1356 1357 eval1 := mock.Eval() 1358 eval1.JobID = jobId 1359 eval1.Namespace = "n1" 1360 b.Enqueue(eval1) 1361 1362 // This eval should not block 1363 eval2 := mock.Eval() 1364 eval2.JobID = jobId 1365 eval2.Namespace = "default" 1366 b.Enqueue(eval2) 1367 1368 // This eval should block 1369 eval3 := mock.Eval() 1370 eval3.JobID = jobId 1371 eval3.Namespace = "default" 1372 b.Enqueue(eval3) 1373 1374 require := require.New(t) 1375 out1, _, err := b.Dequeue(defaultSched, 5*time.Millisecond) 1376 require.Nil(err) 1377 require.Equal(eval1.ID, out1.ID) 1378 1379 out2, _, err := b.Dequeue(defaultSched, 5*time.Millisecond) 1380 require.Nil(err) 1381 require.Equal(eval2.ID, out2.ID) 1382 1383 out3, _, err := b.Dequeue(defaultSched, 5*time.Millisecond) 1384 require.Nil(err) 1385 require.Nil(out3) 1386 1387 require.Equal(1, len(b.pending)) 1388 1389 } 1390 1391 func TestEvalBroker_ReadyEvals_Ordering(t *testing.T) { 1392 1393 ready := ReadyEvaluations{} 1394 1395 newEval := func(jobID, evalID string, priority int, index uint64) *structs.Evaluation { 1396 eval := mock.Eval() 1397 eval.JobID = jobID 1398 eval.ID = evalID 1399 eval.Priority = priority 1400 eval.CreateIndex = uint64(index) 1401 return eval 1402 } 1403 1404 // note: we're intentionally pushing these out-of-order to assert we're 1405 // getting them back out in the intended order and not just as inserted 1406 heap.Push(&ready, newEval("example1", "eval01", 50, 1)) 1407 heap.Push(&ready, newEval("example3", "eval03", 70, 3)) 1408 heap.Push(&ready, newEval("example2", "eval02", 50, 2)) 1409 1410 next := heap.Pop(&ready).(*structs.Evaluation) 1411 test.Eq(t, "eval03", next.ID, 1412 test.Sprint("expected highest Priority to be next ready")) 1413 1414 next = heap.Pop(&ready).(*structs.Evaluation) 1415 test.Eq(t, "eval01", next.ID, 1416 test.Sprint("expected oldest CreateIndex to be next ready")) 1417 1418 heap.Push(&ready, newEval("example4", "eval04", 50, 4)) 1419 1420 next = heap.Pop(&ready).(*structs.Evaluation) 1421 test.Eq(t, "eval02", next.ID, 1422 test.Sprint("expected oldest CreateIndex to be next ready")) 1423 1424 } 1425 1426 func TestEvalBroker_PendingEval_Ordering(t *testing.T) { 1427 pending := PendingEvaluations{} 1428 1429 newEval := func(evalID string, priority int, index uint64) *structs.Evaluation { 1430 eval := mock.Eval() 1431 eval.ID = evalID 1432 eval.Priority = priority 1433 eval.ModifyIndex = uint64(index) 1434 return eval 1435 } 1436 1437 // note: we're intentionally pushing these out-of-order to assert we're 1438 // getting them back out in the intended order and not just as inserted 1439 heap.Push(&pending, newEval("eval03", 50, 3)) 1440 heap.Push(&pending, newEval("eval02", 100, 2)) 1441 heap.Push(&pending, newEval("eval01", 50, 1)) 1442 1443 next := heap.Pop(&pending).(*structs.Evaluation) 1444 test.Eq(t, "eval02", next.ID, 1445 test.Sprint("expected eval with highest priority to be next")) 1446 1447 next = heap.Pop(&pending).(*structs.Evaluation) 1448 test.Eq(t, "eval03", next.ID, 1449 test.Sprint("expected eval with highest modify index to be next")) 1450 1451 heap.Push(&pending, newEval("eval04", 30, 4)) 1452 next = heap.Pop(&pending).(*structs.Evaluation) 1453 test.Eq(t, "eval01", next.ID, 1454 test.Sprint("expected eval with highest priority to be nexct")) 1455 1456 } 1457 1458 func TestEvalBroker_PendingEvals_MarkForCancel(t *testing.T) { 1459 ci.Parallel(t) 1460 1461 pending := PendingEvaluations{} 1462 1463 // note: we're intentionally pushing these out-of-order to assert we're 1464 // getting them back out in the intended order and not just as inserted 1465 for i := 100; i > 0; i -= 10 { 1466 eval := mock.Eval() 1467 eval.JobID = "example" 1468 eval.CreateIndex = uint64(i) 1469 eval.ModifyIndex = uint64(i) 1470 heap.Push(&pending, eval) 1471 } 1472 1473 canceled := pending.MarkForCancel() 1474 must.Eq(t, 9, len(canceled)) 1475 must.Eq(t, 1, pending.Len()) 1476 1477 raw := heap.Pop(&pending) 1478 must.NotNil(t, raw) 1479 eval := raw.(*structs.Evaluation) 1480 must.Eq(t, 100, eval.ModifyIndex) 1481 } 1482 1483 func TestEvalBroker_Cancelable(t *testing.T) { 1484 ci.Parallel(t) 1485 1486 b := testBroker(t, time.Minute) 1487 1488 evals := []*structs.Evaluation{} 1489 for i := 0; i < 20; i++ { 1490 eval := mock.Eval() 1491 evals = append(evals, eval) 1492 } 1493 b.cancelable = evals 1494 b.stats.TotalCancelable = len(b.cancelable) 1495 1496 must.Len(t, 20, b.cancelable) 1497 cancelable := b.Cancelable(10) 1498 must.Len(t, 10, cancelable) 1499 must.Len(t, 10, b.cancelable) 1500 must.Eq(t, 10, b.stats.TotalCancelable) 1501 1502 cancelable = b.Cancelable(20) 1503 must.Len(t, 10, cancelable) 1504 must.Len(t, 0, b.cancelable) 1505 must.Eq(t, 0, b.stats.TotalCancelable) 1506 } 1507 1508 // TestEvalBroker_IntegrationTest exercises the eval broker with realistic 1509 // workflows 1510 func TestEvalBroker_IntegrationTest(t *testing.T) { 1511 ci.Parallel(t) 1512 1513 srv, cleanupS1 := TestServer(t, func(c *Config) { 1514 c.NumSchedulers = 0 // Prevent dequeue 1515 c.EvalReapCancelableInterval = time.Minute * 10 // Prevent sweep-up 1516 }) 1517 1518 defer cleanupS1() 1519 testutil.WaitForLeader(t, srv.RPC) 1520 1521 codec := rpcClient(t, srv) 1522 store := srv.fsm.State() 1523 1524 // create a system job, a node for it to run on, and a set of node up/down 1525 // events that will result in evaluations queued. 1526 1527 job := mock.SystemJob() 1528 jobReq := &structs.JobRegisterRequest{ 1529 Job: job, 1530 EvalPriority: 50, 1531 WriteRequest: structs.WriteRequest{Region: "global"}, 1532 } 1533 var jobResp structs.JobRegisterResponse 1534 err := msgpackrpc.CallWithCodec(codec, "Job.Register", jobReq, &jobResp) 1535 must.NoError(t, err) 1536 1537 node := mock.Node() 1538 nodeReq := &structs.NodeRegisterRequest{ 1539 Node: node, 1540 WriteRequest: structs.WriteRequest{Region: "global"}, 1541 } 1542 var nodeResp structs.NodeUpdateResponse 1543 err = msgpackrpc.CallWithCodec(codec, "Node.Register", nodeReq, &nodeResp) 1544 must.NoError(t, err) 1545 1546 for i := 0; i < 10; i++ { 1547 status := structs.NodeStatusDown 1548 if i%2 == 0 { 1549 status = structs.NodeStatusReady 1550 } 1551 statusReq := &structs.NodeUpdateStatusRequest{ 1552 NodeID: node.ID, 1553 Status: status, 1554 WriteRequest: structs.WriteRequest{Region: "global"}, 1555 } 1556 var statusResp structs.NodeUpdateResponse 1557 err = msgpackrpc.CallWithCodec(codec, "Node.UpdateStatus", statusReq, &statusResp) 1558 must.NoError(t, err) 1559 } 1560 1561 // ensure we have the expected number of evaluations and eval broker state 1562 1563 // retreive the stats from the broker, less some uninteresting ones 1564 getStats := func() BrokerStats { 1565 t.Helper() 1566 stats := srv.evalBroker.Stats() 1567 stats.DelayedEvals = nil 1568 stats.ByScheduler = nil 1569 return *stats 1570 } 1571 1572 getEvalStatuses := func() map[string]int { 1573 t.Helper() 1574 statuses := map[string]int{} 1575 iter, err := store.Evals(nil, state.SortDefault) 1576 must.NoError(t, err) 1577 for { 1578 raw := iter.Next() 1579 if raw == nil { 1580 break 1581 } 1582 eval := raw.(*structs.Evaluation) 1583 statuses[eval.Status] += 1 1584 if eval.Status == structs.EvalStatusCancelled { 1585 must.Eq(t, "canceled after more recent eval was processed", eval.StatusDescription) 1586 } 1587 } 1588 return statuses 1589 } 1590 1591 must.Eq(t, map[string]int{structs.EvalStatusPending: 11}, getEvalStatuses()) 1592 must.Eq(t, BrokerStats{TotalReady: 1, TotalUnacked: 0, 1593 TotalPending: 10, TotalCancelable: 0}, getStats()) 1594 1595 // start schedulers: all the evals are for a single job so there should only 1596 // be one eval processesed at a time no matter how many schedulers we run 1597 1598 config := DefaultConfig() 1599 config.NumSchedulers = 4 1600 must.NoError(t, srv.Reload(config)) 1601 1602 // assert that all but 2 evals were canceled and that the eval broker state 1603 // has been cleared 1604 1605 var got map[string]int 1606 1607 must.Wait(t, wait.InitialSuccess( 1608 wait.Timeout(5*time.Second), 1609 wait.Gap(100*time.Millisecond), 1610 wait.BoolFunc(func() bool { 1611 got = getEvalStatuses() 1612 return got[structs.EvalStatusComplete] == 2 && 1613 got[structs.EvalStatusCancelled] == 9 1614 }), 1615 ), 1616 must.Func(func() string { 1617 return fmt.Sprintf("expected map[complete:2 canceled:9] within timeout, got: %v with broker status=%#v", got, getStats()) 1618 }), 1619 ) 1620 1621 must.Eq(t, BrokerStats{TotalReady: 0, TotalUnacked: 0, 1622 TotalPending: 0, TotalCancelable: 0}, getStats()) 1623 }