github.com/kardianos/nomad@v0.1.3-0.20151022182107-b13df73ee850/nomad/eval_broker_test.go (about) 1 package nomad 2 3 import ( 4 "testing" 5 "time" 6 7 "github.com/hashicorp/nomad/nomad/mock" 8 "github.com/hashicorp/nomad/nomad/structs" 9 ) 10 11 var ( 12 defaultSched = []string{ 13 structs.JobTypeService, 14 structs.JobTypeBatch, 15 } 16 ) 17 18 func testBroker(t *testing.T, timeout time.Duration) *EvalBroker { 19 if timeout == 0 { 20 timeout = 5 * time.Second 21 } 22 b, err := NewEvalBroker(timeout, 3) 23 if err != nil { 24 t.Fatalf("err: %v", err) 25 } 26 return b 27 } 28 29 func TestEvalBroker_Enqueue_Dequeue_Nack_Ack(t *testing.T) { 30 b := testBroker(t, 0) 31 32 // Enqueue, but broker is disabled! 33 eval := mock.Eval() 34 err := b.Enqueue(eval) 35 if err != nil { 36 t.Fatalf("err: %v", err) 37 } 38 39 // Verify nothing was done 40 stats := b.Stats() 41 if stats.TotalReady != 0 { 42 t.Fatalf("bad: %#v", stats) 43 } 44 45 if b.Enabled() { 46 t.Fatalf("should not be enabled") 47 } 48 49 // Enable the broker, and enqueue 50 b.SetEnabled(true) 51 err = b.Enqueue(eval) 52 if err != nil { 53 t.Fatalf("err: %v", err) 54 } 55 56 // Double enqueue is a no-op 57 err = b.Enqueue(eval) 58 if err != nil { 59 t.Fatalf("err: %v", err) 60 } 61 62 if !b.Enabled() { 63 t.Fatalf("should be enabled") 64 } 65 66 // Verify enqueue is done 67 stats = b.Stats() 68 if stats.TotalReady != 1 { 69 t.Fatalf("bad: %#v", stats) 70 } 71 if stats.ByScheduler[eval.Type].Ready != 1 { 72 t.Fatalf("bad: %#v", stats) 73 } 74 75 // Dequeue should work 76 out, token, err := b.Dequeue(defaultSched, time.Second) 77 if err != nil { 78 t.Fatalf("err: %v", err) 79 } 80 if out != eval { 81 t.Fatalf("bad : %#v", out) 82 } 83 84 tokenOut, ok := b.Outstanding(out.ID) 85 if !ok { 86 t.Fatalf("should be outstanding") 87 } 88 if tokenOut != token { 89 t.Fatalf("Bad: %#v %#v", token, tokenOut) 90 } 91 92 // Check the stats 93 stats = b.Stats() 94 if stats.TotalReady != 0 { 95 t.Fatalf("bad: %#v", stats) 96 } 97 if stats.TotalUnacked != 1 { 98 t.Fatalf("bad: %#v", stats) 99 } 100 if stats.ByScheduler[eval.Type].Ready != 0 { 101 t.Fatalf("bad: %#v", stats) 102 } 103 if stats.ByScheduler[eval.Type].Unacked != 1 { 104 t.Fatalf("bad: %#v", stats) 105 } 106 107 // Nack with wrong token should fail 108 err = b.Nack(eval.ID, "foobarbaz") 109 if err == nil { 110 t.Fatalf("should fail to nack") 111 } 112 113 // Nack back into the queue 114 err = b.Nack(eval.ID, token) 115 if err != nil { 116 t.Fatalf("err: %v", err) 117 } 118 119 if _, ok := b.Outstanding(out.ID); ok { 120 t.Fatalf("should not be outstanding") 121 } 122 123 // Check the stats 124 stats = b.Stats() 125 if stats.TotalReady != 1 { 126 t.Fatalf("bad: %#v", stats) 127 } 128 if stats.TotalUnacked != 0 { 129 t.Fatalf("bad: %#v", stats) 130 } 131 if stats.ByScheduler[eval.Type].Ready != 1 { 132 t.Fatalf("bad: %#v", stats) 133 } 134 if stats.ByScheduler[eval.Type].Unacked != 0 { 135 t.Fatalf("bad: %#v", stats) 136 } 137 138 // Dequeue should work again 139 out2, token2, err := b.Dequeue(defaultSched, time.Second) 140 if err != nil { 141 t.Fatalf("err: %v", err) 142 } 143 if out2 != eval { 144 t.Fatalf("bad : %#v", out2) 145 } 146 if token2 == token { 147 t.Fatalf("should get a new token") 148 } 149 150 tokenOut2, ok := b.Outstanding(out.ID) 151 if !ok { 152 t.Fatalf("should be outstanding") 153 } 154 if tokenOut2 != token2 { 155 t.Fatalf("Bad: %#v %#v", token2, tokenOut2) 156 } 157 158 // Ack with wrong token 159 err = b.Ack(eval.ID, "zip") 160 if err == nil { 161 t.Fatalf("should fail to ack") 162 } 163 164 // Ack finally 165 err = b.Ack(eval.ID, token2) 166 if err != nil { 167 t.Fatalf("err: %v", err) 168 } 169 170 if _, ok := b.Outstanding(out.ID); ok { 171 t.Fatalf("should not be outstanding") 172 } 173 174 // Check the stats 175 stats = b.Stats() 176 if stats.TotalReady != 0 { 177 t.Fatalf("bad: %#v", stats) 178 } 179 if stats.TotalUnacked != 0 { 180 t.Fatalf("bad: %#v", stats) 181 } 182 if stats.ByScheduler[eval.Type].Ready != 0 { 183 t.Fatalf("bad: %#v", stats) 184 } 185 if stats.ByScheduler[eval.Type].Unacked != 0 { 186 t.Fatalf("bad: %#v", stats) 187 } 188 } 189 190 func TestEvalBroker_Serialize_DuplicateJobID(t *testing.T) { 191 b := testBroker(t, 0) 192 b.SetEnabled(true) 193 194 eval := mock.Eval() 195 err := b.Enqueue(eval) 196 if err != nil { 197 t.Fatalf("err: %v", err) 198 } 199 200 eval2 := mock.Eval() 201 eval2.JobID = eval.JobID 202 eval2.CreateIndex = eval.CreateIndex + 1 203 err = b.Enqueue(eval2) 204 if err != nil { 205 t.Fatalf("err: %v", err) 206 } 207 208 eval3 := mock.Eval() 209 eval3.JobID = eval.JobID 210 eval3.CreateIndex = eval.CreateIndex + 2 211 err = b.Enqueue(eval3) 212 if err != nil { 213 t.Fatalf("err: %v", err) 214 } 215 216 stats := b.Stats() 217 if stats.TotalReady != 1 { 218 t.Fatalf("bad: %#v", stats) 219 } 220 if stats.TotalBlocked != 2 { 221 t.Fatalf("bad: %#v", stats) 222 } 223 224 // Dequeue should work 225 out, token, err := b.Dequeue(defaultSched, time.Second) 226 if err != nil { 227 t.Fatalf("err: %v", err) 228 } 229 if out != eval { 230 t.Fatalf("bad : %#v", out) 231 } 232 233 // Check the stats 234 stats = b.Stats() 235 if stats.TotalReady != 0 { 236 t.Fatalf("bad: %#v", stats) 237 } 238 if stats.TotalUnacked != 1 { 239 t.Fatalf("bad: %#v", stats) 240 } 241 if stats.TotalBlocked != 2 { 242 t.Fatalf("bad: %#v", stats) 243 } 244 245 // Ack out 246 err = b.Ack(eval.ID, token) 247 if err != nil { 248 t.Fatalf("err: %v", err) 249 } 250 251 // Check the stats 252 stats = b.Stats() 253 if stats.TotalReady != 1 { 254 t.Fatalf("bad: %#v", stats) 255 } 256 if stats.TotalUnacked != 0 { 257 t.Fatalf("bad: %#v", stats) 258 } 259 if stats.TotalBlocked != 1 { 260 t.Fatalf("bad: %#v", stats) 261 } 262 263 // Dequeue should work 264 out, token, err = b.Dequeue(defaultSched, time.Second) 265 if err != nil { 266 t.Fatalf("err: %v", err) 267 } 268 if out != eval2 { 269 t.Fatalf("bad : %#v", out) 270 } 271 272 // Check the stats 273 stats = b.Stats() 274 if stats.TotalReady != 0 { 275 t.Fatalf("bad: %#v", stats) 276 } 277 if stats.TotalUnacked != 1 { 278 t.Fatalf("bad: %#v", stats) 279 } 280 if stats.TotalBlocked != 1 { 281 t.Fatalf("bad: %#v", stats) 282 } 283 284 // Ack out 285 err = b.Ack(eval2.ID, token) 286 if err != nil { 287 t.Fatalf("err: %v", err) 288 } 289 290 // Check the stats 291 stats = b.Stats() 292 if stats.TotalReady != 1 { 293 t.Fatalf("bad: %#v", stats) 294 } 295 if stats.TotalUnacked != 0 { 296 t.Fatalf("bad: %#v", stats) 297 } 298 if stats.TotalBlocked != 0 { 299 t.Fatalf("bad: %#v", stats) 300 } 301 302 // Dequeue should work 303 out, token, err = b.Dequeue(defaultSched, time.Second) 304 if err != nil { 305 t.Fatalf("err: %v", err) 306 } 307 if out != eval3 { 308 t.Fatalf("bad : %#v", out) 309 } 310 311 // Check the stats 312 stats = b.Stats() 313 if stats.TotalReady != 0 { 314 t.Fatalf("bad: %#v", stats) 315 } 316 if stats.TotalUnacked != 1 { 317 t.Fatalf("bad: %#v", stats) 318 } 319 if stats.TotalBlocked != 0 { 320 t.Fatalf("bad: %#v", stats) 321 } 322 323 // Ack out 324 err = b.Ack(eval3.ID, token) 325 if err != nil { 326 t.Fatalf("err: %v", err) 327 } 328 329 // Check the stats 330 stats = b.Stats() 331 if stats.TotalReady != 0 { 332 t.Fatalf("bad: %#v", stats) 333 } 334 if stats.TotalUnacked != 0 { 335 t.Fatalf("bad: %#v", stats) 336 } 337 if stats.TotalBlocked != 0 { 338 t.Fatalf("bad: %#v", stats) 339 } 340 } 341 342 func TestEvalBroker_Enqueue_Disable(t *testing.T) { 343 b := testBroker(t, 0) 344 345 // Enqueue 346 eval := mock.Eval() 347 b.SetEnabled(true) 348 err := b.Enqueue(eval) 349 if err != nil { 350 t.Fatalf("err: %v", err) 351 } 352 353 // Flush via SetEnabled 354 b.SetEnabled(false) 355 356 // Check the stats 357 stats := b.Stats() 358 if stats.TotalReady != 0 { 359 t.Fatalf("bad: %#v", stats) 360 } 361 if stats.TotalUnacked != 0 { 362 t.Fatalf("bad: %#v", stats) 363 } 364 if _, ok := stats.ByScheduler[eval.Type]; ok { 365 t.Fatalf("bad: %#v", stats) 366 } 367 } 368 369 func TestEvalBroker_Dequeue_Timeout(t *testing.T) { 370 b := testBroker(t, 0) 371 b.SetEnabled(true) 372 373 start := time.Now() 374 out, _, err := b.Dequeue(defaultSched, 5*time.Millisecond) 375 end := time.Now() 376 377 if err != nil { 378 t.Fatalf("err: %v", err) 379 } 380 if out != nil { 381 t.Fatalf("unexpected: %#v", out) 382 } 383 384 if diff := end.Sub(start); diff < 5*time.Millisecond { 385 t.Fatalf("bad: %#v", diff) 386 } 387 } 388 389 // Ensure higher priority dequeued first 390 func TestEvalBroker_Dequeue_Priority(t *testing.T) { 391 b := testBroker(t, 0) 392 b.SetEnabled(true) 393 394 eval1 := mock.Eval() 395 eval1.Priority = 10 396 b.Enqueue(eval1) 397 398 eval2 := mock.Eval() 399 eval2.Priority = 30 400 b.Enqueue(eval2) 401 402 eval3 := mock.Eval() 403 eval3.Priority = 20 404 b.Enqueue(eval3) 405 406 out1, _, _ := b.Dequeue(defaultSched, time.Second) 407 if out1 != eval2 { 408 t.Fatalf("bad: %#v", out1) 409 } 410 411 out2, _, _ := b.Dequeue(defaultSched, time.Second) 412 if out2 != eval3 { 413 t.Fatalf("bad: %#v", out2) 414 } 415 416 out3, _, _ := b.Dequeue(defaultSched, time.Second) 417 if out3 != eval1 { 418 t.Fatalf("bad: %#v", out3) 419 } 420 } 421 422 // Ensure FIFO at fixed priority 423 func TestEvalBroker_Dequeue_FIFO(t *testing.T) { 424 b := testBroker(t, 0) 425 b.SetEnabled(true) 426 NUM := 100 427 428 for i := 0; i < NUM; i++ { 429 eval1 := mock.Eval() 430 eval1.CreateIndex = uint64(i) 431 eval1.ModifyIndex = uint64(i) 432 b.Enqueue(eval1) 433 } 434 435 for i := 0; i < NUM; i++ { 436 out1, _, _ := b.Dequeue(defaultSched, time.Second) 437 if out1.CreateIndex != uint64(i) { 438 t.Fatalf("bad: %d %#v", i, out1) 439 } 440 } 441 } 442 443 // Ensure fairness between schedulers 444 func TestEvalBroker_Dequeue_Fairness(t *testing.T) { 445 b := testBroker(t, 0) 446 b.SetEnabled(true) 447 NUM := 100 448 449 for i := 0; i < NUM; i++ { 450 eval1 := mock.Eval() 451 if i < (NUM / 2) { 452 eval1.Type = structs.JobTypeService 453 } else { 454 eval1.Type = structs.JobTypeBatch 455 } 456 b.Enqueue(eval1) 457 } 458 459 counter := 0 460 for i := 0; i < NUM; i++ { 461 out1, _, _ := b.Dequeue(defaultSched, time.Second) 462 463 switch out1.Type { 464 case structs.JobTypeService: 465 if counter < 0 { 466 counter = 0 467 } 468 counter += 1 469 case structs.JobTypeBatch: 470 if counter > 0 { 471 counter = 0 472 } 473 counter -= 1 474 } 475 476 // This will fail randomly at times. It is very hard to 477 // test deterministically that its acting randomly. 478 if counter >= 25 || counter <= -25 { 479 t.Fatalf("unlikely sequence: %d", counter) 480 } 481 } 482 } 483 484 // Ensure we get unblocked 485 func TestEvalBroker_Dequeue_Blocked(t *testing.T) { 486 b := testBroker(t, 0) 487 b.SetEnabled(true) 488 489 // Start with a blocked dequeue 490 outCh := make(chan *structs.Evaluation, 1) 491 go func() { 492 start := time.Now() 493 out, _, err := b.Dequeue(defaultSched, time.Second) 494 end := time.Now() 495 outCh <- out 496 if err != nil { 497 t.Fatalf("err: %v", err) 498 } 499 if d := end.Sub(start); d < 5*time.Millisecond { 500 t.Fatalf("bad: %v", d) 501 } 502 }() 503 504 // Wait for a bit 505 time.Sleep(5 * time.Millisecond) 506 507 // Enqueue 508 eval := mock.Eval() 509 err := b.Enqueue(eval) 510 if err != nil { 511 t.Fatalf("err: %v", err) 512 } 513 514 // Ensure dequeue 515 select { 516 case out := <-outCh: 517 if out != eval { 518 t.Fatalf("bad: %v", out) 519 } 520 case <-time.After(time.Second): 521 t.Fatalf("timeout") 522 } 523 } 524 525 // Ensure we nack in a timely manner 526 func TestEvalBroker_Nack_Timeout(t *testing.T) { 527 b := testBroker(t, 5*time.Millisecond) 528 b.SetEnabled(true) 529 530 // Enqueue 531 eval := mock.Eval() 532 err := b.Enqueue(eval) 533 if err != nil { 534 t.Fatalf("err: %v", err) 535 } 536 537 // Dequeue 538 out, _, err := b.Dequeue(defaultSched, time.Second) 539 start := time.Now() 540 if err != nil { 541 t.Fatalf("err: %v", err) 542 } 543 if out != eval { 544 t.Fatalf("bad: %v", out) 545 } 546 547 // Dequeue, should block on Nack timer 548 out, _, err = b.Dequeue(defaultSched, time.Second) 549 end := time.Now() 550 if err != nil { 551 t.Fatalf("err: %v", err) 552 } 553 if out != eval { 554 t.Fatalf("bad: %v", out) 555 } 556 557 // Check the nack timer 558 if diff := end.Sub(start); diff < 5*time.Millisecond { 559 t.Fatalf("bad: %#v", diff) 560 } 561 } 562 563 func TestEvalBroker_DeliveryLimit(t *testing.T) { 564 b := testBroker(t, 0) 565 b.SetEnabled(true) 566 567 eval := mock.Eval() 568 err := b.Enqueue(eval) 569 if err != nil { 570 t.Fatalf("err: %v", err) 571 } 572 573 for i := 0; i < 3; i++ { 574 // Dequeue should work 575 out, token, err := b.Dequeue(defaultSched, time.Second) 576 if err != nil { 577 t.Fatalf("err: %v", err) 578 } 579 if out != eval { 580 t.Fatalf("bad : %#v", out) 581 } 582 583 // Nack with wrong token should fail 584 err = b.Nack(eval.ID, token) 585 if err != nil { 586 t.Fatalf("err: %v", err) 587 } 588 } 589 590 // Check the stats 591 stats := b.Stats() 592 if stats.TotalReady != 1 { 593 t.Fatalf("bad: %#v", stats) 594 } 595 if stats.TotalUnacked != 0 { 596 t.Fatalf("bad: %#v", stats) 597 } 598 if stats.ByScheduler[failedQueue].Ready != 1 { 599 t.Fatalf("bad: %#v", stats) 600 } 601 if stats.ByScheduler[failedQueue].Unacked != 0 { 602 t.Fatalf("bad: %#v", stats) 603 } 604 605 // Dequeue from failed queue 606 out, token, err := b.Dequeue([]string{failedQueue}, time.Second) 607 if err != nil { 608 t.Fatalf("err: %v", err) 609 } 610 if out != eval { 611 t.Fatalf("bad : %#v", out) 612 } 613 614 // Check the stats 615 stats = b.Stats() 616 if stats.TotalReady != 0 { 617 t.Fatalf("bad: %#v", stats) 618 } 619 if stats.TotalUnacked != 1 { 620 t.Fatalf("bad: %#v", stats) 621 } 622 if stats.ByScheduler[failedQueue].Ready != 0 { 623 t.Fatalf("bad: %#v", stats) 624 } 625 if stats.ByScheduler[failedQueue].Unacked != 1 { 626 t.Fatalf("bad: %#v", stats) 627 } 628 629 // Ack finally 630 err = b.Ack(out.ID, token) 631 if err != nil { 632 t.Fatalf("err: %v", err) 633 } 634 635 if _, ok := b.Outstanding(out.ID); ok { 636 t.Fatalf("should not be outstanding") 637 } 638 639 // Check the stats 640 stats = b.Stats() 641 if stats.TotalReady != 0 { 642 t.Fatalf("bad: %#v", stats) 643 } 644 if stats.TotalUnacked != 0 { 645 t.Fatalf("bad: %#v", stats) 646 } 647 if stats.ByScheduler[failedQueue].Ready != 0 { 648 t.Fatalf("bad: %#v", stats.ByScheduler[failedQueue]) 649 } 650 if stats.ByScheduler[failedQueue].Unacked != 0 { 651 t.Fatalf("bad: %#v", stats.ByScheduler[failedQueue]) 652 } 653 } 654 655 // Ensure fairness between schedulers 656 func TestEvalBroker_Wait(t *testing.T) { 657 b := testBroker(t, 0) 658 b.SetEnabled(true) 659 660 // Create an eval that should wait 661 eval := mock.Eval() 662 eval.Wait = 10 * time.Millisecond 663 err := b.Enqueue(eval) 664 if err != nil { 665 t.Fatalf("err: %v", err) 666 } 667 668 // Verify waiting 669 stats := b.Stats() 670 if stats.TotalReady != 0 { 671 t.Fatalf("bad: %#v", stats) 672 } 673 if stats.TotalWaiting != 1 { 674 t.Fatalf("bad: %#v", stats) 675 } 676 677 // Let the wait elapse 678 time.Sleep(15 * time.Millisecond) 679 680 // Verify ready 681 stats = b.Stats() 682 if stats.TotalReady != 1 { 683 t.Fatalf("bad: %#v", stats) 684 } 685 if stats.TotalWaiting != 0 { 686 t.Fatalf("bad: %#v", stats) 687 } 688 689 // Dequeue should work 690 out, _, err := b.Dequeue(defaultSched, time.Second) 691 if err != nil { 692 t.Fatalf("err: %v", err) 693 } 694 if out != eval { 695 t.Fatalf("bad : %#v", out) 696 } 697 }