github.com/maier/nomad@v0.4.1-0.20161110003312-a9e3d0b8549d/nomad/eval_endpoint_test.go (about) 1 package nomad 2 3 import ( 4 "reflect" 5 "strings" 6 "testing" 7 "time" 8 9 "github.com/hashicorp/net-rpc-msgpackrpc" 10 "github.com/hashicorp/nomad/nomad/mock" 11 "github.com/hashicorp/nomad/nomad/structs" 12 "github.com/hashicorp/nomad/scheduler" 13 "github.com/hashicorp/nomad/testutil" 14 ) 15 16 func TestEvalEndpoint_GetEval(t *testing.T) { 17 s1 := testServer(t, nil) 18 defer s1.Shutdown() 19 codec := rpcClient(t, s1) 20 testutil.WaitForLeader(t, s1.RPC) 21 22 // Create the register request 23 eval1 := mock.Eval() 24 s1.fsm.State().UpsertEvals(1000, []*structs.Evaluation{eval1}) 25 26 // Lookup the eval 27 get := &structs.EvalSpecificRequest{ 28 EvalID: eval1.ID, 29 QueryOptions: structs.QueryOptions{Region: "global"}, 30 } 31 var resp structs.SingleEvalResponse 32 if err := msgpackrpc.CallWithCodec(codec, "Eval.GetEval", get, &resp); err != nil { 33 t.Fatalf("err: %v", err) 34 } 35 if resp.Index != 1000 { 36 t.Fatalf("Bad index: %d %d", resp.Index, 1000) 37 } 38 39 if !reflect.DeepEqual(eval1, resp.Eval) { 40 t.Fatalf("bad: %#v %#v", eval1, resp.Eval) 41 } 42 43 // Lookup non-existing node 44 get.EvalID = structs.GenerateUUID() 45 if err := msgpackrpc.CallWithCodec(codec, "Eval.GetEval", get, &resp); err != nil { 46 t.Fatalf("err: %v", err) 47 } 48 if resp.Index != 1000 { 49 t.Fatalf("Bad index: %d %d", resp.Index, 1000) 50 } 51 if resp.Eval != nil { 52 t.Fatalf("unexpected eval") 53 } 54 } 55 56 func TestEvalEndpoint_GetEval_Blocking(t *testing.T) { 57 s1 := testServer(t, nil) 58 defer s1.Shutdown() 59 state := s1.fsm.State() 60 codec := rpcClient(t, s1) 61 testutil.WaitForLeader(t, s1.RPC) 62 63 // Create the evals 64 eval1 := mock.Eval() 65 eval2 := mock.Eval() 66 67 // First create an unrelated eval 68 time.AfterFunc(100*time.Millisecond, func() { 69 err := state.UpsertEvals(100, []*structs.Evaluation{eval1}) 70 if err != nil { 71 t.Fatalf("err: %v", err) 72 } 73 }) 74 75 // Upsert the eval we are watching later 76 time.AfterFunc(200*time.Millisecond, func() { 77 err := state.UpsertEvals(200, []*structs.Evaluation{eval2}) 78 if err != nil { 79 t.Fatalf("err: %v", err) 80 } 81 }) 82 83 // Lookup the eval 84 req := &structs.EvalSpecificRequest{ 85 EvalID: eval2.ID, 86 QueryOptions: structs.QueryOptions{ 87 Region: "global", 88 MinQueryIndex: 50, 89 }, 90 } 91 var resp structs.SingleEvalResponse 92 start := time.Now() 93 if err := msgpackrpc.CallWithCodec(codec, "Eval.GetEval", req, &resp); err != nil { 94 t.Fatalf("err: %v", err) 95 } 96 97 if elapsed := time.Since(start); elapsed < 200*time.Millisecond { 98 t.Fatalf("should block (returned in %s) %#v", elapsed, resp) 99 } 100 if resp.Index != 200 { 101 t.Fatalf("Bad index: %d %d", resp.Index, 200) 102 } 103 if resp.Eval == nil || resp.Eval.ID != eval2.ID { 104 t.Fatalf("bad: %#v", resp.Eval) 105 } 106 107 // Eval delete triggers watches 108 time.AfterFunc(100*time.Millisecond, func() { 109 err := state.DeleteEval(300, []string{eval2.ID}, []string{}) 110 if err != nil { 111 t.Fatalf("err: %v", err) 112 } 113 }) 114 115 req.QueryOptions.MinQueryIndex = 250 116 var resp2 structs.SingleEvalResponse 117 start = time.Now() 118 if err := msgpackrpc.CallWithCodec(codec, "Eval.GetEval", req, &resp2); err != nil { 119 t.Fatalf("err: %v", err) 120 } 121 122 if elapsed := time.Since(start); elapsed < 100*time.Millisecond { 123 t.Fatalf("should block (returned in %s) %#v", elapsed, resp2) 124 } 125 if resp2.Index != 300 { 126 t.Fatalf("Bad index: %d %d", resp2.Index, 300) 127 } 128 if resp2.Eval != nil { 129 t.Fatalf("bad: %#v", resp2.Eval) 130 } 131 } 132 133 func TestEvalEndpoint_Dequeue(t *testing.T) { 134 s1 := testServer(t, func(c *Config) { 135 c.NumSchedulers = 0 // Prevent automatic dequeue 136 }) 137 defer s1.Shutdown() 138 codec := rpcClient(t, s1) 139 testutil.WaitForLeader(t, s1.RPC) 140 141 // Create the register request 142 eval1 := mock.Eval() 143 s1.evalBroker.Enqueue(eval1) 144 145 // Dequeue the eval 146 get := &structs.EvalDequeueRequest{ 147 Schedulers: defaultSched, 148 SchedulerVersion: scheduler.SchedulerVersion, 149 WriteRequest: structs.WriteRequest{Region: "global"}, 150 } 151 var resp structs.EvalDequeueResponse 152 if err := msgpackrpc.CallWithCodec(codec, "Eval.Dequeue", get, &resp); err != nil { 153 t.Fatalf("err: %v", err) 154 } 155 156 if !reflect.DeepEqual(eval1, resp.Eval) { 157 t.Fatalf("bad: %v %v", eval1, resp.Eval) 158 } 159 160 // Ensure outstanding 161 token, ok := s1.evalBroker.Outstanding(eval1.ID) 162 if !ok { 163 t.Fatalf("should be outstanding") 164 } 165 if token != resp.Token { 166 t.Fatalf("bad token: %#v %#v", token, resp.Token) 167 } 168 } 169 170 func TestEvalEndpoint_Dequeue_Version_Mismatch(t *testing.T) { 171 s1 := testServer(t, func(c *Config) { 172 c.NumSchedulers = 0 // Prevent automatic dequeue 173 }) 174 defer s1.Shutdown() 175 codec := rpcClient(t, s1) 176 testutil.WaitForLeader(t, s1.RPC) 177 178 // Create the register request 179 eval1 := mock.Eval() 180 s1.evalBroker.Enqueue(eval1) 181 182 // Dequeue the eval 183 get := &structs.EvalDequeueRequest{ 184 Schedulers: defaultSched, 185 SchedulerVersion: 0, 186 WriteRequest: structs.WriteRequest{Region: "global"}, 187 } 188 var resp structs.EvalDequeueResponse 189 err := msgpackrpc.CallWithCodec(codec, "Eval.Dequeue", get, &resp) 190 if err == nil || !strings.Contains(err.Error(), "scheduler version is 0") { 191 t.Fatalf("err: %v", err) 192 } 193 } 194 195 func TestEvalEndpoint_Ack(t *testing.T) { 196 s1 := testServer(t, nil) 197 defer s1.Shutdown() 198 codec := rpcClient(t, s1) 199 200 testutil.WaitForResult(func() (bool, error) { 201 return s1.evalBroker.Enabled(), nil 202 }, func(err error) { 203 t.Fatalf("should enable eval broker") 204 }) 205 206 // Create the register request 207 eval1 := mock.Eval() 208 s1.evalBroker.Enqueue(eval1) 209 out, token, err := s1.evalBroker.Dequeue(defaultSched, time.Second) 210 if err != nil { 211 t.Fatalf("err: %v", err) 212 } 213 if out == nil { 214 t.Fatalf("missing eval") 215 } 216 217 // Ack the eval 218 get := &structs.EvalAckRequest{ 219 EvalID: out.ID, 220 Token: token, 221 WriteRequest: structs.WriteRequest{Region: "global"}, 222 } 223 var resp structs.GenericResponse 224 if err := msgpackrpc.CallWithCodec(codec, "Eval.Ack", get, &resp); err != nil { 225 t.Fatalf("err: %v", err) 226 } 227 228 // Ensure outstanding 229 if _, ok := s1.evalBroker.Outstanding(eval1.ID); ok { 230 t.Fatalf("should not be outstanding") 231 } 232 } 233 234 func TestEvalEndpoint_Nack(t *testing.T) { 235 s1 := testServer(t, func(c *Config) { 236 // Disable all of the schedulers so we can manually dequeue 237 // evals and check the queue status 238 c.NumSchedulers = 0 239 }) 240 defer s1.Shutdown() 241 codec := rpcClient(t, s1) 242 243 testutil.WaitForResult(func() (bool, error) { 244 return s1.evalBroker.Enabled(), nil 245 }, func(err error) { 246 t.Fatalf("should enable eval broker") 247 }) 248 249 // Create the register request 250 eval1 := mock.Eval() 251 s1.evalBroker.Enqueue(eval1) 252 out, token, _ := s1.evalBroker.Dequeue(defaultSched, time.Second) 253 if out == nil { 254 t.Fatalf("missing eval") 255 } 256 257 // Nack the eval 258 get := &structs.EvalAckRequest{ 259 EvalID: out.ID, 260 Token: token, 261 WriteRequest: structs.WriteRequest{Region: "global"}, 262 } 263 var resp structs.GenericResponse 264 if err := msgpackrpc.CallWithCodec(codec, "Eval.Nack", get, &resp); err != nil { 265 t.Fatalf("err: %v", err) 266 } 267 268 // Ensure outstanding 269 if _, ok := s1.evalBroker.Outstanding(eval1.ID); ok { 270 t.Fatalf("should not be outstanding") 271 } 272 273 // Should get it back 274 out2, _, _ := s1.evalBroker.Dequeue(defaultSched, time.Second) 275 if out2 != out { 276 t.Fatalf("nack failed") 277 } 278 } 279 280 func TestEvalEndpoint_Update(t *testing.T) { 281 s1 := testServer(t, nil) 282 defer s1.Shutdown() 283 codec := rpcClient(t, s1) 284 285 testutil.WaitForResult(func() (bool, error) { 286 return s1.evalBroker.Enabled(), nil 287 }, func(err error) { 288 t.Fatalf("should enable eval broker") 289 }) 290 291 // Create the register request 292 eval1 := mock.Eval() 293 s1.evalBroker.Enqueue(eval1) 294 out, token, err := s1.evalBroker.Dequeue(defaultSched, time.Second) 295 if err != nil { 296 t.Fatalf("err: %v", err) 297 } 298 if out == nil { 299 t.Fatalf("missing eval") 300 } 301 302 // Update the eval 303 eval2 := eval1.Copy() 304 eval2.Status = structs.EvalStatusComplete 305 306 get := &structs.EvalUpdateRequest{ 307 Evals: []*structs.Evaluation{eval2}, 308 EvalToken: token, 309 WriteRequest: structs.WriteRequest{Region: "global"}, 310 } 311 var resp structs.GenericResponse 312 if err := msgpackrpc.CallWithCodec(codec, "Eval.Update", get, &resp); err != nil { 313 t.Fatalf("err: %v", err) 314 } 315 316 // Ensure updated 317 outE, err := s1.fsm.State().EvalByID(eval2.ID) 318 if err != nil { 319 t.Fatalf("err: %v", err) 320 } 321 if outE.Status != structs.EvalStatusComplete { 322 t.Fatalf("Bad: %#v", out) 323 } 324 } 325 326 func TestEvalEndpoint_Create(t *testing.T) { 327 s1 := testServer(t, func(c *Config) { 328 c.NumSchedulers = 0 // Prevent automatic dequeue 329 }) 330 defer s1.Shutdown() 331 codec := rpcClient(t, s1) 332 333 testutil.WaitForResult(func() (bool, error) { 334 return s1.evalBroker.Enabled(), nil 335 }, func(err error) { 336 t.Fatalf("should enable eval broker") 337 }) 338 339 // Create the register request 340 prev := mock.Eval() 341 s1.evalBroker.Enqueue(prev) 342 out, token, err := s1.evalBroker.Dequeue(defaultSched, time.Second) 343 if err != nil { 344 t.Fatalf("err: %v", err) 345 } 346 if out == nil { 347 t.Fatalf("missing eval") 348 } 349 350 // Create the register request 351 eval1 := mock.Eval() 352 eval1.PreviousEval = prev.ID 353 get := &structs.EvalUpdateRequest{ 354 Evals: []*structs.Evaluation{eval1}, 355 EvalToken: token, 356 WriteRequest: structs.WriteRequest{Region: "global"}, 357 } 358 var resp structs.GenericResponse 359 if err := msgpackrpc.CallWithCodec(codec, "Eval.Create", get, &resp); err != nil { 360 t.Fatalf("err: %v", err) 361 } 362 363 // Ensure created 364 outE, err := s1.fsm.State().EvalByID(eval1.ID) 365 if err != nil { 366 t.Fatalf("err: %v", err) 367 } 368 369 eval1.CreateIndex = resp.Index 370 eval1.ModifyIndex = resp.Index 371 if !reflect.DeepEqual(eval1, outE) { 372 t.Fatalf("Bad: %#v %#v", outE, eval1) 373 } 374 } 375 376 func TestEvalEndpoint_Reap(t *testing.T) { 377 s1 := testServer(t, nil) 378 defer s1.Shutdown() 379 codec := rpcClient(t, s1) 380 testutil.WaitForLeader(t, s1.RPC) 381 382 // Create the register request 383 eval1 := mock.Eval() 384 s1.fsm.State().UpsertEvals(1000, []*structs.Evaluation{eval1}) 385 386 // Reap the eval 387 get := &structs.EvalDeleteRequest{ 388 Evals: []string{eval1.ID}, 389 WriteRequest: structs.WriteRequest{Region: "global"}, 390 } 391 var resp structs.GenericResponse 392 if err := msgpackrpc.CallWithCodec(codec, "Eval.Reap", get, &resp); err != nil { 393 t.Fatalf("err: %v", err) 394 } 395 if resp.Index == 0 { 396 t.Fatalf("Bad index: %d", resp.Index) 397 } 398 399 // Ensure deleted 400 outE, err := s1.fsm.State().EvalByID(eval1.ID) 401 if err != nil { 402 t.Fatalf("err: %v", err) 403 } 404 if outE != nil { 405 t.Fatalf("Bad: %#v", outE) 406 } 407 } 408 409 func TestEvalEndpoint_List(t *testing.T) { 410 s1 := testServer(t, nil) 411 defer s1.Shutdown() 412 codec := rpcClient(t, s1) 413 testutil.WaitForLeader(t, s1.RPC) 414 415 // Create the register request 416 eval1 := mock.Eval() 417 eval1.ID = "aaaaaaaa-3350-4b4b-d185-0e1992ed43e9" 418 eval2 := mock.Eval() 419 eval2.ID = "aaaabbbb-3350-4b4b-d185-0e1992ed43e9" 420 s1.fsm.State().UpsertEvals(1000, []*structs.Evaluation{eval1, eval2}) 421 422 // Lookup the eval 423 get := &structs.EvalListRequest{ 424 QueryOptions: structs.QueryOptions{Region: "global"}, 425 } 426 var resp structs.EvalListResponse 427 if err := msgpackrpc.CallWithCodec(codec, "Eval.List", get, &resp); err != nil { 428 t.Fatalf("err: %v", err) 429 } 430 if resp.Index != 1000 { 431 t.Fatalf("Bad index: %d %d", resp.Index, 1000) 432 } 433 434 if len(resp.Evaluations) != 2 { 435 t.Fatalf("bad: %#v", resp.Evaluations) 436 } 437 438 // Lookup the eval by prefix 439 get = &structs.EvalListRequest{ 440 QueryOptions: structs.QueryOptions{Region: "global", Prefix: "aaaabb"}, 441 } 442 var resp2 structs.EvalListResponse 443 if err := msgpackrpc.CallWithCodec(codec, "Eval.List", get, &resp2); err != nil { 444 t.Fatalf("err: %v", err) 445 } 446 if resp2.Index != 1000 { 447 t.Fatalf("Bad index: %d %d", resp2.Index, 1000) 448 } 449 450 if len(resp2.Evaluations) != 1 { 451 t.Fatalf("bad: %#v", resp2.Evaluations) 452 } 453 454 } 455 456 func TestEvalEndpoint_List_Blocking(t *testing.T) { 457 s1 := testServer(t, nil) 458 defer s1.Shutdown() 459 state := s1.fsm.State() 460 codec := rpcClient(t, s1) 461 testutil.WaitForLeader(t, s1.RPC) 462 463 // Create the ieval 464 eval := mock.Eval() 465 466 // Upsert eval triggers watches 467 time.AfterFunc(100*time.Millisecond, func() { 468 if err := state.UpsertEvals(2, []*structs.Evaluation{eval}); err != nil { 469 t.Fatalf("err: %v", err) 470 } 471 }) 472 473 req := &structs.EvalListRequest{ 474 QueryOptions: structs.QueryOptions{ 475 Region: "global", 476 MinQueryIndex: 1, 477 }, 478 } 479 start := time.Now() 480 var resp structs.EvalListResponse 481 if err := msgpackrpc.CallWithCodec(codec, "Eval.List", req, &resp); err != nil { 482 t.Fatalf("err: %v", err) 483 } 484 485 if elapsed := time.Since(start); elapsed < 100*time.Millisecond { 486 t.Fatalf("should block (returned in %s) %#v", elapsed, resp) 487 } 488 if resp.Index != 2 { 489 t.Fatalf("Bad index: %d %d", resp.Index, 2) 490 } 491 if len(resp.Evaluations) != 1 || resp.Evaluations[0].ID != eval.ID { 492 t.Fatalf("bad: %#v", resp.Evaluations) 493 } 494 495 // Eval deletion triggers watches 496 time.AfterFunc(100*time.Millisecond, func() { 497 if err := state.DeleteEval(3, []string{eval.ID}, nil); err != nil { 498 t.Fatalf("err: %v", err) 499 } 500 }) 501 502 req.MinQueryIndex = 2 503 start = time.Now() 504 var resp2 structs.EvalListResponse 505 if err := msgpackrpc.CallWithCodec(codec, "Eval.List", req, &resp2); err != nil { 506 t.Fatalf("err: %v", err) 507 } 508 509 if elapsed := time.Since(start); elapsed < 100*time.Millisecond { 510 t.Fatalf("should block (returned in %s) %#v", elapsed, resp2) 511 } 512 if resp2.Index != 3 { 513 t.Fatalf("Bad index: %d %d", resp2.Index, 3) 514 } 515 if len(resp2.Evaluations) != 0 { 516 t.Fatalf("bad: %#v", resp2.Evaluations) 517 } 518 } 519 520 func TestEvalEndpoint_Allocations(t *testing.T) { 521 s1 := testServer(t, nil) 522 defer s1.Shutdown() 523 codec := rpcClient(t, s1) 524 testutil.WaitForLeader(t, s1.RPC) 525 526 // Create the register request 527 alloc1 := mock.Alloc() 528 alloc2 := mock.Alloc() 529 alloc2.EvalID = alloc1.EvalID 530 state := s1.fsm.State() 531 state.UpsertJobSummary(998, mock.JobSummary(alloc1.JobID)) 532 state.UpsertJobSummary(999, mock.JobSummary(alloc2.JobID)) 533 err := state.UpsertAllocs(1000, 534 []*structs.Allocation{alloc1, alloc2}) 535 if err != nil { 536 t.Fatalf("err: %v", err) 537 } 538 539 // Lookup the eval 540 get := &structs.EvalSpecificRequest{ 541 EvalID: alloc1.EvalID, 542 QueryOptions: structs.QueryOptions{Region: "global"}, 543 } 544 var resp structs.EvalAllocationsResponse 545 if err := msgpackrpc.CallWithCodec(codec, "Eval.Allocations", get, &resp); err != nil { 546 t.Fatalf("err: %v", err) 547 } 548 if resp.Index != 1000 { 549 t.Fatalf("Bad index: %d %d", resp.Index, 1000) 550 } 551 552 if len(resp.Allocations) != 2 { 553 t.Fatalf("bad: %#v", resp.Allocations) 554 } 555 } 556 557 func TestEvalEndpoint_Allocations_Blocking(t *testing.T) { 558 s1 := testServer(t, nil) 559 defer s1.Shutdown() 560 state := s1.fsm.State() 561 codec := rpcClient(t, s1) 562 testutil.WaitForLeader(t, s1.RPC) 563 564 // Create the allocs 565 alloc1 := mock.Alloc() 566 alloc2 := mock.Alloc() 567 568 // Upsert an unrelated alloc first 569 time.AfterFunc(100*time.Millisecond, func() { 570 state.UpsertJobSummary(99, mock.JobSummary(alloc1.JobID)) 571 err := state.UpsertAllocs(100, []*structs.Allocation{alloc1}) 572 if err != nil { 573 t.Fatalf("err: %v", err) 574 } 575 }) 576 577 // Upsert an alloc which will trigger the watch later 578 time.AfterFunc(200*time.Millisecond, func() { 579 state.UpsertJobSummary(199, mock.JobSummary(alloc2.JobID)) 580 err := state.UpsertAllocs(200, []*structs.Allocation{alloc2}) 581 if err != nil { 582 t.Fatalf("err: %v", err) 583 } 584 }) 585 586 // Lookup the eval 587 get := &structs.EvalSpecificRequest{ 588 EvalID: alloc2.EvalID, 589 QueryOptions: structs.QueryOptions{ 590 Region: "global", 591 MinQueryIndex: 50, 592 }, 593 } 594 var resp structs.EvalAllocationsResponse 595 start := time.Now() 596 if err := msgpackrpc.CallWithCodec(codec, "Eval.Allocations", get, &resp); err != nil { 597 t.Fatalf("err: %v", err) 598 } 599 600 if elapsed := time.Since(start); elapsed < 200*time.Millisecond { 601 t.Fatalf("should block (returned in %s) %#v", elapsed, resp) 602 } 603 if resp.Index != 200 { 604 t.Fatalf("Bad index: %d %d", resp.Index, 200) 605 } 606 if len(resp.Allocations) != 1 || resp.Allocations[0].ID != alloc2.ID { 607 t.Fatalf("bad: %#v", resp.Allocations) 608 } 609 } 610 611 func TestEvalEndpoint_Reblock_NonExistent(t *testing.T) { 612 s1 := testServer(t, func(c *Config) { 613 c.NumSchedulers = 0 // Prevent automatic dequeue 614 }) 615 defer s1.Shutdown() 616 codec := rpcClient(t, s1) 617 618 testutil.WaitForResult(func() (bool, error) { 619 return s1.evalBroker.Enabled(), nil 620 }, func(err error) { 621 t.Fatalf("should enable eval broker") 622 }) 623 624 // Create the register request 625 eval1 := mock.Eval() 626 s1.evalBroker.Enqueue(eval1) 627 out, token, err := s1.evalBroker.Dequeue(defaultSched, time.Second) 628 if err != nil { 629 t.Fatalf("err: %v", err) 630 } 631 if out == nil { 632 t.Fatalf("missing eval") 633 } 634 635 get := &structs.EvalUpdateRequest{ 636 Evals: []*structs.Evaluation{eval1}, 637 EvalToken: token, 638 WriteRequest: structs.WriteRequest{Region: "global"}, 639 } 640 var resp structs.GenericResponse 641 if err := msgpackrpc.CallWithCodec(codec, "Eval.Reblock", get, &resp); err == nil { 642 t.Fatalf("expect error since eval does not exist") 643 } 644 } 645 646 func TestEvalEndpoint_Reblock_NonBlocked(t *testing.T) { 647 s1 := testServer(t, func(c *Config) { 648 c.NumSchedulers = 0 // Prevent automatic dequeue 649 }) 650 defer s1.Shutdown() 651 codec := rpcClient(t, s1) 652 653 testutil.WaitForResult(func() (bool, error) { 654 return s1.evalBroker.Enabled(), nil 655 }, func(err error) { 656 t.Fatalf("should enable eval broker") 657 }) 658 659 // Create the eval 660 eval1 := mock.Eval() 661 s1.evalBroker.Enqueue(eval1) 662 663 // Insert it into the state store 664 if err := s1.fsm.State().UpsertEvals(1000, []*structs.Evaluation{eval1}); err != nil { 665 t.Fatal(err) 666 } 667 668 out, token, err := s1.evalBroker.Dequeue(defaultSched, 2*time.Second) 669 if err != nil { 670 t.Fatalf("err: %v", err) 671 } 672 if out == nil { 673 t.Fatalf("missing eval") 674 } 675 676 get := &structs.EvalUpdateRequest{ 677 Evals: []*structs.Evaluation{eval1}, 678 EvalToken: token, 679 WriteRequest: structs.WriteRequest{Region: "global"}, 680 } 681 var resp structs.GenericResponse 682 if err := msgpackrpc.CallWithCodec(codec, "Eval.Reblock", get, &resp); err == nil { 683 t.Fatalf("should error since eval was not in blocked state", err) 684 } 685 } 686 687 func TestEvalEndpoint_Reblock(t *testing.T) { 688 s1 := testServer(t, func(c *Config) { 689 c.NumSchedulers = 0 // Prevent automatic dequeue 690 }) 691 defer s1.Shutdown() 692 codec := rpcClient(t, s1) 693 694 testutil.WaitForResult(func() (bool, error) { 695 return s1.evalBroker.Enabled(), nil 696 }, func(err error) { 697 t.Fatalf("should enable eval broker") 698 }) 699 700 // Create the eval 701 eval1 := mock.Eval() 702 eval1.Status = structs.EvalStatusBlocked 703 s1.evalBroker.Enqueue(eval1) 704 705 // Insert it into the state store 706 if err := s1.fsm.State().UpsertEvals(1000, []*structs.Evaluation{eval1}); err != nil { 707 t.Fatal(err) 708 } 709 710 out, token, err := s1.evalBroker.Dequeue(defaultSched, 7*time.Second) 711 if err != nil { 712 t.Fatalf("err: %v", err) 713 } 714 if out == nil { 715 t.Fatalf("bad: %v", out) 716 } 717 718 get := &structs.EvalUpdateRequest{ 719 Evals: []*structs.Evaluation{eval1}, 720 EvalToken: token, 721 WriteRequest: structs.WriteRequest{Region: "global"}, 722 } 723 var resp structs.GenericResponse 724 if err := msgpackrpc.CallWithCodec(codec, "Eval.Reblock", get, &resp); err != nil { 725 t.Fatalf("err: %v", err) 726 } 727 728 // Check that it is blocked 729 bStats := s1.blockedEvals.Stats() 730 if bStats.TotalBlocked+bStats.TotalEscaped == 0 { 731 t.Fatalf("ReblockEval didn't insert eval into the blocked eval tracker") 732 } 733 }