github.com/aminovpavel/nomad@v0.11.8/nomad/eval_endpoint_test.go (about) 1 package nomad 2 3 import ( 4 "fmt" 5 "reflect" 6 "strings" 7 "testing" 8 "time" 9 10 memdb "github.com/hashicorp/go-memdb" 11 msgpackrpc "github.com/hashicorp/net-rpc-msgpackrpc" 12 "github.com/hashicorp/nomad/acl" 13 "github.com/hashicorp/nomad/helper/uuid" 14 "github.com/hashicorp/nomad/nomad/mock" 15 "github.com/hashicorp/nomad/nomad/structs" 16 "github.com/hashicorp/nomad/scheduler" 17 "github.com/hashicorp/nomad/testutil" 18 "github.com/stretchr/testify/assert" 19 "github.com/stretchr/testify/require" 20 ) 21 22 func TestEvalEndpoint_GetEval(t *testing.T) { 23 t.Parallel() 24 25 s1, cleanupS1 := TestServer(t, nil) 26 defer cleanupS1() 27 codec := rpcClient(t, s1) 28 testutil.WaitForLeader(t, s1.RPC) 29 30 // Create the register request 31 eval1 := mock.Eval() 32 s1.fsm.State().UpsertEvals(1000, []*structs.Evaluation{eval1}) 33 34 // Lookup the eval 35 get := &structs.EvalSpecificRequest{ 36 EvalID: eval1.ID, 37 QueryOptions: structs.QueryOptions{Region: "global"}, 38 } 39 var resp structs.SingleEvalResponse 40 if err := msgpackrpc.CallWithCodec(codec, "Eval.GetEval", get, &resp); err != nil { 41 t.Fatalf("err: %v", err) 42 } 43 if resp.Index != 1000 { 44 t.Fatalf("Bad index: %d %d", resp.Index, 1000) 45 } 46 47 if !reflect.DeepEqual(eval1, resp.Eval) { 48 t.Fatalf("bad: %#v %#v", eval1, resp.Eval) 49 } 50 51 // Lookup non-existing node 52 get.EvalID = uuid.Generate() 53 if err := msgpackrpc.CallWithCodec(codec, "Eval.GetEval", get, &resp); err != nil { 54 t.Fatalf("err: %v", err) 55 } 56 if resp.Index != 1000 { 57 t.Fatalf("Bad index: %d %d", resp.Index, 1000) 58 } 59 if resp.Eval != nil { 60 t.Fatalf("unexpected eval") 61 } 62 } 63 64 func TestEvalEndpoint_GetEval_ACL(t *testing.T) { 65 t.Parallel() 66 67 s1, root, cleanupS1 := TestACLServer(t, nil) 68 defer cleanupS1() 69 codec := rpcClient(t, s1) 70 testutil.WaitForLeader(t, s1.RPC) 71 assert := assert.New(t) 72 73 // Create the register request 74 eval1 := mock.Eval() 75 state := s1.fsm.State() 76 state.UpsertEvals(1000, []*structs.Evaluation{eval1}) 77 78 // Create ACL tokens 79 validToken := mock.CreatePolicyAndToken(t, state, 1003, "test-valid", 80 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob})) 81 invalidToken := mock.CreatePolicyAndToken(t, state, 1001, "test-invalid", 82 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityListJobs})) 83 84 get := &structs.EvalSpecificRequest{ 85 EvalID: eval1.ID, 86 QueryOptions: structs.QueryOptions{Region: "global"}, 87 } 88 89 // Try with no token and expect permission denied 90 { 91 var resp structs.SingleEvalResponse 92 err := msgpackrpc.CallWithCodec(codec, "Eval.GetEval", get, &resp) 93 assert.NotNil(err) 94 assert.Contains(err.Error(), structs.ErrPermissionDenied.Error()) 95 } 96 97 // Try with an invalid token and expect permission denied 98 { 99 get.AuthToken = invalidToken.SecretID 100 var resp structs.SingleEvalResponse 101 err := msgpackrpc.CallWithCodec(codec, "Eval.GetEval", get, &resp) 102 assert.NotNil(err) 103 assert.Contains(err.Error(), structs.ErrPermissionDenied.Error()) 104 } 105 106 // Lookup the eval using a valid token 107 { 108 get.AuthToken = validToken.SecretID 109 var resp structs.SingleEvalResponse 110 assert.Nil(msgpackrpc.CallWithCodec(codec, "Eval.GetEval", get, &resp)) 111 assert.Equal(uint64(1000), resp.Index, "Bad index: %d %d", resp.Index, 1000) 112 assert.Equal(eval1, resp.Eval) 113 } 114 115 // Lookup the eval using a root token 116 { 117 get.AuthToken = root.SecretID 118 var resp structs.SingleEvalResponse 119 assert.Nil(msgpackrpc.CallWithCodec(codec, "Eval.GetEval", get, &resp)) 120 assert.Equal(uint64(1000), resp.Index, "Bad index: %d %d", resp.Index, 1000) 121 assert.Equal(eval1, resp.Eval) 122 } 123 } 124 125 func TestEvalEndpoint_GetEval_Blocking(t *testing.T) { 126 t.Parallel() 127 128 s1, cleanupS1 := TestServer(t, nil) 129 defer cleanupS1() 130 state := s1.fsm.State() 131 codec := rpcClient(t, s1) 132 testutil.WaitForLeader(t, s1.RPC) 133 134 // Create the evals 135 eval1 := mock.Eval() 136 eval2 := mock.Eval() 137 138 // First create an unrelated eval 139 time.AfterFunc(100*time.Millisecond, func() { 140 err := state.UpsertEvals(100, []*structs.Evaluation{eval1}) 141 if err != nil { 142 t.Fatalf("err: %v", err) 143 } 144 }) 145 146 // Upsert the eval we are watching later 147 time.AfterFunc(200*time.Millisecond, func() { 148 err := state.UpsertEvals(200, []*structs.Evaluation{eval2}) 149 if err != nil { 150 t.Fatalf("err: %v", err) 151 } 152 }) 153 154 // Lookup the eval 155 req := &structs.EvalSpecificRequest{ 156 EvalID: eval2.ID, 157 QueryOptions: structs.QueryOptions{ 158 Region: "global", 159 MinQueryIndex: 150, 160 }, 161 } 162 var resp structs.SingleEvalResponse 163 start := time.Now() 164 if err := msgpackrpc.CallWithCodec(codec, "Eval.GetEval", req, &resp); err != nil { 165 t.Fatalf("err: %v", err) 166 } 167 168 if elapsed := time.Since(start); elapsed < 200*time.Millisecond { 169 t.Fatalf("should block (returned in %s) %#v", elapsed, resp) 170 } 171 if resp.Index != 200 { 172 t.Fatalf("Bad index: %d %d", resp.Index, 200) 173 } 174 if resp.Eval == nil || resp.Eval.ID != eval2.ID { 175 t.Fatalf("bad: %#v", resp.Eval) 176 } 177 178 // Eval delete triggers watches 179 time.AfterFunc(100*time.Millisecond, func() { 180 err := state.DeleteEval(300, []string{eval2.ID}, []string{}) 181 if err != nil { 182 t.Fatalf("err: %v", err) 183 } 184 }) 185 186 req.QueryOptions.MinQueryIndex = 250 187 var resp2 structs.SingleEvalResponse 188 start = time.Now() 189 if err := msgpackrpc.CallWithCodec(codec, "Eval.GetEval", req, &resp2); err != nil { 190 t.Fatalf("err: %v", err) 191 } 192 193 if elapsed := time.Since(start); elapsed < 100*time.Millisecond { 194 t.Fatalf("should block (returned in %s) %#v", elapsed, resp2) 195 } 196 if resp2.Index != 300 { 197 t.Fatalf("Bad index: %d %d", resp2.Index, 300) 198 } 199 if resp2.Eval != nil { 200 t.Fatalf("bad: %#v", resp2.Eval) 201 } 202 } 203 204 func TestEvalEndpoint_Dequeue(t *testing.T) { 205 t.Parallel() 206 207 s1, cleanupS1 := TestServer(t, func(c *Config) { 208 c.NumSchedulers = 0 // Prevent automatic dequeue 209 }) 210 defer cleanupS1() 211 codec := rpcClient(t, s1) 212 testutil.WaitForLeader(t, s1.RPC) 213 214 // Create the register request 215 eval1 := mock.Eval() 216 s1.evalBroker.Enqueue(eval1) 217 218 // Dequeue the eval 219 get := &structs.EvalDequeueRequest{ 220 Schedulers: defaultSched, 221 SchedulerVersion: scheduler.SchedulerVersion, 222 WriteRequest: structs.WriteRequest{Region: "global"}, 223 } 224 var resp structs.EvalDequeueResponse 225 if err := msgpackrpc.CallWithCodec(codec, "Eval.Dequeue", get, &resp); err != nil { 226 t.Fatalf("err: %v", err) 227 } 228 229 if !reflect.DeepEqual(eval1, resp.Eval) { 230 t.Fatalf("bad: %v %v", eval1, resp.Eval) 231 } 232 233 // Ensure outstanding 234 token, ok := s1.evalBroker.Outstanding(eval1.ID) 235 if !ok { 236 t.Fatalf("should be outstanding") 237 } 238 if token != resp.Token { 239 t.Fatalf("bad token: %#v %#v", token, resp.Token) 240 } 241 242 if resp.WaitIndex != eval1.ModifyIndex { 243 t.Fatalf("bad wait index; got %d; want %d", resp.WaitIndex, eval1.ModifyIndex) 244 } 245 } 246 247 // TestEvalEndpoint_Dequeue_WaitIndex_Snapshot asserts that an eval's wait 248 // index will be equal to the highest eval modify index in the state store. 249 func TestEvalEndpoint_Dequeue_WaitIndex_Snapshot(t *testing.T) { 250 t.Parallel() 251 252 s1, cleanupS1 := TestServer(t, func(c *Config) { 253 c.NumSchedulers = 0 // Prevent automatic dequeue 254 }) 255 defer cleanupS1() 256 codec := rpcClient(t, s1) 257 testutil.WaitForLeader(t, s1.RPC) 258 259 // Create the register request 260 eval1 := mock.Eval() 261 eval2 := mock.Eval() 262 eval2.JobID = eval1.JobID 263 s1.fsm.State().UpsertEvals(1000, []*structs.Evaluation{eval1}) 264 s1.evalBroker.Enqueue(eval1) 265 s1.fsm.State().UpsertEvals(1001, []*structs.Evaluation{eval2}) 266 267 // Dequeue the eval 268 get := &structs.EvalDequeueRequest{ 269 Schedulers: defaultSched, 270 SchedulerVersion: scheduler.SchedulerVersion, 271 WriteRequest: structs.WriteRequest{Region: "global"}, 272 } 273 var resp structs.EvalDequeueResponse 274 if err := msgpackrpc.CallWithCodec(codec, "Eval.Dequeue", get, &resp); err != nil { 275 t.Fatalf("err: %v", err) 276 } 277 278 if !reflect.DeepEqual(eval1, resp.Eval) { 279 t.Fatalf("bad: %v %v", eval1, resp.Eval) 280 } 281 282 // Ensure outstanding 283 token, ok := s1.evalBroker.Outstanding(eval1.ID) 284 if !ok { 285 t.Fatalf("should be outstanding") 286 } 287 if token != resp.Token { 288 t.Fatalf("bad token: %#v %#v", token, resp.Token) 289 } 290 291 if resp.WaitIndex != 1001 { 292 t.Fatalf("bad wait index; got %d; want %d", resp.WaitIndex, 1001) 293 } 294 } 295 296 // TestEvalEndpoint_Dequeue_WaitIndex_Eval asserts that an eval's wait index 297 // will be its own modify index if its modify index is greater than all of the 298 // indexes in the state store. This can happen if Dequeue receives an eval that 299 // has not yet been applied from the Raft log to the local node's state store. 300 func TestEvalEndpoint_Dequeue_WaitIndex_Eval(t *testing.T) { 301 t.Parallel() 302 s1, cleanupS1 := TestServer(t, func(c *Config) { 303 c.NumSchedulers = 0 // Prevent automatic dequeue 304 }) 305 defer cleanupS1() 306 codec := rpcClient(t, s1) 307 testutil.WaitForLeader(t, s1.RPC) 308 309 // Create the register request but only upsert 1 into the state store 310 eval1 := mock.Eval() 311 eval2 := mock.Eval() 312 eval2.JobID = eval1.JobID 313 s1.fsm.State().UpsertEvals(1000, []*structs.Evaluation{eval1}) 314 eval2.ModifyIndex = 1001 315 s1.evalBroker.Enqueue(eval2) 316 317 // Dequeue the eval 318 get := &structs.EvalDequeueRequest{ 319 Schedulers: defaultSched, 320 SchedulerVersion: scheduler.SchedulerVersion, 321 WriteRequest: structs.WriteRequest{Region: "global"}, 322 } 323 var resp structs.EvalDequeueResponse 324 require.NoError(t, msgpackrpc.CallWithCodec(codec, "Eval.Dequeue", get, &resp)) 325 require.Equal(t, eval2, resp.Eval) 326 327 // Ensure outstanding 328 token, ok := s1.evalBroker.Outstanding(eval2.ID) 329 require.True(t, ok) 330 require.Equal(t, resp.Token, token) 331 332 // WaitIndex should be equal to the max ModifyIndex - even when that 333 // modify index is of the dequeued eval which has yet to be applied to 334 // the state store. 335 require.Equal(t, eval2.ModifyIndex, resp.WaitIndex) 336 } 337 338 func TestEvalEndpoint_Dequeue_UpdateWaitIndex(t *testing.T) { 339 // test enqueuing an eval, updating a plan result for the same eval and de-queueing the eval 340 t.Parallel() 341 s1, cleanupS1 := TestServer(t, func(c *Config) { 342 c.NumSchedulers = 0 // Prevent automatic dequeue 343 }) 344 defer cleanupS1() 345 codec := rpcClient(t, s1) 346 testutil.WaitForLeader(t, s1.RPC) 347 348 alloc := mock.Alloc() 349 job := alloc.Job 350 alloc.Job = nil 351 352 state := s1.fsm.State() 353 354 if err := state.UpsertJob(999, job); err != nil { 355 t.Fatalf("err: %v", err) 356 } 357 358 eval := mock.Eval() 359 eval.JobID = job.ID 360 361 // Create an eval 362 if err := state.UpsertEvals(1, []*structs.Evaluation{eval}); err != nil { 363 t.Fatalf("err: %v", err) 364 } 365 366 s1.evalBroker.Enqueue(eval) 367 368 // Create a plan result and apply it with a later index 369 res := structs.ApplyPlanResultsRequest{ 370 AllocUpdateRequest: structs.AllocUpdateRequest{ 371 Alloc: []*structs.Allocation{alloc}, 372 Job: job, 373 }, 374 EvalID: eval.ID, 375 } 376 assert := assert.New(t) 377 err := state.UpsertPlanResults(1000, &res) 378 assert.Nil(err) 379 380 // Dequeue the eval 381 get := &structs.EvalDequeueRequest{ 382 Schedulers: defaultSched, 383 SchedulerVersion: scheduler.SchedulerVersion, 384 WriteRequest: structs.WriteRequest{Region: "global"}, 385 } 386 var resp structs.EvalDequeueResponse 387 if err := msgpackrpc.CallWithCodec(codec, "Eval.Dequeue", get, &resp); err != nil { 388 t.Fatalf("err: %v", err) 389 } 390 391 // Ensure outstanding 392 token, ok := s1.evalBroker.Outstanding(eval.ID) 393 if !ok { 394 t.Fatalf("should be outstanding") 395 } 396 if token != resp.Token { 397 t.Fatalf("bad token: %#v %#v", token, resp.Token) 398 } 399 400 if resp.WaitIndex != 1000 { 401 t.Fatalf("bad wait index; got %d; want %d", resp.WaitIndex, 1000) 402 } 403 } 404 405 func TestEvalEndpoint_Dequeue_Version_Mismatch(t *testing.T) { 406 t.Parallel() 407 408 s1, cleanupS1 := TestServer(t, func(c *Config) { 409 c.NumSchedulers = 0 // Prevent automatic dequeue 410 }) 411 defer cleanupS1() 412 codec := rpcClient(t, s1) 413 testutil.WaitForLeader(t, s1.RPC) 414 415 // Create the register request 416 eval1 := mock.Eval() 417 s1.evalBroker.Enqueue(eval1) 418 419 // Dequeue the eval 420 get := &structs.EvalDequeueRequest{ 421 Schedulers: defaultSched, 422 SchedulerVersion: 0, 423 WriteRequest: structs.WriteRequest{Region: "global"}, 424 } 425 var resp structs.EvalDequeueResponse 426 err := msgpackrpc.CallWithCodec(codec, "Eval.Dequeue", get, &resp) 427 if err == nil || !strings.Contains(err.Error(), "scheduler version is 0") { 428 t.Fatalf("err: %v", err) 429 } 430 } 431 432 func TestEvalEndpoint_Ack(t *testing.T) { 433 t.Parallel() 434 435 s1, cleanupS1 := TestServer(t, nil) 436 defer cleanupS1() 437 codec := rpcClient(t, s1) 438 439 testutil.WaitForResult(func() (bool, error) { 440 return s1.evalBroker.Enabled(), nil 441 }, func(err error) { 442 t.Fatalf("should enable eval broker") 443 }) 444 445 // Create the register request 446 eval1 := mock.Eval() 447 s1.evalBroker.Enqueue(eval1) 448 out, token, err := s1.evalBroker.Dequeue(defaultSched, time.Second) 449 if err != nil { 450 t.Fatalf("err: %v", err) 451 } 452 if out == nil { 453 t.Fatalf("missing eval") 454 } 455 456 // Ack the eval 457 get := &structs.EvalAckRequest{ 458 EvalID: out.ID, 459 Token: token, 460 WriteRequest: structs.WriteRequest{Region: "global"}, 461 } 462 var resp structs.GenericResponse 463 if err := msgpackrpc.CallWithCodec(codec, "Eval.Ack", get, &resp); err != nil { 464 t.Fatalf("err: %v", err) 465 } 466 467 // Ensure outstanding 468 if _, ok := s1.evalBroker.Outstanding(eval1.ID); ok { 469 t.Fatalf("should not be outstanding") 470 } 471 } 472 473 func TestEvalEndpoint_Nack(t *testing.T) { 474 t.Parallel() 475 476 s1, cleanupS1 := TestServer(t, func(c *Config) { 477 // Disable all of the schedulers so we can manually dequeue 478 // evals and check the queue status 479 c.NumSchedulers = 0 480 }) 481 defer cleanupS1() 482 codec := rpcClient(t, s1) 483 484 testutil.WaitForResult(func() (bool, error) { 485 return s1.evalBroker.Enabled(), nil 486 }, func(err error) { 487 t.Fatalf("should enable eval broker") 488 }) 489 490 // Create the register request 491 eval1 := mock.Eval() 492 s1.evalBroker.Enqueue(eval1) 493 out, token, _ := s1.evalBroker.Dequeue(defaultSched, time.Second) 494 if out == nil { 495 t.Fatalf("missing eval") 496 } 497 498 // Nack the eval 499 get := &structs.EvalAckRequest{ 500 EvalID: out.ID, 501 Token: token, 502 WriteRequest: structs.WriteRequest{Region: "global"}, 503 } 504 var resp structs.GenericResponse 505 if err := msgpackrpc.CallWithCodec(codec, "Eval.Nack", get, &resp); err != nil { 506 t.Fatalf("err: %v", err) 507 } 508 509 // Ensure outstanding 510 if _, ok := s1.evalBroker.Outstanding(eval1.ID); ok { 511 t.Fatalf("should not be outstanding") 512 } 513 514 // Should get it back 515 testutil.WaitForResult(func() (bool, error) { 516 out2, _, _ := s1.evalBroker.Dequeue(defaultSched, time.Second) 517 if out2 != out { 518 return false, fmt.Errorf("nack failed") 519 } 520 521 return true, nil 522 }, func(err error) { 523 t.Fatal(err) 524 }) 525 } 526 527 func TestEvalEndpoint_Update(t *testing.T) { 528 t.Parallel() 529 530 s1, cleanupS1 := TestServer(t, nil) 531 defer cleanupS1() 532 codec := rpcClient(t, s1) 533 534 testutil.WaitForResult(func() (bool, error) { 535 return s1.evalBroker.Enabled(), nil 536 }, func(err error) { 537 t.Fatalf("should enable eval broker") 538 }) 539 540 // Create the register request 541 eval1 := mock.Eval() 542 s1.evalBroker.Enqueue(eval1) 543 out, token, err := s1.evalBroker.Dequeue(defaultSched, time.Second) 544 if err != nil { 545 t.Fatalf("err: %v", err) 546 } 547 if out == nil { 548 t.Fatalf("missing eval") 549 } 550 551 // Update the eval 552 eval2 := eval1.Copy() 553 eval2.Status = structs.EvalStatusComplete 554 555 get := &structs.EvalUpdateRequest{ 556 Evals: []*structs.Evaluation{eval2}, 557 EvalToken: token, 558 WriteRequest: structs.WriteRequest{Region: "global"}, 559 } 560 var resp structs.GenericResponse 561 if err := msgpackrpc.CallWithCodec(codec, "Eval.Update", get, &resp); err != nil { 562 t.Fatalf("err: %v", err) 563 } 564 565 // Ensure updated 566 ws := memdb.NewWatchSet() 567 outE, err := s1.fsm.State().EvalByID(ws, eval2.ID) 568 if err != nil { 569 t.Fatalf("err: %v", err) 570 } 571 if outE.Status != structs.EvalStatusComplete { 572 t.Fatalf("Bad: %#v", out) 573 } 574 } 575 576 func TestEvalEndpoint_Create(t *testing.T) { 577 t.Parallel() 578 579 s1, cleanupS1 := TestServer(t, func(c *Config) { 580 c.NumSchedulers = 0 // Prevent automatic dequeue 581 }) 582 defer cleanupS1() 583 codec := rpcClient(t, s1) 584 585 testutil.WaitForResult(func() (bool, error) { 586 return s1.evalBroker.Enabled(), nil 587 }, func(err error) { 588 t.Fatalf("should enable eval broker") 589 }) 590 591 // Create the register request 592 prev := mock.Eval() 593 s1.evalBroker.Enqueue(prev) 594 out, token, err := s1.evalBroker.Dequeue(defaultSched, time.Second) 595 if err != nil { 596 t.Fatalf("err: %v", err) 597 } 598 if out == nil { 599 t.Fatalf("missing eval") 600 } 601 602 // Create the register request 603 eval1 := mock.Eval() 604 eval1.PreviousEval = prev.ID 605 get := &structs.EvalUpdateRequest{ 606 Evals: []*structs.Evaluation{eval1}, 607 EvalToken: token, 608 WriteRequest: structs.WriteRequest{Region: "global"}, 609 } 610 var resp structs.GenericResponse 611 if err := msgpackrpc.CallWithCodec(codec, "Eval.Create", get, &resp); err != nil { 612 t.Fatalf("err: %v", err) 613 } 614 615 // Ensure created 616 ws := memdb.NewWatchSet() 617 outE, err := s1.fsm.State().EvalByID(ws, eval1.ID) 618 if err != nil { 619 t.Fatalf("err: %v", err) 620 } 621 622 eval1.CreateIndex = resp.Index 623 eval1.ModifyIndex = resp.Index 624 if !reflect.DeepEqual(eval1, outE) { 625 t.Fatalf("Bad: %#v %#v", outE, eval1) 626 } 627 } 628 629 func TestEvalEndpoint_Reap(t *testing.T) { 630 t.Parallel() 631 632 s1, cleanupS1 := TestServer(t, nil) 633 defer cleanupS1() 634 codec := rpcClient(t, s1) 635 testutil.WaitForLeader(t, s1.RPC) 636 637 // Create the register request 638 eval1 := mock.Eval() 639 s1.fsm.State().UpsertEvals(1000, []*structs.Evaluation{eval1}) 640 641 // Reap the eval 642 get := &structs.EvalDeleteRequest{ 643 Evals: []string{eval1.ID}, 644 WriteRequest: structs.WriteRequest{Region: "global"}, 645 } 646 var resp structs.GenericResponse 647 if err := msgpackrpc.CallWithCodec(codec, "Eval.Reap", get, &resp); err != nil { 648 t.Fatalf("err: %v", err) 649 } 650 if resp.Index == 0 { 651 t.Fatalf("Bad index: %d", resp.Index) 652 } 653 654 // Ensure deleted 655 ws := memdb.NewWatchSet() 656 outE, err := s1.fsm.State().EvalByID(ws, eval1.ID) 657 if err != nil { 658 t.Fatalf("err: %v", err) 659 } 660 if outE != nil { 661 t.Fatalf("Bad: %#v", outE) 662 } 663 } 664 665 func TestEvalEndpoint_List(t *testing.T) { 666 t.Parallel() 667 668 s1, cleanupS1 := TestServer(t, nil) 669 defer cleanupS1() 670 codec := rpcClient(t, s1) 671 testutil.WaitForLeader(t, s1.RPC) 672 673 // Create the register request 674 eval1 := mock.Eval() 675 eval1.ID = "aaaaaaaa-3350-4b4b-d185-0e1992ed43e9" 676 eval2 := mock.Eval() 677 eval2.ID = "aaaabbbb-3350-4b4b-d185-0e1992ed43e9" 678 s1.fsm.State().UpsertEvals(1000, []*structs.Evaluation{eval1, eval2}) 679 680 // Lookup the eval 681 get := &structs.EvalListRequest{ 682 QueryOptions: structs.QueryOptions{ 683 Region: "global", 684 Namespace: structs.DefaultNamespace, 685 }, 686 } 687 var resp structs.EvalListResponse 688 if err := msgpackrpc.CallWithCodec(codec, "Eval.List", get, &resp); err != nil { 689 t.Fatalf("err: %v", err) 690 } 691 if resp.Index != 1000 { 692 t.Fatalf("Bad index: %d %d", resp.Index, 1000) 693 } 694 695 if len(resp.Evaluations) != 2 { 696 t.Fatalf("bad: %#v", resp.Evaluations) 697 } 698 699 // Lookup the eval by prefix 700 get = &structs.EvalListRequest{ 701 QueryOptions: structs.QueryOptions{ 702 Region: "global", 703 Namespace: structs.DefaultNamespace, 704 Prefix: "aaaabb", 705 }, 706 } 707 var resp2 structs.EvalListResponse 708 if err := msgpackrpc.CallWithCodec(codec, "Eval.List", get, &resp2); err != nil { 709 t.Fatalf("err: %v", err) 710 } 711 if resp2.Index != 1000 { 712 t.Fatalf("Bad index: %d %d", resp2.Index, 1000) 713 } 714 715 if len(resp2.Evaluations) != 1 { 716 t.Fatalf("bad: %#v", resp2.Evaluations) 717 } 718 719 } 720 721 func TestEvalEndpoint_List_ACL(t *testing.T) { 722 t.Parallel() 723 724 s1, root, cleanupS1 := TestACLServer(t, nil) 725 defer cleanupS1() 726 codec := rpcClient(t, s1) 727 testutil.WaitForLeader(t, s1.RPC) 728 assert := assert.New(t) 729 730 // Create the register request 731 eval1 := mock.Eval() 732 eval1.ID = "aaaaaaaa-3350-4b4b-d185-0e1992ed43e9" 733 eval2 := mock.Eval() 734 eval2.ID = "aaaabbbb-3350-4b4b-d185-0e1992ed43e9" 735 state := s1.fsm.State() 736 assert.Nil(state.UpsertEvals(1000, []*structs.Evaluation{eval1, eval2})) 737 738 // Create ACL tokens 739 validToken := mock.CreatePolicyAndToken(t, state, 1003, "test-valid", 740 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob})) 741 invalidToken := mock.CreatePolicyAndToken(t, state, 1001, "test-invalid", 742 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityListJobs})) 743 744 get := &structs.EvalListRequest{ 745 QueryOptions: structs.QueryOptions{ 746 Region: "global", 747 Namespace: structs.DefaultNamespace, 748 }, 749 } 750 751 // Try without a token and expect permission denied 752 { 753 var resp structs.EvalListResponse 754 err := msgpackrpc.CallWithCodec(codec, "Eval.List", get, &resp) 755 assert.NotNil(err) 756 assert.Contains(err.Error(), structs.ErrPermissionDenied.Error()) 757 } 758 759 // Try with an invalid token and expect permission denied 760 { 761 get.AuthToken = invalidToken.SecretID 762 var resp structs.EvalListResponse 763 err := msgpackrpc.CallWithCodec(codec, "Eval.List", get, &resp) 764 assert.NotNil(err) 765 assert.Contains(err.Error(), structs.ErrPermissionDenied.Error()) 766 } 767 768 // List evals with a valid token 769 { 770 get.AuthToken = validToken.SecretID 771 var resp structs.EvalListResponse 772 assert.Nil(msgpackrpc.CallWithCodec(codec, "Eval.List", get, &resp)) 773 assert.Equal(uint64(1000), resp.Index, "Bad index: %d %d", resp.Index, 1000) 774 assert.Lenf(resp.Evaluations, 2, "bad: %#v", resp.Evaluations) 775 } 776 777 // List evals with a root token 778 { 779 get.AuthToken = root.SecretID 780 var resp structs.EvalListResponse 781 assert.Nil(msgpackrpc.CallWithCodec(codec, "Eval.List", get, &resp)) 782 assert.Equal(uint64(1000), resp.Index, "Bad index: %d %d", resp.Index, 1000) 783 assert.Lenf(resp.Evaluations, 2, "bad: %#v", resp.Evaluations) 784 } 785 } 786 787 func TestEvalEndpoint_List_Blocking(t *testing.T) { 788 t.Parallel() 789 790 s1, cleanupS1 := TestServer(t, nil) 791 defer cleanupS1() 792 state := s1.fsm.State() 793 codec := rpcClient(t, s1) 794 testutil.WaitForLeader(t, s1.RPC) 795 796 // Create the ieval 797 eval := mock.Eval() 798 799 // Upsert eval triggers watches 800 time.AfterFunc(100*time.Millisecond, func() { 801 if err := state.UpsertEvals(2, []*structs.Evaluation{eval}); err != nil { 802 t.Fatalf("err: %v", err) 803 } 804 }) 805 806 req := &structs.EvalListRequest{ 807 QueryOptions: structs.QueryOptions{ 808 Region: "global", 809 Namespace: structs.DefaultNamespace, 810 MinQueryIndex: 1, 811 }, 812 } 813 start := time.Now() 814 var resp structs.EvalListResponse 815 if err := msgpackrpc.CallWithCodec(codec, "Eval.List", req, &resp); err != nil { 816 t.Fatalf("err: %v", err) 817 } 818 819 if elapsed := time.Since(start); elapsed < 100*time.Millisecond { 820 t.Fatalf("should block (returned in %s) %#v", elapsed, resp) 821 } 822 if resp.Index != 2 { 823 t.Fatalf("Bad index: %d %d", resp.Index, 2) 824 } 825 if len(resp.Evaluations) != 1 || resp.Evaluations[0].ID != eval.ID { 826 t.Fatalf("bad: %#v", resp.Evaluations) 827 } 828 829 // Eval deletion triggers watches 830 time.AfterFunc(100*time.Millisecond, func() { 831 if err := state.DeleteEval(3, []string{eval.ID}, nil); err != nil { 832 t.Fatalf("err: %v", err) 833 } 834 }) 835 836 req.MinQueryIndex = 2 837 start = time.Now() 838 var resp2 structs.EvalListResponse 839 if err := msgpackrpc.CallWithCodec(codec, "Eval.List", req, &resp2); err != nil { 840 t.Fatalf("err: %v", err) 841 } 842 843 if elapsed := time.Since(start); elapsed < 100*time.Millisecond { 844 t.Fatalf("should block (returned in %s) %#v", elapsed, resp2) 845 } 846 if resp2.Index != 3 { 847 t.Fatalf("Bad index: %d %d", resp2.Index, 3) 848 } 849 if len(resp2.Evaluations) != 0 { 850 t.Fatalf("bad: %#v", resp2.Evaluations) 851 } 852 } 853 854 func TestEvalEndpoint_Allocations(t *testing.T) { 855 t.Parallel() 856 857 s1, cleanupS1 := TestServer(t, nil) 858 defer cleanupS1() 859 codec := rpcClient(t, s1) 860 testutil.WaitForLeader(t, s1.RPC) 861 862 // Create the register request 863 alloc1 := mock.Alloc() 864 alloc2 := mock.Alloc() 865 alloc2.EvalID = alloc1.EvalID 866 state := s1.fsm.State() 867 state.UpsertJobSummary(998, mock.JobSummary(alloc1.JobID)) 868 state.UpsertJobSummary(999, mock.JobSummary(alloc2.JobID)) 869 err := state.UpsertAllocs(1000, 870 []*structs.Allocation{alloc1, alloc2}) 871 if err != nil { 872 t.Fatalf("err: %v", err) 873 } 874 875 // Lookup the eval 876 get := &structs.EvalSpecificRequest{ 877 EvalID: alloc1.EvalID, 878 QueryOptions: structs.QueryOptions{Region: "global"}, 879 } 880 var resp structs.EvalAllocationsResponse 881 if err := msgpackrpc.CallWithCodec(codec, "Eval.Allocations", get, &resp); err != nil { 882 t.Fatalf("err: %v", err) 883 } 884 if resp.Index != 1000 { 885 t.Fatalf("Bad index: %d %d", resp.Index, 1000) 886 } 887 888 if len(resp.Allocations) != 2 { 889 t.Fatalf("bad: %#v", resp.Allocations) 890 } 891 } 892 893 func TestEvalEndpoint_Allocations_ACL(t *testing.T) { 894 t.Parallel() 895 896 s1, root, cleanupS1 := TestACLServer(t, nil) 897 defer cleanupS1() 898 codec := rpcClient(t, s1) 899 testutil.WaitForLeader(t, s1.RPC) 900 assert := assert.New(t) 901 902 // Create the register request 903 alloc1 := mock.Alloc() 904 alloc2 := mock.Alloc() 905 alloc2.EvalID = alloc1.EvalID 906 state := s1.fsm.State() 907 assert.Nil(state.UpsertJobSummary(998, mock.JobSummary(alloc1.JobID))) 908 assert.Nil(state.UpsertJobSummary(999, mock.JobSummary(alloc2.JobID))) 909 assert.Nil(state.UpsertAllocs(1000, []*structs.Allocation{alloc1, alloc2})) 910 911 // Create ACL tokens 912 validToken := mock.CreatePolicyAndToken(t, state, 1003, "test-valid", 913 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob})) 914 invalidToken := mock.CreatePolicyAndToken(t, state, 1001, "test-invalid", 915 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityListJobs})) 916 917 get := &structs.EvalSpecificRequest{ 918 EvalID: alloc1.EvalID, 919 QueryOptions: structs.QueryOptions{Region: "global"}, 920 } 921 922 // Try with no token and expect permission denied 923 { 924 var resp structs.EvalAllocationsResponse 925 err := msgpackrpc.CallWithCodec(codec, "Eval.Allocations", get, &resp) 926 assert.NotNil(err) 927 assert.Contains(err.Error(), structs.ErrPermissionDenied.Error()) 928 } 929 930 // Try with an invalid token and expect permission denied 931 { 932 get.AuthToken = invalidToken.SecretID 933 var resp structs.EvalAllocationsResponse 934 err := msgpackrpc.CallWithCodec(codec, "Eval.Allocations", get, &resp) 935 assert.NotNil(err) 936 assert.Contains(err.Error(), structs.ErrPermissionDenied.Error()) 937 } 938 939 // Lookup the eval with a valid token 940 { 941 get.AuthToken = validToken.SecretID 942 var resp structs.EvalAllocationsResponse 943 assert.Nil(msgpackrpc.CallWithCodec(codec, "Eval.Allocations", get, &resp)) 944 assert.Equal(uint64(1000), resp.Index, "Bad index: %d %d", resp.Index, 1000) 945 assert.Lenf(resp.Allocations, 2, "bad: %#v", resp.Allocations) 946 } 947 948 // Lookup the eval with a root token 949 { 950 get.AuthToken = root.SecretID 951 var resp structs.EvalAllocationsResponse 952 assert.Nil(msgpackrpc.CallWithCodec(codec, "Eval.Allocations", get, &resp)) 953 assert.Equal(uint64(1000), resp.Index, "Bad index: %d %d", resp.Index, 1000) 954 assert.Lenf(resp.Allocations, 2, "bad: %#v", resp.Allocations) 955 } 956 } 957 958 func TestEvalEndpoint_Allocations_Blocking(t *testing.T) { 959 t.Parallel() 960 961 s1, cleanupS1 := TestServer(t, nil) 962 defer cleanupS1() 963 state := s1.fsm.State() 964 codec := rpcClient(t, s1) 965 testutil.WaitForLeader(t, s1.RPC) 966 967 // Create the allocs 968 alloc1 := mock.Alloc() 969 alloc2 := mock.Alloc() 970 971 // Upsert an unrelated alloc first 972 time.AfterFunc(100*time.Millisecond, func() { 973 state.UpsertJobSummary(99, mock.JobSummary(alloc1.JobID)) 974 err := state.UpsertAllocs(100, []*structs.Allocation{alloc1}) 975 if err != nil { 976 t.Fatalf("err: %v", err) 977 } 978 }) 979 980 // Upsert an alloc which will trigger the watch later 981 time.AfterFunc(200*time.Millisecond, func() { 982 state.UpsertJobSummary(199, mock.JobSummary(alloc2.JobID)) 983 err := state.UpsertAllocs(200, []*structs.Allocation{alloc2}) 984 if err != nil { 985 t.Fatalf("err: %v", err) 986 } 987 }) 988 989 // Lookup the eval 990 get := &structs.EvalSpecificRequest{ 991 EvalID: alloc2.EvalID, 992 QueryOptions: structs.QueryOptions{ 993 Region: "global", 994 MinQueryIndex: 150, 995 }, 996 } 997 var resp structs.EvalAllocationsResponse 998 start := time.Now() 999 if err := msgpackrpc.CallWithCodec(codec, "Eval.Allocations", get, &resp); err != nil { 1000 t.Fatalf("err: %v", err) 1001 } 1002 1003 if elapsed := time.Since(start); elapsed < 200*time.Millisecond { 1004 t.Fatalf("should block (returned in %s) %#v", elapsed, resp) 1005 } 1006 if resp.Index != 200 { 1007 t.Fatalf("Bad index: %d %d", resp.Index, 200) 1008 } 1009 if len(resp.Allocations) != 1 || resp.Allocations[0].ID != alloc2.ID { 1010 t.Fatalf("bad: %#v", resp.Allocations) 1011 } 1012 } 1013 1014 func TestEvalEndpoint_Reblock_Nonexistent(t *testing.T) { 1015 t.Parallel() 1016 1017 s1, cleanupS1 := TestServer(t, func(c *Config) { 1018 c.NumSchedulers = 0 // Prevent automatic dequeue 1019 }) 1020 defer cleanupS1() 1021 codec := rpcClient(t, s1) 1022 1023 testutil.WaitForResult(func() (bool, error) { 1024 return s1.evalBroker.Enabled(), nil 1025 }, func(err error) { 1026 t.Fatalf("should enable eval broker") 1027 }) 1028 1029 // Create the register request 1030 eval1 := mock.Eval() 1031 s1.evalBroker.Enqueue(eval1) 1032 out, token, err := s1.evalBroker.Dequeue(defaultSched, time.Second) 1033 if err != nil { 1034 t.Fatalf("err: %v", err) 1035 } 1036 if out == nil { 1037 t.Fatalf("missing eval") 1038 } 1039 1040 get := &structs.EvalUpdateRequest{ 1041 Evals: []*structs.Evaluation{eval1}, 1042 EvalToken: token, 1043 WriteRequest: structs.WriteRequest{Region: "global"}, 1044 } 1045 var resp structs.GenericResponse 1046 if err := msgpackrpc.CallWithCodec(codec, "Eval.Reblock", get, &resp); err == nil { 1047 t.Fatalf("expect error since eval does not exist") 1048 } 1049 } 1050 1051 func TestEvalEndpoint_Reblock_NonBlocked(t *testing.T) { 1052 t.Parallel() 1053 1054 s1, cleanupS1 := TestServer(t, func(c *Config) { 1055 c.NumSchedulers = 0 // Prevent automatic dequeue 1056 }) 1057 defer cleanupS1() 1058 codec := rpcClient(t, s1) 1059 1060 testutil.WaitForResult(func() (bool, error) { 1061 return s1.evalBroker.Enabled(), nil 1062 }, func(err error) { 1063 t.Fatalf("should enable eval broker") 1064 }) 1065 1066 // Create the eval 1067 eval1 := mock.Eval() 1068 s1.evalBroker.Enqueue(eval1) 1069 1070 // Insert it into the state store 1071 if err := s1.fsm.State().UpsertEvals(1000, []*structs.Evaluation{eval1}); err != nil { 1072 t.Fatal(err) 1073 } 1074 1075 out, token, err := s1.evalBroker.Dequeue(defaultSched, 2*time.Second) 1076 if err != nil { 1077 t.Fatalf("err: %v", err) 1078 } 1079 if out == nil { 1080 t.Fatalf("missing eval") 1081 } 1082 1083 get := &structs.EvalUpdateRequest{ 1084 Evals: []*structs.Evaluation{eval1}, 1085 EvalToken: token, 1086 WriteRequest: structs.WriteRequest{Region: "global"}, 1087 } 1088 var resp structs.GenericResponse 1089 if err := msgpackrpc.CallWithCodec(codec, "Eval.Reblock", get, &resp); err == nil { 1090 t.Fatalf("should error since eval was not in blocked state: %v", err) 1091 } 1092 } 1093 1094 func TestEvalEndpoint_Reblock(t *testing.T) { 1095 t.Parallel() 1096 1097 s1, cleanupS1 := TestServer(t, func(c *Config) { 1098 c.NumSchedulers = 0 // Prevent automatic dequeue 1099 }) 1100 defer cleanupS1() 1101 codec := rpcClient(t, s1) 1102 1103 testutil.WaitForResult(func() (bool, error) { 1104 return s1.evalBroker.Enabled(), nil 1105 }, func(err error) { 1106 t.Fatalf("should enable eval broker") 1107 }) 1108 1109 // Create the eval 1110 eval1 := mock.Eval() 1111 eval1.Status = structs.EvalStatusBlocked 1112 s1.evalBroker.Enqueue(eval1) 1113 1114 // Insert it into the state store 1115 if err := s1.fsm.State().UpsertEvals(1000, []*structs.Evaluation{eval1}); err != nil { 1116 t.Fatal(err) 1117 } 1118 1119 out, token, err := s1.evalBroker.Dequeue(defaultSched, 7*time.Second) 1120 if err != nil { 1121 t.Fatalf("err: %v", err) 1122 } 1123 if out == nil { 1124 t.Fatalf("bad: %v", out) 1125 } 1126 1127 get := &structs.EvalUpdateRequest{ 1128 Evals: []*structs.Evaluation{eval1}, 1129 EvalToken: token, 1130 WriteRequest: structs.WriteRequest{Region: "global"}, 1131 } 1132 var resp structs.GenericResponse 1133 if err := msgpackrpc.CallWithCodec(codec, "Eval.Reblock", get, &resp); err != nil { 1134 t.Fatalf("err: %v", err) 1135 } 1136 1137 // Check that it is blocked 1138 bStats := s1.blockedEvals.Stats() 1139 if bStats.TotalBlocked+bStats.TotalEscaped == 0 { 1140 t.Fatalf("ReblockEval didn't insert eval into the blocked eval tracker") 1141 } 1142 }