github.com/emate/nomad@v0.8.2-wo-binpacking/nomad/job_endpoint_test.go (about) 1 package nomad 2 3 import ( 4 "fmt" 5 "reflect" 6 "strings" 7 "testing" 8 "time" 9 10 memdb "github.com/hashicorp/go-memdb" 11 msgpackrpc "github.com/hashicorp/net-rpc-msgpackrpc" 12 "github.com/hashicorp/nomad/acl" 13 "github.com/hashicorp/nomad/helper" 14 "github.com/hashicorp/nomad/helper/uuid" 15 "github.com/hashicorp/nomad/nomad/mock" 16 "github.com/hashicorp/nomad/nomad/structs" 17 "github.com/hashicorp/nomad/testutil" 18 "github.com/kr/pretty" 19 "github.com/stretchr/testify/require" 20 ) 21 22 func TestJobEndpoint_Register(t *testing.T) { 23 t.Parallel() 24 s1 := TestServer(t, func(c *Config) { 25 c.NumSchedulers = 0 // Prevent automatic dequeue 26 }) 27 defer s1.Shutdown() 28 codec := rpcClient(t, s1) 29 testutil.WaitForLeader(t, s1.RPC) 30 31 // Create the register request 32 job := mock.Job() 33 req := &structs.JobRegisterRequest{ 34 Job: job, 35 WriteRequest: structs.WriteRequest{ 36 Region: "global", 37 Namespace: job.Namespace, 38 }, 39 } 40 41 // Fetch the response 42 var resp structs.JobRegisterResponse 43 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { 44 t.Fatalf("err: %v", err) 45 } 46 if resp.Index == 0 { 47 t.Fatalf("bad index: %d", resp.Index) 48 } 49 50 // Check for the node in the FSM 51 state := s1.fsm.State() 52 ws := memdb.NewWatchSet() 53 out, err := state.JobByID(ws, job.Namespace, job.ID) 54 if err != nil { 55 t.Fatalf("err: %v", err) 56 } 57 if out == nil { 58 t.Fatalf("expected job") 59 } 60 if out.CreateIndex != resp.JobModifyIndex { 61 t.Fatalf("index mis-match") 62 } 63 serviceName := out.TaskGroups[0].Tasks[0].Services[0].Name 64 expectedServiceName := "web-frontend" 65 if serviceName != expectedServiceName { 66 t.Fatalf("Expected Service Name: %s, Actual: %s", expectedServiceName, serviceName) 67 } 68 69 // Lookup the evaluation 70 eval, err := state.EvalByID(ws, resp.EvalID) 71 if err != nil { 72 t.Fatalf("err: %v", err) 73 } 74 if eval == nil { 75 t.Fatalf("expected eval") 76 } 77 if eval.CreateIndex != resp.EvalCreateIndex { 78 t.Fatalf("index mis-match") 79 } 80 81 if eval.Priority != job.Priority { 82 t.Fatalf("bad: %#v", eval) 83 } 84 if eval.Type != job.Type { 85 t.Fatalf("bad: %#v", eval) 86 } 87 if eval.TriggeredBy != structs.EvalTriggerJobRegister { 88 t.Fatalf("bad: %#v", eval) 89 } 90 if eval.JobID != job.ID { 91 t.Fatalf("bad: %#v", eval) 92 } 93 if eval.JobModifyIndex != resp.JobModifyIndex { 94 t.Fatalf("bad: %#v", eval) 95 } 96 if eval.Status != structs.EvalStatusPending { 97 t.Fatalf("bad: %#v", eval) 98 } 99 } 100 101 func TestJobEndpoint_Register_ACL(t *testing.T) { 102 t.Parallel() 103 s1, root := TestACLServer(t, func(c *Config) { 104 c.NumSchedulers = 0 // Prevent automatic dequeue 105 }) 106 defer s1.Shutdown() 107 codec := rpcClient(t, s1) 108 testutil.WaitForLeader(t, s1.RPC) 109 110 // Create the register request 111 job := mock.Job() 112 req := &structs.JobRegisterRequest{ 113 Job: job, 114 WriteRequest: structs.WriteRequest{Region: "global"}, 115 } 116 117 // Try without a token, expect failure 118 var resp structs.JobRegisterResponse 119 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err == nil { 120 t.Fatalf("expected error") 121 } 122 123 // Try with a token 124 req.AuthToken = root.SecretID 125 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { 126 t.Fatalf("err: %v", err) 127 } 128 if resp.Index == 0 { 129 t.Fatalf("bad index: %d", resp.Index) 130 } 131 132 // Check for the node in the FSM 133 state := s1.fsm.State() 134 ws := memdb.NewWatchSet() 135 out, err := state.JobByID(ws, job.Namespace, job.ID) 136 if err != nil { 137 t.Fatalf("err: %v", err) 138 } 139 if out == nil { 140 t.Fatalf("expected job") 141 } 142 } 143 144 func TestJobEndpoint_Register_InvalidNamespace(t *testing.T) { 145 t.Parallel() 146 s1 := TestServer(t, func(c *Config) { 147 c.NumSchedulers = 0 // Prevent automatic dequeue 148 }) 149 defer s1.Shutdown() 150 codec := rpcClient(t, s1) 151 testutil.WaitForLeader(t, s1.RPC) 152 153 // Create the register request 154 job := mock.Job() 155 job.Namespace = "foo" 156 req := &structs.JobRegisterRequest{ 157 Job: job, 158 WriteRequest: structs.WriteRequest{Region: "global"}, 159 } 160 161 // Try without a token, expect failure 162 var resp structs.JobRegisterResponse 163 err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp) 164 if err == nil || !strings.Contains(err.Error(), "nonexistent namespace") { 165 t.Fatalf("expected namespace error: %v", err) 166 } 167 168 // Check for the job in the FSM 169 state := s1.fsm.State() 170 ws := memdb.NewWatchSet() 171 out, err := state.JobByID(ws, job.Namespace, job.ID) 172 if err != nil { 173 t.Fatalf("err: %v", err) 174 } 175 if out != nil { 176 t.Fatalf("expected no job") 177 } 178 } 179 180 func TestJobEndpoint_Register_InvalidDriverConfig(t *testing.T) { 181 t.Parallel() 182 s1 := TestServer(t, func(c *Config) { 183 c.NumSchedulers = 0 // Prevent automatic dequeue 184 }) 185 defer s1.Shutdown() 186 codec := rpcClient(t, s1) 187 testutil.WaitForLeader(t, s1.RPC) 188 189 // Create the register request with a job containing an invalid driver 190 // config 191 job := mock.Job() 192 job.TaskGroups[0].Tasks[0].Config["foo"] = 1 193 req := &structs.JobRegisterRequest{ 194 Job: job, 195 WriteRequest: structs.WriteRequest{ 196 Region: "global", 197 Namespace: job.Namespace, 198 }, 199 } 200 201 // Fetch the response 202 var resp structs.JobRegisterResponse 203 err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp) 204 if err == nil { 205 t.Fatalf("expected a validation error") 206 } 207 208 if !strings.Contains(err.Error(), "-> config:") { 209 t.Fatalf("expected a driver config validation error but got: %v", err) 210 } 211 } 212 213 func TestJobEndpoint_Register_Payload(t *testing.T) { 214 t.Parallel() 215 s1 := TestServer(t, func(c *Config) { 216 c.NumSchedulers = 0 // Prevent automatic dequeue 217 }) 218 defer s1.Shutdown() 219 codec := rpcClient(t, s1) 220 testutil.WaitForLeader(t, s1.RPC) 221 222 // Create the register request with a job containing an invalid driver 223 // config 224 job := mock.Job() 225 job.Payload = []byte{0x1} 226 req := &structs.JobRegisterRequest{ 227 Job: job, 228 WriteRequest: structs.WriteRequest{ 229 Region: "global", 230 Namespace: job.Namespace, 231 }, 232 } 233 234 // Fetch the response 235 var resp structs.JobRegisterResponse 236 err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp) 237 if err == nil { 238 t.Fatalf("expected a validation error") 239 } 240 241 if !strings.Contains(err.Error(), "payload") { 242 t.Fatalf("expected a payload error but got: %v", err) 243 } 244 } 245 246 func TestJobEndpoint_Register_Existing(t *testing.T) { 247 t.Parallel() 248 s1 := TestServer(t, func(c *Config) { 249 c.NumSchedulers = 0 // Prevent automatic dequeue 250 }) 251 defer s1.Shutdown() 252 codec := rpcClient(t, s1) 253 testutil.WaitForLeader(t, s1.RPC) 254 255 // Create the register request 256 job := mock.Job() 257 req := &structs.JobRegisterRequest{ 258 Job: job, 259 WriteRequest: structs.WriteRequest{ 260 Region: "global", 261 Namespace: job.Namespace, 262 }, 263 } 264 265 // Fetch the response 266 var resp structs.JobRegisterResponse 267 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { 268 t.Fatalf("err: %v", err) 269 } 270 if resp.Index == 0 { 271 t.Fatalf("bad index: %d", resp.Index) 272 } 273 274 // Update the job definition 275 job2 := mock.Job() 276 job2.Priority = 100 277 job2.ID = job.ID 278 req.Job = job2 279 280 // Attempt update 281 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { 282 t.Fatalf("err: %v", err) 283 } 284 if resp.Index == 0 { 285 t.Fatalf("bad index: %d", resp.Index) 286 } 287 288 // Check for the node in the FSM 289 state := s1.fsm.State() 290 ws := memdb.NewWatchSet() 291 out, err := state.JobByID(ws, job.Namespace, job.ID) 292 if err != nil { 293 t.Fatalf("err: %v", err) 294 } 295 if out == nil { 296 t.Fatalf("expected job") 297 } 298 if out.ModifyIndex != resp.JobModifyIndex { 299 t.Fatalf("index mis-match") 300 } 301 if out.Priority != 100 { 302 t.Fatalf("expected update") 303 } 304 if out.Version != 1 { 305 t.Fatalf("expected update") 306 } 307 308 // Lookup the evaluation 309 eval, err := state.EvalByID(ws, resp.EvalID) 310 if err != nil { 311 t.Fatalf("err: %v", err) 312 } 313 if eval == nil { 314 t.Fatalf("expected eval") 315 } 316 if eval.CreateIndex != resp.EvalCreateIndex { 317 t.Fatalf("index mis-match") 318 } 319 320 if eval.Priority != job2.Priority { 321 t.Fatalf("bad: %#v", eval) 322 } 323 if eval.Type != job2.Type { 324 t.Fatalf("bad: %#v", eval) 325 } 326 if eval.TriggeredBy != structs.EvalTriggerJobRegister { 327 t.Fatalf("bad: %#v", eval) 328 } 329 if eval.JobID != job2.ID { 330 t.Fatalf("bad: %#v", eval) 331 } 332 if eval.JobModifyIndex != resp.JobModifyIndex { 333 t.Fatalf("bad: %#v", eval) 334 } 335 if eval.Status != structs.EvalStatusPending { 336 t.Fatalf("bad: %#v", eval) 337 } 338 339 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { 340 t.Fatalf("err: %v", err) 341 } 342 if resp.Index == 0 { 343 t.Fatalf("bad index: %d", resp.Index) 344 } 345 346 // Check to ensure the job version didn't get bumped because we submitted 347 // the same job 348 state = s1.fsm.State() 349 ws = memdb.NewWatchSet() 350 out, err = state.JobByID(ws, job.Namespace, job.ID) 351 if err != nil { 352 t.Fatalf("err: %v", err) 353 } 354 if out == nil { 355 t.Fatalf("expected job") 356 } 357 if out.Version != 1 { 358 t.Fatalf("expected no update; got %v; diff %v", out.Version, pretty.Diff(job2, out)) 359 } 360 } 361 362 func TestJobEndpoint_Register_Periodic(t *testing.T) { 363 t.Parallel() 364 s1 := TestServer(t, func(c *Config) { 365 c.NumSchedulers = 0 // Prevent automatic dequeue 366 }) 367 defer s1.Shutdown() 368 codec := rpcClient(t, s1) 369 testutil.WaitForLeader(t, s1.RPC) 370 371 // Create the register request for a periodic job. 372 job := mock.PeriodicJob() 373 req := &structs.JobRegisterRequest{ 374 Job: job, 375 WriteRequest: structs.WriteRequest{ 376 Region: "global", 377 Namespace: job.Namespace, 378 }, 379 } 380 381 // Fetch the response 382 var resp structs.JobRegisterResponse 383 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { 384 t.Fatalf("err: %v", err) 385 } 386 if resp.JobModifyIndex == 0 { 387 t.Fatalf("bad index: %d", resp.Index) 388 } 389 390 // Check for the node in the FSM 391 state := s1.fsm.State() 392 ws := memdb.NewWatchSet() 393 out, err := state.JobByID(ws, job.Namespace, job.ID) 394 if err != nil { 395 t.Fatalf("err: %v", err) 396 } 397 if out == nil { 398 t.Fatalf("expected job") 399 } 400 if out.CreateIndex != resp.JobModifyIndex { 401 t.Fatalf("index mis-match") 402 } 403 serviceName := out.TaskGroups[0].Tasks[0].Services[0].Name 404 expectedServiceName := "web-frontend" 405 if serviceName != expectedServiceName { 406 t.Fatalf("Expected Service Name: %s, Actual: %s", expectedServiceName, serviceName) 407 } 408 409 if resp.EvalID != "" { 410 t.Fatalf("Register created an eval for a periodic job") 411 } 412 } 413 414 func TestJobEndpoint_Register_ParameterizedJob(t *testing.T) { 415 t.Parallel() 416 s1 := TestServer(t, func(c *Config) { 417 c.NumSchedulers = 0 // Prevent automatic dequeue 418 }) 419 defer s1.Shutdown() 420 codec := rpcClient(t, s1) 421 testutil.WaitForLeader(t, s1.RPC) 422 423 // Create the register request for a parameterized job. 424 job := mock.BatchJob() 425 job.ParameterizedJob = &structs.ParameterizedJobConfig{} 426 req := &structs.JobRegisterRequest{ 427 Job: job, 428 WriteRequest: structs.WriteRequest{ 429 Region: "global", 430 Namespace: job.Namespace, 431 }, 432 } 433 434 // Fetch the response 435 var resp structs.JobRegisterResponse 436 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { 437 t.Fatalf("err: %v", err) 438 } 439 if resp.JobModifyIndex == 0 { 440 t.Fatalf("bad index: %d", resp.Index) 441 } 442 443 // Check for the job in the FSM 444 state := s1.fsm.State() 445 ws := memdb.NewWatchSet() 446 out, err := state.JobByID(ws, job.Namespace, job.ID) 447 if err != nil { 448 t.Fatalf("err: %v", err) 449 } 450 if out == nil { 451 t.Fatalf("expected job") 452 } 453 if out.CreateIndex != resp.JobModifyIndex { 454 t.Fatalf("index mis-match") 455 } 456 if resp.EvalID != "" { 457 t.Fatalf("Register created an eval for a parameterized job") 458 } 459 } 460 461 func TestJobEndpoint_Register_EnforceIndex(t *testing.T) { 462 t.Parallel() 463 s1 := TestServer(t, func(c *Config) { 464 c.NumSchedulers = 0 // Prevent automatic dequeue 465 }) 466 defer s1.Shutdown() 467 codec := rpcClient(t, s1) 468 testutil.WaitForLeader(t, s1.RPC) 469 470 // Create the register request and enforcing an incorrect index 471 job := mock.Job() 472 req := &structs.JobRegisterRequest{ 473 Job: job, 474 EnforceIndex: true, 475 JobModifyIndex: 100, // Not registered yet so not possible 476 WriteRequest: structs.WriteRequest{ 477 Region: "global", 478 Namespace: job.Namespace, 479 }, 480 } 481 482 // Fetch the response 483 var resp structs.JobRegisterResponse 484 err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp) 485 if err == nil || !strings.Contains(err.Error(), RegisterEnforceIndexErrPrefix) { 486 t.Fatalf("expected enforcement error") 487 } 488 489 // Create the register request and enforcing it is new 490 req = &structs.JobRegisterRequest{ 491 Job: job, 492 EnforceIndex: true, 493 JobModifyIndex: 0, 494 WriteRequest: structs.WriteRequest{ 495 Region: "global", 496 Namespace: job.Namespace, 497 }, 498 } 499 500 // Fetch the response 501 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { 502 t.Fatalf("err: %v", err) 503 } 504 if resp.Index == 0 { 505 t.Fatalf("bad index: %d", resp.Index) 506 } 507 508 curIndex := resp.JobModifyIndex 509 510 // Check for the node in the FSM 511 state := s1.fsm.State() 512 ws := memdb.NewWatchSet() 513 out, err := state.JobByID(ws, job.Namespace, job.ID) 514 if err != nil { 515 t.Fatalf("err: %v", err) 516 } 517 if out == nil { 518 t.Fatalf("expected job") 519 } 520 if out.CreateIndex != resp.JobModifyIndex { 521 t.Fatalf("index mis-match") 522 } 523 524 // Reregister request and enforcing it be a new job 525 req = &structs.JobRegisterRequest{ 526 Job: job, 527 EnforceIndex: true, 528 JobModifyIndex: 0, 529 WriteRequest: structs.WriteRequest{ 530 Region: "global", 531 Namespace: job.Namespace, 532 }, 533 } 534 535 // Fetch the response 536 err = msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp) 537 if err == nil || !strings.Contains(err.Error(), RegisterEnforceIndexErrPrefix) { 538 t.Fatalf("expected enforcement error") 539 } 540 541 // Reregister request and enforcing it be at an incorrect index 542 req = &structs.JobRegisterRequest{ 543 Job: job, 544 EnforceIndex: true, 545 JobModifyIndex: curIndex - 1, 546 WriteRequest: structs.WriteRequest{ 547 Region: "global", 548 Namespace: job.Namespace, 549 }, 550 } 551 552 // Fetch the response 553 err = msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp) 554 if err == nil || !strings.Contains(err.Error(), RegisterEnforceIndexErrPrefix) { 555 t.Fatalf("expected enforcement error") 556 } 557 558 // Reregister request and enforcing it be at the correct index 559 job.Priority = job.Priority + 1 560 req = &structs.JobRegisterRequest{ 561 Job: job, 562 EnforceIndex: true, 563 JobModifyIndex: curIndex, 564 WriteRequest: structs.WriteRequest{ 565 Region: "global", 566 Namespace: job.Namespace, 567 }, 568 } 569 570 // Fetch the response 571 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { 572 t.Fatalf("err: %v", err) 573 } 574 if resp.Index == 0 { 575 t.Fatalf("bad index: %d", resp.Index) 576 } 577 578 out, err = state.JobByID(ws, job.Namespace, job.ID) 579 if err != nil { 580 t.Fatalf("err: %v", err) 581 } 582 if out == nil { 583 t.Fatalf("expected job") 584 } 585 if out.Priority != job.Priority { 586 t.Fatalf("priority mis-match") 587 } 588 } 589 590 func TestJobEndpoint_Register_Vault_Disabled(t *testing.T) { 591 t.Parallel() 592 s1 := TestServer(t, func(c *Config) { 593 c.NumSchedulers = 0 // Prevent automatic dequeue 594 f := false 595 c.VaultConfig.Enabled = &f 596 }) 597 defer s1.Shutdown() 598 codec := rpcClient(t, s1) 599 testutil.WaitForLeader(t, s1.RPC) 600 601 // Create the register request with a job asking for a vault policy 602 job := mock.Job() 603 job.TaskGroups[0].Tasks[0].Vault = &structs.Vault{ 604 Policies: []string{"foo"}, 605 ChangeMode: structs.VaultChangeModeRestart, 606 } 607 req := &structs.JobRegisterRequest{ 608 Job: job, 609 WriteRequest: structs.WriteRequest{ 610 Region: "global", 611 Namespace: job.Namespace, 612 }, 613 } 614 615 // Fetch the response 616 var resp structs.JobRegisterResponse 617 err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp) 618 if err == nil || !strings.Contains(err.Error(), "Vault not enabled") { 619 t.Fatalf("expected Vault not enabled error: %v", err) 620 } 621 } 622 623 func TestJobEndpoint_Register_Vault_AllowUnauthenticated(t *testing.T) { 624 t.Parallel() 625 s1 := TestServer(t, func(c *Config) { 626 c.NumSchedulers = 0 // Prevent automatic dequeue 627 }) 628 defer s1.Shutdown() 629 codec := rpcClient(t, s1) 630 testutil.WaitForLeader(t, s1.RPC) 631 632 // Enable vault and allow authenticated 633 tr := true 634 s1.config.VaultConfig.Enabled = &tr 635 s1.config.VaultConfig.AllowUnauthenticated = &tr 636 637 // Replace the Vault Client on the server 638 s1.vault = &TestVaultClient{} 639 640 // Create the register request with a job asking for a vault policy 641 job := mock.Job() 642 job.TaskGroups[0].Tasks[0].Vault = &structs.Vault{ 643 Policies: []string{"foo"}, 644 ChangeMode: structs.VaultChangeModeRestart, 645 } 646 req := &structs.JobRegisterRequest{ 647 Job: job, 648 WriteRequest: structs.WriteRequest{ 649 Region: "global", 650 Namespace: job.Namespace, 651 }, 652 } 653 654 // Fetch the response 655 var resp structs.JobRegisterResponse 656 err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp) 657 if err != nil { 658 t.Fatalf("bad: %v", err) 659 } 660 661 // Check for the job in the FSM 662 state := s1.fsm.State() 663 ws := memdb.NewWatchSet() 664 out, err := state.JobByID(ws, job.Namespace, job.ID) 665 if err != nil { 666 t.Fatalf("err: %v", err) 667 } 668 if out == nil { 669 t.Fatalf("expected job") 670 } 671 if out.CreateIndex != resp.JobModifyIndex { 672 t.Fatalf("index mis-match") 673 } 674 } 675 676 func TestJobEndpoint_Register_Vault_NoToken(t *testing.T) { 677 t.Parallel() 678 s1 := TestServer(t, func(c *Config) { 679 c.NumSchedulers = 0 // Prevent automatic dequeue 680 }) 681 defer s1.Shutdown() 682 codec := rpcClient(t, s1) 683 testutil.WaitForLeader(t, s1.RPC) 684 685 // Enable vault 686 tr, f := true, false 687 s1.config.VaultConfig.Enabled = &tr 688 s1.config.VaultConfig.AllowUnauthenticated = &f 689 690 // Replace the Vault Client on the server 691 s1.vault = &TestVaultClient{} 692 693 // Create the register request with a job asking for a vault policy but 694 // don't send a Vault token 695 job := mock.Job() 696 job.TaskGroups[0].Tasks[0].Vault = &structs.Vault{ 697 Policies: []string{"foo"}, 698 ChangeMode: structs.VaultChangeModeRestart, 699 } 700 req := &structs.JobRegisterRequest{ 701 Job: job, 702 WriteRequest: structs.WriteRequest{ 703 Region: "global", 704 Namespace: job.Namespace, 705 }, 706 } 707 708 // Fetch the response 709 var resp structs.JobRegisterResponse 710 err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp) 711 if err == nil || !strings.Contains(err.Error(), "missing Vault Token") { 712 t.Fatalf("expected Vault not enabled error: %v", err) 713 } 714 } 715 716 func TestJobEndpoint_Register_Vault_Policies(t *testing.T) { 717 t.Parallel() 718 s1 := TestServer(t, func(c *Config) { 719 c.NumSchedulers = 0 // Prevent automatic dequeue 720 }) 721 defer s1.Shutdown() 722 codec := rpcClient(t, s1) 723 testutil.WaitForLeader(t, s1.RPC) 724 725 // Enable vault 726 tr, f := true, false 727 s1.config.VaultConfig.Enabled = &tr 728 s1.config.VaultConfig.AllowUnauthenticated = &f 729 730 // Replace the Vault Client on the server 731 tvc := &TestVaultClient{} 732 s1.vault = tvc 733 734 // Add three tokens: one that allows the requesting policy, one that does 735 // not and one that returns an error 736 policy := "foo" 737 738 badToken := uuid.Generate() 739 badPolicies := []string{"a", "b", "c"} 740 tvc.SetLookupTokenAllowedPolicies(badToken, badPolicies) 741 742 goodToken := uuid.Generate() 743 goodPolicies := []string{"foo", "bar", "baz"} 744 tvc.SetLookupTokenAllowedPolicies(goodToken, goodPolicies) 745 746 rootToken := uuid.Generate() 747 rootPolicies := []string{"root"} 748 tvc.SetLookupTokenAllowedPolicies(rootToken, rootPolicies) 749 750 errToken := uuid.Generate() 751 expectedErr := fmt.Errorf("return errors from vault") 752 tvc.SetLookupTokenError(errToken, expectedErr) 753 754 // Create the register request with a job asking for a vault policy but 755 // send the bad Vault token 756 job := mock.Job() 757 job.VaultToken = badToken 758 job.TaskGroups[0].Tasks[0].Vault = &structs.Vault{ 759 Policies: []string{policy}, 760 ChangeMode: structs.VaultChangeModeRestart, 761 } 762 req := &structs.JobRegisterRequest{ 763 Job: job, 764 WriteRequest: structs.WriteRequest{ 765 Region: "global", 766 Namespace: job.Namespace, 767 }, 768 } 769 770 // Fetch the response 771 var resp structs.JobRegisterResponse 772 err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp) 773 if err == nil || !strings.Contains(err.Error(), 774 "doesn't allow access to the following policies: "+policy) { 775 t.Fatalf("expected permission denied error: %v", err) 776 } 777 778 // Use the err token 779 job.VaultToken = errToken 780 err = msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp) 781 if err == nil || !strings.Contains(err.Error(), expectedErr.Error()) { 782 t.Fatalf("expected permission denied error: %v", err) 783 } 784 785 // Use the good token 786 job.VaultToken = goodToken 787 788 // Fetch the response 789 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { 790 t.Fatalf("bad: %v", err) 791 } 792 793 // Check for the job in the FSM 794 state := s1.fsm.State() 795 ws := memdb.NewWatchSet() 796 out, err := state.JobByID(ws, job.Namespace, job.ID) 797 if err != nil { 798 t.Fatalf("err: %v", err) 799 } 800 if out == nil { 801 t.Fatalf("expected job") 802 } 803 if out.CreateIndex != resp.JobModifyIndex { 804 t.Fatalf("index mis-match") 805 } 806 if out.VaultToken != "" { 807 t.Fatalf("vault token not cleared") 808 } 809 810 // Check that an implicit constraint was created 811 constraints := out.TaskGroups[0].Constraints 812 if l := len(constraints); l != 1 { 813 t.Fatalf("Unexpected number of tests: %v", l) 814 } 815 816 if !constraints[0].Equal(vaultConstraint) { 817 t.Fatalf("bad constraint; got %#v; want %#v", constraints[0], vaultConstraint) 818 } 819 820 // Create the register request with another job asking for a vault policy but 821 // send the root Vault token 822 job2 := mock.Job() 823 job2.VaultToken = rootToken 824 job.TaskGroups[0].Tasks[0].Vault = &structs.Vault{ 825 Policies: []string{policy}, 826 ChangeMode: structs.VaultChangeModeRestart, 827 } 828 req = &structs.JobRegisterRequest{ 829 Job: job2, 830 WriteRequest: structs.WriteRequest{ 831 Region: "global", 832 Namespace: job.Namespace, 833 }, 834 } 835 836 // Fetch the response 837 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { 838 t.Fatalf("bad: %v", err) 839 } 840 841 // Check for the job in the FSM 842 out, err = state.JobByID(ws, job2.Namespace, job2.ID) 843 if err != nil { 844 t.Fatalf("err: %v", err) 845 } 846 if out == nil { 847 t.Fatalf("expected job") 848 } 849 if out.CreateIndex != resp.JobModifyIndex { 850 t.Fatalf("index mis-match") 851 } 852 if out.VaultToken != "" { 853 t.Fatalf("vault token not cleared") 854 } 855 } 856 857 func TestJobEndpoint_Revert(t *testing.T) { 858 t.Parallel() 859 s1 := TestServer(t, func(c *Config) { 860 c.NumSchedulers = 0 // Prevent automatic dequeue 861 }) 862 defer s1.Shutdown() 863 codec := rpcClient(t, s1) 864 testutil.WaitForLeader(t, s1.RPC) 865 866 // Create the initial register request 867 job := mock.Job() 868 job.Priority = 100 869 req := &structs.JobRegisterRequest{ 870 Job: job, 871 WriteRequest: structs.WriteRequest{ 872 Region: "global", 873 Namespace: job.Namespace, 874 }, 875 } 876 877 // Fetch the response 878 var resp structs.JobRegisterResponse 879 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { 880 t.Fatalf("err: %v", err) 881 } 882 if resp.Index == 0 { 883 t.Fatalf("bad index: %d", resp.Index) 884 } 885 886 // Reregister again to get another version 887 job2 := job.Copy() 888 job2.Priority = 1 889 req = &structs.JobRegisterRequest{ 890 Job: job2, 891 WriteRequest: structs.WriteRequest{ 892 Region: "global", 893 Namespace: job.Namespace, 894 }, 895 } 896 897 // Fetch the response 898 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { 899 t.Fatalf("err: %v", err) 900 } 901 if resp.Index == 0 { 902 t.Fatalf("bad index: %d", resp.Index) 903 } 904 905 // Create revert request and enforcing it be at an incorrect version 906 revertReq := &structs.JobRevertRequest{ 907 JobID: job.ID, 908 JobVersion: 0, 909 EnforcePriorVersion: helper.Uint64ToPtr(10), 910 WriteRequest: structs.WriteRequest{ 911 Region: "global", 912 Namespace: job.Namespace, 913 }, 914 } 915 916 // Fetch the response 917 err := msgpackrpc.CallWithCodec(codec, "Job.Revert", revertReq, &resp) 918 if err == nil || !strings.Contains(err.Error(), "enforcing version 10") { 919 t.Fatalf("expected enforcement error") 920 } 921 922 // Create revert request and enforcing it be at the current version 923 revertReq = &structs.JobRevertRequest{ 924 JobID: job.ID, 925 JobVersion: 1, 926 WriteRequest: structs.WriteRequest{ 927 Region: "global", 928 Namespace: job.Namespace, 929 }, 930 } 931 932 // Fetch the response 933 err = msgpackrpc.CallWithCodec(codec, "Job.Revert", revertReq, &resp) 934 if err == nil || !strings.Contains(err.Error(), "current version") { 935 t.Fatalf("expected current version err: %v", err) 936 } 937 938 // Create revert request and enforcing it be at version 1 939 revertReq = &structs.JobRevertRequest{ 940 JobID: job.ID, 941 JobVersion: 0, 942 EnforcePriorVersion: helper.Uint64ToPtr(1), 943 WriteRequest: structs.WriteRequest{ 944 Region: "global", 945 Namespace: job.Namespace, 946 }, 947 } 948 949 // Fetch the response 950 if err := msgpackrpc.CallWithCodec(codec, "Job.Revert", revertReq, &resp); err != nil { 951 t.Fatalf("err: %v", err) 952 } 953 if resp.Index == 0 { 954 t.Fatalf("bad index: %d", resp.Index) 955 } 956 if resp.EvalID == "" || resp.EvalCreateIndex == 0 { 957 t.Fatalf("bad created eval: %+v", resp) 958 } 959 if resp.JobModifyIndex == 0 { 960 t.Fatalf("bad job modify index: %d", resp.JobModifyIndex) 961 } 962 963 // Create revert request and don't enforce. We are at version 2 but it is 964 // the same as version 0 965 revertReq = &structs.JobRevertRequest{ 966 JobID: job.ID, 967 JobVersion: 0, 968 WriteRequest: structs.WriteRequest{ 969 Region: "global", 970 Namespace: job.Namespace, 971 }, 972 } 973 974 // Fetch the response 975 if err := msgpackrpc.CallWithCodec(codec, "Job.Revert", revertReq, &resp); err != nil { 976 t.Fatalf("err: %v", err) 977 } 978 if resp.Index == 0 { 979 t.Fatalf("bad index: %d", resp.Index) 980 } 981 if resp.EvalID == "" || resp.EvalCreateIndex == 0 { 982 t.Fatalf("bad created eval: %+v", resp) 983 } 984 if resp.JobModifyIndex == 0 { 985 t.Fatalf("bad job modify index: %d", resp.JobModifyIndex) 986 } 987 988 // Check that the job is at the correct version and that the eval was 989 // created 990 state := s1.fsm.State() 991 ws := memdb.NewWatchSet() 992 out, err := state.JobByID(ws, job.Namespace, job.ID) 993 if err != nil { 994 t.Fatalf("err: %v", err) 995 } 996 if out == nil { 997 t.Fatalf("expected job") 998 } 999 if out.Priority != job.Priority { 1000 t.Fatalf("priority mis-match") 1001 } 1002 if out.Version != 2 { 1003 t.Fatalf("got version %d; want %d", out.Version, 2) 1004 } 1005 1006 eout, err := state.EvalByID(ws, resp.EvalID) 1007 if err != nil { 1008 t.Fatalf("err: %v", err) 1009 } 1010 if eout == nil { 1011 t.Fatalf("expected eval") 1012 } 1013 if eout.JobID != job.ID { 1014 t.Fatalf("job id mis-match") 1015 } 1016 1017 versions, err := state.JobVersionsByID(ws, job.Namespace, job.ID) 1018 if err != nil { 1019 t.Fatalf("err: %v", err) 1020 } 1021 if len(versions) != 3 { 1022 t.Fatalf("got %d versions; want %d", len(versions), 3) 1023 } 1024 } 1025 1026 func TestJobEndpoint_Revert_ACL(t *testing.T) { 1027 t.Parallel() 1028 require := require.New(t) 1029 1030 s1, root := TestACLServer(t, func(c *Config) { 1031 c.NumSchedulers = 0 // Prevent automatic dequeue 1032 }) 1033 1034 defer s1.Shutdown() 1035 codec := rpcClient(t, s1) 1036 state := s1.fsm.State() 1037 testutil.WaitForLeader(t, s1.RPC) 1038 1039 // Create the job 1040 job := mock.Job() 1041 err := state.UpsertJob(300, job) 1042 require.Nil(err) 1043 1044 job2 := job.Copy() 1045 job2.Priority = 1 1046 err = state.UpsertJob(400, job2) 1047 require.Nil(err) 1048 1049 // Create revert request and enforcing it be at the current version 1050 revertReq := &structs.JobRevertRequest{ 1051 JobID: job.ID, 1052 JobVersion: 0, 1053 WriteRequest: structs.WriteRequest{ 1054 Region: "global", 1055 Namespace: job.Namespace, 1056 }, 1057 } 1058 1059 // Attempt to fetch the response without a valid token 1060 var resp structs.JobRegisterResponse 1061 err = msgpackrpc.CallWithCodec(codec, "Job.Revert", revertReq, &resp) 1062 require.NotNil(err) 1063 require.Contains(err.Error(), "Permission denied") 1064 1065 // Attempt to fetch the response with an invalid token 1066 invalidToken := mock.CreatePolicyAndToken(t, state, 1001, "test-invalid", 1067 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob})) 1068 1069 revertReq.AuthToken = invalidToken.SecretID 1070 var invalidResp structs.JobRegisterResponse 1071 err = msgpackrpc.CallWithCodec(codec, "Job.Revert", revertReq, &invalidResp) 1072 require.NotNil(err) 1073 require.Contains(err.Error(), "Permission denied") 1074 1075 // Fetch the response with a valid management token 1076 revertReq.AuthToken = root.SecretID 1077 var validResp structs.JobRegisterResponse 1078 err = msgpackrpc.CallWithCodec(codec, "Job.Revert", revertReq, &validResp) 1079 require.Nil(err) 1080 1081 // Try with a valid non-management token 1082 validToken := mock.CreatePolicyAndToken(t, state, 1003, "test-valid", 1083 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilitySubmitJob})) 1084 1085 revertReq.AuthToken = validToken.SecretID 1086 var validResp2 structs.JobRegisterResponse 1087 err = msgpackrpc.CallWithCodec(codec, "Job.Revert", revertReq, &validResp2) 1088 require.Nil(err) 1089 } 1090 1091 func TestJobEndpoint_Stable(t *testing.T) { 1092 t.Parallel() 1093 s1 := TestServer(t, func(c *Config) { 1094 c.NumSchedulers = 0 // Prevent automatic dequeue 1095 }) 1096 defer s1.Shutdown() 1097 codec := rpcClient(t, s1) 1098 testutil.WaitForLeader(t, s1.RPC) 1099 1100 // Create the initial register request 1101 job := mock.Job() 1102 req := &structs.JobRegisterRequest{ 1103 Job: job, 1104 WriteRequest: structs.WriteRequest{ 1105 Region: "global", 1106 Namespace: job.Namespace, 1107 }, 1108 } 1109 1110 // Fetch the response 1111 var resp structs.JobRegisterResponse 1112 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { 1113 t.Fatalf("err: %v", err) 1114 } 1115 if resp.Index == 0 { 1116 t.Fatalf("bad index: %d", resp.Index) 1117 } 1118 1119 // Create stability request 1120 stableReq := &structs.JobStabilityRequest{ 1121 JobID: job.ID, 1122 JobVersion: 0, 1123 Stable: true, 1124 WriteRequest: structs.WriteRequest{ 1125 Region: "global", 1126 Namespace: job.Namespace, 1127 }, 1128 } 1129 1130 // Fetch the response 1131 var stableResp structs.JobStabilityResponse 1132 if err := msgpackrpc.CallWithCodec(codec, "Job.Stable", stableReq, &stableResp); err != nil { 1133 t.Fatalf("err: %v", err) 1134 } 1135 if stableResp.Index == 0 { 1136 t.Fatalf("bad index: %d", resp.Index) 1137 } 1138 1139 // Check that the job is marked stable 1140 state := s1.fsm.State() 1141 ws := memdb.NewWatchSet() 1142 out, err := state.JobByID(ws, job.Namespace, job.ID) 1143 if err != nil { 1144 t.Fatalf("err: %v", err) 1145 } 1146 if out == nil { 1147 t.Fatalf("expected job") 1148 } 1149 if !out.Stable { 1150 t.Fatalf("Job is not marked stable") 1151 } 1152 } 1153 1154 func TestJobEndpoint_Stable_ACL(t *testing.T) { 1155 t.Parallel() 1156 require := require.New(t) 1157 1158 s1, root := TestACLServer(t, func(c *Config) { 1159 c.NumSchedulers = 0 // Prevent automatic dequeue 1160 }) 1161 defer s1.Shutdown() 1162 codec := rpcClient(t, s1) 1163 state := s1.fsm.State() 1164 testutil.WaitForLeader(t, s1.RPC) 1165 1166 // Register the job 1167 job := mock.Job() 1168 err := state.UpsertJob(1000, job) 1169 require.Nil(err) 1170 1171 // Create stability request 1172 stableReq := &structs.JobStabilityRequest{ 1173 JobID: job.ID, 1174 JobVersion: 0, 1175 Stable: true, 1176 WriteRequest: structs.WriteRequest{ 1177 Region: "global", 1178 Namespace: job.Namespace, 1179 }, 1180 } 1181 1182 // Attempt to fetch the token without a token 1183 var stableResp structs.JobStabilityResponse 1184 err = msgpackrpc.CallWithCodec(codec, "Job.Stable", stableReq, &stableResp) 1185 require.NotNil(err) 1186 require.Contains("Permission denied", err.Error()) 1187 1188 // Expect failure for request with an invalid token 1189 invalidToken := mock.CreatePolicyAndToken(t, state, 1003, "test-invalid", 1190 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityListJobs})) 1191 1192 stableReq.AuthToken = invalidToken.SecretID 1193 var invalidStableResp structs.JobStabilityResponse 1194 err = msgpackrpc.CallWithCodec(codec, "Job.Stable", stableReq, &invalidStableResp) 1195 require.NotNil(err) 1196 require.Contains("Permission denied", err.Error()) 1197 1198 // Attempt to fetch with a management token 1199 stableReq.AuthToken = root.SecretID 1200 var validStableResp structs.JobStabilityResponse 1201 err = msgpackrpc.CallWithCodec(codec, "Job.Stable", stableReq, &validStableResp) 1202 require.Nil(err) 1203 1204 // Attempt to fetch with a valid token 1205 validToken := mock.CreatePolicyAndToken(t, state, 1005, "test-invalid", 1206 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilitySubmitJob})) 1207 1208 stableReq.AuthToken = validToken.SecretID 1209 var validStableResp2 structs.JobStabilityResponse 1210 err = msgpackrpc.CallWithCodec(codec, "Job.Stable", stableReq, &validStableResp2) 1211 require.Nil(err) 1212 1213 // Check that the job is marked stable 1214 ws := memdb.NewWatchSet() 1215 out, err := state.JobByID(ws, job.Namespace, job.ID) 1216 require.Nil(err) 1217 require.NotNil(job) 1218 require.Equal(true, out.Stable) 1219 } 1220 1221 func TestJobEndpoint_Evaluate(t *testing.T) { 1222 t.Parallel() 1223 s1 := TestServer(t, func(c *Config) { 1224 c.NumSchedulers = 0 // Prevent automatic dequeue 1225 }) 1226 defer s1.Shutdown() 1227 codec := rpcClient(t, s1) 1228 testutil.WaitForLeader(t, s1.RPC) 1229 1230 // Create the register request 1231 job := mock.Job() 1232 req := &structs.JobRegisterRequest{ 1233 Job: job, 1234 WriteRequest: structs.WriteRequest{ 1235 Region: "global", 1236 Namespace: job.Namespace, 1237 }, 1238 } 1239 1240 // Fetch the response 1241 var resp structs.JobRegisterResponse 1242 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { 1243 t.Fatalf("err: %v", err) 1244 } 1245 if resp.Index == 0 { 1246 t.Fatalf("bad index: %d", resp.Index) 1247 } 1248 1249 // Force a re-evaluation 1250 reEval := &structs.JobEvaluateRequest{ 1251 JobID: job.ID, 1252 WriteRequest: structs.WriteRequest{ 1253 Region: "global", 1254 Namespace: job.Namespace, 1255 }, 1256 } 1257 1258 // Fetch the response 1259 if err := msgpackrpc.CallWithCodec(codec, "Job.Evaluate", reEval, &resp); err != nil { 1260 t.Fatalf("err: %v", err) 1261 } 1262 if resp.Index == 0 { 1263 t.Fatalf("bad index: %d", resp.Index) 1264 } 1265 1266 // Lookup the evaluation 1267 state := s1.fsm.State() 1268 ws := memdb.NewWatchSet() 1269 eval, err := state.EvalByID(ws, resp.EvalID) 1270 if err != nil { 1271 t.Fatalf("err: %v", err) 1272 } 1273 if eval == nil { 1274 t.Fatalf("expected eval") 1275 } 1276 if eval.CreateIndex != resp.EvalCreateIndex { 1277 t.Fatalf("index mis-match") 1278 } 1279 1280 if eval.Priority != job.Priority { 1281 t.Fatalf("bad: %#v", eval) 1282 } 1283 if eval.Type != job.Type { 1284 t.Fatalf("bad: %#v", eval) 1285 } 1286 if eval.TriggeredBy != structs.EvalTriggerJobRegister { 1287 t.Fatalf("bad: %#v", eval) 1288 } 1289 if eval.JobID != job.ID { 1290 t.Fatalf("bad: %#v", eval) 1291 } 1292 if eval.JobModifyIndex != resp.JobModifyIndex { 1293 t.Fatalf("bad: %#v", eval) 1294 } 1295 if eval.Status != structs.EvalStatusPending { 1296 t.Fatalf("bad: %#v", eval) 1297 } 1298 } 1299 1300 func TestJobEndpoint_Evaluate_ACL(t *testing.T) { 1301 t.Parallel() 1302 require := require.New(t) 1303 1304 s1, root := TestACLServer(t, func(c *Config) { 1305 c.NumSchedulers = 0 // Prevent automatic dequeue 1306 }) 1307 defer s1.Shutdown() 1308 codec := rpcClient(t, s1) 1309 testutil.WaitForLeader(t, s1.RPC) 1310 state := s1.fsm.State() 1311 1312 // Create the job 1313 job := mock.Job() 1314 err := state.UpsertJob(300, job) 1315 require.Nil(err) 1316 1317 // Force a re-evaluation 1318 reEval := &structs.JobEvaluateRequest{ 1319 JobID: job.ID, 1320 WriteRequest: structs.WriteRequest{ 1321 Region: "global", 1322 Namespace: job.Namespace, 1323 }, 1324 } 1325 1326 // Attempt to fetch the response without a token 1327 var resp structs.JobRegisterResponse 1328 err = msgpackrpc.CallWithCodec(codec, "Job.Evaluate", reEval, &resp) 1329 require.NotNil(err) 1330 require.Contains(err.Error(), "Permission denied") 1331 1332 // Attempt to fetch the response with an invalid token 1333 invalidToken := mock.CreatePolicyAndToken(t, state, 1003, "test-invalid", 1334 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityListJobs})) 1335 1336 reEval.AuthToken = invalidToken.SecretID 1337 var invalidResp structs.JobRegisterResponse 1338 err = msgpackrpc.CallWithCodec(codec, "Job.Evaluate", reEval, &invalidResp) 1339 require.NotNil(err) 1340 require.Contains(err.Error(), "Permission denied") 1341 1342 // Fetch the response with a valid management token 1343 reEval.AuthToken = root.SecretID 1344 var validResp structs.JobRegisterResponse 1345 err = msgpackrpc.CallWithCodec(codec, "Job.Evaluate", reEval, &validResp) 1346 require.Nil(err) 1347 1348 // Fetch the response with a valid token 1349 validToken := mock.CreatePolicyAndToken(t, state, 1005, "test-valid", 1350 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob})) 1351 1352 reEval.AuthToken = validToken.SecretID 1353 var validResp2 structs.JobRegisterResponse 1354 err = msgpackrpc.CallWithCodec(codec, "Job.Evaluate", reEval, &validResp2) 1355 require.Nil(err) 1356 1357 // Lookup the evaluation 1358 ws := memdb.NewWatchSet() 1359 eval, err := state.EvalByID(ws, validResp2.EvalID) 1360 require.Nil(err) 1361 require.NotNil(eval) 1362 1363 require.Equal(eval.CreateIndex, validResp2.EvalCreateIndex) 1364 require.Equal(eval.Priority, job.Priority) 1365 require.Equal(eval.Type, job.Type) 1366 require.Equal(eval.TriggeredBy, structs.EvalTriggerJobRegister) 1367 require.Equal(eval.JobID, job.ID) 1368 require.Equal(eval.JobModifyIndex, validResp2.JobModifyIndex) 1369 require.Equal(eval.Status, structs.EvalStatusPending) 1370 } 1371 1372 func TestJobEndpoint_Evaluate_Periodic(t *testing.T) { 1373 t.Parallel() 1374 s1 := TestServer(t, func(c *Config) { 1375 c.NumSchedulers = 0 // Prevent automatic dequeue 1376 }) 1377 defer s1.Shutdown() 1378 codec := rpcClient(t, s1) 1379 testutil.WaitForLeader(t, s1.RPC) 1380 1381 // Create the register request 1382 job := mock.PeriodicJob() 1383 req := &structs.JobRegisterRequest{ 1384 Job: job, 1385 WriteRequest: structs.WriteRequest{ 1386 Region: "global", 1387 Namespace: job.Namespace, 1388 }, 1389 } 1390 1391 // Fetch the response 1392 var resp structs.JobRegisterResponse 1393 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { 1394 t.Fatalf("err: %v", err) 1395 } 1396 if resp.JobModifyIndex == 0 { 1397 t.Fatalf("bad index: %d", resp.Index) 1398 } 1399 1400 // Force a re-evaluation 1401 reEval := &structs.JobEvaluateRequest{ 1402 JobID: job.ID, 1403 WriteRequest: structs.WriteRequest{ 1404 Region: "global", 1405 Namespace: job.Namespace, 1406 }, 1407 } 1408 1409 // Fetch the response 1410 if err := msgpackrpc.CallWithCodec(codec, "Job.Evaluate", reEval, &resp); err == nil { 1411 t.Fatal("expect an err") 1412 } 1413 } 1414 1415 func TestJobEndpoint_Evaluate_ParameterizedJob(t *testing.T) { 1416 t.Parallel() 1417 s1 := TestServer(t, func(c *Config) { 1418 c.NumSchedulers = 0 // Prevent automatic dequeue 1419 }) 1420 defer s1.Shutdown() 1421 codec := rpcClient(t, s1) 1422 testutil.WaitForLeader(t, s1.RPC) 1423 1424 // Create the register request 1425 job := mock.BatchJob() 1426 job.ParameterizedJob = &structs.ParameterizedJobConfig{} 1427 req := &structs.JobRegisterRequest{ 1428 Job: job, 1429 WriteRequest: structs.WriteRequest{ 1430 Region: "global", 1431 Namespace: job.Namespace, 1432 }, 1433 } 1434 1435 // Fetch the response 1436 var resp structs.JobRegisterResponse 1437 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { 1438 t.Fatalf("err: %v", err) 1439 } 1440 if resp.JobModifyIndex == 0 { 1441 t.Fatalf("bad index: %d", resp.Index) 1442 } 1443 1444 // Force a re-evaluation 1445 reEval := &structs.JobEvaluateRequest{ 1446 JobID: job.ID, 1447 WriteRequest: structs.WriteRequest{ 1448 Region: "global", 1449 Namespace: job.Namespace, 1450 }, 1451 } 1452 1453 // Fetch the response 1454 if err := msgpackrpc.CallWithCodec(codec, "Job.Evaluate", reEval, &resp); err == nil { 1455 t.Fatal("expect an err") 1456 } 1457 } 1458 1459 func TestJobEndpoint_Deregister(t *testing.T) { 1460 t.Parallel() 1461 require := require.New(t) 1462 s1 := TestServer(t, func(c *Config) { 1463 c.NumSchedulers = 0 // Prevent automatic dequeue 1464 }) 1465 defer s1.Shutdown() 1466 codec := rpcClient(t, s1) 1467 testutil.WaitForLeader(t, s1.RPC) 1468 1469 // Create the register requests 1470 job := mock.Job() 1471 reg := &structs.JobRegisterRequest{ 1472 Job: job, 1473 WriteRequest: structs.WriteRequest{ 1474 Region: "global", 1475 Namespace: job.Namespace, 1476 }, 1477 } 1478 1479 // Fetch the response 1480 var resp structs.JobRegisterResponse 1481 require.Nil(msgpackrpc.CallWithCodec(codec, "Job.Register", reg, &resp)) 1482 1483 // Deregister but don't purge 1484 dereg := &structs.JobDeregisterRequest{ 1485 JobID: job.ID, 1486 Purge: false, 1487 WriteRequest: structs.WriteRequest{ 1488 Region: "global", 1489 Namespace: job.Namespace, 1490 }, 1491 } 1492 var resp2 structs.JobDeregisterResponse 1493 require.Nil(msgpackrpc.CallWithCodec(codec, "Job.Deregister", dereg, &resp2)) 1494 require.NotZero(resp2.Index) 1495 1496 // Check for the job in the FSM 1497 state := s1.fsm.State() 1498 out, err := state.JobByID(nil, job.Namespace, job.ID) 1499 require.Nil(err) 1500 require.NotNil(out) 1501 require.True(out.Stop) 1502 1503 // Lookup the evaluation 1504 eval, err := state.EvalByID(nil, resp2.EvalID) 1505 require.Nil(err) 1506 require.NotNil(eval) 1507 require.EqualValues(resp2.EvalCreateIndex, eval.CreateIndex) 1508 require.Equal(job.Priority, eval.Priority) 1509 require.Equal(job.Type, eval.Type) 1510 require.Equal(structs.EvalTriggerJobDeregister, eval.TriggeredBy) 1511 require.Equal(job.ID, eval.JobID) 1512 require.Equal(structs.EvalStatusPending, eval.Status) 1513 1514 // Deregister and purge 1515 dereg2 := &structs.JobDeregisterRequest{ 1516 JobID: job.ID, 1517 Purge: true, 1518 WriteRequest: structs.WriteRequest{ 1519 Region: "global", 1520 Namespace: job.Namespace, 1521 }, 1522 } 1523 var resp3 structs.JobDeregisterResponse 1524 require.Nil(msgpackrpc.CallWithCodec(codec, "Job.Deregister", dereg2, &resp3)) 1525 require.NotZero(resp3.Index) 1526 1527 // Check for the job in the FSM 1528 out, err = state.JobByID(nil, job.Namespace, job.ID) 1529 require.Nil(err) 1530 require.Nil(out) 1531 1532 // Lookup the evaluation 1533 eval, err = state.EvalByID(nil, resp3.EvalID) 1534 require.Nil(err) 1535 require.NotNil(eval) 1536 1537 require.EqualValues(resp3.EvalCreateIndex, eval.CreateIndex) 1538 require.Equal(job.Priority, eval.Priority) 1539 require.Equal(job.Type, eval.Type) 1540 require.Equal(structs.EvalTriggerJobDeregister, eval.TriggeredBy) 1541 require.Equal(job.ID, eval.JobID) 1542 require.Equal(structs.EvalStatusPending, eval.Status) 1543 } 1544 1545 func TestJobEndpoint_Deregister_ACL(t *testing.T) { 1546 t.Parallel() 1547 require := require.New(t) 1548 1549 s1, root := TestACLServer(t, func(c *Config) { 1550 c.NumSchedulers = 0 // Prevent automatic dequeue 1551 }) 1552 defer s1.Shutdown() 1553 codec := rpcClient(t, s1) 1554 testutil.WaitForLeader(t, s1.RPC) 1555 state := s1.fsm.State() 1556 1557 // Create and register a job 1558 job := mock.Job() 1559 err := state.UpsertJob(100, job) 1560 1561 // Deregister and purge 1562 req := &structs.JobDeregisterRequest{ 1563 JobID: job.ID, 1564 Purge: true, 1565 WriteRequest: structs.WriteRequest{ 1566 Region: "global", 1567 Namespace: job.Namespace, 1568 }, 1569 } 1570 1571 // Expect failure for request without a token 1572 var resp structs.JobDeregisterResponse 1573 err = msgpackrpc.CallWithCodec(codec, "Job.Deregister", req, &resp) 1574 require.NotNil(err) 1575 require.Contains(err.Error(), "Permission denied") 1576 1577 // Expect failure for request with an invalid token 1578 invalidToken := mock.CreatePolicyAndToken(t, state, 1003, "test-invalid", 1579 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityListJobs})) 1580 req.AuthToken = invalidToken.SecretID 1581 1582 var invalidResp structs.JobDeregisterResponse 1583 err = msgpackrpc.CallWithCodec(codec, "Job.Deregister", req, &invalidResp) 1584 require.NotNil(err) 1585 require.Contains(err.Error(), "Permission denied") 1586 1587 // Expect success with a valid management token 1588 req.AuthToken = root.SecretID 1589 1590 var validResp structs.JobDeregisterResponse 1591 err = msgpackrpc.CallWithCodec(codec, "Job.Deregister", req, &validResp) 1592 require.Nil(err) 1593 require.NotEqual(validResp.Index, 0) 1594 1595 // Expect success with a valid token 1596 validToken := mock.CreatePolicyAndToken(t, state, 1005, "test-valid", 1597 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilitySubmitJob})) 1598 req.AuthToken = validToken.SecretID 1599 1600 var validResp2 structs.JobDeregisterResponse 1601 err = msgpackrpc.CallWithCodec(codec, "Job.Deregister", req, &validResp2) 1602 require.Nil(err) 1603 require.NotEqual(validResp2.Index, 0) 1604 1605 // Check for the job in the FSM 1606 out, err := state.JobByID(nil, job.Namespace, job.ID) 1607 require.Nil(err) 1608 require.Nil(out) 1609 1610 // Lookup the evaluation 1611 eval, err := state.EvalByID(nil, validResp2.EvalID) 1612 require.Nil(err) 1613 require.NotNil(eval, nil) 1614 1615 require.Equal(eval.CreateIndex, validResp2.EvalCreateIndex) 1616 require.Equal(eval.Priority, structs.JobDefaultPriority) 1617 require.Equal(eval.Type, structs.JobTypeService) 1618 require.Equal(eval.TriggeredBy, structs.EvalTriggerJobDeregister) 1619 require.Equal(eval.JobID, job.ID) 1620 require.Equal(eval.JobModifyIndex, validResp2.JobModifyIndex) 1621 require.Equal(eval.Status, structs.EvalStatusPending) 1622 } 1623 1624 func TestJobEndpoint_Deregister_Nonexistent(t *testing.T) { 1625 t.Parallel() 1626 s1 := TestServer(t, func(c *Config) { 1627 c.NumSchedulers = 0 // Prevent automatic dequeue 1628 }) 1629 defer s1.Shutdown() 1630 codec := rpcClient(t, s1) 1631 testutil.WaitForLeader(t, s1.RPC) 1632 1633 // Deregister 1634 jobID := "foo" 1635 dereg := &structs.JobDeregisterRequest{ 1636 JobID: jobID, 1637 WriteRequest: structs.WriteRequest{ 1638 Region: "global", 1639 Namespace: structs.DefaultNamespace, 1640 }, 1641 } 1642 var resp2 structs.JobDeregisterResponse 1643 if err := msgpackrpc.CallWithCodec(codec, "Job.Deregister", dereg, &resp2); err != nil { 1644 t.Fatalf("err: %v", err) 1645 } 1646 if resp2.JobModifyIndex == 0 { 1647 t.Fatalf("bad index: %d", resp2.Index) 1648 } 1649 1650 // Lookup the evaluation 1651 state := s1.fsm.State() 1652 ws := memdb.NewWatchSet() 1653 eval, err := state.EvalByID(ws, resp2.EvalID) 1654 if err != nil { 1655 t.Fatalf("err: %v", err) 1656 } 1657 if eval == nil { 1658 t.Fatalf("expected eval") 1659 } 1660 if eval.CreateIndex != resp2.EvalCreateIndex { 1661 t.Fatalf("index mis-match") 1662 } 1663 1664 if eval.Priority != structs.JobDefaultPriority { 1665 t.Fatalf("bad: %#v", eval) 1666 } 1667 if eval.Type != structs.JobTypeService { 1668 t.Fatalf("bad: %#v", eval) 1669 } 1670 if eval.TriggeredBy != structs.EvalTriggerJobDeregister { 1671 t.Fatalf("bad: %#v", eval) 1672 } 1673 if eval.JobID != jobID { 1674 t.Fatalf("bad: %#v", eval) 1675 } 1676 if eval.JobModifyIndex != resp2.JobModifyIndex { 1677 t.Fatalf("bad: %#v", eval) 1678 } 1679 if eval.Status != structs.EvalStatusPending { 1680 t.Fatalf("bad: %#v", eval) 1681 } 1682 } 1683 1684 func TestJobEndpoint_Deregister_Periodic(t *testing.T) { 1685 t.Parallel() 1686 s1 := TestServer(t, func(c *Config) { 1687 c.NumSchedulers = 0 // Prevent automatic dequeue 1688 }) 1689 defer s1.Shutdown() 1690 codec := rpcClient(t, s1) 1691 testutil.WaitForLeader(t, s1.RPC) 1692 1693 // Create the register request 1694 job := mock.PeriodicJob() 1695 reg := &structs.JobRegisterRequest{ 1696 Job: job, 1697 WriteRequest: structs.WriteRequest{ 1698 Region: "global", 1699 Namespace: job.Namespace, 1700 }, 1701 } 1702 1703 // Fetch the response 1704 var resp structs.JobRegisterResponse 1705 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", reg, &resp); err != nil { 1706 t.Fatalf("err: %v", err) 1707 } 1708 1709 // Deregister 1710 dereg := &structs.JobDeregisterRequest{ 1711 JobID: job.ID, 1712 Purge: true, 1713 WriteRequest: structs.WriteRequest{ 1714 Region: "global", 1715 Namespace: job.Namespace, 1716 }, 1717 } 1718 var resp2 structs.JobDeregisterResponse 1719 if err := msgpackrpc.CallWithCodec(codec, "Job.Deregister", dereg, &resp2); err != nil { 1720 t.Fatalf("err: %v", err) 1721 } 1722 if resp2.JobModifyIndex == 0 { 1723 t.Fatalf("bad index: %d", resp2.Index) 1724 } 1725 1726 // Check for the node in the FSM 1727 state := s1.fsm.State() 1728 ws := memdb.NewWatchSet() 1729 out, err := state.JobByID(ws, job.Namespace, job.ID) 1730 if err != nil { 1731 t.Fatalf("err: %v", err) 1732 } 1733 if out != nil { 1734 t.Fatalf("unexpected job") 1735 } 1736 1737 if resp.EvalID != "" { 1738 t.Fatalf("Deregister created an eval for a periodic job") 1739 } 1740 } 1741 1742 func TestJobEndpoint_Deregister_ParameterizedJob(t *testing.T) { 1743 t.Parallel() 1744 s1 := TestServer(t, func(c *Config) { 1745 c.NumSchedulers = 0 // Prevent automatic dequeue 1746 }) 1747 defer s1.Shutdown() 1748 codec := rpcClient(t, s1) 1749 testutil.WaitForLeader(t, s1.RPC) 1750 1751 // Create the register request 1752 job := mock.BatchJob() 1753 job.ParameterizedJob = &structs.ParameterizedJobConfig{} 1754 reg := &structs.JobRegisterRequest{ 1755 Job: job, 1756 WriteRequest: structs.WriteRequest{ 1757 Region: "global", 1758 Namespace: job.Namespace, 1759 }, 1760 } 1761 1762 // Fetch the response 1763 var resp structs.JobRegisterResponse 1764 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", reg, &resp); err != nil { 1765 t.Fatalf("err: %v", err) 1766 } 1767 1768 // Deregister 1769 dereg := &structs.JobDeregisterRequest{ 1770 JobID: job.ID, 1771 Purge: true, 1772 WriteRequest: structs.WriteRequest{ 1773 Region: "global", 1774 Namespace: job.Namespace, 1775 }, 1776 } 1777 var resp2 structs.JobDeregisterResponse 1778 if err := msgpackrpc.CallWithCodec(codec, "Job.Deregister", dereg, &resp2); err != nil { 1779 t.Fatalf("err: %v", err) 1780 } 1781 if resp2.JobModifyIndex == 0 { 1782 t.Fatalf("bad index: %d", resp2.Index) 1783 } 1784 1785 // Check for the node in the FSM 1786 state := s1.fsm.State() 1787 ws := memdb.NewWatchSet() 1788 out, err := state.JobByID(ws, job.Namespace, job.ID) 1789 if err != nil { 1790 t.Fatalf("err: %v", err) 1791 } 1792 if out != nil { 1793 t.Fatalf("unexpected job") 1794 } 1795 1796 if resp.EvalID != "" { 1797 t.Fatalf("Deregister created an eval for a parameterized job") 1798 } 1799 } 1800 1801 func TestJobEndpoint_BatchDeregister(t *testing.T) { 1802 t.Parallel() 1803 require := require.New(t) 1804 s1 := TestServer(t, func(c *Config) { 1805 c.NumSchedulers = 0 // Prevent automatic dequeue 1806 }) 1807 defer s1.Shutdown() 1808 codec := rpcClient(t, s1) 1809 testutil.WaitForLeader(t, s1.RPC) 1810 1811 // Create the register requests 1812 job := mock.Job() 1813 reg := &structs.JobRegisterRequest{ 1814 Job: job, 1815 WriteRequest: structs.WriteRequest{ 1816 Region: "global", 1817 Namespace: job.Namespace, 1818 }, 1819 } 1820 1821 // Fetch the response 1822 var resp structs.JobRegisterResponse 1823 require.Nil(msgpackrpc.CallWithCodec(codec, "Job.Register", reg, &resp)) 1824 1825 job2 := mock.Job() 1826 job2.Priority = 1 1827 reg2 := &structs.JobRegisterRequest{ 1828 Job: job2, 1829 WriteRequest: structs.WriteRequest{ 1830 Region: "global", 1831 Namespace: job2.Namespace, 1832 }, 1833 } 1834 1835 // Fetch the response 1836 require.Nil(msgpackrpc.CallWithCodec(codec, "Job.Register", reg2, &resp)) 1837 1838 // Deregister 1839 dereg := &structs.JobBatchDeregisterRequest{ 1840 Jobs: map[structs.NamespacedID]*structs.JobDeregisterOptions{ 1841 { 1842 ID: job.ID, 1843 Namespace: job.Namespace, 1844 }: {}, 1845 { 1846 ID: job2.ID, 1847 Namespace: job2.Namespace, 1848 }: { 1849 Purge: true, 1850 }, 1851 }, 1852 WriteRequest: structs.WriteRequest{ 1853 Region: "global", 1854 Namespace: job.Namespace, 1855 }, 1856 } 1857 var resp2 structs.JobBatchDeregisterResponse 1858 require.Nil(msgpackrpc.CallWithCodec(codec, "Job.BatchDeregister", dereg, &resp2)) 1859 require.NotZero(resp2.Index) 1860 1861 // Check for the job in the FSM 1862 state := s1.fsm.State() 1863 out, err := state.JobByID(nil, job.Namespace, job.ID) 1864 require.Nil(err) 1865 require.NotNil(out) 1866 require.True(out.Stop) 1867 1868 out, err = state.JobByID(nil, job2.Namespace, job2.ID) 1869 require.Nil(err) 1870 require.Nil(out) 1871 1872 // Lookup the evaluation 1873 for jobNS, eval := range resp2.JobEvals { 1874 expectedJob := job 1875 if jobNS.ID != job.ID { 1876 expectedJob = job2 1877 } 1878 1879 eval, err := state.EvalByID(nil, eval) 1880 require.Nil(err) 1881 require.NotNil(eval) 1882 require.EqualValues(resp2.Index, eval.CreateIndex) 1883 require.Equal(expectedJob.Priority, eval.Priority) 1884 require.Equal(expectedJob.Type, eval.Type) 1885 require.Equal(structs.EvalTriggerJobDeregister, eval.TriggeredBy) 1886 require.Equal(expectedJob.ID, eval.JobID) 1887 require.Equal(structs.EvalStatusPending, eval.Status) 1888 } 1889 } 1890 1891 func TestJobEndpoint_BatchDeregister_ACL(t *testing.T) { 1892 t.Parallel() 1893 require := require.New(t) 1894 1895 s1, root := TestACLServer(t, func(c *Config) { 1896 c.NumSchedulers = 0 // Prevent automatic dequeue 1897 }) 1898 defer s1.Shutdown() 1899 codec := rpcClient(t, s1) 1900 testutil.WaitForLeader(t, s1.RPC) 1901 state := s1.fsm.State() 1902 1903 // Create and register a job 1904 job, job2 := mock.Job(), mock.Job() 1905 require.Nil(state.UpsertJob(100, job)) 1906 require.Nil(state.UpsertJob(101, job2)) 1907 1908 // Deregister 1909 req := &structs.JobBatchDeregisterRequest{ 1910 Jobs: map[structs.NamespacedID]*structs.JobDeregisterOptions{ 1911 { 1912 ID: job.ID, 1913 Namespace: job.Namespace, 1914 }: {}, 1915 { 1916 ID: job2.ID, 1917 Namespace: job2.Namespace, 1918 }: {}, 1919 }, 1920 WriteRequest: structs.WriteRequest{ 1921 Region: "global", 1922 }, 1923 } 1924 1925 // Expect failure for request without a token 1926 var resp structs.JobBatchDeregisterResponse 1927 err := msgpackrpc.CallWithCodec(codec, "Job.BatchDeregister", req, &resp) 1928 require.NotNil(err) 1929 require.True(structs.IsErrPermissionDenied(err)) 1930 1931 // Expect failure for request with an invalid token 1932 invalidToken := mock.CreatePolicyAndToken(t, state, 1003, "test-invalid", 1933 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityListJobs})) 1934 req.AuthToken = invalidToken.SecretID 1935 1936 var invalidResp structs.JobDeregisterResponse 1937 err = msgpackrpc.CallWithCodec(codec, "Job.BatchDeregister", req, &invalidResp) 1938 require.NotNil(err) 1939 require.True(structs.IsErrPermissionDenied(err)) 1940 1941 // Expect success with a valid management token 1942 req.AuthToken = root.SecretID 1943 1944 var validResp structs.JobDeregisterResponse 1945 err = msgpackrpc.CallWithCodec(codec, "Job.BatchDeregister", req, &validResp) 1946 require.Nil(err) 1947 require.NotEqual(validResp.Index, 0) 1948 1949 // Expect success with a valid token 1950 validToken := mock.CreatePolicyAndToken(t, state, 1005, "test-valid", 1951 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilitySubmitJob})) 1952 req.AuthToken = validToken.SecretID 1953 1954 var validResp2 structs.JobDeregisterResponse 1955 err = msgpackrpc.CallWithCodec(codec, "Job.BatchDeregister", req, &validResp2) 1956 require.Nil(err) 1957 require.NotEqual(validResp2.Index, 0) 1958 } 1959 1960 func TestJobEndpoint_GetJob(t *testing.T) { 1961 t.Parallel() 1962 s1 := TestServer(t, nil) 1963 defer s1.Shutdown() 1964 codec := rpcClient(t, s1) 1965 testutil.WaitForLeader(t, s1.RPC) 1966 1967 // Create the register request 1968 job := mock.Job() 1969 reg := &structs.JobRegisterRequest{ 1970 Job: job, 1971 WriteRequest: structs.WriteRequest{ 1972 Region: "global", 1973 Namespace: job.Namespace, 1974 }, 1975 } 1976 1977 // Fetch the response 1978 var resp structs.JobRegisterResponse 1979 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", reg, &resp); err != nil { 1980 t.Fatalf("err: %v", err) 1981 } 1982 job.CreateIndex = resp.JobModifyIndex 1983 job.ModifyIndex = resp.JobModifyIndex 1984 job.JobModifyIndex = resp.JobModifyIndex 1985 1986 // Lookup the job 1987 get := &structs.JobSpecificRequest{ 1988 JobID: job.ID, 1989 QueryOptions: structs.QueryOptions{ 1990 Region: "global", 1991 Namespace: job.Namespace, 1992 }, 1993 } 1994 var resp2 structs.SingleJobResponse 1995 if err := msgpackrpc.CallWithCodec(codec, "Job.GetJob", get, &resp2); err != nil { 1996 t.Fatalf("err: %v", err) 1997 } 1998 if resp2.Index != resp.JobModifyIndex { 1999 t.Fatalf("Bad index: %d %d", resp2.Index, resp.Index) 2000 } 2001 2002 // Make a copy of the origin job and change the service name so that we can 2003 // do a deep equal with the response from the GET JOB Api 2004 j := job 2005 j.TaskGroups[0].Tasks[0].Services[0].Name = "web-frontend" 2006 for tgix, tg := range j.TaskGroups { 2007 for tidx, t := range tg.Tasks { 2008 for sidx, service := range t.Services { 2009 for cidx, check := range service.Checks { 2010 check.Name = resp2.Job.TaskGroups[tgix].Tasks[tidx].Services[sidx].Checks[cidx].Name 2011 } 2012 } 2013 } 2014 } 2015 2016 // Clear the submit times 2017 j.SubmitTime = 0 2018 resp2.Job.SubmitTime = 0 2019 2020 if !reflect.DeepEqual(j, resp2.Job) { 2021 t.Fatalf("bad: %#v %#v", job, resp2.Job) 2022 } 2023 2024 // Lookup non-existing job 2025 get.JobID = "foobarbaz" 2026 if err := msgpackrpc.CallWithCodec(codec, "Job.GetJob", get, &resp2); err != nil { 2027 t.Fatalf("err: %v", err) 2028 } 2029 if resp2.Index != resp.JobModifyIndex { 2030 t.Fatalf("Bad index: %d %d", resp2.Index, resp.Index) 2031 } 2032 if resp2.Job != nil { 2033 t.Fatalf("unexpected job") 2034 } 2035 } 2036 2037 func TestJobEndpoint_GetJob_ACL(t *testing.T) { 2038 t.Parallel() 2039 require := require.New(t) 2040 2041 s1, root := TestACLServer(t, nil) 2042 defer s1.Shutdown() 2043 codec := rpcClient(t, s1) 2044 testutil.WaitForLeader(t, s1.RPC) 2045 state := s1.fsm.State() 2046 2047 // Create the job 2048 job := mock.Job() 2049 err := state.UpsertJob(1000, job) 2050 require.Nil(err) 2051 2052 // Lookup the job 2053 get := &structs.JobSpecificRequest{ 2054 JobID: job.ID, 2055 QueryOptions: structs.QueryOptions{ 2056 Region: "global", 2057 Namespace: job.Namespace, 2058 }, 2059 } 2060 2061 // Looking up the job without a token should fail 2062 var resp structs.SingleJobResponse 2063 err = msgpackrpc.CallWithCodec(codec, "Job.GetJob", get, &resp) 2064 require.NotNil(err) 2065 require.Contains(err.Error(), "Permission denied") 2066 2067 // Expect failure for request with an invalid token 2068 invalidToken := mock.CreatePolicyAndToken(t, state, 1003, "test-invalid", 2069 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityListJobs})) 2070 2071 get.AuthToken = invalidToken.SecretID 2072 var invalidResp structs.SingleJobResponse 2073 err = msgpackrpc.CallWithCodec(codec, "Job.GetJob", get, &invalidResp) 2074 require.NotNil(err) 2075 require.Contains(err.Error(), "Permission denied") 2076 2077 // Looking up the job with a management token should succeed 2078 get.AuthToken = root.SecretID 2079 var validResp structs.SingleJobResponse 2080 err = msgpackrpc.CallWithCodec(codec, "Job.GetJob", get, &validResp) 2081 require.Nil(err) 2082 require.Equal(job.ID, validResp.Job.ID) 2083 2084 // Looking up the job with a valid token should succeed 2085 validToken := mock.CreatePolicyAndToken(t, state, 1005, "test-valid", 2086 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob})) 2087 2088 get.AuthToken = validToken.SecretID 2089 var validResp2 structs.SingleJobResponse 2090 err = msgpackrpc.CallWithCodec(codec, "Job.GetJob", get, &validResp2) 2091 require.Nil(err) 2092 require.Equal(job.ID, validResp2.Job.ID) 2093 } 2094 2095 func TestJobEndpoint_GetJob_Blocking(t *testing.T) { 2096 t.Parallel() 2097 s1 := TestServer(t, nil) 2098 defer s1.Shutdown() 2099 state := s1.fsm.State() 2100 codec := rpcClient(t, s1) 2101 testutil.WaitForLeader(t, s1.RPC) 2102 2103 // Create the jobs 2104 job1 := mock.Job() 2105 job2 := mock.Job() 2106 2107 // Upsert a job we are not interested in first. 2108 time.AfterFunc(100*time.Millisecond, func() { 2109 if err := state.UpsertJob(100, job1); err != nil { 2110 t.Fatalf("err: %v", err) 2111 } 2112 }) 2113 2114 // Upsert another job later which should trigger the watch. 2115 time.AfterFunc(200*time.Millisecond, func() { 2116 if err := state.UpsertJob(200, job2); err != nil { 2117 t.Fatalf("err: %v", err) 2118 } 2119 }) 2120 2121 req := &structs.JobSpecificRequest{ 2122 JobID: job2.ID, 2123 QueryOptions: structs.QueryOptions{ 2124 Region: "global", 2125 Namespace: job2.Namespace, 2126 MinQueryIndex: 150, 2127 }, 2128 } 2129 start := time.Now() 2130 var resp structs.SingleJobResponse 2131 if err := msgpackrpc.CallWithCodec(codec, "Job.GetJob", req, &resp); err != nil { 2132 t.Fatalf("err: %v", err) 2133 } 2134 2135 if elapsed := time.Since(start); elapsed < 200*time.Millisecond { 2136 t.Fatalf("should block (returned in %s) %#v", elapsed, resp) 2137 } 2138 if resp.Index != 200 { 2139 t.Fatalf("Bad index: %d %d", resp.Index, 200) 2140 } 2141 if resp.Job == nil || resp.Job.ID != job2.ID { 2142 t.Fatalf("bad: %#v", resp.Job) 2143 } 2144 2145 // Job delete fires watches 2146 time.AfterFunc(100*time.Millisecond, func() { 2147 if err := state.DeleteJob(300, job2.Namespace, job2.ID); err != nil { 2148 t.Fatalf("err: %v", err) 2149 } 2150 }) 2151 2152 req.QueryOptions.MinQueryIndex = 250 2153 start = time.Now() 2154 2155 var resp2 structs.SingleJobResponse 2156 if err := msgpackrpc.CallWithCodec(codec, "Job.GetJob", req, &resp2); err != nil { 2157 t.Fatalf("err: %v", err) 2158 } 2159 2160 if elapsed := time.Since(start); elapsed < 100*time.Millisecond { 2161 t.Fatalf("should block (returned in %s) %#v", elapsed, resp2) 2162 } 2163 if resp2.Index != 300 { 2164 t.Fatalf("Bad index: %d %d", resp2.Index, 300) 2165 } 2166 if resp2.Job != nil { 2167 t.Fatalf("bad: %#v", resp2.Job) 2168 } 2169 } 2170 2171 func TestJobEndpoint_GetJobVersions(t *testing.T) { 2172 t.Parallel() 2173 s1 := TestServer(t, nil) 2174 defer s1.Shutdown() 2175 codec := rpcClient(t, s1) 2176 testutil.WaitForLeader(t, s1.RPC) 2177 2178 // Create the register request 2179 job := mock.Job() 2180 job.Priority = 88 2181 reg := &structs.JobRegisterRequest{ 2182 Job: job, 2183 WriteRequest: structs.WriteRequest{ 2184 Region: "global", 2185 Namespace: job.Namespace, 2186 }, 2187 } 2188 2189 // Fetch the response 2190 var resp structs.JobRegisterResponse 2191 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", reg, &resp); err != nil { 2192 t.Fatalf("err: %v", err) 2193 } 2194 2195 // Register the job again to create another version 2196 job.Priority = 100 2197 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", reg, &resp); err != nil { 2198 t.Fatalf("err: %v", err) 2199 } 2200 2201 // Lookup the job 2202 get := &structs.JobVersionsRequest{ 2203 JobID: job.ID, 2204 QueryOptions: structs.QueryOptions{ 2205 Region: "global", 2206 Namespace: job.Namespace, 2207 }, 2208 } 2209 var versionsResp structs.JobVersionsResponse 2210 if err := msgpackrpc.CallWithCodec(codec, "Job.GetJobVersions", get, &versionsResp); err != nil { 2211 t.Fatalf("err: %v", err) 2212 } 2213 if versionsResp.Index != resp.JobModifyIndex { 2214 t.Fatalf("Bad index: %d %d", versionsResp.Index, resp.Index) 2215 } 2216 2217 // Make sure there are two job versions 2218 versions := versionsResp.Versions 2219 if l := len(versions); l != 2 { 2220 t.Fatalf("Got %d versions; want 2", l) 2221 } 2222 2223 if v := versions[0]; v.Priority != 100 || v.ID != job.ID || v.Version != 1 { 2224 t.Fatalf("bad: %+v", v) 2225 } 2226 if v := versions[1]; v.Priority != 88 || v.ID != job.ID || v.Version != 0 { 2227 t.Fatalf("bad: %+v", v) 2228 } 2229 2230 // Lookup non-existing job 2231 get.JobID = "foobarbaz" 2232 if err := msgpackrpc.CallWithCodec(codec, "Job.GetJobVersions", get, &versionsResp); err != nil { 2233 t.Fatalf("err: %v", err) 2234 } 2235 if versionsResp.Index != resp.JobModifyIndex { 2236 t.Fatalf("Bad index: %d %d", versionsResp.Index, resp.Index) 2237 } 2238 if l := len(versionsResp.Versions); l != 0 { 2239 t.Fatalf("unexpected versions: %d", l) 2240 } 2241 } 2242 2243 func TestJobEndpoint_GetJobVersions_ACL(t *testing.T) { 2244 t.Parallel() 2245 require := require.New(t) 2246 2247 s1, root := TestACLServer(t, nil) 2248 defer s1.Shutdown() 2249 codec := rpcClient(t, s1) 2250 testutil.WaitForLeader(t, s1.RPC) 2251 state := s1.fsm.State() 2252 2253 // Create two versions of a job with different priorities 2254 job := mock.Job() 2255 job.Priority = 88 2256 err := state.UpsertJob(10, job) 2257 require.Nil(err) 2258 2259 job.Priority = 100 2260 err = state.UpsertJob(100, job) 2261 require.Nil(err) 2262 2263 // Lookup the job 2264 get := &structs.JobVersionsRequest{ 2265 JobID: job.ID, 2266 QueryOptions: structs.QueryOptions{ 2267 Region: "global", 2268 Namespace: job.Namespace, 2269 }, 2270 } 2271 2272 // Attempt to fetch without a token should fail 2273 var resp structs.JobVersionsResponse 2274 err = msgpackrpc.CallWithCodec(codec, "Job.GetJobVersions", get, &resp) 2275 require.NotNil(err) 2276 require.Contains(err.Error(), "Permission denied") 2277 2278 // Expect failure for request with an invalid token 2279 invalidToken := mock.CreatePolicyAndToken(t, state, 1003, "test-invalid", 2280 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityListJobs})) 2281 2282 get.AuthToken = invalidToken.SecretID 2283 var invalidResp structs.JobVersionsResponse 2284 err = msgpackrpc.CallWithCodec(codec, "Job.GetJobVersions", get, &invalidResp) 2285 require.NotNil(err) 2286 require.Contains(err.Error(), "Permission denied") 2287 2288 // Expect success for request with a valid management token 2289 get.AuthToken = root.SecretID 2290 var validResp structs.JobVersionsResponse 2291 err = msgpackrpc.CallWithCodec(codec, "Job.GetJobVersions", get, &validResp) 2292 require.Nil(err) 2293 2294 // Expect success for request with a valid token 2295 validToken := mock.CreatePolicyAndToken(t, state, 1005, "test-valid", 2296 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob})) 2297 2298 get.AuthToken = validToken.SecretID 2299 var validResp2 structs.JobVersionsResponse 2300 err = msgpackrpc.CallWithCodec(codec, "Job.GetJobVersions", get, &validResp2) 2301 require.Nil(err) 2302 2303 // Make sure there are two job versions 2304 versions := validResp2.Versions 2305 require.Equal(2, len(versions)) 2306 require.Equal(versions[0].ID, job.ID) 2307 require.Equal(versions[1].ID, job.ID) 2308 } 2309 2310 func TestJobEndpoint_GetJobVersions_Diff(t *testing.T) { 2311 t.Parallel() 2312 s1 := TestServer(t, nil) 2313 defer s1.Shutdown() 2314 codec := rpcClient(t, s1) 2315 testutil.WaitForLeader(t, s1.RPC) 2316 2317 // Create the register request 2318 job := mock.Job() 2319 job.Priority = 88 2320 reg := &structs.JobRegisterRequest{ 2321 Job: job, 2322 WriteRequest: structs.WriteRequest{ 2323 Region: "global", 2324 Namespace: job.Namespace, 2325 }, 2326 } 2327 2328 // Fetch the response 2329 var resp structs.JobRegisterResponse 2330 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", reg, &resp); err != nil { 2331 t.Fatalf("err: %v", err) 2332 } 2333 2334 // Register the job again to create another version 2335 job.Priority = 90 2336 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", reg, &resp); err != nil { 2337 t.Fatalf("err: %v", err) 2338 } 2339 2340 // Register the job again to create another version 2341 job.Priority = 100 2342 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", reg, &resp); err != nil { 2343 t.Fatalf("err: %v", err) 2344 } 2345 2346 // Lookup the job 2347 get := &structs.JobVersionsRequest{ 2348 JobID: job.ID, 2349 Diffs: true, 2350 QueryOptions: structs.QueryOptions{ 2351 Region: "global", 2352 Namespace: job.Namespace, 2353 }, 2354 } 2355 var versionsResp structs.JobVersionsResponse 2356 if err := msgpackrpc.CallWithCodec(codec, "Job.GetJobVersions", get, &versionsResp); err != nil { 2357 t.Fatalf("err: %v", err) 2358 } 2359 if versionsResp.Index != resp.JobModifyIndex { 2360 t.Fatalf("Bad index: %d %d", versionsResp.Index, resp.Index) 2361 } 2362 2363 // Make sure there are two job versions 2364 versions := versionsResp.Versions 2365 if l := len(versions); l != 3 { 2366 t.Fatalf("Got %d versions; want 3", l) 2367 } 2368 2369 if v := versions[0]; v.Priority != 100 || v.ID != job.ID || v.Version != 2 { 2370 t.Fatalf("bad: %+v", v) 2371 } 2372 if v := versions[1]; v.Priority != 90 || v.ID != job.ID || v.Version != 1 { 2373 t.Fatalf("bad: %+v", v) 2374 } 2375 if v := versions[2]; v.Priority != 88 || v.ID != job.ID || v.Version != 0 { 2376 t.Fatalf("bad: %+v", v) 2377 } 2378 2379 // Ensure we got diffs 2380 diffs := versionsResp.Diffs 2381 if l := len(diffs); l != 2 { 2382 t.Fatalf("Got %d diffs; want 2", l) 2383 } 2384 d1 := diffs[0] 2385 if len(d1.Fields) != 1 { 2386 t.Fatalf("Got too many diffs: %#v", d1) 2387 } 2388 if d1.Fields[0].Name != "Priority" { 2389 t.Fatalf("Got wrong field: %#v", d1) 2390 } 2391 if d1.Fields[0].Old != "90" && d1.Fields[0].New != "100" { 2392 t.Fatalf("Got wrong field values: %#v", d1) 2393 } 2394 d2 := diffs[1] 2395 if len(d2.Fields) != 1 { 2396 t.Fatalf("Got too many diffs: %#v", d2) 2397 } 2398 if d2.Fields[0].Name != "Priority" { 2399 t.Fatalf("Got wrong field: %#v", d2) 2400 } 2401 if d2.Fields[0].Old != "88" && d1.Fields[0].New != "90" { 2402 t.Fatalf("Got wrong field values: %#v", d2) 2403 } 2404 } 2405 2406 func TestJobEndpoint_GetJobVersions_Blocking(t *testing.T) { 2407 t.Parallel() 2408 s1 := TestServer(t, nil) 2409 defer s1.Shutdown() 2410 state := s1.fsm.State() 2411 codec := rpcClient(t, s1) 2412 testutil.WaitForLeader(t, s1.RPC) 2413 2414 // Create the jobs 2415 job1 := mock.Job() 2416 job2 := mock.Job() 2417 job3 := mock.Job() 2418 job3.ID = job2.ID 2419 job3.Priority = 1 2420 2421 // Upsert a job we are not interested in first. 2422 time.AfterFunc(100*time.Millisecond, func() { 2423 if err := state.UpsertJob(100, job1); err != nil { 2424 t.Fatalf("err: %v", err) 2425 } 2426 }) 2427 2428 // Upsert another job later which should trigger the watch. 2429 time.AfterFunc(200*time.Millisecond, func() { 2430 if err := state.UpsertJob(200, job2); err != nil { 2431 t.Fatalf("err: %v", err) 2432 } 2433 }) 2434 2435 req := &structs.JobVersionsRequest{ 2436 JobID: job2.ID, 2437 QueryOptions: structs.QueryOptions{ 2438 Region: "global", 2439 Namespace: job2.Namespace, 2440 MinQueryIndex: 150, 2441 }, 2442 } 2443 start := time.Now() 2444 var resp structs.JobVersionsResponse 2445 if err := msgpackrpc.CallWithCodec(codec, "Job.GetJobVersions", req, &resp); err != nil { 2446 t.Fatalf("err: %v", err) 2447 } 2448 2449 if elapsed := time.Since(start); elapsed < 200*time.Millisecond { 2450 t.Fatalf("should block (returned in %s) %#v", elapsed, resp) 2451 } 2452 if resp.Index != 200 { 2453 t.Fatalf("Bad index: %d %d", resp.Index, 200) 2454 } 2455 if len(resp.Versions) != 1 || resp.Versions[0].ID != job2.ID { 2456 t.Fatalf("bad: %#v", resp.Versions) 2457 } 2458 2459 // Upsert the job again which should trigger the watch. 2460 time.AfterFunc(100*time.Millisecond, func() { 2461 if err := state.UpsertJob(300, job3); err != nil { 2462 t.Fatalf("err: %v", err) 2463 } 2464 }) 2465 2466 req2 := &structs.JobVersionsRequest{ 2467 JobID: job3.ID, 2468 QueryOptions: structs.QueryOptions{ 2469 Region: "global", 2470 Namespace: job3.Namespace, 2471 MinQueryIndex: 250, 2472 }, 2473 } 2474 var resp2 structs.JobVersionsResponse 2475 start = time.Now() 2476 if err := msgpackrpc.CallWithCodec(codec, "Job.GetJobVersions", req2, &resp2); err != nil { 2477 t.Fatalf("err: %v", err) 2478 } 2479 2480 if elapsed := time.Since(start); elapsed < 100*time.Millisecond { 2481 t.Fatalf("should block (returned in %s) %#v", elapsed, resp) 2482 } 2483 if resp2.Index != 300 { 2484 t.Fatalf("Bad index: %d %d", resp.Index, 300) 2485 } 2486 if len(resp2.Versions) != 2 { 2487 t.Fatalf("bad: %#v", resp2.Versions) 2488 } 2489 } 2490 2491 func TestJobEndpoint_GetJobSummary(t *testing.T) { 2492 t.Parallel() 2493 s1 := TestServer(t, func(c *Config) { 2494 c.NumSchedulers = 0 // Prevent automatic dequeue 2495 }) 2496 2497 defer s1.Shutdown() 2498 codec := rpcClient(t, s1) 2499 testutil.WaitForLeader(t, s1.RPC) 2500 2501 // Create the register request 2502 job := mock.Job() 2503 reg := &structs.JobRegisterRequest{ 2504 Job: job, 2505 WriteRequest: structs.WriteRequest{ 2506 Region: "global", 2507 Namespace: job.Namespace, 2508 }, 2509 } 2510 2511 // Fetch the response 2512 var resp structs.JobRegisterResponse 2513 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", reg, &resp); err != nil { 2514 t.Fatalf("err: %v", err) 2515 } 2516 job.CreateIndex = resp.JobModifyIndex 2517 job.ModifyIndex = resp.JobModifyIndex 2518 job.JobModifyIndex = resp.JobModifyIndex 2519 2520 // Lookup the job summary 2521 get := &structs.JobSummaryRequest{ 2522 JobID: job.ID, 2523 QueryOptions: structs.QueryOptions{ 2524 Region: "global", 2525 Namespace: job.Namespace, 2526 }, 2527 } 2528 var resp2 structs.JobSummaryResponse 2529 if err := msgpackrpc.CallWithCodec(codec, "Job.Summary", get, &resp2); err != nil { 2530 t.Fatalf("err: %v", err) 2531 } 2532 if resp2.Index != resp.JobModifyIndex { 2533 t.Fatalf("Bad index: %d %d", resp2.Index, resp.Index) 2534 } 2535 2536 expectedJobSummary := structs.JobSummary{ 2537 JobID: job.ID, 2538 Namespace: job.Namespace, 2539 Summary: map[string]structs.TaskGroupSummary{ 2540 "web": {}, 2541 }, 2542 Children: new(structs.JobChildrenSummary), 2543 CreateIndex: job.CreateIndex, 2544 ModifyIndex: job.CreateIndex, 2545 } 2546 2547 if !reflect.DeepEqual(resp2.JobSummary, &expectedJobSummary) { 2548 t.Fatalf("expected: %v, actual: %v", expectedJobSummary, resp2.JobSummary) 2549 } 2550 } 2551 2552 func TestJobEndpoint_Summary_ACL(t *testing.T) { 2553 require := require.New(t) 2554 t.Parallel() 2555 2556 srv, root := TestACLServer(t, func(c *Config) { 2557 c.NumSchedulers = 0 // Prevent automatic dequeue 2558 }) 2559 defer srv.Shutdown() 2560 codec := rpcClient(t, srv) 2561 testutil.WaitForLeader(t, srv.RPC) 2562 2563 // Create the job 2564 job := mock.Job() 2565 reg := &structs.JobRegisterRequest{ 2566 Job: job, 2567 WriteRequest: structs.WriteRequest{ 2568 Region: "global", 2569 Namespace: job.Namespace, 2570 }, 2571 } 2572 reg.AuthToken = root.SecretID 2573 2574 var err error 2575 2576 // Register the job with a valid token 2577 var regResp structs.JobRegisterResponse 2578 err = msgpackrpc.CallWithCodec(codec, "Job.Register", reg, ®Resp) 2579 require.Nil(err) 2580 2581 job.CreateIndex = regResp.JobModifyIndex 2582 job.ModifyIndex = regResp.JobModifyIndex 2583 job.JobModifyIndex = regResp.JobModifyIndex 2584 2585 req := &structs.JobSummaryRequest{ 2586 JobID: job.ID, 2587 QueryOptions: structs.QueryOptions{ 2588 Region: "global", 2589 Namespace: job.Namespace, 2590 }, 2591 } 2592 2593 // Expect failure for request without a token 2594 var resp structs.JobSummaryResponse 2595 err = msgpackrpc.CallWithCodec(codec, "Job.Summary", req, &resp) 2596 require.NotNil(err) 2597 2598 expectedJobSummary := &structs.JobSummary{ 2599 JobID: job.ID, 2600 Namespace: job.Namespace, 2601 Summary: map[string]structs.TaskGroupSummary{ 2602 "web": {}, 2603 }, 2604 Children: new(structs.JobChildrenSummary), 2605 CreateIndex: job.CreateIndex, 2606 ModifyIndex: job.ModifyIndex, 2607 } 2608 2609 // Expect success when using a management token 2610 req.AuthToken = root.SecretID 2611 var mgmtResp structs.JobSummaryResponse 2612 err = msgpackrpc.CallWithCodec(codec, "Job.Summary", req, &mgmtResp) 2613 require.Nil(err) 2614 require.Equal(expectedJobSummary, mgmtResp.JobSummary) 2615 2616 // Create the namespace policy and tokens 2617 state := srv.fsm.State() 2618 2619 // Expect failure for request with an invalid token 2620 invalidToken := mock.CreatePolicyAndToken(t, state, 1003, "test-invalid", 2621 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityListJobs})) 2622 2623 req.AuthToken = invalidToken.SecretID 2624 var invalidResp structs.JobSummaryResponse 2625 err = msgpackrpc.CallWithCodec(codec, "Job.Summary", req, &invalidResp) 2626 require.NotNil(err) 2627 2628 // Try with a valid token 2629 validToken := mock.CreatePolicyAndToken(t, state, 1001, "test-valid", 2630 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob})) 2631 2632 req.AuthToken = validToken.SecretID 2633 var authResp structs.JobSummaryResponse 2634 err = msgpackrpc.CallWithCodec(codec, "Job.Summary", req, &authResp) 2635 require.Nil(err) 2636 require.Equal(expectedJobSummary, authResp.JobSummary) 2637 } 2638 2639 func TestJobEndpoint_GetJobSummary_Blocking(t *testing.T) { 2640 t.Parallel() 2641 s1 := TestServer(t, nil) 2642 defer s1.Shutdown() 2643 state := s1.fsm.State() 2644 codec := rpcClient(t, s1) 2645 testutil.WaitForLeader(t, s1.RPC) 2646 2647 // Create a job and insert it 2648 job1 := mock.Job() 2649 time.AfterFunc(200*time.Millisecond, func() { 2650 if err := state.UpsertJob(100, job1); err != nil { 2651 t.Fatalf("err: %v", err) 2652 } 2653 }) 2654 2655 // Ensure the job summary request gets fired 2656 req := &structs.JobSummaryRequest{ 2657 JobID: job1.ID, 2658 QueryOptions: structs.QueryOptions{ 2659 Region: "global", 2660 Namespace: job1.Namespace, 2661 MinQueryIndex: 50, 2662 }, 2663 } 2664 var resp structs.JobSummaryResponse 2665 start := time.Now() 2666 if err := msgpackrpc.CallWithCodec(codec, "Job.Summary", req, &resp); err != nil { 2667 t.Fatalf("err: %v", err) 2668 } 2669 if elapsed := time.Since(start); elapsed < 200*time.Millisecond { 2670 t.Fatalf("should block (returned in %s) %#v", elapsed, resp) 2671 } 2672 2673 // Upsert an allocation for the job which should trigger the watch. 2674 time.AfterFunc(200*time.Millisecond, func() { 2675 alloc := mock.Alloc() 2676 alloc.JobID = job1.ID 2677 alloc.Job = job1 2678 if err := state.UpsertAllocs(200, []*structs.Allocation{alloc}); err != nil { 2679 t.Fatalf("err: %v", err) 2680 } 2681 }) 2682 req = &structs.JobSummaryRequest{ 2683 JobID: job1.ID, 2684 QueryOptions: structs.QueryOptions{ 2685 Region: "global", 2686 Namespace: job1.Namespace, 2687 MinQueryIndex: 199, 2688 }, 2689 } 2690 start = time.Now() 2691 var resp1 structs.JobSummaryResponse 2692 if err := msgpackrpc.CallWithCodec(codec, "Job.Summary", req, &resp1); err != nil { 2693 t.Fatalf("err: %v", err) 2694 } 2695 2696 if elapsed := time.Since(start); elapsed < 200*time.Millisecond { 2697 t.Fatalf("should block (returned in %s) %#v", elapsed, resp) 2698 } 2699 if resp1.Index != 200 { 2700 t.Fatalf("Bad index: %d %d", resp.Index, 200) 2701 } 2702 if resp1.JobSummary == nil { 2703 t.Fatalf("bad: %#v", resp) 2704 } 2705 2706 // Job delete fires watches 2707 time.AfterFunc(100*time.Millisecond, func() { 2708 if err := state.DeleteJob(300, job1.Namespace, job1.ID); err != nil { 2709 t.Fatalf("err: %v", err) 2710 } 2711 }) 2712 2713 req.QueryOptions.MinQueryIndex = 250 2714 start = time.Now() 2715 2716 var resp2 structs.SingleJobResponse 2717 if err := msgpackrpc.CallWithCodec(codec, "Job.Summary", req, &resp2); err != nil { 2718 t.Fatalf("err: %v", err) 2719 } 2720 2721 if elapsed := time.Since(start); elapsed < 100*time.Millisecond { 2722 t.Fatalf("should block (returned in %s) %#v", elapsed, resp2) 2723 } 2724 if resp2.Index != 300 { 2725 t.Fatalf("Bad index: %d %d", resp2.Index, 300) 2726 } 2727 if resp2.Job != nil { 2728 t.Fatalf("bad: %#v", resp2.Job) 2729 } 2730 } 2731 2732 func TestJobEndpoint_ListJobs(t *testing.T) { 2733 t.Parallel() 2734 s1 := TestServer(t, nil) 2735 defer s1.Shutdown() 2736 codec := rpcClient(t, s1) 2737 testutil.WaitForLeader(t, s1.RPC) 2738 2739 // Create the register request 2740 job := mock.Job() 2741 state := s1.fsm.State() 2742 err := state.UpsertJob(1000, job) 2743 if err != nil { 2744 t.Fatalf("err: %v", err) 2745 } 2746 2747 // Lookup the jobs 2748 get := &structs.JobListRequest{ 2749 QueryOptions: structs.QueryOptions{ 2750 Region: "global", 2751 Namespace: job.Namespace, 2752 }, 2753 } 2754 var resp2 structs.JobListResponse 2755 if err := msgpackrpc.CallWithCodec(codec, "Job.List", get, &resp2); err != nil { 2756 t.Fatalf("err: %v", err) 2757 } 2758 if resp2.Index != 1000 { 2759 t.Fatalf("Bad index: %d %d", resp2.Index, 1000) 2760 } 2761 2762 if len(resp2.Jobs) != 1 { 2763 t.Fatalf("bad: %#v", resp2.Jobs) 2764 } 2765 if resp2.Jobs[0].ID != job.ID { 2766 t.Fatalf("bad: %#v", resp2.Jobs[0]) 2767 } 2768 2769 // Lookup the jobs by prefix 2770 get = &structs.JobListRequest{ 2771 QueryOptions: structs.QueryOptions{ 2772 Region: "global", 2773 Namespace: job.Namespace, 2774 Prefix: resp2.Jobs[0].ID[:4], 2775 }, 2776 } 2777 var resp3 structs.JobListResponse 2778 if err := msgpackrpc.CallWithCodec(codec, "Job.List", get, &resp3); err != nil { 2779 t.Fatalf("err: %v", err) 2780 } 2781 if resp3.Index != 1000 { 2782 t.Fatalf("Bad index: %d %d", resp3.Index, 1000) 2783 } 2784 2785 if len(resp3.Jobs) != 1 { 2786 t.Fatalf("bad: %#v", resp3.Jobs) 2787 } 2788 if resp3.Jobs[0].ID != job.ID { 2789 t.Fatalf("bad: %#v", resp3.Jobs[0]) 2790 } 2791 } 2792 2793 func TestJobEndpoint_ListJobs_WithACL(t *testing.T) { 2794 require := require.New(t) 2795 t.Parallel() 2796 2797 srv, root := TestACLServer(t, func(c *Config) { 2798 c.NumSchedulers = 0 // Prevent automatic dequeue 2799 }) 2800 defer srv.Shutdown() 2801 codec := rpcClient(t, srv) 2802 testutil.WaitForLeader(t, srv.RPC) 2803 state := srv.fsm.State() 2804 2805 var err error 2806 2807 // Create the register request 2808 job := mock.Job() 2809 err = state.UpsertJob(1000, job) 2810 require.Nil(err) 2811 2812 req := &structs.JobListRequest{ 2813 QueryOptions: structs.QueryOptions{ 2814 Region: "global", 2815 Namespace: job.Namespace, 2816 }, 2817 } 2818 2819 // Expect failure for request without a token 2820 var resp structs.JobListResponse 2821 err = msgpackrpc.CallWithCodec(codec, "Job.List", req, &resp) 2822 require.NotNil(err) 2823 2824 // Expect success for request with a management token 2825 var mgmtResp structs.JobListResponse 2826 req.AuthToken = root.SecretID 2827 err = msgpackrpc.CallWithCodec(codec, "Job.List", req, &mgmtResp) 2828 require.Nil(err) 2829 require.Equal(1, len(mgmtResp.Jobs)) 2830 require.Equal(job.ID, mgmtResp.Jobs[0].ID) 2831 2832 // Expect failure for request with a token that has incorrect permissions 2833 invalidToken := mock.CreatePolicyAndToken(t, state, 1003, "test-invalid", 2834 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob})) 2835 2836 req.AuthToken = invalidToken.SecretID 2837 var invalidResp structs.JobListResponse 2838 err = msgpackrpc.CallWithCodec(codec, "Job.List", req, &invalidResp) 2839 require.NotNil(err) 2840 2841 // Try with a valid token with correct permissions 2842 validToken := mock.CreatePolicyAndToken(t, state, 1001, "test-valid", 2843 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityListJobs})) 2844 var validResp structs.JobListResponse 2845 req.AuthToken = validToken.SecretID 2846 2847 err = msgpackrpc.CallWithCodec(codec, "Job.List", req, &validResp) 2848 require.Nil(err) 2849 require.Equal(1, len(validResp.Jobs)) 2850 require.Equal(job.ID, validResp.Jobs[0].ID) 2851 } 2852 2853 func TestJobEndpoint_ListJobs_Blocking(t *testing.T) { 2854 t.Parallel() 2855 s1 := TestServer(t, nil) 2856 defer s1.Shutdown() 2857 state := s1.fsm.State() 2858 codec := rpcClient(t, s1) 2859 testutil.WaitForLeader(t, s1.RPC) 2860 2861 // Create the job 2862 job := mock.Job() 2863 2864 // Upsert job triggers watches 2865 time.AfterFunc(100*time.Millisecond, func() { 2866 if err := state.UpsertJob(100, job); err != nil { 2867 t.Fatalf("err: %v", err) 2868 } 2869 }) 2870 2871 req := &structs.JobListRequest{ 2872 QueryOptions: structs.QueryOptions{ 2873 Region: "global", 2874 Namespace: job.Namespace, 2875 MinQueryIndex: 50, 2876 }, 2877 } 2878 start := time.Now() 2879 var resp structs.JobListResponse 2880 if err := msgpackrpc.CallWithCodec(codec, "Job.List", req, &resp); err != nil { 2881 t.Fatalf("err: %v", err) 2882 } 2883 2884 if elapsed := time.Since(start); elapsed < 100*time.Millisecond { 2885 t.Fatalf("should block (returned in %s) %#v", elapsed, resp) 2886 } 2887 if resp.Index != 100 { 2888 t.Fatalf("Bad index: %d %d", resp.Index, 100) 2889 } 2890 if len(resp.Jobs) != 1 || resp.Jobs[0].ID != job.ID { 2891 t.Fatalf("bad: %#v", resp) 2892 } 2893 2894 // Job deletion triggers watches 2895 time.AfterFunc(100*time.Millisecond, func() { 2896 if err := state.DeleteJob(200, job.Namespace, job.ID); err != nil { 2897 t.Fatalf("err: %v", err) 2898 } 2899 }) 2900 2901 req.MinQueryIndex = 150 2902 start = time.Now() 2903 var resp2 structs.JobListResponse 2904 if err := msgpackrpc.CallWithCodec(codec, "Job.List", req, &resp2); err != nil { 2905 t.Fatalf("err: %v", err) 2906 } 2907 2908 if elapsed := time.Since(start); elapsed < 100*time.Millisecond { 2909 t.Fatalf("should block (returned in %s) %#v", elapsed, resp2) 2910 } 2911 if resp2.Index != 200 { 2912 t.Fatalf("Bad index: %d %d", resp2.Index, 200) 2913 } 2914 if len(resp2.Jobs) != 0 { 2915 t.Fatalf("bad: %#v", resp2) 2916 } 2917 } 2918 2919 func TestJobEndpoint_Allocations(t *testing.T) { 2920 t.Parallel() 2921 s1 := TestServer(t, nil) 2922 defer s1.Shutdown() 2923 codec := rpcClient(t, s1) 2924 testutil.WaitForLeader(t, s1.RPC) 2925 2926 // Create the register request 2927 alloc1 := mock.Alloc() 2928 alloc2 := mock.Alloc() 2929 alloc2.JobID = alloc1.JobID 2930 state := s1.fsm.State() 2931 state.UpsertJobSummary(998, mock.JobSummary(alloc1.JobID)) 2932 state.UpsertJobSummary(999, mock.JobSummary(alloc2.JobID)) 2933 err := state.UpsertAllocs(1000, 2934 []*structs.Allocation{alloc1, alloc2}) 2935 if err != nil { 2936 t.Fatalf("err: %v", err) 2937 } 2938 2939 // Lookup the jobs 2940 get := &structs.JobSpecificRequest{ 2941 JobID: alloc1.JobID, 2942 QueryOptions: structs.QueryOptions{ 2943 Region: "global", 2944 Namespace: alloc1.Job.Namespace, 2945 }, 2946 } 2947 var resp2 structs.JobAllocationsResponse 2948 if err := msgpackrpc.CallWithCodec(codec, "Job.Allocations", get, &resp2); err != nil { 2949 t.Fatalf("err: %v", err) 2950 } 2951 if resp2.Index != 1000 { 2952 t.Fatalf("Bad index: %d %d", resp2.Index, 1000) 2953 } 2954 2955 if len(resp2.Allocations) != 2 { 2956 t.Fatalf("bad: %#v", resp2.Allocations) 2957 } 2958 } 2959 2960 func TestJobEndpoint_Allocations_ACL(t *testing.T) { 2961 t.Parallel() 2962 require := require.New(t) 2963 2964 s1, root := TestACLServer(t, nil) 2965 defer s1.Shutdown() 2966 codec := rpcClient(t, s1) 2967 testutil.WaitForLeader(t, s1.RPC) 2968 state := s1.fsm.State() 2969 2970 // Create allocations for a job 2971 alloc1 := mock.Alloc() 2972 alloc2 := mock.Alloc() 2973 alloc2.JobID = alloc1.JobID 2974 state.UpsertJobSummary(998, mock.JobSummary(alloc1.JobID)) 2975 state.UpsertJobSummary(999, mock.JobSummary(alloc2.JobID)) 2976 err := state.UpsertAllocs(1000, 2977 []*structs.Allocation{alloc1, alloc2}) 2978 require.Nil(err) 2979 2980 // Look up allocations for that job 2981 get := &structs.JobSpecificRequest{ 2982 JobID: alloc1.JobID, 2983 QueryOptions: structs.QueryOptions{ 2984 Region: "global", 2985 Namespace: alloc1.Job.Namespace, 2986 }, 2987 } 2988 2989 // Attempt to fetch the response without a token should fail 2990 var resp structs.JobAllocationsResponse 2991 err = msgpackrpc.CallWithCodec(codec, "Job.Allocations", get, &resp) 2992 require.NotNil(err) 2993 require.Contains(err.Error(), "Permission denied") 2994 2995 // Attempt to fetch the response with an invalid token should fail 2996 invalidToken := mock.CreatePolicyAndToken(t, state, 1001, "test-invalid", 2997 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityListJobs})) 2998 2999 get.AuthToken = invalidToken.SecretID 3000 var invalidResp structs.JobAllocationsResponse 3001 err = msgpackrpc.CallWithCodec(codec, "Job.Allocations", get, &invalidResp) 3002 require.NotNil(err) 3003 require.Contains(err.Error(), "Permission denied") 3004 3005 // Attempt to fetch the response with valid management token should succeed 3006 get.AuthToken = root.SecretID 3007 var validResp structs.JobAllocationsResponse 3008 err = msgpackrpc.CallWithCodec(codec, "Job.Allocations", get, &validResp) 3009 require.Nil(err) 3010 3011 // Attempt to fetch the response with valid management token should succeed 3012 validToken := mock.CreatePolicyAndToken(t, state, 1005, "test-valid", 3013 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob})) 3014 3015 get.AuthToken = validToken.SecretID 3016 var validResp2 structs.JobAllocationsResponse 3017 err = msgpackrpc.CallWithCodec(codec, "Job.Allocations", get, &validResp2) 3018 require.Nil(err) 3019 3020 require.Equal(2, len(validResp2.Allocations)) 3021 } 3022 3023 func TestJobEndpoint_Allocations_Blocking(t *testing.T) { 3024 t.Parallel() 3025 s1 := TestServer(t, nil) 3026 defer s1.Shutdown() 3027 codec := rpcClient(t, s1) 3028 testutil.WaitForLeader(t, s1.RPC) 3029 3030 // Create the register request 3031 alloc1 := mock.Alloc() 3032 alloc2 := mock.Alloc() 3033 alloc2.JobID = "job1" 3034 state := s1.fsm.State() 3035 3036 // First upsert an unrelated alloc 3037 time.AfterFunc(100*time.Millisecond, func() { 3038 state.UpsertJobSummary(99, mock.JobSummary(alloc1.JobID)) 3039 err := state.UpsertAllocs(100, []*structs.Allocation{alloc1}) 3040 if err != nil { 3041 t.Fatalf("err: %v", err) 3042 } 3043 }) 3044 3045 // Upsert an alloc for the job we are interested in later 3046 time.AfterFunc(200*time.Millisecond, func() { 3047 state.UpsertJobSummary(199, mock.JobSummary(alloc2.JobID)) 3048 err := state.UpsertAllocs(200, []*structs.Allocation{alloc2}) 3049 if err != nil { 3050 t.Fatalf("err: %v", err) 3051 } 3052 }) 3053 3054 // Lookup the jobs 3055 get := &structs.JobSpecificRequest{ 3056 JobID: "job1", 3057 QueryOptions: structs.QueryOptions{ 3058 Region: "global", 3059 Namespace: alloc1.Job.Namespace, 3060 MinQueryIndex: 150, 3061 }, 3062 } 3063 var resp structs.JobAllocationsResponse 3064 start := time.Now() 3065 if err := msgpackrpc.CallWithCodec(codec, "Job.Allocations", get, &resp); err != nil { 3066 t.Fatalf("err: %v", err) 3067 } 3068 3069 if elapsed := time.Since(start); elapsed < 200*time.Millisecond { 3070 t.Fatalf("should block (returned in %s) %#v", elapsed, resp) 3071 } 3072 if resp.Index != 200 { 3073 t.Fatalf("Bad index: %d %d", resp.Index, 200) 3074 } 3075 if len(resp.Allocations) != 1 || resp.Allocations[0].JobID != "job1" { 3076 t.Fatalf("bad: %#v", resp.Allocations) 3077 } 3078 } 3079 3080 func TestJobEndpoint_Evaluations(t *testing.T) { 3081 t.Parallel() 3082 s1 := TestServer(t, nil) 3083 defer s1.Shutdown() 3084 codec := rpcClient(t, s1) 3085 testutil.WaitForLeader(t, s1.RPC) 3086 3087 // Create the register request 3088 eval1 := mock.Eval() 3089 eval2 := mock.Eval() 3090 eval2.JobID = eval1.JobID 3091 state := s1.fsm.State() 3092 err := state.UpsertEvals(1000, 3093 []*structs.Evaluation{eval1, eval2}) 3094 if err != nil { 3095 t.Fatalf("err: %v", err) 3096 } 3097 3098 // Lookup the jobs 3099 get := &structs.JobSpecificRequest{ 3100 JobID: eval1.JobID, 3101 QueryOptions: structs.QueryOptions{ 3102 Region: "global", 3103 Namespace: eval1.Namespace, 3104 }, 3105 } 3106 var resp2 structs.JobEvaluationsResponse 3107 if err := msgpackrpc.CallWithCodec(codec, "Job.Evaluations", get, &resp2); err != nil { 3108 t.Fatalf("err: %v", err) 3109 } 3110 if resp2.Index != 1000 { 3111 t.Fatalf("Bad index: %d %d", resp2.Index, 1000) 3112 } 3113 3114 if len(resp2.Evaluations) != 2 { 3115 t.Fatalf("bad: %#v", resp2.Evaluations) 3116 } 3117 } 3118 3119 func TestJobEndpoint_Evaluations_ACL(t *testing.T) { 3120 t.Parallel() 3121 require := require.New(t) 3122 3123 s1, root := TestACLServer(t, nil) 3124 defer s1.Shutdown() 3125 codec := rpcClient(t, s1) 3126 testutil.WaitForLeader(t, s1.RPC) 3127 state := s1.fsm.State() 3128 3129 // Create evaluations for the same job 3130 eval1 := mock.Eval() 3131 eval2 := mock.Eval() 3132 eval2.JobID = eval1.JobID 3133 err := state.UpsertEvals(1000, 3134 []*structs.Evaluation{eval1, eval2}) 3135 require.Nil(err) 3136 3137 // Lookup the jobs 3138 get := &structs.JobSpecificRequest{ 3139 JobID: eval1.JobID, 3140 QueryOptions: structs.QueryOptions{ 3141 Region: "global", 3142 Namespace: eval1.Namespace, 3143 }, 3144 } 3145 3146 // Attempt to fetch without providing a token 3147 var resp structs.JobEvaluationsResponse 3148 err = msgpackrpc.CallWithCodec(codec, "Job.Evaluations", get, &resp) 3149 require.NotNil(err) 3150 require.Contains(err.Error(), "Permission denied") 3151 3152 // Attempt to fetch the response with an invalid token 3153 invalidToken := mock.CreatePolicyAndToken(t, state, 1001, "test-invalid", 3154 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityListJobs})) 3155 3156 get.AuthToken = invalidToken.SecretID 3157 var invalidResp structs.JobEvaluationsResponse 3158 err = msgpackrpc.CallWithCodec(codec, "Job.Evaluations", get, &invalidResp) 3159 require.NotNil(err) 3160 require.Contains(err.Error(), "Permission denied") 3161 3162 // Attempt to fetch with valid management token should succeed 3163 get.AuthToken = root.SecretID 3164 var validResp structs.JobEvaluationsResponse 3165 err = msgpackrpc.CallWithCodec(codec, "Job.Evaluations", get, &validResp) 3166 require.Nil(err) 3167 require.Equal(2, len(validResp.Evaluations)) 3168 3169 // Attempt to fetch with valid token should succeed 3170 validToken := mock.CreatePolicyAndToken(t, state, 1003, "test-valid", 3171 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob})) 3172 3173 get.AuthToken = validToken.SecretID 3174 var validResp2 structs.JobEvaluationsResponse 3175 err = msgpackrpc.CallWithCodec(codec, "Job.Evaluations", get, &validResp2) 3176 require.Nil(err) 3177 require.Equal(2, len(validResp2.Evaluations)) 3178 } 3179 3180 func TestJobEndpoint_Evaluations_Blocking(t *testing.T) { 3181 t.Parallel() 3182 s1 := TestServer(t, nil) 3183 defer s1.Shutdown() 3184 codec := rpcClient(t, s1) 3185 testutil.WaitForLeader(t, s1.RPC) 3186 3187 // Create the register request 3188 eval1 := mock.Eval() 3189 eval2 := mock.Eval() 3190 eval2.JobID = "job1" 3191 state := s1.fsm.State() 3192 3193 // First upsert an unrelated eval 3194 time.AfterFunc(100*time.Millisecond, func() { 3195 err := state.UpsertEvals(100, []*structs.Evaluation{eval1}) 3196 if err != nil { 3197 t.Fatalf("err: %v", err) 3198 } 3199 }) 3200 3201 // Upsert an eval for the job we are interested in later 3202 time.AfterFunc(200*time.Millisecond, func() { 3203 err := state.UpsertEvals(200, []*structs.Evaluation{eval2}) 3204 if err != nil { 3205 t.Fatalf("err: %v", err) 3206 } 3207 }) 3208 3209 // Lookup the jobs 3210 get := &structs.JobSpecificRequest{ 3211 JobID: "job1", 3212 QueryOptions: structs.QueryOptions{ 3213 Region: "global", 3214 Namespace: eval1.Namespace, 3215 MinQueryIndex: 150, 3216 }, 3217 } 3218 var resp structs.JobEvaluationsResponse 3219 start := time.Now() 3220 if err := msgpackrpc.CallWithCodec(codec, "Job.Evaluations", get, &resp); err != nil { 3221 t.Fatalf("err: %v", err) 3222 } 3223 3224 if elapsed := time.Since(start); elapsed < 200*time.Millisecond { 3225 t.Fatalf("should block (returned in %s) %#v", elapsed, resp) 3226 } 3227 if resp.Index != 200 { 3228 t.Fatalf("Bad index: %d %d", resp.Index, 200) 3229 } 3230 if len(resp.Evaluations) != 1 || resp.Evaluations[0].JobID != "job1" { 3231 t.Fatalf("bad: %#v", resp.Evaluations) 3232 } 3233 } 3234 3235 func TestJobEndpoint_Deployments(t *testing.T) { 3236 t.Parallel() 3237 s1 := TestServer(t, nil) 3238 defer s1.Shutdown() 3239 codec := rpcClient(t, s1) 3240 testutil.WaitForLeader(t, s1.RPC) 3241 state := s1.fsm.State() 3242 require := require.New(t) 3243 3244 // Create the register request 3245 j := mock.Job() 3246 d1 := mock.Deployment() 3247 d2 := mock.Deployment() 3248 d1.JobID = j.ID 3249 d2.JobID = j.ID 3250 require.Nil(state.UpsertJob(1000, j), "UpsertJob") 3251 require.Nil(state.UpsertDeployment(1001, d1), "UpsertDeployment") 3252 require.Nil(state.UpsertDeployment(1002, d2), "UpsertDeployment") 3253 3254 // Lookup the jobs 3255 get := &structs.JobSpecificRequest{ 3256 JobID: j.ID, 3257 QueryOptions: structs.QueryOptions{ 3258 Region: "global", 3259 Namespace: j.Namespace, 3260 }, 3261 } 3262 var resp structs.DeploymentListResponse 3263 require.Nil(msgpackrpc.CallWithCodec(codec, "Job.Deployments", get, &resp), "RPC") 3264 require.EqualValues(1002, resp.Index, "response index") 3265 require.Len(resp.Deployments, 2, "deployments for job") 3266 } 3267 3268 func TestJobEndpoint_Deployments_ACL(t *testing.T) { 3269 t.Parallel() 3270 require := require.New(t) 3271 3272 s1, root := TestACLServer(t, nil) 3273 defer s1.Shutdown() 3274 codec := rpcClient(t, s1) 3275 testutil.WaitForLeader(t, s1.RPC) 3276 state := s1.fsm.State() 3277 3278 // Create a job and corresponding deployments 3279 j := mock.Job() 3280 d1 := mock.Deployment() 3281 d2 := mock.Deployment() 3282 d1.JobID = j.ID 3283 d2.JobID = j.ID 3284 require.Nil(state.UpsertJob(1000, j), "UpsertJob") 3285 require.Nil(state.UpsertDeployment(1001, d1), "UpsertDeployment") 3286 require.Nil(state.UpsertDeployment(1002, d2), "UpsertDeployment") 3287 3288 // Lookup the jobs 3289 get := &structs.JobSpecificRequest{ 3290 JobID: j.ID, 3291 QueryOptions: structs.QueryOptions{ 3292 Region: "global", 3293 Namespace: j.Namespace, 3294 }, 3295 } 3296 // Lookup with no token should fail 3297 var resp structs.DeploymentListResponse 3298 err := msgpackrpc.CallWithCodec(codec, "Job.Deployments", get, &resp) 3299 require.NotNil(err) 3300 require.Contains(err.Error(), "Permission denied") 3301 3302 // Attempt to fetch the response with an invalid token 3303 invalidToken := mock.CreatePolicyAndToken(t, state, 1001, "test-invalid", 3304 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityListJobs})) 3305 3306 get.AuthToken = invalidToken.SecretID 3307 var invalidResp structs.DeploymentListResponse 3308 err = msgpackrpc.CallWithCodec(codec, "Job.Deployments", get, &invalidResp) 3309 require.NotNil(err) 3310 require.Contains(err.Error(), "Permission denied") 3311 3312 // Lookup with valid management token should succeed 3313 get.AuthToken = root.SecretID 3314 var validResp structs.DeploymentListResponse 3315 require.Nil(msgpackrpc.CallWithCodec(codec, "Job.Deployments", get, &validResp), "RPC") 3316 require.EqualValues(1002, validResp.Index, "response index") 3317 require.Len(validResp.Deployments, 2, "deployments for job") 3318 3319 // Lookup with valid token should succeed 3320 validToken := mock.CreatePolicyAndToken(t, state, 1005, "test-valid", 3321 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob})) 3322 3323 get.AuthToken = validToken.SecretID 3324 var validResp2 structs.DeploymentListResponse 3325 require.Nil(msgpackrpc.CallWithCodec(codec, "Job.Deployments", get, &validResp2), "RPC") 3326 require.EqualValues(1002, validResp2.Index, "response index") 3327 require.Len(validResp2.Deployments, 2, "deployments for job") 3328 } 3329 3330 func TestJobEndpoint_Deployments_Blocking(t *testing.T) { 3331 t.Parallel() 3332 s1 := TestServer(t, nil) 3333 defer s1.Shutdown() 3334 codec := rpcClient(t, s1) 3335 testutil.WaitForLeader(t, s1.RPC) 3336 state := s1.fsm.State() 3337 require := require.New(t) 3338 3339 // Create the register request 3340 j := mock.Job() 3341 d1 := mock.Deployment() 3342 d2 := mock.Deployment() 3343 d2.JobID = j.ID 3344 require.Nil(state.UpsertJob(50, j), "UpsertJob") 3345 3346 // First upsert an unrelated eval 3347 time.AfterFunc(100*time.Millisecond, func() { 3348 require.Nil(state.UpsertDeployment(100, d1), "UpsertDeployment") 3349 }) 3350 3351 // Upsert an eval for the job we are interested in later 3352 time.AfterFunc(200*time.Millisecond, func() { 3353 require.Nil(state.UpsertDeployment(200, d2), "UpsertDeployment") 3354 }) 3355 3356 // Lookup the jobs 3357 get := &structs.JobSpecificRequest{ 3358 JobID: d2.JobID, 3359 QueryOptions: structs.QueryOptions{ 3360 Region: "global", 3361 Namespace: d2.Namespace, 3362 MinQueryIndex: 150, 3363 }, 3364 } 3365 var resp structs.DeploymentListResponse 3366 start := time.Now() 3367 require.Nil(msgpackrpc.CallWithCodec(codec, "Job.Deployments", get, &resp), "RPC") 3368 require.EqualValues(200, resp.Index, "response index") 3369 require.Len(resp.Deployments, 1, "deployments for job") 3370 require.Equal(d2.ID, resp.Deployments[0].ID, "returned deployment") 3371 if elapsed := time.Since(start); elapsed < 200*time.Millisecond { 3372 t.Fatalf("should block (returned in %s) %#v", elapsed, resp) 3373 } 3374 } 3375 3376 func TestJobEndpoint_LatestDeployment(t *testing.T) { 3377 t.Parallel() 3378 s1 := TestServer(t, nil) 3379 defer s1.Shutdown() 3380 codec := rpcClient(t, s1) 3381 testutil.WaitForLeader(t, s1.RPC) 3382 state := s1.fsm.State() 3383 require := require.New(t) 3384 3385 // Create the register request 3386 j := mock.Job() 3387 d1 := mock.Deployment() 3388 d2 := mock.Deployment() 3389 d1.JobID = j.ID 3390 d2.JobID = j.ID 3391 d2.CreateIndex = d1.CreateIndex + 100 3392 d2.ModifyIndex = d2.CreateIndex + 100 3393 require.Nil(state.UpsertJob(1000, j), "UpsertJob") 3394 require.Nil(state.UpsertDeployment(1001, d1), "UpsertDeployment") 3395 require.Nil(state.UpsertDeployment(1002, d2), "UpsertDeployment") 3396 3397 // Lookup the jobs 3398 get := &structs.JobSpecificRequest{ 3399 JobID: j.ID, 3400 QueryOptions: structs.QueryOptions{ 3401 Region: "global", 3402 Namespace: j.Namespace, 3403 }, 3404 } 3405 var resp structs.SingleDeploymentResponse 3406 require.Nil(msgpackrpc.CallWithCodec(codec, "Job.LatestDeployment", get, &resp), "RPC") 3407 require.EqualValues(1002, resp.Index, "response index") 3408 require.NotNil(resp.Deployment, "want a deployment") 3409 require.Equal(d2.ID, resp.Deployment.ID, "latest deployment for job") 3410 } 3411 3412 func TestJobEndpoint_LatestDeployment_ACL(t *testing.T) { 3413 t.Parallel() 3414 require := require.New(t) 3415 3416 s1, root := TestACLServer(t, nil) 3417 defer s1.Shutdown() 3418 codec := rpcClient(t, s1) 3419 testutil.WaitForLeader(t, s1.RPC) 3420 state := s1.fsm.State() 3421 3422 // Create a job and deployments 3423 j := mock.Job() 3424 d1 := mock.Deployment() 3425 d2 := mock.Deployment() 3426 d1.JobID = j.ID 3427 d2.JobID = j.ID 3428 d2.CreateIndex = d1.CreateIndex + 100 3429 d2.ModifyIndex = d2.CreateIndex + 100 3430 require.Nil(state.UpsertJob(1000, j), "UpsertJob") 3431 require.Nil(state.UpsertDeployment(1001, d1), "UpsertDeployment") 3432 require.Nil(state.UpsertDeployment(1002, d2), "UpsertDeployment") 3433 3434 // Lookup the jobs 3435 get := &structs.JobSpecificRequest{ 3436 JobID: j.ID, 3437 QueryOptions: structs.QueryOptions{ 3438 Region: "global", 3439 Namespace: j.Namespace, 3440 }, 3441 } 3442 3443 // Attempt to fetch the response without a token should fail 3444 var resp structs.SingleDeploymentResponse 3445 err := msgpackrpc.CallWithCodec(codec, "Job.LatestDeployment", get, &resp) 3446 require.NotNil(err) 3447 require.Contains(err.Error(), "Permission denied") 3448 3449 // Attempt to fetch the response with an invalid token should fail 3450 invalidToken := mock.CreatePolicyAndToken(t, state, 1001, "test-invalid", 3451 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityListJobs})) 3452 3453 get.AuthToken = invalidToken.SecretID 3454 var invalidResp structs.SingleDeploymentResponse 3455 err = msgpackrpc.CallWithCodec(codec, "Job.LatestDeployment", get, &invalidResp) 3456 require.NotNil(err) 3457 require.Contains(err.Error(), "Permission denied") 3458 3459 // Fetching latest deployment with a valid management token should succeed 3460 get.AuthToken = root.SecretID 3461 var validResp structs.SingleDeploymentResponse 3462 require.Nil(msgpackrpc.CallWithCodec(codec, "Job.LatestDeployment", get, &validResp), "RPC") 3463 require.EqualValues(1002, validResp.Index, "response index") 3464 require.NotNil(validResp.Deployment, "want a deployment") 3465 require.Equal(d2.ID, validResp.Deployment.ID, "latest deployment for job") 3466 3467 // Fetching latest deployment with a valid token should succeed 3468 validToken := mock.CreatePolicyAndToken(t, state, 1004, "test-valid", 3469 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob})) 3470 3471 get.AuthToken = validToken.SecretID 3472 var validResp2 structs.SingleDeploymentResponse 3473 require.Nil(msgpackrpc.CallWithCodec(codec, "Job.LatestDeployment", get, &validResp2), "RPC") 3474 require.EqualValues(1002, validResp2.Index, "response index") 3475 require.NotNil(validResp2.Deployment, "want a deployment") 3476 require.Equal(d2.ID, validResp2.Deployment.ID, "latest deployment for job") 3477 } 3478 3479 func TestJobEndpoint_LatestDeployment_Blocking(t *testing.T) { 3480 t.Parallel() 3481 s1 := TestServer(t, nil) 3482 defer s1.Shutdown() 3483 codec := rpcClient(t, s1) 3484 testutil.WaitForLeader(t, s1.RPC) 3485 state := s1.fsm.State() 3486 require := require.New(t) 3487 3488 // Create the register request 3489 j := mock.Job() 3490 d1 := mock.Deployment() 3491 d2 := mock.Deployment() 3492 d2.JobID = j.ID 3493 require.Nil(state.UpsertJob(50, j), "UpsertJob") 3494 3495 // First upsert an unrelated eval 3496 time.AfterFunc(100*time.Millisecond, func() { 3497 require.Nil(state.UpsertDeployment(100, d1), "UpsertDeployment") 3498 }) 3499 3500 // Upsert an eval for the job we are interested in later 3501 time.AfterFunc(200*time.Millisecond, func() { 3502 require.Nil(state.UpsertDeployment(200, d2), "UpsertDeployment") 3503 }) 3504 3505 // Lookup the jobs 3506 get := &structs.JobSpecificRequest{ 3507 JobID: d2.JobID, 3508 QueryOptions: structs.QueryOptions{ 3509 Region: "global", 3510 Namespace: d2.Namespace, 3511 MinQueryIndex: 150, 3512 }, 3513 } 3514 var resp structs.SingleDeploymentResponse 3515 start := time.Now() 3516 require.Nil(msgpackrpc.CallWithCodec(codec, "Job.LatestDeployment", get, &resp), "RPC") 3517 require.EqualValues(200, resp.Index, "response index") 3518 require.NotNil(resp.Deployment, "deployment for job") 3519 require.Equal(d2.ID, resp.Deployment.ID, "returned deployment") 3520 if elapsed := time.Since(start); elapsed < 200*time.Millisecond { 3521 t.Fatalf("should block (returned in %s) %#v", elapsed, resp) 3522 } 3523 } 3524 3525 func TestJobEndpoint_Plan_ACL(t *testing.T) { 3526 t.Parallel() 3527 s1, root := TestACLServer(t, func(c *Config) { 3528 c.NumSchedulers = 0 // Prevent automatic dequeue 3529 }) 3530 defer s1.Shutdown() 3531 codec := rpcClient(t, s1) 3532 testutil.WaitForLeader(t, s1.RPC) 3533 3534 // Create a plan request 3535 job := mock.Job() 3536 planReq := &structs.JobPlanRequest{ 3537 Job: job, 3538 Diff: true, 3539 WriteRequest: structs.WriteRequest{ 3540 Region: "global", 3541 Namespace: job.Namespace, 3542 }, 3543 } 3544 3545 // Try without a token, expect failure 3546 var planResp structs.JobPlanResponse 3547 if err := msgpackrpc.CallWithCodec(codec, "Job.Plan", planReq, &planResp); err == nil { 3548 t.Fatalf("expected error") 3549 } 3550 3551 // Try with a token 3552 planReq.AuthToken = root.SecretID 3553 if err := msgpackrpc.CallWithCodec(codec, "Job.Plan", planReq, &planResp); err != nil { 3554 t.Fatalf("err: %v", err) 3555 } 3556 } 3557 3558 func TestJobEndpoint_Plan_WithDiff(t *testing.T) { 3559 t.Parallel() 3560 s1 := TestServer(t, func(c *Config) { 3561 c.NumSchedulers = 0 // Prevent automatic dequeue 3562 }) 3563 defer s1.Shutdown() 3564 codec := rpcClient(t, s1) 3565 testutil.WaitForLeader(t, s1.RPC) 3566 3567 // Create the register request 3568 job := mock.Job() 3569 req := &structs.JobRegisterRequest{ 3570 Job: job, 3571 WriteRequest: structs.WriteRequest{ 3572 Region: "global", 3573 Namespace: job.Namespace, 3574 }, 3575 } 3576 3577 // Fetch the response 3578 var resp structs.JobRegisterResponse 3579 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { 3580 t.Fatalf("err: %v", err) 3581 } 3582 if resp.Index == 0 { 3583 t.Fatalf("bad index: %d", resp.Index) 3584 } 3585 3586 // Create a plan request 3587 planReq := &structs.JobPlanRequest{ 3588 Job: job, 3589 Diff: true, 3590 WriteRequest: structs.WriteRequest{ 3591 Region: "global", 3592 Namespace: job.Namespace, 3593 }, 3594 } 3595 3596 // Fetch the response 3597 var planResp structs.JobPlanResponse 3598 if err := msgpackrpc.CallWithCodec(codec, "Job.Plan", planReq, &planResp); err != nil { 3599 t.Fatalf("err: %v", err) 3600 } 3601 3602 // Check the response 3603 if planResp.JobModifyIndex == 0 { 3604 t.Fatalf("bad cas: %d", planResp.JobModifyIndex) 3605 } 3606 if planResp.Annotations == nil { 3607 t.Fatalf("no annotations") 3608 } 3609 if planResp.Diff == nil { 3610 t.Fatalf("no diff") 3611 } 3612 if len(planResp.FailedTGAllocs) == 0 { 3613 t.Fatalf("no failed task group alloc metrics") 3614 } 3615 } 3616 3617 func TestJobEndpoint_Plan_NoDiff(t *testing.T) { 3618 t.Parallel() 3619 s1 := TestServer(t, func(c *Config) { 3620 c.NumSchedulers = 0 // Prevent automatic dequeue 3621 }) 3622 defer s1.Shutdown() 3623 codec := rpcClient(t, s1) 3624 testutil.WaitForLeader(t, s1.RPC) 3625 3626 // Create the register request 3627 job := mock.Job() 3628 req := &structs.JobRegisterRequest{ 3629 Job: job, 3630 WriteRequest: structs.WriteRequest{ 3631 Region: "global", 3632 Namespace: job.Namespace, 3633 }, 3634 } 3635 3636 // Fetch the response 3637 var resp structs.JobRegisterResponse 3638 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { 3639 t.Fatalf("err: %v", err) 3640 } 3641 if resp.Index == 0 { 3642 t.Fatalf("bad index: %d", resp.Index) 3643 } 3644 3645 // Create a plan request 3646 planReq := &structs.JobPlanRequest{ 3647 Job: job, 3648 Diff: false, 3649 WriteRequest: structs.WriteRequest{ 3650 Region: "global", 3651 Namespace: job.Namespace, 3652 }, 3653 } 3654 3655 // Fetch the response 3656 var planResp structs.JobPlanResponse 3657 if err := msgpackrpc.CallWithCodec(codec, "Job.Plan", planReq, &planResp); err != nil { 3658 t.Fatalf("err: %v", err) 3659 } 3660 3661 // Check the response 3662 if planResp.JobModifyIndex == 0 { 3663 t.Fatalf("bad cas: %d", planResp.JobModifyIndex) 3664 } 3665 if planResp.Annotations == nil { 3666 t.Fatalf("no annotations") 3667 } 3668 if planResp.Diff != nil { 3669 t.Fatalf("got diff") 3670 } 3671 if len(planResp.FailedTGAllocs) == 0 { 3672 t.Fatalf("no failed task group alloc metrics") 3673 } 3674 } 3675 3676 func TestJobEndpoint_ImplicitConstraints_Vault(t *testing.T) { 3677 t.Parallel() 3678 s1 := TestServer(t, func(c *Config) { 3679 c.NumSchedulers = 0 // Prevent automatic dequeue 3680 }) 3681 defer s1.Shutdown() 3682 codec := rpcClient(t, s1) 3683 testutil.WaitForLeader(t, s1.RPC) 3684 3685 // Enable vault 3686 tr, f := true, false 3687 s1.config.VaultConfig.Enabled = &tr 3688 s1.config.VaultConfig.AllowUnauthenticated = &f 3689 3690 // Replace the Vault Client on the server 3691 tvc := &TestVaultClient{} 3692 s1.vault = tvc 3693 3694 policy := "foo" 3695 goodToken := uuid.Generate() 3696 goodPolicies := []string{"foo", "bar", "baz"} 3697 tvc.SetLookupTokenAllowedPolicies(goodToken, goodPolicies) 3698 3699 // Create the register request with a job asking for a vault policy 3700 job := mock.Job() 3701 job.VaultToken = goodToken 3702 job.TaskGroups[0].Tasks[0].Vault = &structs.Vault{ 3703 Policies: []string{policy}, 3704 ChangeMode: structs.VaultChangeModeRestart, 3705 } 3706 req := &structs.JobRegisterRequest{ 3707 Job: job, 3708 WriteRequest: structs.WriteRequest{ 3709 Region: "global", 3710 Namespace: job.Namespace, 3711 }, 3712 } 3713 3714 // Fetch the response 3715 var resp structs.JobRegisterResponse 3716 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { 3717 t.Fatalf("bad: %v", err) 3718 } 3719 3720 // Check for the job in the FSM 3721 state := s1.fsm.State() 3722 ws := memdb.NewWatchSet() 3723 out, err := state.JobByID(ws, job.Namespace, job.ID) 3724 if err != nil { 3725 t.Fatalf("err: %v", err) 3726 } 3727 if out == nil { 3728 t.Fatalf("expected job") 3729 } 3730 if out.CreateIndex != resp.JobModifyIndex { 3731 t.Fatalf("index mis-match") 3732 } 3733 3734 // Check that there is an implicit vault constraint 3735 constraints := out.TaskGroups[0].Constraints 3736 if len(constraints) != 1 { 3737 t.Fatalf("Expected an implicit constraint") 3738 } 3739 3740 if !constraints[0].Equal(vaultConstraint) { 3741 t.Fatalf("Expected implicit vault constraint") 3742 } 3743 } 3744 3745 func TestJobEndpoint_ImplicitConstraints_Signals(t *testing.T) { 3746 t.Parallel() 3747 s1 := TestServer(t, func(c *Config) { 3748 c.NumSchedulers = 0 // Prevent automatic dequeue 3749 }) 3750 defer s1.Shutdown() 3751 codec := rpcClient(t, s1) 3752 testutil.WaitForLeader(t, s1.RPC) 3753 3754 // Create the register request with a job asking for a template that sends a 3755 // signal 3756 job := mock.Job() 3757 signal1 := "SIGUSR1" 3758 signal2 := "SIGHUP" 3759 job.TaskGroups[0].Tasks[0].Templates = []*structs.Template{ 3760 { 3761 SourcePath: "foo", 3762 DestPath: "bar", 3763 ChangeMode: structs.TemplateChangeModeSignal, 3764 ChangeSignal: signal1, 3765 }, 3766 { 3767 SourcePath: "foo", 3768 DestPath: "baz", 3769 ChangeMode: structs.TemplateChangeModeSignal, 3770 ChangeSignal: signal2, 3771 }, 3772 } 3773 req := &structs.JobRegisterRequest{ 3774 Job: job, 3775 WriteRequest: structs.WriteRequest{ 3776 Region: "global", 3777 Namespace: job.Namespace, 3778 }, 3779 } 3780 3781 // Fetch the response 3782 var resp structs.JobRegisterResponse 3783 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { 3784 t.Fatalf("bad: %v", err) 3785 } 3786 3787 // Check for the job in the FSM 3788 state := s1.fsm.State() 3789 ws := memdb.NewWatchSet() 3790 out, err := state.JobByID(ws, job.Namespace, job.ID) 3791 if err != nil { 3792 t.Fatalf("err: %v", err) 3793 } 3794 if out == nil { 3795 t.Fatalf("expected job") 3796 } 3797 if out.CreateIndex != resp.JobModifyIndex { 3798 t.Fatalf("index mis-match") 3799 } 3800 3801 // Check that there is an implicit signal constraint 3802 constraints := out.TaskGroups[0].Constraints 3803 if len(constraints) != 1 { 3804 t.Fatalf("Expected an implicit constraint") 3805 } 3806 3807 sigConstraint := getSignalConstraint([]string{signal1, signal2}) 3808 if !strings.HasPrefix(sigConstraint.RTarget, "SIGHUP") { 3809 t.Fatalf("signals not sorted: %v", sigConstraint.RTarget) 3810 } 3811 3812 if !constraints[0].Equal(sigConstraint) { 3813 t.Fatalf("Expected implicit vault constraint") 3814 } 3815 } 3816 3817 func TestJobEndpoint_ValidateJob_InvalidDriverConf(t *testing.T) { 3818 t.Parallel() 3819 // Create a mock job with an invalid config 3820 job := mock.Job() 3821 job.TaskGroups[0].Tasks[0].Config = map[string]interface{}{ 3822 "foo": "bar", 3823 } 3824 3825 err, warnings := validateJob(job) 3826 if err == nil || !strings.Contains(err.Error(), "-> config") { 3827 t.Fatalf("Expected config error; got %v", err) 3828 } 3829 3830 if warnings != nil { 3831 t.Fatalf("got unexpected warnings: %v", warnings) 3832 } 3833 } 3834 3835 func TestJobEndpoint_ValidateJob_InvalidSignals(t *testing.T) { 3836 t.Parallel() 3837 // Create a mock job that wants to send a signal to a driver that can't 3838 job := mock.Job() 3839 job.TaskGroups[0].Tasks[0].Driver = "qemu" 3840 job.TaskGroups[0].Tasks[0].Vault = &structs.Vault{ 3841 Policies: []string{"foo"}, 3842 ChangeMode: structs.VaultChangeModeSignal, 3843 ChangeSignal: "SIGUSR1", 3844 } 3845 3846 err, warnings := validateJob(job) 3847 if err == nil || !strings.Contains(err.Error(), "support sending signals") { 3848 t.Fatalf("Expected signal feasibility error; got %v", err) 3849 } 3850 3851 if warnings != nil { 3852 t.Fatalf("got unexpected warnings: %v", warnings) 3853 } 3854 } 3855 3856 func TestJobEndpoint_ValidateJob_KillSignal(t *testing.T) { 3857 require := require.New(t) 3858 t.Parallel() 3859 3860 // test validate fails if the driver does not support sending signals, but a 3861 // stop_signal has been specified 3862 { 3863 job := mock.Job() 3864 job.TaskGroups[0].Tasks[0].Driver = "qemu" // qemu does not support sending signals 3865 job.TaskGroups[0].Tasks[0].KillSignal = "SIGINT" 3866 3867 err, warnings := validateJob(job) 3868 require.NotNil(err) 3869 require.True(strings.Contains(err.Error(), "support sending signals")) 3870 require.Nil(warnings) 3871 } 3872 3873 // test validate succeeds if the driver does support sending signals, and 3874 // a stop_signal has been specified 3875 { 3876 job := mock.Job() 3877 job.TaskGroups[0].Tasks[0].KillSignal = "SIGINT" 3878 3879 err, warnings := validateJob(job) 3880 require.Nil(err) 3881 require.Nil(warnings) 3882 } 3883 } 3884 3885 func TestJobEndpoint_ValidateJobUpdate(t *testing.T) { 3886 t.Parallel() 3887 old := mock.Job() 3888 new := mock.Job() 3889 3890 if err := validateJobUpdate(old, new); err != nil { 3891 t.Errorf("expected update to be valid but got: %v", err) 3892 } 3893 3894 new.Type = "batch" 3895 if err := validateJobUpdate(old, new); err == nil { 3896 t.Errorf("expected err when setting new job to a different type") 3897 } else { 3898 t.Log(err) 3899 } 3900 3901 new = mock.Job() 3902 new.Periodic = &structs.PeriodicConfig{Enabled: true} 3903 if err := validateJobUpdate(old, new); err == nil { 3904 t.Errorf("expected err when setting new job to periodic") 3905 } else { 3906 t.Log(err) 3907 } 3908 3909 new = mock.Job() 3910 new.ParameterizedJob = &structs.ParameterizedJobConfig{} 3911 if err := validateJobUpdate(old, new); err == nil { 3912 t.Errorf("expected err when setting new job to parameterized") 3913 } else { 3914 t.Log(err) 3915 } 3916 } 3917 3918 func TestJobEndpoint_ValidateJobUpdate_ACL(t *testing.T) { 3919 t.Parallel() 3920 require := require.New(t) 3921 3922 s1, root := TestACLServer(t, func(c *Config) { 3923 c.NumSchedulers = 0 // Prevent automatic dequeue 3924 }) 3925 defer s1.Shutdown() 3926 codec := rpcClient(t, s1) 3927 testutil.WaitForLeader(t, s1.RPC) 3928 3929 job := mock.Job() 3930 3931 req := &structs.JobValidateRequest{ 3932 Job: job, 3933 WriteRequest: structs.WriteRequest{ 3934 Region: "global", 3935 Namespace: job.Namespace, 3936 }, 3937 } 3938 3939 // Attempt to update without providing a valid token 3940 var resp structs.JobValidateResponse 3941 err := msgpackrpc.CallWithCodec(codec, "Job.Validate", req, &resp) 3942 require.NotNil(err) 3943 3944 // Update with a valid token 3945 req.AuthToken = root.SecretID 3946 var validResp structs.JobValidateResponse 3947 err = msgpackrpc.CallWithCodec(codec, "Job.Validate", req, &validResp) 3948 require.Nil(err) 3949 3950 require.Equal("", validResp.Error) 3951 require.Equal("", validResp.Warnings) 3952 } 3953 3954 func TestJobEndpoint_Dispatch_ACL(t *testing.T) { 3955 t.Parallel() 3956 require := require.New(t) 3957 3958 s1, root := TestACLServer(t, func(c *Config) { 3959 c.NumSchedulers = 0 // Prevent automatic dequeue 3960 }) 3961 3962 defer s1.Shutdown() 3963 codec := rpcClient(t, s1) 3964 testutil.WaitForLeader(t, s1.RPC) 3965 state := s1.fsm.State() 3966 3967 // Create a parameterized job 3968 job := mock.BatchJob() 3969 job.ParameterizedJob = &structs.ParameterizedJobConfig{} 3970 err := state.UpsertJob(400, job) 3971 require.Nil(err) 3972 3973 req := &structs.JobDispatchRequest{ 3974 JobID: job.ID, 3975 WriteRequest: structs.WriteRequest{ 3976 Region: "global", 3977 Namespace: job.Namespace, 3978 }, 3979 } 3980 3981 // Attempt to fetch the response without a token should fail 3982 var resp structs.JobDispatchResponse 3983 err = msgpackrpc.CallWithCodec(codec, "Job.Dispatch", req, &resp) 3984 require.NotNil(err) 3985 require.Contains(err.Error(), "Permission denied") 3986 3987 // Attempt to fetch the response with an invalid token should fail 3988 invalidToken := mock.CreatePolicyAndToken(t, state, 1001, "test-invalid", 3989 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityListJobs})) 3990 req.AuthToken = invalidToken.SecretID 3991 3992 var invalidResp structs.JobDispatchResponse 3993 err = msgpackrpc.CallWithCodec(codec, "Job.Dispatch", req, &invalidResp) 3994 require.NotNil(err) 3995 require.Contains(err.Error(), "Permission denied") 3996 3997 // Dispatch with a valid management token should succeed 3998 req.AuthToken = root.SecretID 3999 4000 var validResp structs.JobDispatchResponse 4001 err = msgpackrpc.CallWithCodec(codec, "Job.Dispatch", req, &validResp) 4002 require.Nil(err) 4003 require.NotNil(validResp.EvalID) 4004 require.NotNil(validResp.DispatchedJobID) 4005 require.NotEqual(validResp.DispatchedJobID, "") 4006 4007 // Dispatch with a valid token should succeed 4008 validToken := mock.CreatePolicyAndToken(t, state, 1003, "test-valid", 4009 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityDispatchJob})) 4010 req.AuthToken = validToken.SecretID 4011 4012 var validResp2 structs.JobDispatchResponse 4013 err = msgpackrpc.CallWithCodec(codec, "Job.Dispatch", req, &validResp2) 4014 require.Nil(err) 4015 require.NotNil(validResp2.EvalID) 4016 require.NotNil(validResp2.DispatchedJobID) 4017 require.NotEqual(validResp2.DispatchedJobID, "") 4018 4019 ws := memdb.NewWatchSet() 4020 out, err := state.JobByID(ws, job.Namespace, validResp2.DispatchedJobID) 4021 require.Nil(err) 4022 require.NotNil(out) 4023 require.Equal(out.ParentID, job.ID) 4024 4025 // Look up the evaluation 4026 eval, err := state.EvalByID(ws, validResp2.EvalID) 4027 require.Nil(err) 4028 require.NotNil(eval) 4029 require.Equal(eval.CreateIndex, validResp2.EvalCreateIndex) 4030 } 4031 4032 func TestJobEndpoint_Dispatch(t *testing.T) { 4033 t.Parallel() 4034 4035 // No requirements 4036 d1 := mock.BatchJob() 4037 d1.ParameterizedJob = &structs.ParameterizedJobConfig{} 4038 4039 // Require input data 4040 d2 := mock.BatchJob() 4041 d2.ParameterizedJob = &structs.ParameterizedJobConfig{ 4042 Payload: structs.DispatchPayloadRequired, 4043 } 4044 4045 // Disallow input data 4046 d3 := mock.BatchJob() 4047 d3.ParameterizedJob = &structs.ParameterizedJobConfig{ 4048 Payload: structs.DispatchPayloadForbidden, 4049 } 4050 4051 // Require meta 4052 d4 := mock.BatchJob() 4053 d4.ParameterizedJob = &structs.ParameterizedJobConfig{ 4054 MetaRequired: []string{"foo", "bar"}, 4055 } 4056 4057 // Optional meta 4058 d5 := mock.BatchJob() 4059 d5.ParameterizedJob = &structs.ParameterizedJobConfig{ 4060 MetaOptional: []string{"foo", "bar"}, 4061 } 4062 4063 // Periodic dispatch job 4064 d6 := mock.PeriodicJob() 4065 d6.ParameterizedJob = &structs.ParameterizedJobConfig{} 4066 4067 d7 := mock.BatchJob() 4068 d7.ParameterizedJob = &structs.ParameterizedJobConfig{} 4069 d7.Stop = true 4070 4071 reqNoInputNoMeta := &structs.JobDispatchRequest{} 4072 reqInputDataNoMeta := &structs.JobDispatchRequest{ 4073 Payload: []byte("hello world"), 4074 } 4075 reqNoInputDataMeta := &structs.JobDispatchRequest{ 4076 Meta: map[string]string{ 4077 "foo": "f1", 4078 "bar": "f2", 4079 }, 4080 } 4081 reqInputDataMeta := &structs.JobDispatchRequest{ 4082 Payload: []byte("hello world"), 4083 Meta: map[string]string{ 4084 "foo": "f1", 4085 "bar": "f2", 4086 }, 4087 } 4088 reqBadMeta := &structs.JobDispatchRequest{ 4089 Payload: []byte("hello world"), 4090 Meta: map[string]string{ 4091 "foo": "f1", 4092 "bar": "f2", 4093 "baz": "f3", 4094 }, 4095 } 4096 reqInputDataTooLarge := &structs.JobDispatchRequest{ 4097 Payload: make([]byte, DispatchPayloadSizeLimit+100), 4098 } 4099 4100 type testCase struct { 4101 name string 4102 parameterizedJob *structs.Job 4103 dispatchReq *structs.JobDispatchRequest 4104 noEval bool 4105 err bool 4106 errStr string 4107 } 4108 cases := []testCase{ 4109 { 4110 name: "optional input data w/ data", 4111 parameterizedJob: d1, 4112 dispatchReq: reqInputDataNoMeta, 4113 err: false, 4114 }, 4115 { 4116 name: "optional input data w/o data", 4117 parameterizedJob: d1, 4118 dispatchReq: reqNoInputNoMeta, 4119 err: false, 4120 }, 4121 { 4122 name: "require input data w/ data", 4123 parameterizedJob: d2, 4124 dispatchReq: reqInputDataNoMeta, 4125 err: false, 4126 }, 4127 { 4128 name: "require input data w/o data", 4129 parameterizedJob: d2, 4130 dispatchReq: reqNoInputNoMeta, 4131 err: true, 4132 errStr: "not provided but required", 4133 }, 4134 { 4135 name: "disallow input data w/o data", 4136 parameterizedJob: d3, 4137 dispatchReq: reqNoInputNoMeta, 4138 err: false, 4139 }, 4140 { 4141 name: "disallow input data w/ data", 4142 parameterizedJob: d3, 4143 dispatchReq: reqInputDataNoMeta, 4144 err: true, 4145 errStr: "provided but forbidden", 4146 }, 4147 { 4148 name: "require meta w/ meta", 4149 parameterizedJob: d4, 4150 dispatchReq: reqInputDataMeta, 4151 err: false, 4152 }, 4153 { 4154 name: "require meta w/o meta", 4155 parameterizedJob: d4, 4156 dispatchReq: reqNoInputNoMeta, 4157 err: true, 4158 errStr: "did not provide required meta keys", 4159 }, 4160 { 4161 name: "optional meta w/ meta", 4162 parameterizedJob: d5, 4163 dispatchReq: reqNoInputDataMeta, 4164 err: false, 4165 }, 4166 { 4167 name: "optional meta w/o meta", 4168 parameterizedJob: d5, 4169 dispatchReq: reqNoInputNoMeta, 4170 err: false, 4171 }, 4172 { 4173 name: "optional meta w/ bad meta", 4174 parameterizedJob: d5, 4175 dispatchReq: reqBadMeta, 4176 err: true, 4177 errStr: "unpermitted metadata keys", 4178 }, 4179 { 4180 name: "optional input w/ too big of input", 4181 parameterizedJob: d1, 4182 dispatchReq: reqInputDataTooLarge, 4183 err: true, 4184 errStr: "Payload exceeds maximum size", 4185 }, 4186 { 4187 name: "periodic job dispatched, ensure no eval", 4188 parameterizedJob: d6, 4189 dispatchReq: reqNoInputNoMeta, 4190 noEval: true, 4191 }, 4192 { 4193 name: "periodic job stopped, ensure error", 4194 parameterizedJob: d7, 4195 dispatchReq: reqNoInputNoMeta, 4196 err: true, 4197 errStr: "stopped", 4198 }, 4199 } 4200 4201 for _, tc := range cases { 4202 t.Run(tc.name, func(t *testing.T) { 4203 s1 := TestServer(t, func(c *Config) { 4204 c.NumSchedulers = 0 // Prevent automatic dequeue 4205 }) 4206 defer s1.Shutdown() 4207 codec := rpcClient(t, s1) 4208 testutil.WaitForLeader(t, s1.RPC) 4209 4210 // Create the register request 4211 regReq := &structs.JobRegisterRequest{ 4212 Job: tc.parameterizedJob, 4213 WriteRequest: structs.WriteRequest{ 4214 Region: "global", 4215 Namespace: tc.parameterizedJob.Namespace, 4216 }, 4217 } 4218 4219 // Fetch the response 4220 var regResp structs.JobRegisterResponse 4221 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", regReq, ®Resp); err != nil { 4222 t.Fatalf("err: %v", err) 4223 } 4224 4225 // Now try to dispatch 4226 tc.dispatchReq.JobID = tc.parameterizedJob.ID 4227 tc.dispatchReq.WriteRequest = structs.WriteRequest{ 4228 Region: "global", 4229 Namespace: tc.parameterizedJob.Namespace, 4230 } 4231 4232 var dispatchResp structs.JobDispatchResponse 4233 dispatchErr := msgpackrpc.CallWithCodec(codec, "Job.Dispatch", tc.dispatchReq, &dispatchResp) 4234 4235 if dispatchErr == nil { 4236 if tc.err { 4237 t.Fatalf("Expected error: %v", dispatchErr) 4238 } 4239 4240 // Check that we got an eval and job id back 4241 switch dispatchResp.EvalID { 4242 case "": 4243 if !tc.noEval { 4244 t.Fatalf("Bad response") 4245 } 4246 default: 4247 if tc.noEval { 4248 t.Fatalf("Got eval %q", dispatchResp.EvalID) 4249 } 4250 } 4251 4252 if dispatchResp.DispatchedJobID == "" { 4253 t.Fatalf("Bad response") 4254 } 4255 4256 state := s1.fsm.State() 4257 ws := memdb.NewWatchSet() 4258 out, err := state.JobByID(ws, tc.parameterizedJob.Namespace, dispatchResp.DispatchedJobID) 4259 if err != nil { 4260 t.Fatalf("err: %v", err) 4261 } 4262 if out == nil { 4263 t.Fatalf("expected job") 4264 } 4265 if out.CreateIndex != dispatchResp.JobCreateIndex { 4266 t.Fatalf("index mis-match") 4267 } 4268 if out.ParentID != tc.parameterizedJob.ID { 4269 t.Fatalf("bad parent ID") 4270 } 4271 4272 if tc.noEval { 4273 return 4274 } 4275 4276 // Lookup the evaluation 4277 eval, err := state.EvalByID(ws, dispatchResp.EvalID) 4278 if err != nil { 4279 t.Fatalf("err: %v", err) 4280 } 4281 4282 if eval == nil { 4283 t.Fatalf("expected eval") 4284 } 4285 if eval.CreateIndex != dispatchResp.EvalCreateIndex { 4286 t.Fatalf("index mis-match") 4287 } 4288 } else { 4289 if !tc.err { 4290 t.Fatalf("Got unexpected error: %v", dispatchErr) 4291 } else if !strings.Contains(dispatchErr.Error(), tc.errStr) { 4292 t.Fatalf("Expected err to include %q; got %v", tc.errStr, dispatchErr) 4293 } 4294 } 4295 }) 4296 } 4297 }