github.com/manicqin/nomad@v0.9.5/nomad/job_endpoint_test.go (about) 1 package nomad 2 3 import ( 4 "errors" 5 "fmt" 6 "reflect" 7 "strings" 8 "testing" 9 "time" 10 11 memdb "github.com/hashicorp/go-memdb" 12 msgpackrpc "github.com/hashicorp/net-rpc-msgpackrpc" 13 "github.com/hashicorp/nomad/acl" 14 "github.com/hashicorp/nomad/helper" 15 "github.com/hashicorp/nomad/helper/uuid" 16 "github.com/hashicorp/nomad/nomad/mock" 17 "github.com/hashicorp/nomad/nomad/structs" 18 "github.com/hashicorp/nomad/testutil" 19 "github.com/kr/pretty" 20 "github.com/stretchr/testify/require" 21 ) 22 23 func TestJobEndpoint_Register(t *testing.T) { 24 t.Parallel() 25 26 s1, cleanupS1 := TestServer(t, func(c *Config) { 27 c.NumSchedulers = 0 // Prevent automatic dequeue 28 }) 29 defer cleanupS1() 30 codec := rpcClient(t, s1) 31 testutil.WaitForLeader(t, s1.RPC) 32 33 // Create the register request 34 job := mock.Job() 35 req := &structs.JobRegisterRequest{ 36 Job: job, 37 WriteRequest: structs.WriteRequest{ 38 Region: "global", 39 Namespace: job.Namespace, 40 }, 41 } 42 43 // Fetch the response 44 var resp structs.JobRegisterResponse 45 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { 46 t.Fatalf("err: %v", err) 47 } 48 if resp.Index == 0 { 49 t.Fatalf("bad index: %d", resp.Index) 50 } 51 52 // Check for the node in the FSM 53 state := s1.fsm.State() 54 ws := memdb.NewWatchSet() 55 out, err := state.JobByID(ws, job.Namespace, job.ID) 56 if err != nil { 57 t.Fatalf("err: %v", err) 58 } 59 if out == nil { 60 t.Fatalf("expected job") 61 } 62 if out.CreateIndex != resp.JobModifyIndex { 63 t.Fatalf("index mis-match") 64 } 65 serviceName := out.TaskGroups[0].Tasks[0].Services[0].Name 66 expectedServiceName := "web-frontend" 67 if serviceName != expectedServiceName { 68 t.Fatalf("Expected Service Name: %s, Actual: %s", expectedServiceName, serviceName) 69 } 70 71 // Lookup the evaluation 72 eval, err := state.EvalByID(ws, resp.EvalID) 73 if err != nil { 74 t.Fatalf("err: %v", err) 75 } 76 if eval == nil { 77 t.Fatalf("expected eval") 78 } 79 if eval.CreateIndex != resp.EvalCreateIndex { 80 t.Fatalf("index mis-match") 81 } 82 83 if eval.Priority != job.Priority { 84 t.Fatalf("bad: %#v", eval) 85 } 86 if eval.Type != job.Type { 87 t.Fatalf("bad: %#v", eval) 88 } 89 if eval.TriggeredBy != structs.EvalTriggerJobRegister { 90 t.Fatalf("bad: %#v", eval) 91 } 92 if eval.JobID != job.ID { 93 t.Fatalf("bad: %#v", eval) 94 } 95 if eval.JobModifyIndex != resp.JobModifyIndex { 96 t.Fatalf("bad: %#v", eval) 97 } 98 if eval.Status != structs.EvalStatusPending { 99 t.Fatalf("bad: %#v", eval) 100 } 101 if eval.CreateTime == 0 { 102 t.Fatalf("eval CreateTime is unset: %#v", eval) 103 } 104 if eval.ModifyTime == 0 { 105 t.Fatalf("eval ModifyTime is unset: %#v", eval) 106 } 107 } 108 109 func TestJobEndpoint_Register_Connect(t *testing.T) { 110 t.Parallel() 111 require := require.New(t) 112 113 s1, cleanupS1 := TestServer(t, func(c *Config) { 114 c.NumSchedulers = 0 // Prevent automatic dequeue 115 }) 116 defer cleanupS1() 117 codec := rpcClient(t, s1) 118 testutil.WaitForLeader(t, s1.RPC) 119 120 // Create the register request 121 job := mock.Job() 122 job.TaskGroups[0].Networks = structs.Networks{ 123 { 124 Mode: "bridge", 125 }, 126 } 127 job.TaskGroups[0].Services = []*structs.Service{ 128 { 129 Name: "backend", 130 PortLabel: "8080", 131 Connect: &structs.ConsulConnect{ 132 SidecarService: &structs.ConsulSidecarService{}, 133 }, 134 }, 135 } 136 req := &structs.JobRegisterRequest{ 137 Job: job, 138 WriteRequest: structs.WriteRequest{ 139 Region: "global", 140 Namespace: job.Namespace, 141 }, 142 } 143 144 // Fetch the response 145 var resp structs.JobRegisterResponse 146 require.NoError(msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp)) 147 require.NotZero(resp.Index) 148 149 // Check for the node in the FSM 150 state := s1.fsm.State() 151 ws := memdb.NewWatchSet() 152 out, err := state.JobByID(ws, job.Namespace, job.ID) 153 require.NoError(err) 154 require.NotNil(out) 155 require.Equal(resp.JobModifyIndex, out.CreateIndex) 156 157 // Check that the sidecar task was injected 158 require.Len(out.TaskGroups[0].Tasks, 2) 159 sidecarTask := out.TaskGroups[0].Tasks[1] 160 require.Equal("connect-proxy-backend", sidecarTask.Name) 161 require.Equal("connect-proxy:backend", string(sidecarTask.Kind)) 162 require.Equal("connect-proxy-backend", out.TaskGroups[0].Networks[0].DynamicPorts[0].Label) 163 164 // Check that round tripping the job doesn't change the sidecarTask 165 out.Meta["test"] = "abc" 166 req.Job = out 167 require.NoError(msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp)) 168 require.NotZero(resp.Index) 169 // Check for the new node in the FSM 170 state = s1.fsm.State() 171 ws = memdb.NewWatchSet() 172 out, err = state.JobByID(ws, job.Namespace, job.ID) 173 require.NoError(err) 174 require.NotNil(out) 175 require.Equal(resp.JobModifyIndex, out.CreateIndex) 176 177 require.Len(out.TaskGroups[0].Tasks, 2) 178 require.Exactly(sidecarTask, out.TaskGroups[0].Tasks[1]) 179 180 } 181 182 func TestJobEndpoint_Register_ConnectWithSidecarTask(t *testing.T) { 183 t.Parallel() 184 require := require.New(t) 185 186 s1, cleanupS1 := TestServer(t, func(c *Config) { 187 c.NumSchedulers = 0 // Prevent automatic dequeue 188 }) 189 defer cleanupS1() 190 codec := rpcClient(t, s1) 191 testutil.WaitForLeader(t, s1.RPC) 192 193 // Create the register request 194 job := mock.Job() 195 job.TaskGroups[0].Networks = structs.Networks{ 196 { 197 Mode: "bridge", 198 }, 199 } 200 job.TaskGroups[0].Services = []*structs.Service{ 201 { 202 Name: "backend", 203 PortLabel: "8080", 204 Connect: &structs.ConsulConnect{ 205 SidecarService: &structs.ConsulSidecarService{}, 206 SidecarTask: &structs.SidecarTask{ 207 Meta: map[string]string{ 208 "source": "test", 209 }, 210 Resources: &structs.Resources{ 211 CPU: 500, 212 }, 213 Config: map[string]interface{}{ 214 "labels": map[string]string{ 215 "foo": "bar", 216 }, 217 }, 218 }, 219 }, 220 }, 221 } 222 req := &structs.JobRegisterRequest{ 223 Job: job, 224 WriteRequest: structs.WriteRequest{ 225 Region: "global", 226 Namespace: job.Namespace, 227 }, 228 } 229 230 // Fetch the response 231 var resp structs.JobRegisterResponse 232 require.NoError(msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp)) 233 require.NotZero(resp.Index) 234 235 // Check for the node in the FSM 236 state := s1.fsm.State() 237 ws := memdb.NewWatchSet() 238 out, err := state.JobByID(ws, job.Namespace, job.ID) 239 require.NoError(err) 240 require.NotNil(out) 241 require.Equal(resp.JobModifyIndex, out.CreateIndex) 242 243 // Check that the sidecar task was injected 244 require.Len(out.TaskGroups[0].Tasks, 2) 245 sidecarTask := out.TaskGroups[0].Tasks[1] 246 require.Equal("connect-proxy-backend", sidecarTask.Name) 247 require.Equal("connect-proxy:backend", string(sidecarTask.Kind)) 248 require.Equal("connect-proxy-backend", out.TaskGroups[0].Networks[0].DynamicPorts[0].Label) 249 250 // Check that the correct fields were overridden from the sidecar_task stanza 251 require.Equal("test", sidecarTask.Meta["source"]) 252 require.Equal(500, sidecarTask.Resources.CPU) 253 require.Equal(connectSidecarResources().MemoryMB, sidecarTask.Resources.MemoryMB) 254 cfg := connectDriverConfig 255 cfg["labels"] = map[string]interface{}{ 256 "foo": "bar", 257 } 258 require.Equal(cfg, sidecarTask.Config) 259 260 // Check that round tripping the job doesn't change the sidecarTask 261 out.Meta["test"] = "abc" 262 req.Job = out 263 require.NoError(msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp)) 264 require.NotZero(resp.Index) 265 // Check for the new node in the FSM 266 state = s1.fsm.State() 267 ws = memdb.NewWatchSet() 268 out, err = state.JobByID(ws, job.Namespace, job.ID) 269 require.NoError(err) 270 require.NotNil(out) 271 require.Equal(resp.JobModifyIndex, out.CreateIndex) 272 273 require.Len(out.TaskGroups[0].Tasks, 2) 274 require.Exactly(sidecarTask, out.TaskGroups[0].Tasks[1]) 275 276 } 277 278 func TestJobEndpoint_Register_ACL(t *testing.T) { 279 t.Parallel() 280 281 s1, root, cleanupS1 := TestACLServer(t, func(c *Config) { 282 c.NumSchedulers = 0 // Prevent automatic dequeue 283 }) 284 defer cleanupS1() 285 testutil.WaitForLeader(t, s1.RPC) 286 287 newVolumeJob := func(readonlyVolume bool) *structs.Job { 288 j := mock.Job() 289 tg := j.TaskGroups[0] 290 tg.Volumes = map[string]*structs.VolumeRequest{ 291 "ca-certs": { 292 Type: structs.VolumeTypeHost, 293 Source: "prod-ca-certs", 294 ReadOnly: readonlyVolume, 295 }, 296 } 297 298 tg.Tasks[0].VolumeMounts = []*structs.VolumeMount{ 299 { 300 Volume: "ca-certs", 301 Destination: "/etc/ca-certificates", 302 // Task readonly does not effect acls 303 ReadOnly: true, 304 }, 305 } 306 307 return j 308 } 309 310 submitJobPolicy := mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob, acl.NamespaceCapabilitySubmitJob}) 311 312 submitJobToken := mock.CreatePolicyAndToken(t, s1.State(), 1001, "test-submit-job", submitJobPolicy) 313 314 volumesPolicyReadWrite := mock.HostVolumePolicy("prod-*", "", []string{acl.HostVolumeCapabilityMountReadWrite}) 315 316 submitJobWithVolumesReadWriteToken := mock.CreatePolicyAndToken(t, s1.State(), 1002, "test-submit-volumes", submitJobPolicy+"\n"+volumesPolicyReadWrite) 317 318 volumesPolicyReadOnly := mock.HostVolumePolicy("prod-*", "", []string{acl.HostVolumeCapabilityMountReadOnly}) 319 320 submitJobWithVolumesReadOnlyToken := mock.CreatePolicyAndToken(t, s1.State(), 1003, "test-submit-volumes-readonly", submitJobPolicy+"\n"+volumesPolicyReadOnly) 321 322 cases := []struct { 323 Name string 324 Job *structs.Job 325 Token string 326 ErrExpected bool 327 }{ 328 { 329 Name: "without a token", 330 Job: mock.Job(), 331 Token: "", 332 ErrExpected: true, 333 }, 334 { 335 Name: "with a token", 336 Job: mock.Job(), 337 Token: root.SecretID, 338 ErrExpected: false, 339 }, 340 { 341 Name: "with a token that can submit a job, but not use a required volume", 342 Job: newVolumeJob(false), 343 Token: submitJobToken.SecretID, 344 ErrExpected: true, 345 }, 346 { 347 Name: "with a token that can submit a job, and use all required volumes", 348 Job: newVolumeJob(false), 349 Token: submitJobWithVolumesReadWriteToken.SecretID, 350 ErrExpected: false, 351 }, 352 { 353 Name: "with a token that can submit a job, but only has readonly access", 354 Job: newVolumeJob(false), 355 Token: submitJobWithVolumesReadOnlyToken.SecretID, 356 ErrExpected: true, 357 }, 358 { 359 Name: "with a token that can submit a job, and readonly volume access is enough", 360 Job: newVolumeJob(true), 361 Token: submitJobWithVolumesReadOnlyToken.SecretID, 362 ErrExpected: false, 363 }, 364 } 365 366 for _, tt := range cases { 367 t.Run(tt.Name, func(t *testing.T) { 368 codec := rpcClient(t, s1) 369 req := &structs.JobRegisterRequest{ 370 Job: tt.Job, 371 WriteRequest: structs.WriteRequest{ 372 Region: "global", 373 Namespace: tt.Job.Namespace, 374 }, 375 } 376 req.AuthToken = tt.Token 377 378 // Try without a token, expect failure 379 var resp structs.JobRegisterResponse 380 err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp) 381 382 // If we expected an error, then the job should _not_ be registered. 383 if tt.ErrExpected { 384 require.Error(t, err, "expected error") 385 return 386 } 387 388 if !tt.ErrExpected { 389 require.NoError(t, err, "unexpected error") 390 } 391 392 require.NotEqual(t, 0, resp.Index) 393 394 state := s1.fsm.State() 395 ws := memdb.NewWatchSet() 396 out, err := state.JobByID(ws, tt.Job.Namespace, tt.Job.ID) 397 require.NoError(t, err) 398 require.NotNil(t, out) 399 require.Equal(t, tt.Job.TaskGroups, out.TaskGroups) 400 }) 401 } 402 } 403 404 func TestJobEndpoint_Register_InvalidNamespace(t *testing.T) { 405 t.Parallel() 406 407 s1, cleanupS1 := TestServer(t, func(c *Config) { 408 c.NumSchedulers = 0 // Prevent automatic dequeue 409 }) 410 defer cleanupS1() 411 codec := rpcClient(t, s1) 412 testutil.WaitForLeader(t, s1.RPC) 413 414 // Create the register request 415 job := mock.Job() 416 job.Namespace = "foo" 417 req := &structs.JobRegisterRequest{ 418 Job: job, 419 WriteRequest: structs.WriteRequest{ 420 Region: "global", 421 Namespace: job.Namespace, 422 }, 423 } 424 425 // Try without a token, expect failure 426 var resp structs.JobRegisterResponse 427 err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp) 428 if err == nil || !strings.Contains(err.Error(), "nonexistent namespace") { 429 t.Fatalf("expected namespace error: %v", err) 430 } 431 432 // Check for the job in the FSM 433 state := s1.fsm.State() 434 ws := memdb.NewWatchSet() 435 out, err := state.JobByID(ws, job.Namespace, job.ID) 436 if err != nil { 437 t.Fatalf("err: %v", err) 438 } 439 if out != nil { 440 t.Fatalf("expected no job") 441 } 442 } 443 444 func TestJobEndpoint_Register_Payload(t *testing.T) { 445 t.Parallel() 446 447 s1, cleanupS1 := TestServer(t, func(c *Config) { 448 c.NumSchedulers = 0 // Prevent automatic dequeue 449 }) 450 defer cleanupS1() 451 codec := rpcClient(t, s1) 452 testutil.WaitForLeader(t, s1.RPC) 453 454 // Create the register request with a job containing an invalid driver 455 // config 456 job := mock.Job() 457 job.Payload = []byte{0x1} 458 req := &structs.JobRegisterRequest{ 459 Job: job, 460 WriteRequest: structs.WriteRequest{ 461 Region: "global", 462 Namespace: job.Namespace, 463 }, 464 } 465 466 // Fetch the response 467 var resp structs.JobRegisterResponse 468 err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp) 469 if err == nil { 470 t.Fatalf("expected a validation error") 471 } 472 473 if !strings.Contains(err.Error(), "payload") { 474 t.Fatalf("expected a payload error but got: %v", err) 475 } 476 } 477 478 func TestJobEndpoint_Register_Existing(t *testing.T) { 479 t.Parallel() 480 481 s1, cleanupS1 := TestServer(t, func(c *Config) { 482 c.NumSchedulers = 0 // Prevent automatic dequeue 483 }) 484 defer cleanupS1() 485 codec := rpcClient(t, s1) 486 testutil.WaitForLeader(t, s1.RPC) 487 488 // Create the register request 489 job := mock.Job() 490 req := &structs.JobRegisterRequest{ 491 Job: job, 492 WriteRequest: structs.WriteRequest{ 493 Region: "global", 494 Namespace: job.Namespace, 495 }, 496 } 497 498 // Fetch the response 499 var resp structs.JobRegisterResponse 500 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { 501 t.Fatalf("err: %v", err) 502 } 503 if resp.Index == 0 { 504 t.Fatalf("bad index: %d", resp.Index) 505 } 506 507 // Update the job definition 508 job2 := mock.Job() 509 job2.Priority = 100 510 job2.ID = job.ID 511 req.Job = job2 512 513 // Attempt update 514 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { 515 t.Fatalf("err: %v", err) 516 } 517 if resp.Index == 0 { 518 t.Fatalf("bad index: %d", resp.Index) 519 } 520 521 // Check for the node in the FSM 522 state := s1.fsm.State() 523 ws := memdb.NewWatchSet() 524 out, err := state.JobByID(ws, job.Namespace, job.ID) 525 if err != nil { 526 t.Fatalf("err: %v", err) 527 } 528 if out == nil { 529 t.Fatalf("expected job") 530 } 531 if out.ModifyIndex != resp.JobModifyIndex { 532 t.Fatalf("index mis-match") 533 } 534 if out.Priority != 100 { 535 t.Fatalf("expected update") 536 } 537 if out.Version != 1 { 538 t.Fatalf("expected update") 539 } 540 541 // Lookup the evaluation 542 eval, err := state.EvalByID(ws, resp.EvalID) 543 if err != nil { 544 t.Fatalf("err: %v", err) 545 } 546 if eval == nil { 547 t.Fatalf("expected eval") 548 } 549 if eval.CreateIndex != resp.EvalCreateIndex { 550 t.Fatalf("index mis-match") 551 } 552 553 if eval.Priority != job2.Priority { 554 t.Fatalf("bad: %#v", eval) 555 } 556 if eval.Type != job2.Type { 557 t.Fatalf("bad: %#v", eval) 558 } 559 if eval.TriggeredBy != structs.EvalTriggerJobRegister { 560 t.Fatalf("bad: %#v", eval) 561 } 562 if eval.JobID != job2.ID { 563 t.Fatalf("bad: %#v", eval) 564 } 565 if eval.JobModifyIndex != resp.JobModifyIndex { 566 t.Fatalf("bad: %#v", eval) 567 } 568 if eval.Status != structs.EvalStatusPending { 569 t.Fatalf("bad: %#v", eval) 570 } 571 if eval.CreateTime == 0 { 572 t.Fatalf("eval CreateTime is unset: %#v", eval) 573 } 574 if eval.ModifyTime == 0 { 575 t.Fatalf("eval ModifyTime is unset: %#v", eval) 576 } 577 578 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { 579 t.Fatalf("err: %v", err) 580 } 581 if resp.Index == 0 { 582 t.Fatalf("bad index: %d", resp.Index) 583 } 584 585 // Check to ensure the job version didn't get bumped because we submitted 586 // the same job 587 state = s1.fsm.State() 588 ws = memdb.NewWatchSet() 589 out, err = state.JobByID(ws, job.Namespace, job.ID) 590 if err != nil { 591 t.Fatalf("err: %v", err) 592 } 593 if out == nil { 594 t.Fatalf("expected job") 595 } 596 if out.Version != 1 { 597 t.Fatalf("expected no update; got %v; diff %v", out.Version, pretty.Diff(job2, out)) 598 } 599 } 600 601 func TestJobEndpoint_Register_Periodic(t *testing.T) { 602 t.Parallel() 603 604 s1, cleanupS1 := TestServer(t, func(c *Config) { 605 c.NumSchedulers = 0 // Prevent automatic dequeue 606 }) 607 defer cleanupS1() 608 codec := rpcClient(t, s1) 609 testutil.WaitForLeader(t, s1.RPC) 610 611 // Create the register request for a periodic job. 612 job := mock.PeriodicJob() 613 req := &structs.JobRegisterRequest{ 614 Job: job, 615 WriteRequest: structs.WriteRequest{ 616 Region: "global", 617 Namespace: job.Namespace, 618 }, 619 } 620 621 // Fetch the response 622 var resp structs.JobRegisterResponse 623 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { 624 t.Fatalf("err: %v", err) 625 } 626 if resp.JobModifyIndex == 0 { 627 t.Fatalf("bad index: %d", resp.Index) 628 } 629 630 // Check for the node in the FSM 631 state := s1.fsm.State() 632 ws := memdb.NewWatchSet() 633 out, err := state.JobByID(ws, job.Namespace, job.ID) 634 if err != nil { 635 t.Fatalf("err: %v", err) 636 } 637 if out == nil { 638 t.Fatalf("expected job") 639 } 640 if out.CreateIndex != resp.JobModifyIndex { 641 t.Fatalf("index mis-match") 642 } 643 serviceName := out.TaskGroups[0].Tasks[0].Services[0].Name 644 expectedServiceName := "web-frontend" 645 if serviceName != expectedServiceName { 646 t.Fatalf("Expected Service Name: %s, Actual: %s", expectedServiceName, serviceName) 647 } 648 649 if resp.EvalID != "" { 650 t.Fatalf("Register created an eval for a periodic job") 651 } 652 } 653 654 func TestJobEndpoint_Register_ParameterizedJob(t *testing.T) { 655 t.Parallel() 656 657 s1, cleanupS1 := TestServer(t, func(c *Config) { 658 c.NumSchedulers = 0 // Prevent automatic dequeue 659 }) 660 defer cleanupS1() 661 codec := rpcClient(t, s1) 662 testutil.WaitForLeader(t, s1.RPC) 663 664 // Create the register request for a parameterized job. 665 job := mock.BatchJob() 666 job.ParameterizedJob = &structs.ParameterizedJobConfig{} 667 req := &structs.JobRegisterRequest{ 668 Job: job, 669 WriteRequest: structs.WriteRequest{ 670 Region: "global", 671 Namespace: job.Namespace, 672 }, 673 } 674 675 // Fetch the response 676 var resp structs.JobRegisterResponse 677 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { 678 t.Fatalf("err: %v", err) 679 } 680 if resp.JobModifyIndex == 0 { 681 t.Fatalf("bad index: %d", resp.Index) 682 } 683 684 // Check for the job in the FSM 685 state := s1.fsm.State() 686 ws := memdb.NewWatchSet() 687 out, err := state.JobByID(ws, job.Namespace, job.ID) 688 if err != nil { 689 t.Fatalf("err: %v", err) 690 } 691 if out == nil { 692 t.Fatalf("expected job") 693 } 694 if out.CreateIndex != resp.JobModifyIndex { 695 t.Fatalf("index mis-match") 696 } 697 if resp.EvalID != "" { 698 t.Fatalf("Register created an eval for a parameterized job") 699 } 700 } 701 702 func TestJobEndpoint_Register_Dispatched(t *testing.T) { 703 t.Parallel() 704 require := require.New(t) 705 706 s1, cleanupS1 := TestServer(t, func(c *Config) { 707 c.NumSchedulers = 0 // Prevent automatic dequeue 708 }) 709 defer cleanupS1() 710 codec := rpcClient(t, s1) 711 testutil.WaitForLeader(t, s1.RPC) 712 713 // Create the register request with a job with 'Dispatch' set to true 714 job := mock.Job() 715 job.Dispatched = true 716 req := &structs.JobRegisterRequest{ 717 Job: job, 718 WriteRequest: structs.WriteRequest{ 719 Region: "global", 720 Namespace: job.Namespace, 721 }, 722 } 723 724 // Fetch the response 725 var resp structs.JobRegisterResponse 726 err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp) 727 require.Error(err) 728 require.Contains(err.Error(), "job can't be submitted with 'Dispatched'") 729 } 730 731 func TestJobEndpoint_Register_EnforceIndex(t *testing.T) { 732 t.Parallel() 733 734 s1, cleanupS1 := TestServer(t, func(c *Config) { 735 c.NumSchedulers = 0 // Prevent automatic dequeue 736 }) 737 defer cleanupS1() 738 codec := rpcClient(t, s1) 739 testutil.WaitForLeader(t, s1.RPC) 740 741 // Create the register request and enforcing an incorrect index 742 job := mock.Job() 743 req := &structs.JobRegisterRequest{ 744 Job: job, 745 EnforceIndex: true, 746 JobModifyIndex: 100, // Not registered yet so not possible 747 WriteRequest: structs.WriteRequest{ 748 Region: "global", 749 Namespace: job.Namespace, 750 }, 751 } 752 753 // Fetch the response 754 var resp structs.JobRegisterResponse 755 err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp) 756 if err == nil || !strings.Contains(err.Error(), RegisterEnforceIndexErrPrefix) { 757 t.Fatalf("expected enforcement error") 758 } 759 760 // Create the register request and enforcing it is new 761 req = &structs.JobRegisterRequest{ 762 Job: job, 763 EnforceIndex: true, 764 JobModifyIndex: 0, 765 WriteRequest: structs.WriteRequest{ 766 Region: "global", 767 Namespace: job.Namespace, 768 }, 769 } 770 771 // Fetch the response 772 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { 773 t.Fatalf("err: %v", err) 774 } 775 if resp.Index == 0 { 776 t.Fatalf("bad index: %d", resp.Index) 777 } 778 779 curIndex := resp.JobModifyIndex 780 781 // Check for the node in the FSM 782 state := s1.fsm.State() 783 ws := memdb.NewWatchSet() 784 out, err := state.JobByID(ws, job.Namespace, job.ID) 785 if err != nil { 786 t.Fatalf("err: %v", err) 787 } 788 if out == nil { 789 t.Fatalf("expected job") 790 } 791 if out.CreateIndex != resp.JobModifyIndex { 792 t.Fatalf("index mis-match") 793 } 794 795 // Reregister request and enforcing it be a new job 796 req = &structs.JobRegisterRequest{ 797 Job: job, 798 EnforceIndex: true, 799 JobModifyIndex: 0, 800 WriteRequest: structs.WriteRequest{ 801 Region: "global", 802 Namespace: job.Namespace, 803 }, 804 } 805 806 // Fetch the response 807 err = msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp) 808 if err == nil || !strings.Contains(err.Error(), RegisterEnforceIndexErrPrefix) { 809 t.Fatalf("expected enforcement error") 810 } 811 812 // Reregister request and enforcing it be at an incorrect index 813 req = &structs.JobRegisterRequest{ 814 Job: job, 815 EnforceIndex: true, 816 JobModifyIndex: curIndex - 1, 817 WriteRequest: structs.WriteRequest{ 818 Region: "global", 819 Namespace: job.Namespace, 820 }, 821 } 822 823 // Fetch the response 824 err = msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp) 825 if err == nil || !strings.Contains(err.Error(), RegisterEnforceIndexErrPrefix) { 826 t.Fatalf("expected enforcement error") 827 } 828 829 // Reregister request and enforcing it be at the correct index 830 job.Priority = job.Priority + 1 831 req = &structs.JobRegisterRequest{ 832 Job: job, 833 EnforceIndex: true, 834 JobModifyIndex: curIndex, 835 WriteRequest: structs.WriteRequest{ 836 Region: "global", 837 Namespace: job.Namespace, 838 }, 839 } 840 841 // Fetch the response 842 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { 843 t.Fatalf("err: %v", err) 844 } 845 if resp.Index == 0 { 846 t.Fatalf("bad index: %d", resp.Index) 847 } 848 849 out, err = state.JobByID(ws, job.Namespace, job.ID) 850 if err != nil { 851 t.Fatalf("err: %v", err) 852 } 853 if out == nil { 854 t.Fatalf("expected job") 855 } 856 if out.Priority != job.Priority { 857 t.Fatalf("priority mis-match") 858 } 859 } 860 861 // TestJobEndpoint_Register_Vault_Disabled asserts that submitting a job that 862 // uses Vault when Vault is *disabled* results in an error. 863 func TestJobEndpoint_Register_Vault_Disabled(t *testing.T) { 864 t.Parallel() 865 866 s1, cleanupS1 := TestServer(t, func(c *Config) { 867 c.NumSchedulers = 0 // Prevent automatic dequeue 868 f := false 869 c.VaultConfig.Enabled = &f 870 }) 871 defer cleanupS1() 872 codec := rpcClient(t, s1) 873 testutil.WaitForLeader(t, s1.RPC) 874 875 // Create the register request with a job asking for a vault policy 876 job := mock.Job() 877 job.TaskGroups[0].Tasks[0].Vault = &structs.Vault{ 878 Policies: []string{"foo"}, 879 ChangeMode: structs.VaultChangeModeRestart, 880 } 881 req := &structs.JobRegisterRequest{ 882 Job: job, 883 WriteRequest: structs.WriteRequest{ 884 Region: "global", 885 Namespace: job.Namespace, 886 }, 887 } 888 889 // Fetch the response 890 var resp structs.JobRegisterResponse 891 err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp) 892 if err == nil || !strings.Contains(err.Error(), "Vault not enabled") { 893 t.Fatalf("expected Vault not enabled error: %v", err) 894 } 895 } 896 897 // TestJobEndpoint_Register_Vault_AllowUnauthenticated asserts submitting a job 898 // with a Vault policy but without a Vault token is *succeeds* if 899 // allow_unauthenticated=true. 900 func TestJobEndpoint_Register_Vault_AllowUnauthenticated(t *testing.T) { 901 t.Parallel() 902 903 s1, cleanupS1 := TestServer(t, func(c *Config) { 904 c.NumSchedulers = 0 // Prevent automatic dequeue 905 }) 906 defer cleanupS1() 907 codec := rpcClient(t, s1) 908 testutil.WaitForLeader(t, s1.RPC) 909 910 // Enable vault and allow authenticated 911 tr := true 912 s1.config.VaultConfig.Enabled = &tr 913 s1.config.VaultConfig.AllowUnauthenticated = &tr 914 915 // Replace the Vault Client on the server 916 s1.vault = &TestVaultClient{} 917 918 // Create the register request with a job asking for a vault policy 919 job := mock.Job() 920 job.TaskGroups[0].Tasks[0].Vault = &structs.Vault{ 921 Policies: []string{"foo"}, 922 ChangeMode: structs.VaultChangeModeRestart, 923 } 924 req := &structs.JobRegisterRequest{ 925 Job: job, 926 WriteRequest: structs.WriteRequest{ 927 Region: "global", 928 Namespace: job.Namespace, 929 }, 930 } 931 932 // Fetch the response 933 var resp structs.JobRegisterResponse 934 err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp) 935 if err != nil { 936 t.Fatalf("bad: %v", err) 937 } 938 939 // Check for the job in the FSM 940 state := s1.fsm.State() 941 ws := memdb.NewWatchSet() 942 out, err := state.JobByID(ws, job.Namespace, job.ID) 943 if err != nil { 944 t.Fatalf("err: %v", err) 945 } 946 if out == nil { 947 t.Fatalf("expected job") 948 } 949 if out.CreateIndex != resp.JobModifyIndex { 950 t.Fatalf("index mis-match") 951 } 952 } 953 954 // TestJobEndpoint_Register_Vault_OverrideConstraint asserts that job 955 // submitters can specify their own Vault constraint to override the 956 // automatically injected one. 957 func TestJobEndpoint_Register_Vault_OverrideConstraint(t *testing.T) { 958 t.Parallel() 959 960 s1, cleanupS1 := TestServer(t, func(c *Config) { 961 c.NumSchedulers = 0 // Prevent automatic dequeue 962 }) 963 defer cleanupS1() 964 codec := rpcClient(t, s1) 965 testutil.WaitForLeader(t, s1.RPC) 966 967 // Enable vault and allow authenticated 968 tr := true 969 s1.config.VaultConfig.Enabled = &tr 970 s1.config.VaultConfig.AllowUnauthenticated = &tr 971 972 // Replace the Vault Client on the server 973 s1.vault = &TestVaultClient{} 974 975 // Create the register request with a job asking for a vault policy 976 job := mock.Job() 977 job.TaskGroups[0].Tasks[0].Vault = &structs.Vault{ 978 Policies: []string{"foo"}, 979 ChangeMode: structs.VaultChangeModeRestart, 980 } 981 job.TaskGroups[0].Tasks[0].Constraints = []*structs.Constraint{ 982 { 983 LTarget: "${attr.vault.version}", 984 Operand: "is_set", 985 }, 986 } 987 req := &structs.JobRegisterRequest{ 988 Job: job, 989 WriteRequest: structs.WriteRequest{ 990 Region: "global", 991 Namespace: job.Namespace, 992 }, 993 } 994 995 // Fetch the response 996 var resp structs.JobRegisterResponse 997 err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp) 998 require.NoError(t, err) 999 1000 // Check for the job in the FSM 1001 state := s1.fsm.State() 1002 ws := memdb.NewWatchSet() 1003 out, err := state.JobByID(ws, job.Namespace, job.ID) 1004 require.NoError(t, err) 1005 require.NotNil(t, out) 1006 require.Equal(t, resp.JobModifyIndex, out.CreateIndex) 1007 1008 // Assert constraint was not overridden by the server 1009 outConstraints := out.TaskGroups[0].Tasks[0].Constraints 1010 require.Len(t, outConstraints, 1) 1011 require.True(t, job.TaskGroups[0].Tasks[0].Constraints[0].Equals(outConstraints[0])) 1012 } 1013 1014 func TestJobEndpoint_Register_Vault_NoToken(t *testing.T) { 1015 t.Parallel() 1016 1017 s1, cleanupS1 := TestServer(t, func(c *Config) { 1018 c.NumSchedulers = 0 // Prevent automatic dequeue 1019 }) 1020 defer cleanupS1() 1021 codec := rpcClient(t, s1) 1022 testutil.WaitForLeader(t, s1.RPC) 1023 1024 // Enable vault 1025 tr, f := true, false 1026 s1.config.VaultConfig.Enabled = &tr 1027 s1.config.VaultConfig.AllowUnauthenticated = &f 1028 1029 // Replace the Vault Client on the server 1030 s1.vault = &TestVaultClient{} 1031 1032 // Create the register request with a job asking for a vault policy but 1033 // don't send a Vault token 1034 job := mock.Job() 1035 job.TaskGroups[0].Tasks[0].Vault = &structs.Vault{ 1036 Policies: []string{"foo"}, 1037 ChangeMode: structs.VaultChangeModeRestart, 1038 } 1039 req := &structs.JobRegisterRequest{ 1040 Job: job, 1041 WriteRequest: structs.WriteRequest{ 1042 Region: "global", 1043 Namespace: job.Namespace, 1044 }, 1045 } 1046 1047 // Fetch the response 1048 var resp structs.JobRegisterResponse 1049 err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp) 1050 if err == nil || !strings.Contains(err.Error(), "missing Vault Token") { 1051 t.Fatalf("expected Vault not enabled error: %v", err) 1052 } 1053 } 1054 1055 func TestJobEndpoint_Register_Vault_Policies(t *testing.T) { 1056 t.Parallel() 1057 1058 s1, cleanupS1 := TestServer(t, func(c *Config) { 1059 c.NumSchedulers = 0 // Prevent automatic dequeue 1060 }) 1061 defer cleanupS1() 1062 codec := rpcClient(t, s1) 1063 testutil.WaitForLeader(t, s1.RPC) 1064 1065 // Enable vault 1066 tr, f := true, false 1067 s1.config.VaultConfig.Enabled = &tr 1068 s1.config.VaultConfig.AllowUnauthenticated = &f 1069 1070 // Replace the Vault Client on the server 1071 tvc := &TestVaultClient{} 1072 s1.vault = tvc 1073 1074 // Add three tokens: one that allows the requesting policy, one that does 1075 // not and one that returns an error 1076 policy := "foo" 1077 1078 badToken := uuid.Generate() 1079 badPolicies := []string{"a", "b", "c"} 1080 tvc.SetLookupTokenAllowedPolicies(badToken, badPolicies) 1081 1082 goodToken := uuid.Generate() 1083 goodPolicies := []string{"foo", "bar", "baz"} 1084 tvc.SetLookupTokenAllowedPolicies(goodToken, goodPolicies) 1085 1086 rootToken := uuid.Generate() 1087 rootPolicies := []string{"root"} 1088 tvc.SetLookupTokenAllowedPolicies(rootToken, rootPolicies) 1089 1090 errToken := uuid.Generate() 1091 expectedErr := fmt.Errorf("return errors from vault") 1092 tvc.SetLookupTokenError(errToken, expectedErr) 1093 1094 // Create the register request with a job asking for a vault policy but 1095 // send the bad Vault token 1096 job := mock.Job() 1097 job.VaultToken = badToken 1098 job.TaskGroups[0].Tasks[0].Vault = &structs.Vault{ 1099 Policies: []string{policy}, 1100 ChangeMode: structs.VaultChangeModeRestart, 1101 } 1102 req := &structs.JobRegisterRequest{ 1103 Job: job, 1104 WriteRequest: structs.WriteRequest{ 1105 Region: "global", 1106 Namespace: job.Namespace, 1107 }, 1108 } 1109 1110 // Fetch the response 1111 var resp structs.JobRegisterResponse 1112 err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp) 1113 if err == nil || !strings.Contains(err.Error(), 1114 "doesn't allow access to the following policies: "+policy) { 1115 t.Fatalf("expected permission denied error: %v", err) 1116 } 1117 1118 // Use the err token 1119 job.VaultToken = errToken 1120 err = msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp) 1121 if err == nil || !strings.Contains(err.Error(), expectedErr.Error()) { 1122 t.Fatalf("expected permission denied error: %v", err) 1123 } 1124 1125 // Use the good token 1126 job.VaultToken = goodToken 1127 1128 // Fetch the response 1129 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { 1130 t.Fatalf("bad: %v", err) 1131 } 1132 1133 // Check for the job in the FSM 1134 state := s1.fsm.State() 1135 ws := memdb.NewWatchSet() 1136 out, err := state.JobByID(ws, job.Namespace, job.ID) 1137 if err != nil { 1138 t.Fatalf("err: %v", err) 1139 } 1140 if out == nil { 1141 t.Fatalf("expected job") 1142 } 1143 if out.CreateIndex != resp.JobModifyIndex { 1144 t.Fatalf("index mis-match") 1145 } 1146 if out.VaultToken != "" { 1147 t.Fatalf("vault token not cleared") 1148 } 1149 1150 // Check that an implicit constraint was created 1151 constraints := out.TaskGroups[0].Constraints 1152 if l := len(constraints); l != 1 { 1153 t.Fatalf("Unexpected number of tests: %v", l) 1154 } 1155 1156 if !constraints[0].Equal(vaultConstraint) { 1157 t.Fatalf("bad constraint; got %#v; want %#v", constraints[0], vaultConstraint) 1158 } 1159 1160 // Create the register request with another job asking for a vault policy but 1161 // send the root Vault token 1162 job2 := mock.Job() 1163 job2.VaultToken = rootToken 1164 job.TaskGroups[0].Tasks[0].Vault = &structs.Vault{ 1165 Policies: []string{policy}, 1166 ChangeMode: structs.VaultChangeModeRestart, 1167 } 1168 req = &structs.JobRegisterRequest{ 1169 Job: job2, 1170 WriteRequest: structs.WriteRequest{ 1171 Region: "global", 1172 Namespace: job.Namespace, 1173 }, 1174 } 1175 1176 // Fetch the response 1177 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { 1178 t.Fatalf("bad: %v", err) 1179 } 1180 1181 // Check for the job in the FSM 1182 out, err = state.JobByID(ws, job2.Namespace, job2.ID) 1183 if err != nil { 1184 t.Fatalf("err: %v", err) 1185 } 1186 if out == nil { 1187 t.Fatalf("expected job") 1188 } 1189 if out.CreateIndex != resp.JobModifyIndex { 1190 t.Fatalf("index mis-match") 1191 } 1192 if out.VaultToken != "" { 1193 t.Fatalf("vault token not cleared") 1194 } 1195 } 1196 1197 // TestJobEndpoint_Register_SemverConstraint asserts that semver ordering is 1198 // used when evaluating semver constraints. 1199 func TestJobEndpoint_Register_SemverConstraint(t *testing.T) { 1200 t.Parallel() 1201 1202 s1, cleanupS1 := TestServer(t, nil) 1203 defer cleanupS1() 1204 codec := rpcClient(t, s1) 1205 testutil.WaitForLeader(t, s1.RPC) 1206 state := s1.State() 1207 1208 // Create a job with a semver constraint 1209 job := mock.Job() 1210 job.Constraints = []*structs.Constraint{ 1211 { 1212 LTarget: "${attr.vault.version}", 1213 RTarget: ">= 0.6.1", 1214 Operand: structs.ConstraintSemver, 1215 }, 1216 } 1217 job.TaskGroups[0].Count = 1 1218 1219 // Insert 2 Nodes, 1 matching the constraint, 1 not 1220 node1 := mock.Node() 1221 node1.Attributes["vault.version"] = "1.3.0-beta1+ent" 1222 node1.ComputeClass() 1223 require.NoError(t, state.UpsertNode(1, node1)) 1224 1225 node2 := mock.Node() 1226 delete(node2.Attributes, "vault.version") 1227 node2.ComputeClass() 1228 require.NoError(t, state.UpsertNode(2, node2)) 1229 1230 // Create the register request 1231 req := &structs.JobRegisterRequest{ 1232 Job: job, 1233 WriteRequest: structs.WriteRequest{ 1234 Region: "global", 1235 Namespace: job.Namespace, 1236 }, 1237 } 1238 1239 // Fetch the response 1240 var resp structs.JobRegisterResponse 1241 require.NoError(t, msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp)) 1242 require.NotZero(t, resp.Index) 1243 1244 // Wait for placements 1245 allocReq := &structs.JobSpecificRequest{ 1246 JobID: job.ID, 1247 QueryOptions: structs.QueryOptions{ 1248 Region: "global", 1249 Namespace: structs.DefaultNamespace, 1250 }, 1251 } 1252 1253 testutil.WaitForResult(func() (bool, error) { 1254 resp := structs.JobAllocationsResponse{} 1255 err := msgpackrpc.CallWithCodec(codec, "Job.Allocations", allocReq, &resp) 1256 if err != nil { 1257 return false, err 1258 } 1259 if n := len(resp.Allocations); n != 1 { 1260 return false, fmt.Errorf("expected 1 alloc, found %d", n) 1261 } 1262 alloc := resp.Allocations[0] 1263 if alloc.NodeID != node1.ID { 1264 return false, fmt.Errorf("expected alloc to be one node=%q but found node=%q", 1265 node1.ID, alloc.NodeID) 1266 } 1267 return true, nil 1268 }, func(waitErr error) { 1269 evals, err := state.EvalsByJob(nil, structs.DefaultNamespace, job.ID) 1270 require.NoError(t, err) 1271 for i, e := range evals { 1272 t.Logf("%d Eval: %s", i, pretty.Sprint(e)) 1273 } 1274 1275 require.NoError(t, waitErr) 1276 }) 1277 } 1278 1279 func TestJobEndpoint_Revert(t *testing.T) { 1280 t.Parallel() 1281 1282 s1, cleanupS1 := TestServer(t, func(c *Config) { 1283 c.NumSchedulers = 0 // Prevent automatic dequeue 1284 }) 1285 defer cleanupS1() 1286 codec := rpcClient(t, s1) 1287 testutil.WaitForLeader(t, s1.RPC) 1288 1289 // Create the initial register request 1290 job := mock.Job() 1291 job.Priority = 100 1292 req := &structs.JobRegisterRequest{ 1293 Job: job, 1294 WriteRequest: structs.WriteRequest{ 1295 Region: "global", 1296 Namespace: job.Namespace, 1297 }, 1298 } 1299 1300 // Fetch the response 1301 var resp structs.JobRegisterResponse 1302 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { 1303 t.Fatalf("err: %v", err) 1304 } 1305 if resp.Index == 0 { 1306 t.Fatalf("bad index: %d", resp.Index) 1307 } 1308 1309 // Reregister again to get another version 1310 job2 := job.Copy() 1311 job2.Priority = 1 1312 req = &structs.JobRegisterRequest{ 1313 Job: job2, 1314 WriteRequest: structs.WriteRequest{ 1315 Region: "global", 1316 Namespace: job.Namespace, 1317 }, 1318 } 1319 1320 // Fetch the response 1321 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { 1322 t.Fatalf("err: %v", err) 1323 } 1324 if resp.Index == 0 { 1325 t.Fatalf("bad index: %d", resp.Index) 1326 } 1327 1328 // Create revert request and enforcing it be at an incorrect version 1329 revertReq := &structs.JobRevertRequest{ 1330 JobID: job.ID, 1331 JobVersion: 0, 1332 EnforcePriorVersion: helper.Uint64ToPtr(10), 1333 WriteRequest: structs.WriteRequest{ 1334 Region: "global", 1335 Namespace: job.Namespace, 1336 }, 1337 } 1338 1339 // Fetch the response 1340 err := msgpackrpc.CallWithCodec(codec, "Job.Revert", revertReq, &resp) 1341 if err == nil || !strings.Contains(err.Error(), "enforcing version 10") { 1342 t.Fatalf("expected enforcement error") 1343 } 1344 1345 // Create revert request and enforcing it be at the current version 1346 revertReq = &structs.JobRevertRequest{ 1347 JobID: job.ID, 1348 JobVersion: 1, 1349 WriteRequest: structs.WriteRequest{ 1350 Region: "global", 1351 Namespace: job.Namespace, 1352 }, 1353 } 1354 1355 // Fetch the response 1356 err = msgpackrpc.CallWithCodec(codec, "Job.Revert", revertReq, &resp) 1357 if err == nil || !strings.Contains(err.Error(), "current version") { 1358 t.Fatalf("expected current version err: %v", err) 1359 } 1360 1361 // Create revert request and enforcing it be at version 1 1362 revertReq = &structs.JobRevertRequest{ 1363 JobID: job.ID, 1364 JobVersion: 0, 1365 EnforcePriorVersion: helper.Uint64ToPtr(1), 1366 WriteRequest: structs.WriteRequest{ 1367 Region: "global", 1368 Namespace: job.Namespace, 1369 }, 1370 } 1371 1372 // Fetch the response 1373 if err := msgpackrpc.CallWithCodec(codec, "Job.Revert", revertReq, &resp); err != nil { 1374 t.Fatalf("err: %v", err) 1375 } 1376 if resp.Index == 0 { 1377 t.Fatalf("bad index: %d", resp.Index) 1378 } 1379 if resp.EvalID == "" || resp.EvalCreateIndex == 0 { 1380 t.Fatalf("bad created eval: %+v", resp) 1381 } 1382 if resp.JobModifyIndex == 0 { 1383 t.Fatalf("bad job modify index: %d", resp.JobModifyIndex) 1384 } 1385 1386 // Create revert request and don't enforce. We are at version 2 but it is 1387 // the same as version 0 1388 revertReq = &structs.JobRevertRequest{ 1389 JobID: job.ID, 1390 JobVersion: 0, 1391 WriteRequest: structs.WriteRequest{ 1392 Region: "global", 1393 Namespace: job.Namespace, 1394 }, 1395 } 1396 1397 // Fetch the response 1398 if err := msgpackrpc.CallWithCodec(codec, "Job.Revert", revertReq, &resp); err != nil { 1399 t.Fatalf("err: %v", err) 1400 } 1401 if resp.Index == 0 { 1402 t.Fatalf("bad index: %d", resp.Index) 1403 } 1404 if resp.EvalID == "" || resp.EvalCreateIndex == 0 { 1405 t.Fatalf("bad created eval: %+v", resp) 1406 } 1407 if resp.JobModifyIndex == 0 { 1408 t.Fatalf("bad job modify index: %d", resp.JobModifyIndex) 1409 } 1410 1411 // Check that the job is at the correct version and that the eval was 1412 // created 1413 state := s1.fsm.State() 1414 ws := memdb.NewWatchSet() 1415 out, err := state.JobByID(ws, job.Namespace, job.ID) 1416 if err != nil { 1417 t.Fatalf("err: %v", err) 1418 } 1419 if out == nil { 1420 t.Fatalf("expected job") 1421 } 1422 if out.Priority != job.Priority { 1423 t.Fatalf("priority mis-match") 1424 } 1425 if out.Version != 2 { 1426 t.Fatalf("got version %d; want %d", out.Version, 2) 1427 } 1428 1429 eout, err := state.EvalByID(ws, resp.EvalID) 1430 if err != nil { 1431 t.Fatalf("err: %v", err) 1432 } 1433 if eout == nil { 1434 t.Fatalf("expected eval") 1435 } 1436 if eout.JobID != job.ID { 1437 t.Fatalf("job id mis-match") 1438 } 1439 1440 versions, err := state.JobVersionsByID(ws, job.Namespace, job.ID) 1441 if err != nil { 1442 t.Fatalf("err: %v", err) 1443 } 1444 if len(versions) != 3 { 1445 t.Fatalf("got %d versions; want %d", len(versions), 3) 1446 } 1447 } 1448 1449 func TestJobEndpoint_Revert_Vault_NoToken(t *testing.T) { 1450 t.Parallel() 1451 1452 s1, cleanupS1 := TestServer(t, func(c *Config) { 1453 c.NumSchedulers = 0 // Prevent automatic dequeue 1454 }) 1455 defer cleanupS1() 1456 codec := rpcClient(t, s1) 1457 testutil.WaitForLeader(t, s1.RPC) 1458 1459 // Enable vault 1460 tr, f := true, false 1461 s1.config.VaultConfig.Enabled = &tr 1462 s1.config.VaultConfig.AllowUnauthenticated = &f 1463 1464 // Replace the Vault Client on the server 1465 tvc := &TestVaultClient{} 1466 s1.vault = tvc 1467 1468 // Add three tokens: one that allows the requesting policy, one that does 1469 // not and one that returns an error 1470 policy := "foo" 1471 1472 goodToken := uuid.Generate() 1473 goodPolicies := []string{"foo", "bar", "baz"} 1474 tvc.SetLookupTokenAllowedPolicies(goodToken, goodPolicies) 1475 1476 // Create the initial register request 1477 job := mock.Job() 1478 job.VaultToken = goodToken 1479 job.Priority = 100 1480 job.TaskGroups[0].Tasks[0].Vault = &structs.Vault{ 1481 Policies: []string{policy}, 1482 ChangeMode: structs.VaultChangeModeRestart, 1483 } 1484 req := &structs.JobRegisterRequest{ 1485 Job: job, 1486 WriteRequest: structs.WriteRequest{ 1487 Region: "global", 1488 Namespace: job.Namespace, 1489 }, 1490 } 1491 1492 // Fetch the response 1493 var resp structs.JobRegisterResponse 1494 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { 1495 t.Fatalf("err: %v", err) 1496 } 1497 1498 // Reregister again to get another version 1499 job2 := job.Copy() 1500 job2.Priority = 1 1501 req = &structs.JobRegisterRequest{ 1502 Job: job2, 1503 WriteRequest: structs.WriteRequest{ 1504 Region: "global", 1505 Namespace: job.Namespace, 1506 }, 1507 } 1508 1509 // Fetch the response 1510 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { 1511 t.Fatalf("err: %v", err) 1512 } 1513 1514 revertReq := &structs.JobRevertRequest{ 1515 JobID: job.ID, 1516 JobVersion: 1, 1517 WriteRequest: structs.WriteRequest{ 1518 Region: "global", 1519 Namespace: job.Namespace, 1520 }, 1521 } 1522 1523 // Fetch the response 1524 err := msgpackrpc.CallWithCodec(codec, "Job.Revert", revertReq, &resp) 1525 if err == nil || !strings.Contains(err.Error(), "current version") { 1526 t.Fatalf("expected current version err: %v", err) 1527 } 1528 1529 // Create revert request and enforcing it be at version 1 1530 revertReq = &structs.JobRevertRequest{ 1531 JobID: job.ID, 1532 JobVersion: 0, 1533 EnforcePriorVersion: helper.Uint64ToPtr(1), 1534 WriteRequest: structs.WriteRequest{ 1535 Region: "global", 1536 Namespace: job.Namespace, 1537 }, 1538 } 1539 1540 // Fetch the response 1541 err = msgpackrpc.CallWithCodec(codec, "Job.Revert", revertReq, &resp) 1542 if err == nil || !strings.Contains(err.Error(), "missing Vault Token") { 1543 t.Fatalf("expected Vault not enabled error: %v", err) 1544 } 1545 } 1546 1547 // TestJobEndpoint_Revert_Vault_Policies asserts that job revert uses the 1548 // revert request's Vault token when authorizing policies. 1549 func TestJobEndpoint_Revert_Vault_Policies(t *testing.T) { 1550 t.Parallel() 1551 1552 s1, cleanupS1 := TestServer(t, func(c *Config) { 1553 c.NumSchedulers = 0 // Prevent automatic dequeue 1554 }) 1555 defer cleanupS1() 1556 codec := rpcClient(t, s1) 1557 testutil.WaitForLeader(t, s1.RPC) 1558 1559 // Enable vault 1560 tr, f := true, false 1561 s1.config.VaultConfig.Enabled = &tr 1562 s1.config.VaultConfig.AllowUnauthenticated = &f 1563 1564 // Replace the Vault Client on the server 1565 tvc := &TestVaultClient{} 1566 s1.vault = tvc 1567 1568 // Add three tokens: one that allows the requesting policy, one that does 1569 // not and one that returns an error 1570 policy := "foo" 1571 1572 badToken := uuid.Generate() 1573 badPolicies := []string{"a", "b", "c"} 1574 tvc.SetLookupTokenAllowedPolicies(badToken, badPolicies) 1575 1576 registerGoodToken := uuid.Generate() 1577 goodPolicies := []string{"foo", "bar", "baz"} 1578 tvc.SetLookupTokenAllowedPolicies(registerGoodToken, goodPolicies) 1579 1580 revertGoodToken := uuid.Generate() 1581 revertGoodPolicies := []string{"foo", "bar_revert", "baz_revert"} 1582 tvc.SetLookupTokenAllowedPolicies(revertGoodToken, revertGoodPolicies) 1583 1584 rootToken := uuid.Generate() 1585 rootPolicies := []string{"root"} 1586 tvc.SetLookupTokenAllowedPolicies(rootToken, rootPolicies) 1587 1588 errToken := uuid.Generate() 1589 expectedErr := fmt.Errorf("return errors from vault") 1590 tvc.SetLookupTokenError(errToken, expectedErr) 1591 1592 // Create the initial register request 1593 job := mock.Job() 1594 job.VaultToken = registerGoodToken 1595 job.Priority = 100 1596 job.TaskGroups[0].Tasks[0].Vault = &structs.Vault{ 1597 Policies: []string{policy}, 1598 ChangeMode: structs.VaultChangeModeRestart, 1599 } 1600 req := &structs.JobRegisterRequest{ 1601 Job: job, 1602 WriteRequest: structs.WriteRequest{ 1603 Region: "global", 1604 Namespace: job.Namespace, 1605 }, 1606 } 1607 1608 // Fetch the response 1609 var resp structs.JobRegisterResponse 1610 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { 1611 t.Fatalf("err: %v", err) 1612 } 1613 1614 // Reregister again to get another version 1615 job2 := job.Copy() 1616 job2.Priority = 1 1617 req = &structs.JobRegisterRequest{ 1618 Job: job2, 1619 WriteRequest: structs.WriteRequest{ 1620 Region: "global", 1621 Namespace: job.Namespace, 1622 }, 1623 } 1624 1625 // Fetch the response 1626 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { 1627 t.Fatalf("err: %v", err) 1628 } 1629 1630 // Create the revert request with the bad Vault token 1631 revertReq := &structs.JobRevertRequest{ 1632 JobID: job.ID, 1633 JobVersion: 0, 1634 WriteRequest: structs.WriteRequest{ 1635 Region: "global", 1636 Namespace: job.Namespace, 1637 }, 1638 VaultToken: badToken, 1639 } 1640 1641 // Fetch the response 1642 err := msgpackrpc.CallWithCodec(codec, "Job.Revert", revertReq, &resp) 1643 if err == nil || !strings.Contains(err.Error(), 1644 "doesn't allow access to the following policies: "+policy) { 1645 t.Fatalf("expected permission denied error: %v", err) 1646 } 1647 1648 // Use the err token 1649 revertReq.VaultToken = errToken 1650 err = msgpackrpc.CallWithCodec(codec, "Job.Revert", revertReq, &resp) 1651 if err == nil || !strings.Contains(err.Error(), expectedErr.Error()) { 1652 t.Fatalf("expected permission denied error: %v", err) 1653 } 1654 1655 // Use a good token 1656 revertReq.VaultToken = revertGoodToken 1657 if err := msgpackrpc.CallWithCodec(codec, "Job.Revert", revertReq, &resp); err != nil { 1658 t.Fatalf("bad: %v", err) 1659 } 1660 } 1661 1662 func TestJobEndpoint_Revert_ACL(t *testing.T) { 1663 t.Parallel() 1664 require := require.New(t) 1665 1666 s1, root, cleanupS1 := TestACLServer(t, func(c *Config) { 1667 c.NumSchedulers = 0 // Prevent automatic dequeue 1668 }) 1669 1670 defer cleanupS1() 1671 codec := rpcClient(t, s1) 1672 state := s1.fsm.State() 1673 testutil.WaitForLeader(t, s1.RPC) 1674 1675 // Create the jobs 1676 job := mock.Job() 1677 err := state.UpsertJob(300, job) 1678 require.Nil(err) 1679 1680 job2 := job.Copy() 1681 job2.Priority = 1 1682 err = state.UpsertJob(400, job2) 1683 require.Nil(err) 1684 1685 // Create revert request and enforcing it be at the current version 1686 revertReq := &structs.JobRevertRequest{ 1687 JobID: job.ID, 1688 JobVersion: 0, 1689 WriteRequest: structs.WriteRequest{ 1690 Region: "global", 1691 Namespace: job.Namespace, 1692 }, 1693 } 1694 1695 // Attempt to fetch the response without a valid token 1696 var resp structs.JobRegisterResponse 1697 err = msgpackrpc.CallWithCodec(codec, "Job.Revert", revertReq, &resp) 1698 require.NotNil(err) 1699 require.Contains(err.Error(), "Permission denied") 1700 1701 // Attempt to fetch the response with an invalid token 1702 invalidToken := mock.CreatePolicyAndToken(t, state, 1001, "test-invalid", 1703 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob})) 1704 1705 revertReq.AuthToken = invalidToken.SecretID 1706 var invalidResp structs.JobRegisterResponse 1707 err = msgpackrpc.CallWithCodec(codec, "Job.Revert", revertReq, &invalidResp) 1708 require.NotNil(err) 1709 require.Contains(err.Error(), "Permission denied") 1710 1711 // Fetch the response with a valid management token 1712 revertReq.AuthToken = root.SecretID 1713 var validResp structs.JobRegisterResponse 1714 err = msgpackrpc.CallWithCodec(codec, "Job.Revert", revertReq, &validResp) 1715 require.Nil(err) 1716 1717 // Try with a valid non-management token 1718 validToken := mock.CreatePolicyAndToken(t, state, 1003, "test-valid", 1719 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilitySubmitJob})) 1720 1721 revertReq.AuthToken = validToken.SecretID 1722 var validResp2 structs.JobRegisterResponse 1723 err = msgpackrpc.CallWithCodec(codec, "Job.Revert", revertReq, &validResp2) 1724 require.Nil(err) 1725 } 1726 1727 func TestJobEndpoint_Stable(t *testing.T) { 1728 t.Parallel() 1729 1730 s1, cleanupS1 := TestServer(t, func(c *Config) { 1731 c.NumSchedulers = 0 // Prevent automatic dequeue 1732 }) 1733 defer cleanupS1() 1734 codec := rpcClient(t, s1) 1735 testutil.WaitForLeader(t, s1.RPC) 1736 1737 // Create the initial register request 1738 job := mock.Job() 1739 req := &structs.JobRegisterRequest{ 1740 Job: job, 1741 WriteRequest: structs.WriteRequest{ 1742 Region: "global", 1743 Namespace: job.Namespace, 1744 }, 1745 } 1746 1747 // Fetch the response 1748 var resp structs.JobRegisterResponse 1749 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { 1750 t.Fatalf("err: %v", err) 1751 } 1752 if resp.Index == 0 { 1753 t.Fatalf("bad index: %d", resp.Index) 1754 } 1755 1756 // Create stability request 1757 stableReq := &structs.JobStabilityRequest{ 1758 JobID: job.ID, 1759 JobVersion: 0, 1760 Stable: true, 1761 WriteRequest: structs.WriteRequest{ 1762 Region: "global", 1763 Namespace: job.Namespace, 1764 }, 1765 } 1766 1767 // Fetch the response 1768 var stableResp structs.JobStabilityResponse 1769 if err := msgpackrpc.CallWithCodec(codec, "Job.Stable", stableReq, &stableResp); err != nil { 1770 t.Fatalf("err: %v", err) 1771 } 1772 if stableResp.Index == 0 { 1773 t.Fatalf("bad index: %d", resp.Index) 1774 } 1775 1776 // Check that the job is marked stable 1777 state := s1.fsm.State() 1778 ws := memdb.NewWatchSet() 1779 out, err := state.JobByID(ws, job.Namespace, job.ID) 1780 if err != nil { 1781 t.Fatalf("err: %v", err) 1782 } 1783 if out == nil { 1784 t.Fatalf("expected job") 1785 } 1786 if !out.Stable { 1787 t.Fatalf("Job is not marked stable") 1788 } 1789 } 1790 1791 func TestJobEndpoint_Stable_ACL(t *testing.T) { 1792 t.Parallel() 1793 require := require.New(t) 1794 1795 s1, root, cleanupS1 := TestACLServer(t, func(c *Config) { 1796 c.NumSchedulers = 0 // Prevent automatic dequeue 1797 }) 1798 defer cleanupS1() 1799 codec := rpcClient(t, s1) 1800 state := s1.fsm.State() 1801 testutil.WaitForLeader(t, s1.RPC) 1802 1803 // Register the job 1804 job := mock.Job() 1805 err := state.UpsertJob(1000, job) 1806 require.Nil(err) 1807 1808 // Create stability request 1809 stableReq := &structs.JobStabilityRequest{ 1810 JobID: job.ID, 1811 JobVersion: 0, 1812 Stable: true, 1813 WriteRequest: structs.WriteRequest{ 1814 Region: "global", 1815 Namespace: job.Namespace, 1816 }, 1817 } 1818 1819 // Attempt to fetch the token without a token 1820 var stableResp structs.JobStabilityResponse 1821 err = msgpackrpc.CallWithCodec(codec, "Job.Stable", stableReq, &stableResp) 1822 require.NotNil(err) 1823 require.Contains("Permission denied", err.Error()) 1824 1825 // Expect failure for request with an invalid token 1826 invalidToken := mock.CreatePolicyAndToken(t, state, 1003, "test-invalid", 1827 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityListJobs})) 1828 1829 stableReq.AuthToken = invalidToken.SecretID 1830 var invalidStableResp structs.JobStabilityResponse 1831 err = msgpackrpc.CallWithCodec(codec, "Job.Stable", stableReq, &invalidStableResp) 1832 require.NotNil(err) 1833 require.Contains("Permission denied", err.Error()) 1834 1835 // Attempt to fetch with a management token 1836 stableReq.AuthToken = root.SecretID 1837 var validStableResp structs.JobStabilityResponse 1838 err = msgpackrpc.CallWithCodec(codec, "Job.Stable", stableReq, &validStableResp) 1839 require.Nil(err) 1840 1841 // Attempt to fetch with a valid token 1842 validToken := mock.CreatePolicyAndToken(t, state, 1005, "test-invalid", 1843 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilitySubmitJob})) 1844 1845 stableReq.AuthToken = validToken.SecretID 1846 var validStableResp2 structs.JobStabilityResponse 1847 err = msgpackrpc.CallWithCodec(codec, "Job.Stable", stableReq, &validStableResp2) 1848 require.Nil(err) 1849 1850 // Check that the job is marked stable 1851 ws := memdb.NewWatchSet() 1852 out, err := state.JobByID(ws, job.Namespace, job.ID) 1853 require.Nil(err) 1854 require.NotNil(job) 1855 require.Equal(true, out.Stable) 1856 } 1857 1858 func TestJobEndpoint_Evaluate(t *testing.T) { 1859 t.Parallel() 1860 1861 s1, cleanupS1 := TestServer(t, func(c *Config) { 1862 c.NumSchedulers = 0 // Prevent automatic dequeue 1863 }) 1864 defer cleanupS1() 1865 codec := rpcClient(t, s1) 1866 testutil.WaitForLeader(t, s1.RPC) 1867 1868 // Create the register request 1869 job := mock.Job() 1870 req := &structs.JobRegisterRequest{ 1871 Job: job, 1872 WriteRequest: structs.WriteRequest{ 1873 Region: "global", 1874 Namespace: job.Namespace, 1875 }, 1876 } 1877 1878 // Fetch the response 1879 var resp structs.JobRegisterResponse 1880 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { 1881 t.Fatalf("err: %v", err) 1882 } 1883 if resp.Index == 0 { 1884 t.Fatalf("bad index: %d", resp.Index) 1885 } 1886 1887 // Force a re-evaluation 1888 reEval := &structs.JobEvaluateRequest{ 1889 JobID: job.ID, 1890 WriteRequest: structs.WriteRequest{ 1891 Region: "global", 1892 Namespace: job.Namespace, 1893 }, 1894 } 1895 1896 // Fetch the response 1897 if err := msgpackrpc.CallWithCodec(codec, "Job.Evaluate", reEval, &resp); err != nil { 1898 t.Fatalf("err: %v", err) 1899 } 1900 if resp.Index == 0 { 1901 t.Fatalf("bad index: %d", resp.Index) 1902 } 1903 1904 // Lookup the evaluation 1905 state := s1.fsm.State() 1906 ws := memdb.NewWatchSet() 1907 eval, err := state.EvalByID(ws, resp.EvalID) 1908 if err != nil { 1909 t.Fatalf("err: %v", err) 1910 } 1911 if eval == nil { 1912 t.Fatalf("expected eval") 1913 } 1914 if eval.CreateIndex != resp.EvalCreateIndex { 1915 t.Fatalf("index mis-match") 1916 } 1917 1918 if eval.Priority != job.Priority { 1919 t.Fatalf("bad: %#v", eval) 1920 } 1921 if eval.Type != job.Type { 1922 t.Fatalf("bad: %#v", eval) 1923 } 1924 if eval.TriggeredBy != structs.EvalTriggerJobRegister { 1925 t.Fatalf("bad: %#v", eval) 1926 } 1927 if eval.JobID != job.ID { 1928 t.Fatalf("bad: %#v", eval) 1929 } 1930 if eval.JobModifyIndex != resp.JobModifyIndex { 1931 t.Fatalf("bad: %#v", eval) 1932 } 1933 if eval.Status != structs.EvalStatusPending { 1934 t.Fatalf("bad: %#v", eval) 1935 } 1936 if eval.CreateTime == 0 { 1937 t.Fatalf("eval CreateTime is unset: %#v", eval) 1938 } 1939 if eval.ModifyTime == 0 { 1940 t.Fatalf("eval ModifyTime is unset: %#v", eval) 1941 } 1942 } 1943 1944 func TestJobEndpoint_ForceRescheduleEvaluate(t *testing.T) { 1945 t.Parallel() 1946 require := require.New(t) 1947 1948 s1, cleanupS1 := TestServer(t, func(c *Config) { 1949 c.NumSchedulers = 0 // Prevent automatic dequeue 1950 }) 1951 defer cleanupS1() 1952 codec := rpcClient(t, s1) 1953 testutil.WaitForLeader(t, s1.RPC) 1954 1955 // Create the register request 1956 job := mock.Job() 1957 req := &structs.JobRegisterRequest{ 1958 Job: job, 1959 WriteRequest: structs.WriteRequest{ 1960 Region: "global", 1961 Namespace: job.Namespace, 1962 }, 1963 } 1964 1965 // Fetch the response 1966 var resp structs.JobRegisterResponse 1967 err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp) 1968 require.Nil(err) 1969 require.NotEqual(0, resp.Index) 1970 1971 state := s1.fsm.State() 1972 job, err = state.JobByID(nil, structs.DefaultNamespace, job.ID) 1973 require.Nil(err) 1974 1975 // Create a failed alloc 1976 alloc := mock.Alloc() 1977 alloc.Job = job 1978 alloc.JobID = job.ID 1979 alloc.TaskGroup = job.TaskGroups[0].Name 1980 alloc.Namespace = job.Namespace 1981 alloc.ClientStatus = structs.AllocClientStatusFailed 1982 err = s1.State().UpsertAllocs(resp.Index+1, []*structs.Allocation{alloc}) 1983 require.Nil(err) 1984 1985 // Force a re-evaluation 1986 reEval := &structs.JobEvaluateRequest{ 1987 JobID: job.ID, 1988 EvalOptions: structs.EvalOptions{ForceReschedule: true}, 1989 WriteRequest: structs.WriteRequest{ 1990 Region: "global", 1991 Namespace: job.Namespace, 1992 }, 1993 } 1994 1995 // Fetch the response 1996 err = msgpackrpc.CallWithCodec(codec, "Job.Evaluate", reEval, &resp) 1997 require.Nil(err) 1998 require.NotEqual(0, resp.Index) 1999 2000 // Lookup the evaluation 2001 ws := memdb.NewWatchSet() 2002 eval, err := state.EvalByID(ws, resp.EvalID) 2003 require.Nil(err) 2004 require.NotNil(eval) 2005 require.Equal(eval.CreateIndex, resp.EvalCreateIndex) 2006 require.Equal(eval.Priority, job.Priority) 2007 require.Equal(eval.Type, job.Type) 2008 require.Equal(eval.TriggeredBy, structs.EvalTriggerJobRegister) 2009 require.Equal(eval.JobID, job.ID) 2010 require.Equal(eval.JobModifyIndex, resp.JobModifyIndex) 2011 require.Equal(eval.Status, structs.EvalStatusPending) 2012 require.NotZero(eval.CreateTime) 2013 require.NotZero(eval.ModifyTime) 2014 2015 // Lookup the alloc, verify DesiredTransition ForceReschedule 2016 alloc, err = state.AllocByID(ws, alloc.ID) 2017 require.NotNil(alloc) 2018 require.Nil(err) 2019 require.True(*alloc.DesiredTransition.ForceReschedule) 2020 } 2021 2022 func TestJobEndpoint_Evaluate_ACL(t *testing.T) { 2023 t.Parallel() 2024 require := require.New(t) 2025 2026 s1, root, cleanupS1 := TestACLServer(t, func(c *Config) { 2027 c.NumSchedulers = 0 // Prevent automatic dequeue 2028 }) 2029 defer cleanupS1() 2030 codec := rpcClient(t, s1) 2031 testutil.WaitForLeader(t, s1.RPC) 2032 state := s1.fsm.State() 2033 2034 // Create the job 2035 job := mock.Job() 2036 err := state.UpsertJob(300, job) 2037 require.Nil(err) 2038 2039 // Force a re-evaluation 2040 reEval := &structs.JobEvaluateRequest{ 2041 JobID: job.ID, 2042 WriteRequest: structs.WriteRequest{ 2043 Region: "global", 2044 Namespace: job.Namespace, 2045 }, 2046 } 2047 2048 // Attempt to fetch the response without a token 2049 var resp structs.JobRegisterResponse 2050 err = msgpackrpc.CallWithCodec(codec, "Job.Evaluate", reEval, &resp) 2051 require.NotNil(err) 2052 require.Contains(err.Error(), "Permission denied") 2053 2054 // Attempt to fetch the response with an invalid token 2055 invalidToken := mock.CreatePolicyAndToken(t, state, 1003, "test-invalid", 2056 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityListJobs})) 2057 2058 reEval.AuthToken = invalidToken.SecretID 2059 var invalidResp structs.JobRegisterResponse 2060 err = msgpackrpc.CallWithCodec(codec, "Job.Evaluate", reEval, &invalidResp) 2061 require.NotNil(err) 2062 require.Contains(err.Error(), "Permission denied") 2063 2064 // Fetch the response with a valid management token 2065 reEval.AuthToken = root.SecretID 2066 var validResp structs.JobRegisterResponse 2067 err = msgpackrpc.CallWithCodec(codec, "Job.Evaluate", reEval, &validResp) 2068 require.Nil(err) 2069 2070 // Fetch the response with a valid token 2071 validToken := mock.CreatePolicyAndToken(t, state, 1005, "test-valid", 2072 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob})) 2073 2074 reEval.AuthToken = validToken.SecretID 2075 var validResp2 structs.JobRegisterResponse 2076 err = msgpackrpc.CallWithCodec(codec, "Job.Evaluate", reEval, &validResp2) 2077 require.Nil(err) 2078 2079 // Lookup the evaluation 2080 ws := memdb.NewWatchSet() 2081 eval, err := state.EvalByID(ws, validResp2.EvalID) 2082 require.Nil(err) 2083 require.NotNil(eval) 2084 2085 require.Equal(eval.CreateIndex, validResp2.EvalCreateIndex) 2086 require.Equal(eval.Priority, job.Priority) 2087 require.Equal(eval.Type, job.Type) 2088 require.Equal(eval.TriggeredBy, structs.EvalTriggerJobRegister) 2089 require.Equal(eval.JobID, job.ID) 2090 require.Equal(eval.JobModifyIndex, validResp2.JobModifyIndex) 2091 require.Equal(eval.Status, structs.EvalStatusPending) 2092 require.NotZero(eval.CreateTime) 2093 require.NotZero(eval.ModifyTime) 2094 } 2095 2096 func TestJobEndpoint_Evaluate_Periodic(t *testing.T) { 2097 t.Parallel() 2098 2099 s1, cleanupS1 := TestServer(t, func(c *Config) { 2100 c.NumSchedulers = 0 // Prevent automatic dequeue 2101 }) 2102 defer cleanupS1() 2103 codec := rpcClient(t, s1) 2104 testutil.WaitForLeader(t, s1.RPC) 2105 2106 // Create the register request 2107 job := mock.PeriodicJob() 2108 req := &structs.JobRegisterRequest{ 2109 Job: job, 2110 WriteRequest: structs.WriteRequest{ 2111 Region: "global", 2112 Namespace: job.Namespace, 2113 }, 2114 } 2115 2116 // Fetch the response 2117 var resp structs.JobRegisterResponse 2118 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { 2119 t.Fatalf("err: %v", err) 2120 } 2121 if resp.JobModifyIndex == 0 { 2122 t.Fatalf("bad index: %d", resp.Index) 2123 } 2124 2125 // Force a re-evaluation 2126 reEval := &structs.JobEvaluateRequest{ 2127 JobID: job.ID, 2128 WriteRequest: structs.WriteRequest{ 2129 Region: "global", 2130 Namespace: job.Namespace, 2131 }, 2132 } 2133 2134 // Fetch the response 2135 if err := msgpackrpc.CallWithCodec(codec, "Job.Evaluate", reEval, &resp); err == nil { 2136 t.Fatal("expect an err") 2137 } 2138 } 2139 2140 func TestJobEndpoint_Evaluate_ParameterizedJob(t *testing.T) { 2141 t.Parallel() 2142 2143 s1, cleanupS1 := TestServer(t, func(c *Config) { 2144 c.NumSchedulers = 0 // Prevent automatic dequeue 2145 }) 2146 defer cleanupS1() 2147 codec := rpcClient(t, s1) 2148 testutil.WaitForLeader(t, s1.RPC) 2149 2150 // Create the register request 2151 job := mock.BatchJob() 2152 job.ParameterizedJob = &structs.ParameterizedJobConfig{} 2153 req := &structs.JobRegisterRequest{ 2154 Job: job, 2155 WriteRequest: structs.WriteRequest{ 2156 Region: "global", 2157 Namespace: job.Namespace, 2158 }, 2159 } 2160 2161 // Fetch the response 2162 var resp structs.JobRegisterResponse 2163 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { 2164 t.Fatalf("err: %v", err) 2165 } 2166 if resp.JobModifyIndex == 0 { 2167 t.Fatalf("bad index: %d", resp.Index) 2168 } 2169 2170 // Force a re-evaluation 2171 reEval := &structs.JobEvaluateRequest{ 2172 JobID: job.ID, 2173 WriteRequest: structs.WriteRequest{ 2174 Region: "global", 2175 Namespace: job.Namespace, 2176 }, 2177 } 2178 2179 // Fetch the response 2180 if err := msgpackrpc.CallWithCodec(codec, "Job.Evaluate", reEval, &resp); err == nil { 2181 t.Fatal("expect an err") 2182 } 2183 } 2184 2185 func TestJobEndpoint_Deregister(t *testing.T) { 2186 t.Parallel() 2187 require := require.New(t) 2188 2189 s1, cleanupS1 := TestServer(t, func(c *Config) { 2190 c.NumSchedulers = 0 // Prevent automatic dequeue 2191 }) 2192 defer cleanupS1() 2193 codec := rpcClient(t, s1) 2194 testutil.WaitForLeader(t, s1.RPC) 2195 2196 // Create the register requests 2197 job := mock.Job() 2198 reg := &structs.JobRegisterRequest{ 2199 Job: job, 2200 WriteRequest: structs.WriteRequest{ 2201 Region: "global", 2202 Namespace: job.Namespace, 2203 }, 2204 } 2205 2206 // Fetch the response 2207 var resp structs.JobRegisterResponse 2208 require.Nil(msgpackrpc.CallWithCodec(codec, "Job.Register", reg, &resp)) 2209 2210 // Deregister but don't purge 2211 dereg := &structs.JobDeregisterRequest{ 2212 JobID: job.ID, 2213 Purge: false, 2214 WriteRequest: structs.WriteRequest{ 2215 Region: "global", 2216 Namespace: job.Namespace, 2217 }, 2218 } 2219 var resp2 structs.JobDeregisterResponse 2220 require.Nil(msgpackrpc.CallWithCodec(codec, "Job.Deregister", dereg, &resp2)) 2221 require.NotZero(resp2.Index) 2222 2223 // Check for the job in the FSM 2224 state := s1.fsm.State() 2225 out, err := state.JobByID(nil, job.Namespace, job.ID) 2226 require.Nil(err) 2227 require.NotNil(out) 2228 require.True(out.Stop) 2229 2230 // Lookup the evaluation 2231 eval, err := state.EvalByID(nil, resp2.EvalID) 2232 require.Nil(err) 2233 require.NotNil(eval) 2234 require.EqualValues(resp2.EvalCreateIndex, eval.CreateIndex) 2235 require.Equal(job.Priority, eval.Priority) 2236 require.Equal(job.Type, eval.Type) 2237 require.Equal(structs.EvalTriggerJobDeregister, eval.TriggeredBy) 2238 require.Equal(job.ID, eval.JobID) 2239 require.Equal(structs.EvalStatusPending, eval.Status) 2240 require.NotZero(eval.CreateTime) 2241 require.NotZero(eval.ModifyTime) 2242 2243 // Deregister and purge 2244 dereg2 := &structs.JobDeregisterRequest{ 2245 JobID: job.ID, 2246 Purge: true, 2247 WriteRequest: structs.WriteRequest{ 2248 Region: "global", 2249 Namespace: job.Namespace, 2250 }, 2251 } 2252 var resp3 structs.JobDeregisterResponse 2253 require.Nil(msgpackrpc.CallWithCodec(codec, "Job.Deregister", dereg2, &resp3)) 2254 require.NotZero(resp3.Index) 2255 2256 // Check for the job in the FSM 2257 out, err = state.JobByID(nil, job.Namespace, job.ID) 2258 require.Nil(err) 2259 require.Nil(out) 2260 2261 // Lookup the evaluation 2262 eval, err = state.EvalByID(nil, resp3.EvalID) 2263 require.Nil(err) 2264 require.NotNil(eval) 2265 2266 require.EqualValues(resp3.EvalCreateIndex, eval.CreateIndex) 2267 require.Equal(job.Priority, eval.Priority) 2268 require.Equal(job.Type, eval.Type) 2269 require.Equal(structs.EvalTriggerJobDeregister, eval.TriggeredBy) 2270 require.Equal(job.ID, eval.JobID) 2271 require.Equal(structs.EvalStatusPending, eval.Status) 2272 require.NotZero(eval.CreateTime) 2273 require.NotZero(eval.ModifyTime) 2274 } 2275 2276 func TestJobEndpoint_Deregister_ACL(t *testing.T) { 2277 t.Parallel() 2278 require := require.New(t) 2279 2280 s1, root, cleanupS1 := TestACLServer(t, func(c *Config) { 2281 c.NumSchedulers = 0 // Prevent automatic dequeue 2282 }) 2283 defer cleanupS1() 2284 codec := rpcClient(t, s1) 2285 testutil.WaitForLeader(t, s1.RPC) 2286 state := s1.fsm.State() 2287 2288 // Create and register a job 2289 job := mock.Job() 2290 err := state.UpsertJob(100, job) 2291 require.Nil(err) 2292 2293 // Deregister and purge 2294 req := &structs.JobDeregisterRequest{ 2295 JobID: job.ID, 2296 Purge: true, 2297 WriteRequest: structs.WriteRequest{ 2298 Region: "global", 2299 Namespace: job.Namespace, 2300 }, 2301 } 2302 2303 // Expect failure for request without a token 2304 var resp structs.JobDeregisterResponse 2305 err = msgpackrpc.CallWithCodec(codec, "Job.Deregister", req, &resp) 2306 require.NotNil(err) 2307 require.Contains(err.Error(), "Permission denied") 2308 2309 // Expect failure for request with an invalid token 2310 invalidToken := mock.CreatePolicyAndToken(t, state, 1003, "test-invalid", 2311 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityListJobs})) 2312 req.AuthToken = invalidToken.SecretID 2313 2314 var invalidResp structs.JobDeregisterResponse 2315 err = msgpackrpc.CallWithCodec(codec, "Job.Deregister", req, &invalidResp) 2316 require.NotNil(err) 2317 require.Contains(err.Error(), "Permission denied") 2318 2319 // Expect success with a valid management token 2320 req.AuthToken = root.SecretID 2321 2322 var validResp structs.JobDeregisterResponse 2323 err = msgpackrpc.CallWithCodec(codec, "Job.Deregister", req, &validResp) 2324 require.Nil(err) 2325 require.NotEqual(validResp.Index, 0) 2326 2327 // Expect success with a valid token 2328 validToken := mock.CreatePolicyAndToken(t, state, 1005, "test-valid", 2329 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilitySubmitJob})) 2330 req.AuthToken = validToken.SecretID 2331 2332 var validResp2 structs.JobDeregisterResponse 2333 err = msgpackrpc.CallWithCodec(codec, "Job.Deregister", req, &validResp2) 2334 require.Nil(err) 2335 require.NotEqual(validResp2.Index, 0) 2336 2337 // Check for the job in the FSM 2338 out, err := state.JobByID(nil, job.Namespace, job.ID) 2339 require.Nil(err) 2340 require.Nil(out) 2341 2342 // Lookup the evaluation 2343 eval, err := state.EvalByID(nil, validResp2.EvalID) 2344 require.Nil(err) 2345 require.NotNil(eval, nil) 2346 2347 require.Equal(eval.CreateIndex, validResp2.EvalCreateIndex) 2348 require.Equal(eval.Priority, structs.JobDefaultPriority) 2349 require.Equal(eval.Type, structs.JobTypeService) 2350 require.Equal(eval.TriggeredBy, structs.EvalTriggerJobDeregister) 2351 require.Equal(eval.JobID, job.ID) 2352 require.Equal(eval.JobModifyIndex, validResp2.JobModifyIndex) 2353 require.Equal(eval.Status, structs.EvalStatusPending) 2354 require.NotZero(eval.CreateTime) 2355 require.NotZero(eval.ModifyTime) 2356 } 2357 2358 func TestJobEndpoint_Deregister_Nonexistent(t *testing.T) { 2359 t.Parallel() 2360 2361 s1, cleanupS1 := TestServer(t, func(c *Config) { 2362 c.NumSchedulers = 0 // Prevent automatic dequeue 2363 }) 2364 defer cleanupS1() 2365 codec := rpcClient(t, s1) 2366 testutil.WaitForLeader(t, s1.RPC) 2367 2368 // Deregister 2369 jobID := "foo" 2370 dereg := &structs.JobDeregisterRequest{ 2371 JobID: jobID, 2372 WriteRequest: structs.WriteRequest{ 2373 Region: "global", 2374 Namespace: structs.DefaultNamespace, 2375 }, 2376 } 2377 var resp2 structs.JobDeregisterResponse 2378 if err := msgpackrpc.CallWithCodec(codec, "Job.Deregister", dereg, &resp2); err != nil { 2379 t.Fatalf("err: %v", err) 2380 } 2381 if resp2.JobModifyIndex == 0 { 2382 t.Fatalf("bad index: %d", resp2.Index) 2383 } 2384 2385 // Lookup the evaluation 2386 state := s1.fsm.State() 2387 ws := memdb.NewWatchSet() 2388 eval, err := state.EvalByID(ws, resp2.EvalID) 2389 if err != nil { 2390 t.Fatalf("err: %v", err) 2391 } 2392 if eval == nil { 2393 t.Fatalf("expected eval") 2394 } 2395 if eval.CreateIndex != resp2.EvalCreateIndex { 2396 t.Fatalf("index mis-match") 2397 } 2398 2399 if eval.Priority != structs.JobDefaultPriority { 2400 t.Fatalf("bad: %#v", eval) 2401 } 2402 if eval.Type != structs.JobTypeService { 2403 t.Fatalf("bad: %#v", eval) 2404 } 2405 if eval.TriggeredBy != structs.EvalTriggerJobDeregister { 2406 t.Fatalf("bad: %#v", eval) 2407 } 2408 if eval.JobID != jobID { 2409 t.Fatalf("bad: %#v", eval) 2410 } 2411 if eval.JobModifyIndex != resp2.JobModifyIndex { 2412 t.Fatalf("bad: %#v", eval) 2413 } 2414 if eval.Status != structs.EvalStatusPending { 2415 t.Fatalf("bad: %#v", eval) 2416 } 2417 if eval.CreateTime == 0 { 2418 t.Fatalf("eval CreateTime is unset: %#v", eval) 2419 } 2420 if eval.ModifyTime == 0 { 2421 t.Fatalf("eval ModifyTime is unset: %#v", eval) 2422 } 2423 } 2424 2425 func TestJobEndpoint_Deregister_Periodic(t *testing.T) { 2426 t.Parallel() 2427 2428 s1, cleanupS1 := TestServer(t, func(c *Config) { 2429 c.NumSchedulers = 0 // Prevent automatic dequeue 2430 }) 2431 defer cleanupS1() 2432 codec := rpcClient(t, s1) 2433 testutil.WaitForLeader(t, s1.RPC) 2434 2435 // Create the register request 2436 job := mock.PeriodicJob() 2437 reg := &structs.JobRegisterRequest{ 2438 Job: job, 2439 WriteRequest: structs.WriteRequest{ 2440 Region: "global", 2441 Namespace: job.Namespace, 2442 }, 2443 } 2444 2445 // Fetch the response 2446 var resp structs.JobRegisterResponse 2447 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", reg, &resp); err != nil { 2448 t.Fatalf("err: %v", err) 2449 } 2450 2451 // Deregister 2452 dereg := &structs.JobDeregisterRequest{ 2453 JobID: job.ID, 2454 Purge: true, 2455 WriteRequest: structs.WriteRequest{ 2456 Region: "global", 2457 Namespace: job.Namespace, 2458 }, 2459 } 2460 var resp2 structs.JobDeregisterResponse 2461 if err := msgpackrpc.CallWithCodec(codec, "Job.Deregister", dereg, &resp2); err != nil { 2462 t.Fatalf("err: %v", err) 2463 } 2464 if resp2.JobModifyIndex == 0 { 2465 t.Fatalf("bad index: %d", resp2.Index) 2466 } 2467 2468 // Check for the node in the FSM 2469 state := s1.fsm.State() 2470 ws := memdb.NewWatchSet() 2471 out, err := state.JobByID(ws, job.Namespace, job.ID) 2472 if err != nil { 2473 t.Fatalf("err: %v", err) 2474 } 2475 if out != nil { 2476 t.Fatalf("unexpected job") 2477 } 2478 2479 if resp.EvalID != "" { 2480 t.Fatalf("Deregister created an eval for a periodic job") 2481 } 2482 } 2483 2484 func TestJobEndpoint_Deregister_ParameterizedJob(t *testing.T) { 2485 t.Parallel() 2486 2487 s1, cleanupS1 := TestServer(t, func(c *Config) { 2488 c.NumSchedulers = 0 // Prevent automatic dequeue 2489 }) 2490 defer cleanupS1() 2491 codec := rpcClient(t, s1) 2492 testutil.WaitForLeader(t, s1.RPC) 2493 2494 // Create the register request 2495 job := mock.BatchJob() 2496 job.ParameterizedJob = &structs.ParameterizedJobConfig{} 2497 reg := &structs.JobRegisterRequest{ 2498 Job: job, 2499 WriteRequest: structs.WriteRequest{ 2500 Region: "global", 2501 Namespace: job.Namespace, 2502 }, 2503 } 2504 2505 // Fetch the response 2506 var resp structs.JobRegisterResponse 2507 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", reg, &resp); err != nil { 2508 t.Fatalf("err: %v", err) 2509 } 2510 2511 // Deregister 2512 dereg := &structs.JobDeregisterRequest{ 2513 JobID: job.ID, 2514 Purge: true, 2515 WriteRequest: structs.WriteRequest{ 2516 Region: "global", 2517 Namespace: job.Namespace, 2518 }, 2519 } 2520 var resp2 structs.JobDeregisterResponse 2521 if err := msgpackrpc.CallWithCodec(codec, "Job.Deregister", dereg, &resp2); err != nil { 2522 t.Fatalf("err: %v", err) 2523 } 2524 if resp2.JobModifyIndex == 0 { 2525 t.Fatalf("bad index: %d", resp2.Index) 2526 } 2527 2528 // Check for the node in the FSM 2529 state := s1.fsm.State() 2530 ws := memdb.NewWatchSet() 2531 out, err := state.JobByID(ws, job.Namespace, job.ID) 2532 if err != nil { 2533 t.Fatalf("err: %v", err) 2534 } 2535 if out != nil { 2536 t.Fatalf("unexpected job") 2537 } 2538 2539 if resp.EvalID != "" { 2540 t.Fatalf("Deregister created an eval for a parameterized job") 2541 } 2542 } 2543 2544 func TestJobEndpoint_BatchDeregister(t *testing.T) { 2545 t.Parallel() 2546 require := require.New(t) 2547 2548 s1, cleanupS1 := TestServer(t, func(c *Config) { 2549 c.NumSchedulers = 0 // Prevent automatic dequeue 2550 }) 2551 defer cleanupS1() 2552 codec := rpcClient(t, s1) 2553 testutil.WaitForLeader(t, s1.RPC) 2554 2555 // Create the register requests 2556 job := mock.Job() 2557 reg := &structs.JobRegisterRequest{ 2558 Job: job, 2559 WriteRequest: structs.WriteRequest{ 2560 Region: "global", 2561 Namespace: job.Namespace, 2562 }, 2563 } 2564 2565 // Fetch the response 2566 var resp structs.JobRegisterResponse 2567 require.Nil(msgpackrpc.CallWithCodec(codec, "Job.Register", reg, &resp)) 2568 2569 job2 := mock.Job() 2570 job2.Priority = 1 2571 reg2 := &structs.JobRegisterRequest{ 2572 Job: job2, 2573 WriteRequest: structs.WriteRequest{ 2574 Region: "global", 2575 Namespace: job2.Namespace, 2576 }, 2577 } 2578 2579 // Fetch the response 2580 require.Nil(msgpackrpc.CallWithCodec(codec, "Job.Register", reg2, &resp)) 2581 2582 // Deregister 2583 dereg := &structs.JobBatchDeregisterRequest{ 2584 Jobs: map[structs.NamespacedID]*structs.JobDeregisterOptions{ 2585 { 2586 ID: job.ID, 2587 Namespace: job.Namespace, 2588 }: {}, 2589 { 2590 ID: job2.ID, 2591 Namespace: job2.Namespace, 2592 }: { 2593 Purge: true, 2594 }, 2595 }, 2596 WriteRequest: structs.WriteRequest{ 2597 Region: "global", 2598 Namespace: job.Namespace, 2599 }, 2600 } 2601 var resp2 structs.JobBatchDeregisterResponse 2602 require.Nil(msgpackrpc.CallWithCodec(codec, "Job.BatchDeregister", dereg, &resp2)) 2603 require.NotZero(resp2.Index) 2604 2605 // Check for the job in the FSM 2606 state := s1.fsm.State() 2607 out, err := state.JobByID(nil, job.Namespace, job.ID) 2608 require.Nil(err) 2609 require.NotNil(out) 2610 require.True(out.Stop) 2611 2612 out, err = state.JobByID(nil, job2.Namespace, job2.ID) 2613 require.Nil(err) 2614 require.Nil(out) 2615 2616 // Lookup the evaluation 2617 for jobNS, eval := range resp2.JobEvals { 2618 expectedJob := job 2619 if jobNS.ID != job.ID { 2620 expectedJob = job2 2621 } 2622 2623 eval, err := state.EvalByID(nil, eval) 2624 require.Nil(err) 2625 require.NotNil(eval) 2626 require.EqualValues(resp2.Index, eval.CreateIndex) 2627 require.Equal(expectedJob.Priority, eval.Priority) 2628 require.Equal(expectedJob.Type, eval.Type) 2629 require.Equal(structs.EvalTriggerJobDeregister, eval.TriggeredBy) 2630 require.Equal(expectedJob.ID, eval.JobID) 2631 require.Equal(structs.EvalStatusPending, eval.Status) 2632 require.NotZero(eval.CreateTime) 2633 require.NotZero(eval.ModifyTime) 2634 } 2635 } 2636 2637 func TestJobEndpoint_BatchDeregister_ACL(t *testing.T) { 2638 t.Parallel() 2639 require := require.New(t) 2640 2641 s1, root, cleanupS1 := TestACLServer(t, func(c *Config) { 2642 c.NumSchedulers = 0 // Prevent automatic dequeue 2643 }) 2644 defer cleanupS1() 2645 codec := rpcClient(t, s1) 2646 testutil.WaitForLeader(t, s1.RPC) 2647 state := s1.fsm.State() 2648 2649 // Create and register a job 2650 job, job2 := mock.Job(), mock.Job() 2651 require.Nil(state.UpsertJob(100, job)) 2652 require.Nil(state.UpsertJob(101, job2)) 2653 2654 // Deregister 2655 req := &structs.JobBatchDeregisterRequest{ 2656 Jobs: map[structs.NamespacedID]*structs.JobDeregisterOptions{ 2657 { 2658 ID: job.ID, 2659 Namespace: job.Namespace, 2660 }: {}, 2661 { 2662 ID: job2.ID, 2663 Namespace: job2.Namespace, 2664 }: {}, 2665 }, 2666 WriteRequest: structs.WriteRequest{ 2667 Region: "global", 2668 }, 2669 } 2670 2671 // Expect failure for request without a token 2672 var resp structs.JobBatchDeregisterResponse 2673 err := msgpackrpc.CallWithCodec(codec, "Job.BatchDeregister", req, &resp) 2674 require.NotNil(err) 2675 require.True(structs.IsErrPermissionDenied(err)) 2676 2677 // Expect failure for request with an invalid token 2678 invalidToken := mock.CreatePolicyAndToken(t, state, 1003, "test-invalid", 2679 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityListJobs})) 2680 req.AuthToken = invalidToken.SecretID 2681 2682 var invalidResp structs.JobDeregisterResponse 2683 err = msgpackrpc.CallWithCodec(codec, "Job.BatchDeregister", req, &invalidResp) 2684 require.NotNil(err) 2685 require.True(structs.IsErrPermissionDenied(err)) 2686 2687 // Expect success with a valid management token 2688 req.AuthToken = root.SecretID 2689 2690 var validResp structs.JobDeregisterResponse 2691 err = msgpackrpc.CallWithCodec(codec, "Job.BatchDeregister", req, &validResp) 2692 require.Nil(err) 2693 require.NotEqual(validResp.Index, 0) 2694 2695 // Expect success with a valid token 2696 validToken := mock.CreatePolicyAndToken(t, state, 1005, "test-valid", 2697 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilitySubmitJob})) 2698 req.AuthToken = validToken.SecretID 2699 2700 var validResp2 structs.JobDeregisterResponse 2701 err = msgpackrpc.CallWithCodec(codec, "Job.BatchDeregister", req, &validResp2) 2702 require.Nil(err) 2703 require.NotEqual(validResp2.Index, 0) 2704 } 2705 2706 func TestJobEndpoint_GetJob(t *testing.T) { 2707 t.Parallel() 2708 2709 s1, cleanupS1 := TestServer(t, nil) 2710 defer cleanupS1() 2711 codec := rpcClient(t, s1) 2712 testutil.WaitForLeader(t, s1.RPC) 2713 2714 // Create the register request 2715 job := mock.Job() 2716 reg := &structs.JobRegisterRequest{ 2717 Job: job, 2718 WriteRequest: structs.WriteRequest{ 2719 Region: "global", 2720 Namespace: job.Namespace, 2721 }, 2722 } 2723 2724 // Fetch the response 2725 var resp structs.JobRegisterResponse 2726 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", reg, &resp); err != nil { 2727 t.Fatalf("err: %v", err) 2728 } 2729 job.CreateIndex = resp.JobModifyIndex 2730 job.ModifyIndex = resp.JobModifyIndex 2731 job.JobModifyIndex = resp.JobModifyIndex 2732 2733 // Lookup the job 2734 get := &structs.JobSpecificRequest{ 2735 JobID: job.ID, 2736 QueryOptions: structs.QueryOptions{ 2737 Region: "global", 2738 Namespace: job.Namespace, 2739 }, 2740 } 2741 var resp2 structs.SingleJobResponse 2742 if err := msgpackrpc.CallWithCodec(codec, "Job.GetJob", get, &resp2); err != nil { 2743 t.Fatalf("err: %v", err) 2744 } 2745 if resp2.Index != resp.JobModifyIndex { 2746 t.Fatalf("Bad index: %d %d", resp2.Index, resp.Index) 2747 } 2748 2749 // Make a copy of the origin job and change the service name so that we can 2750 // do a deep equal with the response from the GET JOB Api 2751 j := job 2752 j.TaskGroups[0].Tasks[0].Services[0].Name = "web-frontend" 2753 for tgix, tg := range j.TaskGroups { 2754 for tidx, t := range tg.Tasks { 2755 for sidx, service := range t.Services { 2756 for cidx, check := range service.Checks { 2757 check.Name = resp2.Job.TaskGroups[tgix].Tasks[tidx].Services[sidx].Checks[cidx].Name 2758 } 2759 } 2760 } 2761 } 2762 2763 // Clear the submit times 2764 j.SubmitTime = 0 2765 resp2.Job.SubmitTime = 0 2766 2767 if !reflect.DeepEqual(j, resp2.Job) { 2768 t.Fatalf("bad: %#v %#v", job, resp2.Job) 2769 } 2770 2771 // Lookup non-existing job 2772 get.JobID = "foobarbaz" 2773 if err := msgpackrpc.CallWithCodec(codec, "Job.GetJob", get, &resp2); err != nil { 2774 t.Fatalf("err: %v", err) 2775 } 2776 if resp2.Index != resp.JobModifyIndex { 2777 t.Fatalf("Bad index: %d %d", resp2.Index, resp.Index) 2778 } 2779 if resp2.Job != nil { 2780 t.Fatalf("unexpected job") 2781 } 2782 } 2783 2784 func TestJobEndpoint_GetJob_ACL(t *testing.T) { 2785 t.Parallel() 2786 require := require.New(t) 2787 2788 s1, root, cleanupS1 := TestACLServer(t, nil) 2789 defer cleanupS1() 2790 codec := rpcClient(t, s1) 2791 testutil.WaitForLeader(t, s1.RPC) 2792 state := s1.fsm.State() 2793 2794 // Create the job 2795 job := mock.Job() 2796 err := state.UpsertJob(1000, job) 2797 require.Nil(err) 2798 2799 // Lookup the job 2800 get := &structs.JobSpecificRequest{ 2801 JobID: job.ID, 2802 QueryOptions: structs.QueryOptions{ 2803 Region: "global", 2804 Namespace: job.Namespace, 2805 }, 2806 } 2807 2808 // Looking up the job without a token should fail 2809 var resp structs.SingleJobResponse 2810 err = msgpackrpc.CallWithCodec(codec, "Job.GetJob", get, &resp) 2811 require.NotNil(err) 2812 require.Contains(err.Error(), "Permission denied") 2813 2814 // Expect failure for request with an invalid token 2815 invalidToken := mock.CreatePolicyAndToken(t, state, 1003, "test-invalid", 2816 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityListJobs})) 2817 2818 get.AuthToken = invalidToken.SecretID 2819 var invalidResp structs.SingleJobResponse 2820 err = msgpackrpc.CallWithCodec(codec, "Job.GetJob", get, &invalidResp) 2821 require.NotNil(err) 2822 require.Contains(err.Error(), "Permission denied") 2823 2824 // Looking up the job with a management token should succeed 2825 get.AuthToken = root.SecretID 2826 var validResp structs.SingleJobResponse 2827 err = msgpackrpc.CallWithCodec(codec, "Job.GetJob", get, &validResp) 2828 require.Nil(err) 2829 require.Equal(job.ID, validResp.Job.ID) 2830 2831 // Looking up the job with a valid token should succeed 2832 validToken := mock.CreatePolicyAndToken(t, state, 1005, "test-valid", 2833 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob})) 2834 2835 get.AuthToken = validToken.SecretID 2836 var validResp2 structs.SingleJobResponse 2837 err = msgpackrpc.CallWithCodec(codec, "Job.GetJob", get, &validResp2) 2838 require.Nil(err) 2839 require.Equal(job.ID, validResp2.Job.ID) 2840 } 2841 2842 func TestJobEndpoint_GetJob_Blocking(t *testing.T) { 2843 t.Parallel() 2844 2845 s1, cleanupS1 := TestServer(t, nil) 2846 defer cleanupS1() 2847 state := s1.fsm.State() 2848 codec := rpcClient(t, s1) 2849 testutil.WaitForLeader(t, s1.RPC) 2850 2851 // Create the jobs 2852 job1 := mock.Job() 2853 job2 := mock.Job() 2854 2855 // Upsert a job we are not interested in first. 2856 time.AfterFunc(100*time.Millisecond, func() { 2857 if err := state.UpsertJob(100, job1); err != nil { 2858 t.Fatalf("err: %v", err) 2859 } 2860 }) 2861 2862 // Upsert another job later which should trigger the watch. 2863 time.AfterFunc(200*time.Millisecond, func() { 2864 if err := state.UpsertJob(200, job2); err != nil { 2865 t.Fatalf("err: %v", err) 2866 } 2867 }) 2868 2869 req := &structs.JobSpecificRequest{ 2870 JobID: job2.ID, 2871 QueryOptions: structs.QueryOptions{ 2872 Region: "global", 2873 Namespace: job2.Namespace, 2874 MinQueryIndex: 150, 2875 }, 2876 } 2877 start := time.Now() 2878 var resp structs.SingleJobResponse 2879 if err := msgpackrpc.CallWithCodec(codec, "Job.GetJob", req, &resp); err != nil { 2880 t.Fatalf("err: %v", err) 2881 } 2882 2883 if elapsed := time.Since(start); elapsed < 200*time.Millisecond { 2884 t.Fatalf("should block (returned in %s) %#v", elapsed, resp) 2885 } 2886 if resp.Index != 200 { 2887 t.Fatalf("Bad index: %d %d", resp.Index, 200) 2888 } 2889 if resp.Job == nil || resp.Job.ID != job2.ID { 2890 t.Fatalf("bad: %#v", resp.Job) 2891 } 2892 2893 // Job delete fires watches 2894 time.AfterFunc(100*time.Millisecond, func() { 2895 if err := state.DeleteJob(300, job2.Namespace, job2.ID); err != nil { 2896 t.Fatalf("err: %v", err) 2897 } 2898 }) 2899 2900 req.QueryOptions.MinQueryIndex = 250 2901 start = time.Now() 2902 2903 var resp2 structs.SingleJobResponse 2904 if err := msgpackrpc.CallWithCodec(codec, "Job.GetJob", req, &resp2); err != nil { 2905 t.Fatalf("err: %v", err) 2906 } 2907 2908 if elapsed := time.Since(start); elapsed < 100*time.Millisecond { 2909 t.Fatalf("should block (returned in %s) %#v", elapsed, resp2) 2910 } 2911 if resp2.Index != 300 { 2912 t.Fatalf("Bad index: %d %d", resp2.Index, 300) 2913 } 2914 if resp2.Job != nil { 2915 t.Fatalf("bad: %#v", resp2.Job) 2916 } 2917 } 2918 2919 func TestJobEndpoint_GetJobVersions(t *testing.T) { 2920 t.Parallel() 2921 2922 s1, cleanupS1 := TestServer(t, nil) 2923 defer cleanupS1() 2924 codec := rpcClient(t, s1) 2925 testutil.WaitForLeader(t, s1.RPC) 2926 2927 // Create the register request 2928 job := mock.Job() 2929 job.Priority = 88 2930 reg := &structs.JobRegisterRequest{ 2931 Job: job, 2932 WriteRequest: structs.WriteRequest{ 2933 Region: "global", 2934 Namespace: job.Namespace, 2935 }, 2936 } 2937 2938 // Fetch the response 2939 var resp structs.JobRegisterResponse 2940 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", reg, &resp); err != nil { 2941 t.Fatalf("err: %v", err) 2942 } 2943 2944 // Register the job again to create another version 2945 job.Priority = 100 2946 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", reg, &resp); err != nil { 2947 t.Fatalf("err: %v", err) 2948 } 2949 2950 // Lookup the job 2951 get := &structs.JobVersionsRequest{ 2952 JobID: job.ID, 2953 QueryOptions: structs.QueryOptions{ 2954 Region: "global", 2955 Namespace: job.Namespace, 2956 }, 2957 } 2958 var versionsResp structs.JobVersionsResponse 2959 if err := msgpackrpc.CallWithCodec(codec, "Job.GetJobVersions", get, &versionsResp); err != nil { 2960 t.Fatalf("err: %v", err) 2961 } 2962 if versionsResp.Index != resp.JobModifyIndex { 2963 t.Fatalf("Bad index: %d %d", versionsResp.Index, resp.Index) 2964 } 2965 2966 // Make sure there are two job versions 2967 versions := versionsResp.Versions 2968 if l := len(versions); l != 2 { 2969 t.Fatalf("Got %d versions; want 2", l) 2970 } 2971 2972 if v := versions[0]; v.Priority != 100 || v.ID != job.ID || v.Version != 1 { 2973 t.Fatalf("bad: %+v", v) 2974 } 2975 if v := versions[1]; v.Priority != 88 || v.ID != job.ID || v.Version != 0 { 2976 t.Fatalf("bad: %+v", v) 2977 } 2978 2979 // Lookup non-existing job 2980 get.JobID = "foobarbaz" 2981 if err := msgpackrpc.CallWithCodec(codec, "Job.GetJobVersions", get, &versionsResp); err != nil { 2982 t.Fatalf("err: %v", err) 2983 } 2984 if versionsResp.Index != resp.JobModifyIndex { 2985 t.Fatalf("Bad index: %d %d", versionsResp.Index, resp.Index) 2986 } 2987 if l := len(versionsResp.Versions); l != 0 { 2988 t.Fatalf("unexpected versions: %d", l) 2989 } 2990 } 2991 2992 func TestJobEndpoint_GetJobVersions_ACL(t *testing.T) { 2993 t.Parallel() 2994 require := require.New(t) 2995 2996 s1, root, cleanupS1 := TestACLServer(t, nil) 2997 defer cleanupS1() 2998 codec := rpcClient(t, s1) 2999 testutil.WaitForLeader(t, s1.RPC) 3000 state := s1.fsm.State() 3001 3002 // Create two versions of a job with different priorities 3003 job := mock.Job() 3004 job.Priority = 88 3005 err := state.UpsertJob(10, job) 3006 require.Nil(err) 3007 3008 job.Priority = 100 3009 err = state.UpsertJob(100, job) 3010 require.Nil(err) 3011 3012 // Lookup the job 3013 get := &structs.JobVersionsRequest{ 3014 JobID: job.ID, 3015 QueryOptions: structs.QueryOptions{ 3016 Region: "global", 3017 Namespace: job.Namespace, 3018 }, 3019 } 3020 3021 // Attempt to fetch without a token should fail 3022 var resp structs.JobVersionsResponse 3023 err = msgpackrpc.CallWithCodec(codec, "Job.GetJobVersions", get, &resp) 3024 require.NotNil(err) 3025 require.Contains(err.Error(), "Permission denied") 3026 3027 // Expect failure for request with an invalid token 3028 invalidToken := mock.CreatePolicyAndToken(t, state, 1003, "test-invalid", 3029 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityListJobs})) 3030 3031 get.AuthToken = invalidToken.SecretID 3032 var invalidResp structs.JobVersionsResponse 3033 err = msgpackrpc.CallWithCodec(codec, "Job.GetJobVersions", get, &invalidResp) 3034 require.NotNil(err) 3035 require.Contains(err.Error(), "Permission denied") 3036 3037 // Expect success for request with a valid management token 3038 get.AuthToken = root.SecretID 3039 var validResp structs.JobVersionsResponse 3040 err = msgpackrpc.CallWithCodec(codec, "Job.GetJobVersions", get, &validResp) 3041 require.Nil(err) 3042 3043 // Expect success for request with a valid token 3044 validToken := mock.CreatePolicyAndToken(t, state, 1005, "test-valid", 3045 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob})) 3046 3047 get.AuthToken = validToken.SecretID 3048 var validResp2 structs.JobVersionsResponse 3049 err = msgpackrpc.CallWithCodec(codec, "Job.GetJobVersions", get, &validResp2) 3050 require.Nil(err) 3051 3052 // Make sure there are two job versions 3053 versions := validResp2.Versions 3054 require.Equal(2, len(versions)) 3055 require.Equal(versions[0].ID, job.ID) 3056 require.Equal(versions[1].ID, job.ID) 3057 } 3058 3059 func TestJobEndpoint_GetJobVersions_Diff(t *testing.T) { 3060 t.Parallel() 3061 3062 s1, cleanupS1 := TestServer(t, nil) 3063 defer cleanupS1() 3064 codec := rpcClient(t, s1) 3065 testutil.WaitForLeader(t, s1.RPC) 3066 3067 // Create the register request 3068 job := mock.Job() 3069 job.Priority = 88 3070 reg := &structs.JobRegisterRequest{ 3071 Job: job, 3072 WriteRequest: structs.WriteRequest{ 3073 Region: "global", 3074 Namespace: job.Namespace, 3075 }, 3076 } 3077 3078 // Fetch the response 3079 var resp structs.JobRegisterResponse 3080 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", reg, &resp); err != nil { 3081 t.Fatalf("err: %v", err) 3082 } 3083 3084 // Register the job again to create another version 3085 job.Priority = 90 3086 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", reg, &resp); err != nil { 3087 t.Fatalf("err: %v", err) 3088 } 3089 3090 // Register the job again to create another version 3091 job.Priority = 100 3092 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", reg, &resp); err != nil { 3093 t.Fatalf("err: %v", err) 3094 } 3095 3096 // Lookup the job 3097 get := &structs.JobVersionsRequest{ 3098 JobID: job.ID, 3099 Diffs: true, 3100 QueryOptions: structs.QueryOptions{ 3101 Region: "global", 3102 Namespace: job.Namespace, 3103 }, 3104 } 3105 var versionsResp structs.JobVersionsResponse 3106 if err := msgpackrpc.CallWithCodec(codec, "Job.GetJobVersions", get, &versionsResp); err != nil { 3107 t.Fatalf("err: %v", err) 3108 } 3109 if versionsResp.Index != resp.JobModifyIndex { 3110 t.Fatalf("Bad index: %d %d", versionsResp.Index, resp.Index) 3111 } 3112 3113 // Make sure there are two job versions 3114 versions := versionsResp.Versions 3115 if l := len(versions); l != 3 { 3116 t.Fatalf("Got %d versions; want 3", l) 3117 } 3118 3119 if v := versions[0]; v.Priority != 100 || v.ID != job.ID || v.Version != 2 { 3120 t.Fatalf("bad: %+v", v) 3121 } 3122 if v := versions[1]; v.Priority != 90 || v.ID != job.ID || v.Version != 1 { 3123 t.Fatalf("bad: %+v", v) 3124 } 3125 if v := versions[2]; v.Priority != 88 || v.ID != job.ID || v.Version != 0 { 3126 t.Fatalf("bad: %+v", v) 3127 } 3128 3129 // Ensure we got diffs 3130 diffs := versionsResp.Diffs 3131 if l := len(diffs); l != 2 { 3132 t.Fatalf("Got %d diffs; want 2", l) 3133 } 3134 d1 := diffs[0] 3135 if len(d1.Fields) != 1 { 3136 t.Fatalf("Got too many diffs: %#v", d1) 3137 } 3138 if d1.Fields[0].Name != "Priority" { 3139 t.Fatalf("Got wrong field: %#v", d1) 3140 } 3141 if d1.Fields[0].Old != "90" && d1.Fields[0].New != "100" { 3142 t.Fatalf("Got wrong field values: %#v", d1) 3143 } 3144 d2 := diffs[1] 3145 if len(d2.Fields) != 1 { 3146 t.Fatalf("Got too many diffs: %#v", d2) 3147 } 3148 if d2.Fields[0].Name != "Priority" { 3149 t.Fatalf("Got wrong field: %#v", d2) 3150 } 3151 if d2.Fields[0].Old != "88" && d1.Fields[0].New != "90" { 3152 t.Fatalf("Got wrong field values: %#v", d2) 3153 } 3154 } 3155 3156 func TestJobEndpoint_GetJobVersions_Blocking(t *testing.T) { 3157 t.Parallel() 3158 3159 s1, cleanupS1 := TestServer(t, nil) 3160 defer cleanupS1() 3161 state := s1.fsm.State() 3162 codec := rpcClient(t, s1) 3163 testutil.WaitForLeader(t, s1.RPC) 3164 3165 // Create the jobs 3166 job1 := mock.Job() 3167 job2 := mock.Job() 3168 job3 := mock.Job() 3169 job3.ID = job2.ID 3170 job3.Priority = 1 3171 3172 // Upsert a job we are not interested in first. 3173 time.AfterFunc(100*time.Millisecond, func() { 3174 if err := state.UpsertJob(100, job1); err != nil { 3175 t.Fatalf("err: %v", err) 3176 } 3177 }) 3178 3179 // Upsert another job later which should trigger the watch. 3180 time.AfterFunc(200*time.Millisecond, func() { 3181 if err := state.UpsertJob(200, job2); err != nil { 3182 t.Fatalf("err: %v", err) 3183 } 3184 }) 3185 3186 req := &structs.JobVersionsRequest{ 3187 JobID: job2.ID, 3188 QueryOptions: structs.QueryOptions{ 3189 Region: "global", 3190 Namespace: job2.Namespace, 3191 MinQueryIndex: 150, 3192 }, 3193 } 3194 start := time.Now() 3195 var resp structs.JobVersionsResponse 3196 if err := msgpackrpc.CallWithCodec(codec, "Job.GetJobVersions", req, &resp); err != nil { 3197 t.Fatalf("err: %v", err) 3198 } 3199 3200 if elapsed := time.Since(start); elapsed < 200*time.Millisecond { 3201 t.Fatalf("should block (returned in %s) %#v", elapsed, resp) 3202 } 3203 if resp.Index != 200 { 3204 t.Fatalf("Bad index: %d %d", resp.Index, 200) 3205 } 3206 if len(resp.Versions) != 1 || resp.Versions[0].ID != job2.ID { 3207 t.Fatalf("bad: %#v", resp.Versions) 3208 } 3209 3210 // Upsert the job again which should trigger the watch. 3211 time.AfterFunc(100*time.Millisecond, func() { 3212 if err := state.UpsertJob(300, job3); err != nil { 3213 t.Fatalf("err: %v", err) 3214 } 3215 }) 3216 3217 req2 := &structs.JobVersionsRequest{ 3218 JobID: job3.ID, 3219 QueryOptions: structs.QueryOptions{ 3220 Region: "global", 3221 Namespace: job3.Namespace, 3222 MinQueryIndex: 250, 3223 }, 3224 } 3225 var resp2 structs.JobVersionsResponse 3226 start = time.Now() 3227 if err := msgpackrpc.CallWithCodec(codec, "Job.GetJobVersions", req2, &resp2); err != nil { 3228 t.Fatalf("err: %v", err) 3229 } 3230 3231 if elapsed := time.Since(start); elapsed < 100*time.Millisecond { 3232 t.Fatalf("should block (returned in %s) %#v", elapsed, resp) 3233 } 3234 if resp2.Index != 300 { 3235 t.Fatalf("Bad index: %d %d", resp.Index, 300) 3236 } 3237 if len(resp2.Versions) != 2 { 3238 t.Fatalf("bad: %#v", resp2.Versions) 3239 } 3240 } 3241 3242 func TestJobEndpoint_GetJobSummary(t *testing.T) { 3243 t.Parallel() 3244 3245 s1, cleanupS1 := TestServer(t, func(c *Config) { 3246 c.NumSchedulers = 0 // Prevent automatic dequeue 3247 }) 3248 3249 defer cleanupS1() 3250 codec := rpcClient(t, s1) 3251 testutil.WaitForLeader(t, s1.RPC) 3252 3253 // Create the register request 3254 job := mock.Job() 3255 reg := &structs.JobRegisterRequest{ 3256 Job: job, 3257 WriteRequest: structs.WriteRequest{ 3258 Region: "global", 3259 Namespace: job.Namespace, 3260 }, 3261 } 3262 3263 // Fetch the response 3264 var resp structs.JobRegisterResponse 3265 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", reg, &resp); err != nil { 3266 t.Fatalf("err: %v", err) 3267 } 3268 job.CreateIndex = resp.JobModifyIndex 3269 job.ModifyIndex = resp.JobModifyIndex 3270 job.JobModifyIndex = resp.JobModifyIndex 3271 3272 // Lookup the job summary 3273 get := &structs.JobSummaryRequest{ 3274 JobID: job.ID, 3275 QueryOptions: structs.QueryOptions{ 3276 Region: "global", 3277 Namespace: job.Namespace, 3278 }, 3279 } 3280 var resp2 structs.JobSummaryResponse 3281 if err := msgpackrpc.CallWithCodec(codec, "Job.Summary", get, &resp2); err != nil { 3282 t.Fatalf("err: %v", err) 3283 } 3284 if resp2.Index != resp.JobModifyIndex { 3285 t.Fatalf("Bad index: %d %d", resp2.Index, resp.Index) 3286 } 3287 3288 expectedJobSummary := structs.JobSummary{ 3289 JobID: job.ID, 3290 Namespace: job.Namespace, 3291 Summary: map[string]structs.TaskGroupSummary{ 3292 "web": {}, 3293 }, 3294 Children: new(structs.JobChildrenSummary), 3295 CreateIndex: job.CreateIndex, 3296 ModifyIndex: job.CreateIndex, 3297 } 3298 3299 if !reflect.DeepEqual(resp2.JobSummary, &expectedJobSummary) { 3300 t.Fatalf("expected: %v, actual: %v", expectedJobSummary, resp2.JobSummary) 3301 } 3302 } 3303 3304 func TestJobEndpoint_Summary_ACL(t *testing.T) { 3305 t.Parallel() 3306 require := require.New(t) 3307 3308 s1, root, cleanupS1 := TestACLServer(t, func(c *Config) { 3309 c.NumSchedulers = 0 // Prevent automatic dequeue 3310 }) 3311 defer cleanupS1() 3312 codec := rpcClient(t, s1) 3313 testutil.WaitForLeader(t, s1.RPC) 3314 3315 // Create the job 3316 job := mock.Job() 3317 reg := &structs.JobRegisterRequest{ 3318 Job: job, 3319 WriteRequest: structs.WriteRequest{ 3320 Region: "global", 3321 Namespace: job.Namespace, 3322 }, 3323 } 3324 reg.AuthToken = root.SecretID 3325 3326 var err error 3327 3328 // Register the job with a valid token 3329 var regResp structs.JobRegisterResponse 3330 err = msgpackrpc.CallWithCodec(codec, "Job.Register", reg, ®Resp) 3331 require.Nil(err) 3332 3333 job.CreateIndex = regResp.JobModifyIndex 3334 job.ModifyIndex = regResp.JobModifyIndex 3335 job.JobModifyIndex = regResp.JobModifyIndex 3336 3337 req := &structs.JobSummaryRequest{ 3338 JobID: job.ID, 3339 QueryOptions: structs.QueryOptions{ 3340 Region: "global", 3341 Namespace: job.Namespace, 3342 }, 3343 } 3344 3345 // Expect failure for request without a token 3346 var resp structs.JobSummaryResponse 3347 err = msgpackrpc.CallWithCodec(codec, "Job.Summary", req, &resp) 3348 require.NotNil(err) 3349 3350 expectedJobSummary := &structs.JobSummary{ 3351 JobID: job.ID, 3352 Namespace: job.Namespace, 3353 Summary: map[string]structs.TaskGroupSummary{ 3354 "web": {}, 3355 }, 3356 Children: new(structs.JobChildrenSummary), 3357 CreateIndex: job.CreateIndex, 3358 ModifyIndex: job.ModifyIndex, 3359 } 3360 3361 // Expect success when using a management token 3362 req.AuthToken = root.SecretID 3363 var mgmtResp structs.JobSummaryResponse 3364 err = msgpackrpc.CallWithCodec(codec, "Job.Summary", req, &mgmtResp) 3365 require.Nil(err) 3366 require.Equal(expectedJobSummary, mgmtResp.JobSummary) 3367 3368 // Create the namespace policy and tokens 3369 state := s1.fsm.State() 3370 3371 // Expect failure for request with an invalid token 3372 invalidToken := mock.CreatePolicyAndToken(t, state, 1003, "test-invalid", 3373 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityListJobs})) 3374 3375 req.AuthToken = invalidToken.SecretID 3376 var invalidResp structs.JobSummaryResponse 3377 err = msgpackrpc.CallWithCodec(codec, "Job.Summary", req, &invalidResp) 3378 require.NotNil(err) 3379 3380 // Try with a valid token 3381 validToken := mock.CreatePolicyAndToken(t, state, 1001, "test-valid", 3382 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob})) 3383 3384 req.AuthToken = validToken.SecretID 3385 var authResp structs.JobSummaryResponse 3386 err = msgpackrpc.CallWithCodec(codec, "Job.Summary", req, &authResp) 3387 require.Nil(err) 3388 require.Equal(expectedJobSummary, authResp.JobSummary) 3389 } 3390 3391 func TestJobEndpoint_GetJobSummary_Blocking(t *testing.T) { 3392 t.Parallel() 3393 3394 s1, cleanupS1 := TestServer(t, nil) 3395 defer cleanupS1() 3396 state := s1.fsm.State() 3397 codec := rpcClient(t, s1) 3398 testutil.WaitForLeader(t, s1.RPC) 3399 3400 // Create a job and insert it 3401 job1 := mock.Job() 3402 time.AfterFunc(200*time.Millisecond, func() { 3403 if err := state.UpsertJob(100, job1); err != nil { 3404 t.Fatalf("err: %v", err) 3405 } 3406 }) 3407 3408 // Ensure the job summary request gets fired 3409 req := &structs.JobSummaryRequest{ 3410 JobID: job1.ID, 3411 QueryOptions: structs.QueryOptions{ 3412 Region: "global", 3413 Namespace: job1.Namespace, 3414 MinQueryIndex: 50, 3415 }, 3416 } 3417 var resp structs.JobSummaryResponse 3418 start := time.Now() 3419 if err := msgpackrpc.CallWithCodec(codec, "Job.Summary", req, &resp); err != nil { 3420 t.Fatalf("err: %v", err) 3421 } 3422 if elapsed := time.Since(start); elapsed < 200*time.Millisecond { 3423 t.Fatalf("should block (returned in %s) %#v", elapsed, resp) 3424 } 3425 3426 // Upsert an allocation for the job which should trigger the watch. 3427 time.AfterFunc(200*time.Millisecond, func() { 3428 alloc := mock.Alloc() 3429 alloc.JobID = job1.ID 3430 alloc.Job = job1 3431 if err := state.UpsertAllocs(200, []*structs.Allocation{alloc}); err != nil { 3432 t.Fatalf("err: %v", err) 3433 } 3434 }) 3435 req = &structs.JobSummaryRequest{ 3436 JobID: job1.ID, 3437 QueryOptions: structs.QueryOptions{ 3438 Region: "global", 3439 Namespace: job1.Namespace, 3440 MinQueryIndex: 199, 3441 }, 3442 } 3443 start = time.Now() 3444 var resp1 structs.JobSummaryResponse 3445 if err := msgpackrpc.CallWithCodec(codec, "Job.Summary", req, &resp1); err != nil { 3446 t.Fatalf("err: %v", err) 3447 } 3448 3449 if elapsed := time.Since(start); elapsed < 200*time.Millisecond { 3450 t.Fatalf("should block (returned in %s) %#v", elapsed, resp) 3451 } 3452 if resp1.Index != 200 { 3453 t.Fatalf("Bad index: %d %d", resp.Index, 200) 3454 } 3455 if resp1.JobSummary == nil { 3456 t.Fatalf("bad: %#v", resp) 3457 } 3458 3459 // Job delete fires watches 3460 time.AfterFunc(100*time.Millisecond, func() { 3461 if err := state.DeleteJob(300, job1.Namespace, job1.ID); err != nil { 3462 t.Fatalf("err: %v", err) 3463 } 3464 }) 3465 3466 req.QueryOptions.MinQueryIndex = 250 3467 start = time.Now() 3468 3469 var resp2 structs.SingleJobResponse 3470 if err := msgpackrpc.CallWithCodec(codec, "Job.Summary", req, &resp2); err != nil { 3471 t.Fatalf("err: %v", err) 3472 } 3473 3474 if elapsed := time.Since(start); elapsed < 100*time.Millisecond { 3475 t.Fatalf("should block (returned in %s) %#v", elapsed, resp2) 3476 } 3477 if resp2.Index != 300 { 3478 t.Fatalf("Bad index: %d %d", resp2.Index, 300) 3479 } 3480 if resp2.Job != nil { 3481 t.Fatalf("bad: %#v", resp2.Job) 3482 } 3483 } 3484 3485 func TestJobEndpoint_ListJobs(t *testing.T) { 3486 t.Parallel() 3487 3488 s1, cleanupS1 := TestServer(t, nil) 3489 defer cleanupS1() 3490 codec := rpcClient(t, s1) 3491 testutil.WaitForLeader(t, s1.RPC) 3492 3493 // Create the register request 3494 job := mock.Job() 3495 state := s1.fsm.State() 3496 err := state.UpsertJob(1000, job) 3497 if err != nil { 3498 t.Fatalf("err: %v", err) 3499 } 3500 3501 // Lookup the jobs 3502 get := &structs.JobListRequest{ 3503 QueryOptions: structs.QueryOptions{ 3504 Region: "global", 3505 Namespace: job.Namespace, 3506 }, 3507 } 3508 var resp2 structs.JobListResponse 3509 if err := msgpackrpc.CallWithCodec(codec, "Job.List", get, &resp2); err != nil { 3510 t.Fatalf("err: %v", err) 3511 } 3512 if resp2.Index != 1000 { 3513 t.Fatalf("Bad index: %d %d", resp2.Index, 1000) 3514 } 3515 3516 if len(resp2.Jobs) != 1 { 3517 t.Fatalf("bad: %#v", resp2.Jobs) 3518 } 3519 if resp2.Jobs[0].ID != job.ID { 3520 t.Fatalf("bad: %#v", resp2.Jobs[0]) 3521 } 3522 3523 // Lookup the jobs by prefix 3524 get = &structs.JobListRequest{ 3525 QueryOptions: structs.QueryOptions{ 3526 Region: "global", 3527 Namespace: job.Namespace, 3528 Prefix: resp2.Jobs[0].ID[:4], 3529 }, 3530 } 3531 var resp3 structs.JobListResponse 3532 if err := msgpackrpc.CallWithCodec(codec, "Job.List", get, &resp3); err != nil { 3533 t.Fatalf("err: %v", err) 3534 } 3535 if resp3.Index != 1000 { 3536 t.Fatalf("Bad index: %d %d", resp3.Index, 1000) 3537 } 3538 3539 if len(resp3.Jobs) != 1 { 3540 t.Fatalf("bad: %#v", resp3.Jobs) 3541 } 3542 if resp3.Jobs[0].ID != job.ID { 3543 t.Fatalf("bad: %#v", resp3.Jobs[0]) 3544 } 3545 } 3546 3547 func TestJobEndpoint_ListJobs_WithACL(t *testing.T) { 3548 t.Parallel() 3549 require := require.New(t) 3550 3551 s1, root, cleanupS1 := TestACLServer(t, func(c *Config) { 3552 c.NumSchedulers = 0 // Prevent automatic dequeue 3553 }) 3554 defer cleanupS1() 3555 codec := rpcClient(t, s1) 3556 testutil.WaitForLeader(t, s1.RPC) 3557 state := s1.fsm.State() 3558 3559 var err error 3560 3561 // Create the register request 3562 job := mock.Job() 3563 err = state.UpsertJob(1000, job) 3564 require.Nil(err) 3565 3566 req := &structs.JobListRequest{ 3567 QueryOptions: structs.QueryOptions{ 3568 Region: "global", 3569 Namespace: job.Namespace, 3570 }, 3571 } 3572 3573 // Expect failure for request without a token 3574 var resp structs.JobListResponse 3575 err = msgpackrpc.CallWithCodec(codec, "Job.List", req, &resp) 3576 require.NotNil(err) 3577 3578 // Expect success for request with a management token 3579 var mgmtResp structs.JobListResponse 3580 req.AuthToken = root.SecretID 3581 err = msgpackrpc.CallWithCodec(codec, "Job.List", req, &mgmtResp) 3582 require.Nil(err) 3583 require.Equal(1, len(mgmtResp.Jobs)) 3584 require.Equal(job.ID, mgmtResp.Jobs[0].ID) 3585 3586 // Expect failure for request with a token that has incorrect permissions 3587 invalidToken := mock.CreatePolicyAndToken(t, state, 1003, "test-invalid", 3588 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob})) 3589 3590 req.AuthToken = invalidToken.SecretID 3591 var invalidResp structs.JobListResponse 3592 err = msgpackrpc.CallWithCodec(codec, "Job.List", req, &invalidResp) 3593 require.NotNil(err) 3594 3595 // Try with a valid token with correct permissions 3596 validToken := mock.CreatePolicyAndToken(t, state, 1001, "test-valid", 3597 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityListJobs})) 3598 var validResp structs.JobListResponse 3599 req.AuthToken = validToken.SecretID 3600 3601 err = msgpackrpc.CallWithCodec(codec, "Job.List", req, &validResp) 3602 require.Nil(err) 3603 require.Equal(1, len(validResp.Jobs)) 3604 require.Equal(job.ID, validResp.Jobs[0].ID) 3605 } 3606 3607 func TestJobEndpoint_ListJobs_Blocking(t *testing.T) { 3608 t.Parallel() 3609 3610 s1, cleanupS1 := TestServer(t, nil) 3611 defer cleanupS1() 3612 state := s1.fsm.State() 3613 codec := rpcClient(t, s1) 3614 testutil.WaitForLeader(t, s1.RPC) 3615 3616 // Create the job 3617 job := mock.Job() 3618 3619 // Upsert job triggers watches 3620 time.AfterFunc(100*time.Millisecond, func() { 3621 if err := state.UpsertJob(100, job); err != nil { 3622 t.Fatalf("err: %v", err) 3623 } 3624 }) 3625 3626 req := &structs.JobListRequest{ 3627 QueryOptions: structs.QueryOptions{ 3628 Region: "global", 3629 Namespace: job.Namespace, 3630 MinQueryIndex: 50, 3631 }, 3632 } 3633 start := time.Now() 3634 var resp structs.JobListResponse 3635 if err := msgpackrpc.CallWithCodec(codec, "Job.List", req, &resp); err != nil { 3636 t.Fatalf("err: %v", err) 3637 } 3638 3639 if elapsed := time.Since(start); elapsed < 100*time.Millisecond { 3640 t.Fatalf("should block (returned in %s) %#v", elapsed, resp) 3641 } 3642 if resp.Index != 100 { 3643 t.Fatalf("Bad index: %d %d", resp.Index, 100) 3644 } 3645 if len(resp.Jobs) != 1 || resp.Jobs[0].ID != job.ID { 3646 t.Fatalf("bad: %#v", resp) 3647 } 3648 3649 // Job deletion triggers watches 3650 time.AfterFunc(100*time.Millisecond, func() { 3651 if err := state.DeleteJob(200, job.Namespace, job.ID); err != nil { 3652 t.Fatalf("err: %v", err) 3653 } 3654 }) 3655 3656 req.MinQueryIndex = 150 3657 start = time.Now() 3658 var resp2 structs.JobListResponse 3659 if err := msgpackrpc.CallWithCodec(codec, "Job.List", req, &resp2); err != nil { 3660 t.Fatalf("err: %v", err) 3661 } 3662 3663 if elapsed := time.Since(start); elapsed < 100*time.Millisecond { 3664 t.Fatalf("should block (returned in %s) %#v", elapsed, resp2) 3665 } 3666 if resp2.Index != 200 { 3667 t.Fatalf("Bad index: %d %d", resp2.Index, 200) 3668 } 3669 if len(resp2.Jobs) != 0 { 3670 t.Fatalf("bad: %#v", resp2) 3671 } 3672 } 3673 3674 func TestJobEndpoint_Allocations(t *testing.T) { 3675 t.Parallel() 3676 3677 s1, cleanupS1 := TestServer(t, nil) 3678 defer cleanupS1() 3679 codec := rpcClient(t, s1) 3680 testutil.WaitForLeader(t, s1.RPC) 3681 3682 // Create the register request 3683 alloc1 := mock.Alloc() 3684 alloc2 := mock.Alloc() 3685 alloc2.JobID = alloc1.JobID 3686 state := s1.fsm.State() 3687 state.UpsertJobSummary(998, mock.JobSummary(alloc1.JobID)) 3688 state.UpsertJobSummary(999, mock.JobSummary(alloc2.JobID)) 3689 err := state.UpsertAllocs(1000, 3690 []*structs.Allocation{alloc1, alloc2}) 3691 if err != nil { 3692 t.Fatalf("err: %v", err) 3693 } 3694 3695 // Lookup the jobs 3696 get := &structs.JobSpecificRequest{ 3697 JobID: alloc1.JobID, 3698 QueryOptions: structs.QueryOptions{ 3699 Region: "global", 3700 Namespace: alloc1.Job.Namespace, 3701 }, 3702 } 3703 var resp2 structs.JobAllocationsResponse 3704 if err := msgpackrpc.CallWithCodec(codec, "Job.Allocations", get, &resp2); err != nil { 3705 t.Fatalf("err: %v", err) 3706 } 3707 if resp2.Index != 1000 { 3708 t.Fatalf("Bad index: %d %d", resp2.Index, 1000) 3709 } 3710 3711 if len(resp2.Allocations) != 2 { 3712 t.Fatalf("bad: %#v", resp2.Allocations) 3713 } 3714 } 3715 3716 func TestJobEndpoint_Allocations_ACL(t *testing.T) { 3717 t.Parallel() 3718 require := require.New(t) 3719 3720 s1, root, cleanupS1 := TestACLServer(t, nil) 3721 defer cleanupS1() 3722 codec := rpcClient(t, s1) 3723 testutil.WaitForLeader(t, s1.RPC) 3724 state := s1.fsm.State() 3725 3726 // Create allocations for a job 3727 alloc1 := mock.Alloc() 3728 alloc2 := mock.Alloc() 3729 alloc2.JobID = alloc1.JobID 3730 state.UpsertJobSummary(998, mock.JobSummary(alloc1.JobID)) 3731 state.UpsertJobSummary(999, mock.JobSummary(alloc2.JobID)) 3732 err := state.UpsertAllocs(1000, 3733 []*structs.Allocation{alloc1, alloc2}) 3734 require.Nil(err) 3735 3736 // Look up allocations for that job 3737 get := &structs.JobSpecificRequest{ 3738 JobID: alloc1.JobID, 3739 QueryOptions: structs.QueryOptions{ 3740 Region: "global", 3741 Namespace: alloc1.Job.Namespace, 3742 }, 3743 } 3744 3745 // Attempt to fetch the response without a token should fail 3746 var resp structs.JobAllocationsResponse 3747 err = msgpackrpc.CallWithCodec(codec, "Job.Allocations", get, &resp) 3748 require.NotNil(err) 3749 require.Contains(err.Error(), "Permission denied") 3750 3751 // Attempt to fetch the response with an invalid token should fail 3752 invalidToken := mock.CreatePolicyAndToken(t, state, 1001, "test-invalid", 3753 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityListJobs})) 3754 3755 get.AuthToken = invalidToken.SecretID 3756 var invalidResp structs.JobAllocationsResponse 3757 err = msgpackrpc.CallWithCodec(codec, "Job.Allocations", get, &invalidResp) 3758 require.NotNil(err) 3759 require.Contains(err.Error(), "Permission denied") 3760 3761 // Attempt to fetch the response with valid management token should succeed 3762 get.AuthToken = root.SecretID 3763 var validResp structs.JobAllocationsResponse 3764 err = msgpackrpc.CallWithCodec(codec, "Job.Allocations", get, &validResp) 3765 require.Nil(err) 3766 3767 // Attempt to fetch the response with valid management token should succeed 3768 validToken := mock.CreatePolicyAndToken(t, state, 1005, "test-valid", 3769 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob})) 3770 3771 get.AuthToken = validToken.SecretID 3772 var validResp2 structs.JobAllocationsResponse 3773 err = msgpackrpc.CallWithCodec(codec, "Job.Allocations", get, &validResp2) 3774 require.Nil(err) 3775 3776 require.Equal(2, len(validResp2.Allocations)) 3777 } 3778 3779 func TestJobEndpoint_Allocations_Blocking(t *testing.T) { 3780 t.Parallel() 3781 3782 s1, cleanupS1 := TestServer(t, nil) 3783 defer cleanupS1() 3784 codec := rpcClient(t, s1) 3785 testutil.WaitForLeader(t, s1.RPC) 3786 3787 // Create the register request 3788 alloc1 := mock.Alloc() 3789 alloc2 := mock.Alloc() 3790 alloc2.JobID = "job1" 3791 state := s1.fsm.State() 3792 3793 // First upsert an unrelated alloc 3794 time.AfterFunc(100*time.Millisecond, func() { 3795 state.UpsertJobSummary(99, mock.JobSummary(alloc1.JobID)) 3796 err := state.UpsertAllocs(100, []*structs.Allocation{alloc1}) 3797 if err != nil { 3798 t.Fatalf("err: %v", err) 3799 } 3800 }) 3801 3802 // Upsert an alloc for the job we are interested in later 3803 time.AfterFunc(200*time.Millisecond, func() { 3804 state.UpsertJobSummary(199, mock.JobSummary(alloc2.JobID)) 3805 err := state.UpsertAllocs(200, []*structs.Allocation{alloc2}) 3806 if err != nil { 3807 t.Fatalf("err: %v", err) 3808 } 3809 }) 3810 3811 // Lookup the jobs 3812 get := &structs.JobSpecificRequest{ 3813 JobID: "job1", 3814 QueryOptions: structs.QueryOptions{ 3815 Region: "global", 3816 Namespace: alloc1.Job.Namespace, 3817 MinQueryIndex: 150, 3818 }, 3819 } 3820 var resp structs.JobAllocationsResponse 3821 start := time.Now() 3822 if err := msgpackrpc.CallWithCodec(codec, "Job.Allocations", get, &resp); err != nil { 3823 t.Fatalf("err: %v", err) 3824 } 3825 3826 if elapsed := time.Since(start); elapsed < 200*time.Millisecond { 3827 t.Fatalf("should block (returned in %s) %#v", elapsed, resp) 3828 } 3829 if resp.Index != 200 { 3830 t.Fatalf("Bad index: %d %d", resp.Index, 200) 3831 } 3832 if len(resp.Allocations) != 1 || resp.Allocations[0].JobID != "job1" { 3833 t.Fatalf("bad: %#v", resp.Allocations) 3834 } 3835 } 3836 3837 // TestJobEndpoint_Allocations_NoJobID asserts not setting a JobID in the 3838 // request returns an error. 3839 func TestJobEndpoint_Allocations_NoJobID(t *testing.T) { 3840 t.Parallel() 3841 3842 s1, cleanupS1 := TestServer(t, nil) 3843 defer cleanupS1() 3844 codec := rpcClient(t, s1) 3845 testutil.WaitForLeader(t, s1.RPC) 3846 3847 get := &structs.JobSpecificRequest{ 3848 JobID: "", 3849 QueryOptions: structs.QueryOptions{ 3850 Region: "global", 3851 Namespace: structs.DefaultNamespace, 3852 }, 3853 } 3854 3855 var resp structs.JobAllocationsResponse 3856 err := msgpackrpc.CallWithCodec(codec, "Job.Allocations", get, &resp) 3857 require.Error(t, err) 3858 require.Contains(t, err.Error(), "missing job ID") 3859 } 3860 3861 func TestJobEndpoint_Evaluations(t *testing.T) { 3862 t.Parallel() 3863 3864 s1, cleanupS1 := TestServer(t, nil) 3865 defer cleanupS1() 3866 codec := rpcClient(t, s1) 3867 testutil.WaitForLeader(t, s1.RPC) 3868 3869 // Create the register request 3870 eval1 := mock.Eval() 3871 eval2 := mock.Eval() 3872 eval2.JobID = eval1.JobID 3873 state := s1.fsm.State() 3874 err := state.UpsertEvals(1000, 3875 []*structs.Evaluation{eval1, eval2}) 3876 if err != nil { 3877 t.Fatalf("err: %v", err) 3878 } 3879 3880 // Lookup the jobs 3881 get := &structs.JobSpecificRequest{ 3882 JobID: eval1.JobID, 3883 QueryOptions: structs.QueryOptions{ 3884 Region: "global", 3885 Namespace: eval1.Namespace, 3886 }, 3887 } 3888 var resp2 structs.JobEvaluationsResponse 3889 if err := msgpackrpc.CallWithCodec(codec, "Job.Evaluations", get, &resp2); err != nil { 3890 t.Fatalf("err: %v", err) 3891 } 3892 if resp2.Index != 1000 { 3893 t.Fatalf("Bad index: %d %d", resp2.Index, 1000) 3894 } 3895 3896 if len(resp2.Evaluations) != 2 { 3897 t.Fatalf("bad: %#v", resp2.Evaluations) 3898 } 3899 } 3900 3901 func TestJobEndpoint_Evaluations_ACL(t *testing.T) { 3902 t.Parallel() 3903 require := require.New(t) 3904 3905 s1, root, cleanupS1 := TestACLServer(t, nil) 3906 defer cleanupS1() 3907 codec := rpcClient(t, s1) 3908 testutil.WaitForLeader(t, s1.RPC) 3909 state := s1.fsm.State() 3910 3911 // Create evaluations for the same job 3912 eval1 := mock.Eval() 3913 eval2 := mock.Eval() 3914 eval2.JobID = eval1.JobID 3915 err := state.UpsertEvals(1000, 3916 []*structs.Evaluation{eval1, eval2}) 3917 require.Nil(err) 3918 3919 // Lookup the jobs 3920 get := &structs.JobSpecificRequest{ 3921 JobID: eval1.JobID, 3922 QueryOptions: structs.QueryOptions{ 3923 Region: "global", 3924 Namespace: eval1.Namespace, 3925 }, 3926 } 3927 3928 // Attempt to fetch without providing a token 3929 var resp structs.JobEvaluationsResponse 3930 err = msgpackrpc.CallWithCodec(codec, "Job.Evaluations", get, &resp) 3931 require.NotNil(err) 3932 require.Contains(err.Error(), "Permission denied") 3933 3934 // Attempt to fetch the response with an invalid token 3935 invalidToken := mock.CreatePolicyAndToken(t, state, 1001, "test-invalid", 3936 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityListJobs})) 3937 3938 get.AuthToken = invalidToken.SecretID 3939 var invalidResp structs.JobEvaluationsResponse 3940 err = msgpackrpc.CallWithCodec(codec, "Job.Evaluations", get, &invalidResp) 3941 require.NotNil(err) 3942 require.Contains(err.Error(), "Permission denied") 3943 3944 // Attempt to fetch with valid management token should succeed 3945 get.AuthToken = root.SecretID 3946 var validResp structs.JobEvaluationsResponse 3947 err = msgpackrpc.CallWithCodec(codec, "Job.Evaluations", get, &validResp) 3948 require.Nil(err) 3949 require.Equal(2, len(validResp.Evaluations)) 3950 3951 // Attempt to fetch with valid token should succeed 3952 validToken := mock.CreatePolicyAndToken(t, state, 1003, "test-valid", 3953 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob})) 3954 3955 get.AuthToken = validToken.SecretID 3956 var validResp2 structs.JobEvaluationsResponse 3957 err = msgpackrpc.CallWithCodec(codec, "Job.Evaluations", get, &validResp2) 3958 require.Nil(err) 3959 require.Equal(2, len(validResp2.Evaluations)) 3960 } 3961 3962 func TestJobEndpoint_Evaluations_Blocking(t *testing.T) { 3963 t.Parallel() 3964 3965 s1, cleanupS1 := TestServer(t, nil) 3966 defer cleanupS1() 3967 codec := rpcClient(t, s1) 3968 testutil.WaitForLeader(t, s1.RPC) 3969 3970 // Create the register request 3971 eval1 := mock.Eval() 3972 eval2 := mock.Eval() 3973 eval2.JobID = "job1" 3974 state := s1.fsm.State() 3975 3976 // First upsert an unrelated eval 3977 time.AfterFunc(100*time.Millisecond, func() { 3978 err := state.UpsertEvals(100, []*structs.Evaluation{eval1}) 3979 if err != nil { 3980 t.Fatalf("err: %v", err) 3981 } 3982 }) 3983 3984 // Upsert an eval for the job we are interested in later 3985 time.AfterFunc(200*time.Millisecond, func() { 3986 err := state.UpsertEvals(200, []*structs.Evaluation{eval2}) 3987 if err != nil { 3988 t.Fatalf("err: %v", err) 3989 } 3990 }) 3991 3992 // Lookup the jobs 3993 get := &structs.JobSpecificRequest{ 3994 JobID: "job1", 3995 QueryOptions: structs.QueryOptions{ 3996 Region: "global", 3997 Namespace: eval1.Namespace, 3998 MinQueryIndex: 150, 3999 }, 4000 } 4001 var resp structs.JobEvaluationsResponse 4002 start := time.Now() 4003 if err := msgpackrpc.CallWithCodec(codec, "Job.Evaluations", get, &resp); err != nil { 4004 t.Fatalf("err: %v", err) 4005 } 4006 4007 if elapsed := time.Since(start); elapsed < 200*time.Millisecond { 4008 t.Fatalf("should block (returned in %s) %#v", elapsed, resp) 4009 } 4010 if resp.Index != 200 { 4011 t.Fatalf("Bad index: %d %d", resp.Index, 200) 4012 } 4013 if len(resp.Evaluations) != 1 || resp.Evaluations[0].JobID != "job1" { 4014 t.Fatalf("bad: %#v", resp.Evaluations) 4015 } 4016 } 4017 4018 func TestJobEndpoint_Deployments(t *testing.T) { 4019 t.Parallel() 4020 4021 s1, cleanupS1 := TestServer(t, nil) 4022 defer cleanupS1() 4023 codec := rpcClient(t, s1) 4024 testutil.WaitForLeader(t, s1.RPC) 4025 state := s1.fsm.State() 4026 require := require.New(t) 4027 4028 // Create the register request 4029 j := mock.Job() 4030 d1 := mock.Deployment() 4031 d2 := mock.Deployment() 4032 d1.JobID = j.ID 4033 d2.JobID = j.ID 4034 require.Nil(state.UpsertJob(1000, j), "UpsertJob") 4035 d1.JobCreateIndex = j.CreateIndex 4036 d2.JobCreateIndex = j.CreateIndex 4037 4038 require.Nil(state.UpsertDeployment(1001, d1), "UpsertDeployment") 4039 require.Nil(state.UpsertDeployment(1002, d2), "UpsertDeployment") 4040 4041 // Lookup the jobs 4042 get := &structs.JobSpecificRequest{ 4043 JobID: j.ID, 4044 QueryOptions: structs.QueryOptions{ 4045 Region: "global", 4046 Namespace: j.Namespace, 4047 }, 4048 } 4049 var resp structs.DeploymentListResponse 4050 require.Nil(msgpackrpc.CallWithCodec(codec, "Job.Deployments", get, &resp), "RPC") 4051 require.EqualValues(1002, resp.Index, "response index") 4052 require.Len(resp.Deployments, 2, "deployments for job") 4053 } 4054 4055 func TestJobEndpoint_Deployments_ACL(t *testing.T) { 4056 t.Parallel() 4057 require := require.New(t) 4058 4059 s1, root, cleanupS1 := TestACLServer(t, nil) 4060 defer cleanupS1() 4061 codec := rpcClient(t, s1) 4062 testutil.WaitForLeader(t, s1.RPC) 4063 state := s1.fsm.State() 4064 4065 // Create a job and corresponding deployments 4066 j := mock.Job() 4067 d1 := mock.Deployment() 4068 d2 := mock.Deployment() 4069 d1.JobID = j.ID 4070 d2.JobID = j.ID 4071 require.Nil(state.UpsertJob(1000, j), "UpsertJob") 4072 d1.JobCreateIndex = j.CreateIndex 4073 d2.JobCreateIndex = j.CreateIndex 4074 require.Nil(state.UpsertDeployment(1001, d1), "UpsertDeployment") 4075 require.Nil(state.UpsertDeployment(1002, d2), "UpsertDeployment") 4076 4077 // Lookup the jobs 4078 get := &structs.JobSpecificRequest{ 4079 JobID: j.ID, 4080 QueryOptions: structs.QueryOptions{ 4081 Region: "global", 4082 Namespace: j.Namespace, 4083 }, 4084 } 4085 // Lookup with no token should fail 4086 var resp structs.DeploymentListResponse 4087 err := msgpackrpc.CallWithCodec(codec, "Job.Deployments", get, &resp) 4088 require.NotNil(err) 4089 require.Contains(err.Error(), "Permission denied") 4090 4091 // Attempt to fetch the response with an invalid token 4092 invalidToken := mock.CreatePolicyAndToken(t, state, 1001, "test-invalid", 4093 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityListJobs})) 4094 4095 get.AuthToken = invalidToken.SecretID 4096 var invalidResp structs.DeploymentListResponse 4097 err = msgpackrpc.CallWithCodec(codec, "Job.Deployments", get, &invalidResp) 4098 require.NotNil(err) 4099 require.Contains(err.Error(), "Permission denied") 4100 4101 // Lookup with valid management token should succeed 4102 get.AuthToken = root.SecretID 4103 var validResp structs.DeploymentListResponse 4104 require.Nil(msgpackrpc.CallWithCodec(codec, "Job.Deployments", get, &validResp), "RPC") 4105 require.EqualValues(1002, validResp.Index, "response index") 4106 require.Len(validResp.Deployments, 2, "deployments for job") 4107 4108 // Lookup with valid token should succeed 4109 validToken := mock.CreatePolicyAndToken(t, state, 1005, "test-valid", 4110 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob})) 4111 4112 get.AuthToken = validToken.SecretID 4113 var validResp2 structs.DeploymentListResponse 4114 require.Nil(msgpackrpc.CallWithCodec(codec, "Job.Deployments", get, &validResp2), "RPC") 4115 require.EqualValues(1002, validResp2.Index, "response index") 4116 require.Len(validResp2.Deployments, 2, "deployments for job") 4117 } 4118 4119 func TestJobEndpoint_Deployments_Blocking(t *testing.T) { 4120 t.Parallel() 4121 4122 s1, cleanupS1 := TestServer(t, nil) 4123 defer cleanupS1() 4124 codec := rpcClient(t, s1) 4125 testutil.WaitForLeader(t, s1.RPC) 4126 state := s1.fsm.State() 4127 require := require.New(t) 4128 4129 // Create the register request 4130 j := mock.Job() 4131 d1 := mock.Deployment() 4132 d2 := mock.Deployment() 4133 d2.JobID = j.ID 4134 require.Nil(state.UpsertJob(50, j), "UpsertJob") 4135 d2.JobCreateIndex = j.CreateIndex 4136 // First upsert an unrelated eval 4137 time.AfterFunc(100*time.Millisecond, func() { 4138 require.Nil(state.UpsertDeployment(100, d1), "UpsertDeployment") 4139 }) 4140 4141 // Upsert an eval for the job we are interested in later 4142 time.AfterFunc(200*time.Millisecond, func() { 4143 require.Nil(state.UpsertDeployment(200, d2), "UpsertDeployment") 4144 }) 4145 4146 // Lookup the jobs 4147 get := &structs.JobSpecificRequest{ 4148 JobID: d2.JobID, 4149 QueryOptions: structs.QueryOptions{ 4150 Region: "global", 4151 Namespace: d2.Namespace, 4152 MinQueryIndex: 150, 4153 }, 4154 } 4155 var resp structs.DeploymentListResponse 4156 start := time.Now() 4157 require.Nil(msgpackrpc.CallWithCodec(codec, "Job.Deployments", get, &resp), "RPC") 4158 require.EqualValues(200, resp.Index, "response index") 4159 require.Len(resp.Deployments, 1, "deployments for job") 4160 require.Equal(d2.ID, resp.Deployments[0].ID, "returned deployment") 4161 if elapsed := time.Since(start); elapsed < 200*time.Millisecond { 4162 t.Fatalf("should block (returned in %s) %#v", elapsed, resp) 4163 } 4164 } 4165 4166 func TestJobEndpoint_LatestDeployment(t *testing.T) { 4167 t.Parallel() 4168 4169 s1, cleanupS1 := TestServer(t, nil) 4170 defer cleanupS1() 4171 codec := rpcClient(t, s1) 4172 testutil.WaitForLeader(t, s1.RPC) 4173 state := s1.fsm.State() 4174 require := require.New(t) 4175 4176 // Create the register request 4177 j := mock.Job() 4178 d1 := mock.Deployment() 4179 d2 := mock.Deployment() 4180 d1.JobID = j.ID 4181 d2.JobID = j.ID 4182 d2.CreateIndex = d1.CreateIndex + 100 4183 d2.ModifyIndex = d2.CreateIndex + 100 4184 require.Nil(state.UpsertJob(1000, j), "UpsertJob") 4185 d1.JobCreateIndex = j.CreateIndex 4186 d2.JobCreateIndex = j.CreateIndex 4187 require.Nil(state.UpsertDeployment(1001, d1), "UpsertDeployment") 4188 require.Nil(state.UpsertDeployment(1002, d2), "UpsertDeployment") 4189 4190 // Lookup the jobs 4191 get := &structs.JobSpecificRequest{ 4192 JobID: j.ID, 4193 QueryOptions: structs.QueryOptions{ 4194 Region: "global", 4195 Namespace: j.Namespace, 4196 }, 4197 } 4198 var resp structs.SingleDeploymentResponse 4199 require.Nil(msgpackrpc.CallWithCodec(codec, "Job.LatestDeployment", get, &resp), "RPC") 4200 require.EqualValues(1002, resp.Index, "response index") 4201 require.NotNil(resp.Deployment, "want a deployment") 4202 require.Equal(d2.ID, resp.Deployment.ID, "latest deployment for job") 4203 } 4204 4205 func TestJobEndpoint_LatestDeployment_ACL(t *testing.T) { 4206 t.Parallel() 4207 require := require.New(t) 4208 4209 s1, root, cleanupS1 := TestACLServer(t, nil) 4210 defer cleanupS1() 4211 codec := rpcClient(t, s1) 4212 testutil.WaitForLeader(t, s1.RPC) 4213 state := s1.fsm.State() 4214 4215 // Create a job and deployments 4216 j := mock.Job() 4217 d1 := mock.Deployment() 4218 d2 := mock.Deployment() 4219 d1.JobID = j.ID 4220 d2.JobID = j.ID 4221 d2.CreateIndex = d1.CreateIndex + 100 4222 d2.ModifyIndex = d2.CreateIndex + 100 4223 require.Nil(state.UpsertJob(1000, j), "UpsertJob") 4224 d1.JobCreateIndex = j.CreateIndex 4225 d2.JobCreateIndex = j.CreateIndex 4226 require.Nil(state.UpsertDeployment(1001, d1), "UpsertDeployment") 4227 require.Nil(state.UpsertDeployment(1002, d2), "UpsertDeployment") 4228 4229 // Lookup the jobs 4230 get := &structs.JobSpecificRequest{ 4231 JobID: j.ID, 4232 QueryOptions: structs.QueryOptions{ 4233 Region: "global", 4234 Namespace: j.Namespace, 4235 }, 4236 } 4237 4238 // Attempt to fetch the response without a token should fail 4239 var resp structs.SingleDeploymentResponse 4240 err := msgpackrpc.CallWithCodec(codec, "Job.LatestDeployment", get, &resp) 4241 require.NotNil(err) 4242 require.Contains(err.Error(), "Permission denied") 4243 4244 // Attempt to fetch the response with an invalid token should fail 4245 invalidToken := mock.CreatePolicyAndToken(t, state, 1001, "test-invalid", 4246 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityListJobs})) 4247 4248 get.AuthToken = invalidToken.SecretID 4249 var invalidResp structs.SingleDeploymentResponse 4250 err = msgpackrpc.CallWithCodec(codec, "Job.LatestDeployment", get, &invalidResp) 4251 require.NotNil(err) 4252 require.Contains(err.Error(), "Permission denied") 4253 4254 // Fetching latest deployment with a valid management token should succeed 4255 get.AuthToken = root.SecretID 4256 var validResp structs.SingleDeploymentResponse 4257 require.Nil(msgpackrpc.CallWithCodec(codec, "Job.LatestDeployment", get, &validResp), "RPC") 4258 require.EqualValues(1002, validResp.Index, "response index") 4259 require.NotNil(validResp.Deployment, "want a deployment") 4260 require.Equal(d2.ID, validResp.Deployment.ID, "latest deployment for job") 4261 4262 // Fetching latest deployment with a valid token should succeed 4263 validToken := mock.CreatePolicyAndToken(t, state, 1004, "test-valid", 4264 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob})) 4265 4266 get.AuthToken = validToken.SecretID 4267 var validResp2 structs.SingleDeploymentResponse 4268 require.Nil(msgpackrpc.CallWithCodec(codec, "Job.LatestDeployment", get, &validResp2), "RPC") 4269 require.EqualValues(1002, validResp2.Index, "response index") 4270 require.NotNil(validResp2.Deployment, "want a deployment") 4271 require.Equal(d2.ID, validResp2.Deployment.ID, "latest deployment for job") 4272 } 4273 4274 func TestJobEndpoint_LatestDeployment_Blocking(t *testing.T) { 4275 t.Parallel() 4276 4277 s1, cleanupS1 := TestServer(t, nil) 4278 defer cleanupS1() 4279 codec := rpcClient(t, s1) 4280 testutil.WaitForLeader(t, s1.RPC) 4281 state := s1.fsm.State() 4282 require := require.New(t) 4283 4284 // Create the register request 4285 j := mock.Job() 4286 d1 := mock.Deployment() 4287 d2 := mock.Deployment() 4288 d2.JobID = j.ID 4289 require.Nil(state.UpsertJob(50, j), "UpsertJob") 4290 d2.JobCreateIndex = j.CreateIndex 4291 4292 // First upsert an unrelated eval 4293 time.AfterFunc(100*time.Millisecond, func() { 4294 require.Nil(state.UpsertDeployment(100, d1), "UpsertDeployment") 4295 }) 4296 4297 // Upsert an eval for the job we are interested in later 4298 time.AfterFunc(200*time.Millisecond, func() { 4299 require.Nil(state.UpsertDeployment(200, d2), "UpsertDeployment") 4300 }) 4301 4302 // Lookup the jobs 4303 get := &structs.JobSpecificRequest{ 4304 JobID: d2.JobID, 4305 QueryOptions: structs.QueryOptions{ 4306 Region: "global", 4307 Namespace: d2.Namespace, 4308 MinQueryIndex: 150, 4309 }, 4310 } 4311 var resp structs.SingleDeploymentResponse 4312 start := time.Now() 4313 require.Nil(msgpackrpc.CallWithCodec(codec, "Job.LatestDeployment", get, &resp), "RPC") 4314 require.EqualValues(200, resp.Index, "response index") 4315 require.NotNil(resp.Deployment, "deployment for job") 4316 require.Equal(d2.ID, resp.Deployment.ID, "returned deployment") 4317 if elapsed := time.Since(start); elapsed < 200*time.Millisecond { 4318 t.Fatalf("should block (returned in %s) %#v", elapsed, resp) 4319 } 4320 } 4321 4322 func TestJobEndpoint_Plan_ACL(t *testing.T) { 4323 t.Parallel() 4324 4325 s1, root, cleanupS1 := TestACLServer(t, func(c *Config) { 4326 c.NumSchedulers = 0 // Prevent automatic dequeue 4327 }) 4328 defer cleanupS1() 4329 codec := rpcClient(t, s1) 4330 testutil.WaitForLeader(t, s1.RPC) 4331 4332 // Create a plan request 4333 job := mock.Job() 4334 planReq := &structs.JobPlanRequest{ 4335 Job: job, 4336 Diff: true, 4337 WriteRequest: structs.WriteRequest{ 4338 Region: "global", 4339 Namespace: job.Namespace, 4340 }, 4341 } 4342 4343 // Try without a token, expect failure 4344 var planResp structs.JobPlanResponse 4345 if err := msgpackrpc.CallWithCodec(codec, "Job.Plan", planReq, &planResp); err == nil { 4346 t.Fatalf("expected error") 4347 } 4348 4349 // Try with a token 4350 planReq.AuthToken = root.SecretID 4351 if err := msgpackrpc.CallWithCodec(codec, "Job.Plan", planReq, &planResp); err != nil { 4352 t.Fatalf("err: %v", err) 4353 } 4354 } 4355 4356 func TestJobEndpoint_Plan_WithDiff(t *testing.T) { 4357 t.Parallel() 4358 4359 s1, cleanupS1 := TestServer(t, func(c *Config) { 4360 c.NumSchedulers = 0 // Prevent automatic dequeue 4361 }) 4362 defer cleanupS1() 4363 codec := rpcClient(t, s1) 4364 testutil.WaitForLeader(t, s1.RPC) 4365 4366 // Create the register request 4367 job := mock.Job() 4368 req := &structs.JobRegisterRequest{ 4369 Job: job, 4370 WriteRequest: structs.WriteRequest{ 4371 Region: "global", 4372 Namespace: job.Namespace, 4373 }, 4374 } 4375 4376 // Fetch the response 4377 var resp structs.JobRegisterResponse 4378 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { 4379 t.Fatalf("err: %v", err) 4380 } 4381 if resp.Index == 0 { 4382 t.Fatalf("bad index: %d", resp.Index) 4383 } 4384 4385 // Create a plan request 4386 planReq := &structs.JobPlanRequest{ 4387 Job: job, 4388 Diff: true, 4389 WriteRequest: structs.WriteRequest{ 4390 Region: "global", 4391 Namespace: job.Namespace, 4392 }, 4393 } 4394 4395 // Fetch the response 4396 var planResp structs.JobPlanResponse 4397 if err := msgpackrpc.CallWithCodec(codec, "Job.Plan", planReq, &planResp); err != nil { 4398 t.Fatalf("err: %v", err) 4399 } 4400 4401 // Check the response 4402 if planResp.JobModifyIndex == 0 { 4403 t.Fatalf("bad cas: %d", planResp.JobModifyIndex) 4404 } 4405 if planResp.Annotations == nil { 4406 t.Fatalf("no annotations") 4407 } 4408 if planResp.Diff == nil { 4409 t.Fatalf("no diff") 4410 } 4411 if len(planResp.FailedTGAllocs) == 0 { 4412 t.Fatalf("no failed task group alloc metrics") 4413 } 4414 } 4415 4416 func TestJobEndpoint_Plan_NoDiff(t *testing.T) { 4417 t.Parallel() 4418 4419 s1, cleanupS1 := TestServer(t, func(c *Config) { 4420 c.NumSchedulers = 0 // Prevent automatic dequeue 4421 }) 4422 defer cleanupS1() 4423 codec := rpcClient(t, s1) 4424 testutil.WaitForLeader(t, s1.RPC) 4425 4426 // Create the register request 4427 job := mock.Job() 4428 req := &structs.JobRegisterRequest{ 4429 Job: job, 4430 WriteRequest: structs.WriteRequest{ 4431 Region: "global", 4432 Namespace: job.Namespace, 4433 }, 4434 } 4435 4436 // Fetch the response 4437 var resp structs.JobRegisterResponse 4438 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { 4439 t.Fatalf("err: %v", err) 4440 } 4441 if resp.Index == 0 { 4442 t.Fatalf("bad index: %d", resp.Index) 4443 } 4444 4445 // Create a plan request 4446 planReq := &structs.JobPlanRequest{ 4447 Job: job, 4448 Diff: false, 4449 WriteRequest: structs.WriteRequest{ 4450 Region: "global", 4451 Namespace: job.Namespace, 4452 }, 4453 } 4454 4455 // Fetch the response 4456 var planResp structs.JobPlanResponse 4457 if err := msgpackrpc.CallWithCodec(codec, "Job.Plan", planReq, &planResp); err != nil { 4458 t.Fatalf("err: %v", err) 4459 } 4460 4461 // Check the response 4462 if planResp.JobModifyIndex == 0 { 4463 t.Fatalf("bad cas: %d", planResp.JobModifyIndex) 4464 } 4465 if planResp.Annotations == nil { 4466 t.Fatalf("no annotations") 4467 } 4468 if planResp.Diff != nil { 4469 t.Fatalf("got diff") 4470 } 4471 if len(planResp.FailedTGAllocs) == 0 { 4472 t.Fatalf("no failed task group alloc metrics") 4473 } 4474 } 4475 4476 func TestJobEndpoint_ImplicitConstraints_Vault(t *testing.T) { 4477 t.Parallel() 4478 4479 s1, cleanupS1 := TestServer(t, func(c *Config) { 4480 c.NumSchedulers = 0 // Prevent automatic dequeue 4481 }) 4482 defer cleanupS1() 4483 codec := rpcClient(t, s1) 4484 testutil.WaitForLeader(t, s1.RPC) 4485 4486 // Enable vault 4487 tr, f := true, false 4488 s1.config.VaultConfig.Enabled = &tr 4489 s1.config.VaultConfig.AllowUnauthenticated = &f 4490 4491 // Replace the Vault Client on the server 4492 tvc := &TestVaultClient{} 4493 s1.vault = tvc 4494 4495 policy := "foo" 4496 goodToken := uuid.Generate() 4497 goodPolicies := []string{"foo", "bar", "baz"} 4498 tvc.SetLookupTokenAllowedPolicies(goodToken, goodPolicies) 4499 4500 // Create the register request with a job asking for a vault policy 4501 job := mock.Job() 4502 job.VaultToken = goodToken 4503 job.TaskGroups[0].Tasks[0].Vault = &structs.Vault{ 4504 Policies: []string{policy}, 4505 ChangeMode: structs.VaultChangeModeRestart, 4506 } 4507 req := &structs.JobRegisterRequest{ 4508 Job: job, 4509 WriteRequest: structs.WriteRequest{ 4510 Region: "global", 4511 Namespace: job.Namespace, 4512 }, 4513 } 4514 4515 // Fetch the response 4516 var resp structs.JobRegisterResponse 4517 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { 4518 t.Fatalf("bad: %v", err) 4519 } 4520 4521 // Check for the job in the FSM 4522 state := s1.fsm.State() 4523 ws := memdb.NewWatchSet() 4524 out, err := state.JobByID(ws, job.Namespace, job.ID) 4525 if err != nil { 4526 t.Fatalf("err: %v", err) 4527 } 4528 if out == nil { 4529 t.Fatalf("expected job") 4530 } 4531 if out.CreateIndex != resp.JobModifyIndex { 4532 t.Fatalf("index mis-match") 4533 } 4534 4535 // Check that there is an implicit vault constraint 4536 constraints := out.TaskGroups[0].Constraints 4537 if len(constraints) != 1 { 4538 t.Fatalf("Expected an implicit constraint") 4539 } 4540 4541 if !constraints[0].Equal(vaultConstraint) { 4542 t.Fatalf("Expected implicit vault constraint") 4543 } 4544 } 4545 4546 func TestJobEndpoint_ValidateJob_ConsulConnect(t *testing.T) { 4547 t.Parallel() 4548 4549 s1, cleanupS1 := TestServer(t, nil) 4550 defer cleanupS1() 4551 codec := rpcClient(t, s1) 4552 testutil.WaitForLeader(t, s1.RPC) 4553 4554 validateJob := func(j *structs.Job) error { 4555 req := &structs.JobRegisterRequest{ 4556 Job: j, 4557 WriteRequest: structs.WriteRequest{ 4558 Region: "global", 4559 Namespace: j.Namespace, 4560 }, 4561 } 4562 var resp structs.JobValidateResponse 4563 if err := msgpackrpc.CallWithCodec(codec, "Job.Validate", req, &resp); err != nil { 4564 return err 4565 } 4566 4567 if resp.Error != "" { 4568 return errors.New(resp.Error) 4569 } 4570 4571 if len(resp.ValidationErrors) != 0 { 4572 return errors.New(strings.Join(resp.ValidationErrors, ",")) 4573 } 4574 4575 if resp.Warnings != "" { 4576 return errors.New(resp.Warnings) 4577 } 4578 4579 return nil 4580 } 4581 4582 tgServices := []*structs.Service{ 4583 { 4584 Name: "count-api", 4585 PortLabel: "9001", 4586 Connect: &structs.ConsulConnect{ 4587 SidecarService: &structs.ConsulSidecarService{}, 4588 }, 4589 }, 4590 } 4591 4592 t.Run("plain job", func(t *testing.T) { 4593 j := mock.Job() 4594 require.NoError(t, validateJob(j)) 4595 }) 4596 t.Run("valid consul connect", func(t *testing.T) { 4597 j := mock.Job() 4598 4599 tg := j.TaskGroups[0] 4600 tg.Services = tgServices 4601 tg.Networks = structs.Networks{ 4602 {Mode: "bridge"}, 4603 } 4604 4605 err := validateJob(j) 4606 require.NoError(t, err) 4607 }) 4608 4609 t.Run("consul connect but missing network", func(t *testing.T) { 4610 j := mock.Job() 4611 4612 tg := j.TaskGroups[0] 4613 tg.Services = tgServices 4614 tg.Networks = nil 4615 4616 err := validateJob(j) 4617 require.Error(t, err) 4618 require.Contains(t, err.Error(), `Consul Connect sidecars require exactly 1 network`) 4619 }) 4620 4621 t.Run("consul connect but non bridge network", func(t *testing.T) { 4622 j := mock.Job() 4623 4624 tg := j.TaskGroups[0] 4625 tg.Services = tgServices 4626 4627 tg.Networks = structs.Networks{ 4628 {Mode: "host"}, 4629 } 4630 4631 err := validateJob(j) 4632 require.Error(t, err) 4633 require.Contains(t, err.Error(), `Consul Connect sidecar requires bridge network, found "host" in group "web"`) 4634 }) 4635 4636 } 4637 4638 func TestJobEndpoint_ImplicitConstraints_Signals(t *testing.T) { 4639 t.Parallel() 4640 4641 s1, cleanupS1 := TestServer(t, func(c *Config) { 4642 c.NumSchedulers = 0 // Prevent automatic dequeue 4643 }) 4644 defer cleanupS1() 4645 codec := rpcClient(t, s1) 4646 testutil.WaitForLeader(t, s1.RPC) 4647 4648 // Create the register request with a job asking for a template that sends a 4649 // signal 4650 job := mock.Job() 4651 signal1 := "SIGUSR1" 4652 signal2 := "SIGHUP" 4653 job.TaskGroups[0].Tasks[0].Templates = []*structs.Template{ 4654 { 4655 SourcePath: "foo", 4656 DestPath: "bar", 4657 ChangeMode: structs.TemplateChangeModeSignal, 4658 ChangeSignal: signal1, 4659 }, 4660 { 4661 SourcePath: "foo", 4662 DestPath: "baz", 4663 ChangeMode: structs.TemplateChangeModeSignal, 4664 ChangeSignal: signal2, 4665 }, 4666 } 4667 req := &structs.JobRegisterRequest{ 4668 Job: job, 4669 WriteRequest: structs.WriteRequest{ 4670 Region: "global", 4671 Namespace: job.Namespace, 4672 }, 4673 } 4674 4675 // Fetch the response 4676 var resp structs.JobRegisterResponse 4677 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { 4678 t.Fatalf("bad: %v", err) 4679 } 4680 4681 // Check for the job in the FSM 4682 state := s1.fsm.State() 4683 ws := memdb.NewWatchSet() 4684 out, err := state.JobByID(ws, job.Namespace, job.ID) 4685 if err != nil { 4686 t.Fatalf("err: %v", err) 4687 } 4688 if out == nil { 4689 t.Fatalf("expected job") 4690 } 4691 if out.CreateIndex != resp.JobModifyIndex { 4692 t.Fatalf("index mis-match") 4693 } 4694 4695 // Check that there is an implicit signal constraint 4696 constraints := out.TaskGroups[0].Constraints 4697 if len(constraints) != 1 { 4698 t.Fatalf("Expected an implicit constraint") 4699 } 4700 4701 sigConstraint := getSignalConstraint([]string{signal1, signal2}) 4702 if !strings.HasPrefix(sigConstraint.RTarget, "SIGHUP") { 4703 t.Fatalf("signals not sorted: %v", sigConstraint.RTarget) 4704 } 4705 4706 if !constraints[0].Equal(sigConstraint) { 4707 t.Fatalf("Expected implicit vault constraint") 4708 } 4709 } 4710 4711 func TestJobEndpoint_ValidateJobUpdate(t *testing.T) { 4712 t.Parallel() 4713 require := require.New(t) 4714 old := mock.Job() 4715 new := mock.Job() 4716 4717 if err := validateJobUpdate(old, new); err != nil { 4718 t.Errorf("expected update to be valid but got: %v", err) 4719 } 4720 4721 new.Type = "batch" 4722 if err := validateJobUpdate(old, new); err == nil { 4723 t.Errorf("expected err when setting new job to a different type") 4724 } else { 4725 t.Log(err) 4726 } 4727 4728 new = mock.Job() 4729 new.Periodic = &structs.PeriodicConfig{Enabled: true} 4730 if err := validateJobUpdate(old, new); err == nil { 4731 t.Errorf("expected err when setting new job to periodic") 4732 } else { 4733 t.Log(err) 4734 } 4735 4736 new = mock.Job() 4737 new.ParameterizedJob = &structs.ParameterizedJobConfig{} 4738 if err := validateJobUpdate(old, new); err == nil { 4739 t.Errorf("expected err when setting new job to parameterized") 4740 } else { 4741 t.Log(err) 4742 } 4743 4744 new = mock.Job() 4745 new.Dispatched = true 4746 require.Error(validateJobUpdate(old, new), 4747 "expected err when setting new job to dispatched") 4748 require.Error(validateJobUpdate(nil, new), 4749 "expected err when setting new job to dispatched") 4750 require.Error(validateJobUpdate(new, old), 4751 "expected err when setting dispatched to false") 4752 require.NoError(validateJobUpdate(nil, old)) 4753 } 4754 4755 func TestJobEndpoint_ValidateJobUpdate_ACL(t *testing.T) { 4756 t.Parallel() 4757 require := require.New(t) 4758 4759 s1, root, cleanupS1 := TestACLServer(t, func(c *Config) { 4760 c.NumSchedulers = 0 // Prevent automatic dequeue 4761 }) 4762 defer cleanupS1() 4763 codec := rpcClient(t, s1) 4764 testutil.WaitForLeader(t, s1.RPC) 4765 4766 job := mock.Job() 4767 4768 req := &structs.JobValidateRequest{ 4769 Job: job, 4770 WriteRequest: structs.WriteRequest{ 4771 Region: "global", 4772 Namespace: job.Namespace, 4773 }, 4774 } 4775 4776 // Attempt to update without providing a valid token 4777 var resp structs.JobValidateResponse 4778 err := msgpackrpc.CallWithCodec(codec, "Job.Validate", req, &resp) 4779 require.NotNil(err) 4780 4781 // Update with a valid token 4782 req.AuthToken = root.SecretID 4783 var validResp structs.JobValidateResponse 4784 err = msgpackrpc.CallWithCodec(codec, "Job.Validate", req, &validResp) 4785 require.Nil(err) 4786 4787 require.Equal("", validResp.Error) 4788 require.Equal("", validResp.Warnings) 4789 } 4790 4791 func TestJobEndpoint_Dispatch_ACL(t *testing.T) { 4792 t.Parallel() 4793 require := require.New(t) 4794 4795 s1, root, cleanupS1 := TestACLServer(t, func(c *Config) { 4796 c.NumSchedulers = 0 // Prevent automatic dequeue 4797 }) 4798 defer cleanupS1() 4799 codec := rpcClient(t, s1) 4800 testutil.WaitForLeader(t, s1.RPC) 4801 state := s1.fsm.State() 4802 4803 // Create a parameterized job 4804 job := mock.BatchJob() 4805 job.ParameterizedJob = &structs.ParameterizedJobConfig{} 4806 err := state.UpsertJob(400, job) 4807 require.Nil(err) 4808 4809 req := &structs.JobDispatchRequest{ 4810 JobID: job.ID, 4811 WriteRequest: structs.WriteRequest{ 4812 Region: "global", 4813 Namespace: job.Namespace, 4814 }, 4815 } 4816 4817 // Attempt to fetch the response without a token should fail 4818 var resp structs.JobDispatchResponse 4819 err = msgpackrpc.CallWithCodec(codec, "Job.Dispatch", req, &resp) 4820 require.NotNil(err) 4821 require.Contains(err.Error(), "Permission denied") 4822 4823 // Attempt to fetch the response with an invalid token should fail 4824 invalidToken := mock.CreatePolicyAndToken(t, state, 1001, "test-invalid", 4825 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityListJobs})) 4826 req.AuthToken = invalidToken.SecretID 4827 4828 var invalidResp structs.JobDispatchResponse 4829 err = msgpackrpc.CallWithCodec(codec, "Job.Dispatch", req, &invalidResp) 4830 require.NotNil(err) 4831 require.Contains(err.Error(), "Permission denied") 4832 4833 // Dispatch with a valid management token should succeed 4834 req.AuthToken = root.SecretID 4835 4836 var validResp structs.JobDispatchResponse 4837 err = msgpackrpc.CallWithCodec(codec, "Job.Dispatch", req, &validResp) 4838 require.Nil(err) 4839 require.NotNil(validResp.EvalID) 4840 require.NotNil(validResp.DispatchedJobID) 4841 require.NotEqual(validResp.DispatchedJobID, "") 4842 4843 // Dispatch with a valid token should succeed 4844 validToken := mock.CreatePolicyAndToken(t, state, 1003, "test-valid", 4845 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityDispatchJob})) 4846 req.AuthToken = validToken.SecretID 4847 4848 var validResp2 structs.JobDispatchResponse 4849 err = msgpackrpc.CallWithCodec(codec, "Job.Dispatch", req, &validResp2) 4850 require.Nil(err) 4851 require.NotNil(validResp2.EvalID) 4852 require.NotNil(validResp2.DispatchedJobID) 4853 require.NotEqual(validResp2.DispatchedJobID, "") 4854 4855 ws := memdb.NewWatchSet() 4856 out, err := state.JobByID(ws, job.Namespace, validResp2.DispatchedJobID) 4857 require.Nil(err) 4858 require.NotNil(out) 4859 require.Equal(out.ParentID, job.ID) 4860 4861 // Look up the evaluation 4862 eval, err := state.EvalByID(ws, validResp2.EvalID) 4863 require.Nil(err) 4864 require.NotNil(eval) 4865 require.Equal(eval.CreateIndex, validResp2.EvalCreateIndex) 4866 } 4867 4868 func TestJobEndpoint_Dispatch(t *testing.T) { 4869 t.Parallel() 4870 4871 // No requirements 4872 d1 := mock.BatchJob() 4873 d1.ParameterizedJob = &structs.ParameterizedJobConfig{} 4874 4875 // Require input data 4876 d2 := mock.BatchJob() 4877 d2.ParameterizedJob = &structs.ParameterizedJobConfig{ 4878 Payload: structs.DispatchPayloadRequired, 4879 } 4880 4881 // Disallow input data 4882 d3 := mock.BatchJob() 4883 d3.ParameterizedJob = &structs.ParameterizedJobConfig{ 4884 Payload: structs.DispatchPayloadForbidden, 4885 } 4886 4887 // Require meta 4888 d4 := mock.BatchJob() 4889 d4.ParameterizedJob = &structs.ParameterizedJobConfig{ 4890 MetaRequired: []string{"foo", "bar"}, 4891 } 4892 4893 // Optional meta 4894 d5 := mock.BatchJob() 4895 d5.ParameterizedJob = &structs.ParameterizedJobConfig{ 4896 MetaOptional: []string{"foo", "bar"}, 4897 } 4898 4899 // Periodic dispatch job 4900 d6 := mock.PeriodicJob() 4901 d6.ParameterizedJob = &structs.ParameterizedJobConfig{} 4902 4903 d7 := mock.BatchJob() 4904 d7.ParameterizedJob = &structs.ParameterizedJobConfig{} 4905 d7.Stop = true 4906 4907 reqNoInputNoMeta := &structs.JobDispatchRequest{} 4908 reqInputDataNoMeta := &structs.JobDispatchRequest{ 4909 Payload: []byte("hello world"), 4910 } 4911 reqNoInputDataMeta := &structs.JobDispatchRequest{ 4912 Meta: map[string]string{ 4913 "foo": "f1", 4914 "bar": "f2", 4915 }, 4916 } 4917 reqInputDataMeta := &structs.JobDispatchRequest{ 4918 Payload: []byte("hello world"), 4919 Meta: map[string]string{ 4920 "foo": "f1", 4921 "bar": "f2", 4922 }, 4923 } 4924 reqBadMeta := &structs.JobDispatchRequest{ 4925 Payload: []byte("hello world"), 4926 Meta: map[string]string{ 4927 "foo": "f1", 4928 "bar": "f2", 4929 "baz": "f3", 4930 }, 4931 } 4932 reqInputDataTooLarge := &structs.JobDispatchRequest{ 4933 Payload: make([]byte, DispatchPayloadSizeLimit+100), 4934 } 4935 4936 type testCase struct { 4937 name string 4938 parameterizedJob *structs.Job 4939 dispatchReq *structs.JobDispatchRequest 4940 noEval bool 4941 err bool 4942 errStr string 4943 } 4944 cases := []testCase{ 4945 { 4946 name: "optional input data w/ data", 4947 parameterizedJob: d1, 4948 dispatchReq: reqInputDataNoMeta, 4949 err: false, 4950 }, 4951 { 4952 name: "optional input data w/o data", 4953 parameterizedJob: d1, 4954 dispatchReq: reqNoInputNoMeta, 4955 err: false, 4956 }, 4957 { 4958 name: "require input data w/ data", 4959 parameterizedJob: d2, 4960 dispatchReq: reqInputDataNoMeta, 4961 err: false, 4962 }, 4963 { 4964 name: "require input data w/o data", 4965 parameterizedJob: d2, 4966 dispatchReq: reqNoInputNoMeta, 4967 err: true, 4968 errStr: "not provided but required", 4969 }, 4970 { 4971 name: "disallow input data w/o data", 4972 parameterizedJob: d3, 4973 dispatchReq: reqNoInputNoMeta, 4974 err: false, 4975 }, 4976 { 4977 name: "disallow input data w/ data", 4978 parameterizedJob: d3, 4979 dispatchReq: reqInputDataNoMeta, 4980 err: true, 4981 errStr: "provided but forbidden", 4982 }, 4983 { 4984 name: "require meta w/ meta", 4985 parameterizedJob: d4, 4986 dispatchReq: reqInputDataMeta, 4987 err: false, 4988 }, 4989 { 4990 name: "require meta w/o meta", 4991 parameterizedJob: d4, 4992 dispatchReq: reqNoInputNoMeta, 4993 err: true, 4994 errStr: "did not provide required meta keys", 4995 }, 4996 { 4997 name: "optional meta w/ meta", 4998 parameterizedJob: d5, 4999 dispatchReq: reqNoInputDataMeta, 5000 err: false, 5001 }, 5002 { 5003 name: "optional meta w/o meta", 5004 parameterizedJob: d5, 5005 dispatchReq: reqNoInputNoMeta, 5006 err: false, 5007 }, 5008 { 5009 name: "optional meta w/ bad meta", 5010 parameterizedJob: d5, 5011 dispatchReq: reqBadMeta, 5012 err: true, 5013 errStr: "unpermitted metadata keys", 5014 }, 5015 { 5016 name: "optional input w/ too big of input", 5017 parameterizedJob: d1, 5018 dispatchReq: reqInputDataTooLarge, 5019 err: true, 5020 errStr: "Payload exceeds maximum size", 5021 }, 5022 { 5023 name: "periodic job dispatched, ensure no eval", 5024 parameterizedJob: d6, 5025 dispatchReq: reqNoInputNoMeta, 5026 noEval: true, 5027 }, 5028 { 5029 name: "periodic job stopped, ensure error", 5030 parameterizedJob: d7, 5031 dispatchReq: reqNoInputNoMeta, 5032 err: true, 5033 errStr: "stopped", 5034 }, 5035 } 5036 5037 for _, tc := range cases { 5038 t.Run(tc.name, func(t *testing.T) { 5039 s1, cleanupS1 := TestServer(t, func(c *Config) { 5040 c.NumSchedulers = 0 // Prevent automatic dequeue 5041 }) 5042 defer cleanupS1() 5043 codec := rpcClient(t, s1) 5044 testutil.WaitForLeader(t, s1.RPC) 5045 5046 // Create the register request 5047 regReq := &structs.JobRegisterRequest{ 5048 Job: tc.parameterizedJob, 5049 WriteRequest: structs.WriteRequest{ 5050 Region: "global", 5051 Namespace: tc.parameterizedJob.Namespace, 5052 }, 5053 } 5054 5055 // Fetch the response 5056 var regResp structs.JobRegisterResponse 5057 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", regReq, ®Resp); err != nil { 5058 t.Fatalf("err: %v", err) 5059 } 5060 5061 // Now try to dispatch 5062 tc.dispatchReq.JobID = tc.parameterizedJob.ID 5063 tc.dispatchReq.WriteRequest = structs.WriteRequest{ 5064 Region: "global", 5065 Namespace: tc.parameterizedJob.Namespace, 5066 } 5067 5068 var dispatchResp structs.JobDispatchResponse 5069 dispatchErr := msgpackrpc.CallWithCodec(codec, "Job.Dispatch", tc.dispatchReq, &dispatchResp) 5070 5071 if dispatchErr == nil { 5072 if tc.err { 5073 t.Fatalf("Expected error: %v", dispatchErr) 5074 } 5075 5076 // Check that we got an eval and job id back 5077 switch dispatchResp.EvalID { 5078 case "": 5079 if !tc.noEval { 5080 t.Fatalf("Bad response") 5081 } 5082 default: 5083 if tc.noEval { 5084 t.Fatalf("Got eval %q", dispatchResp.EvalID) 5085 } 5086 } 5087 5088 if dispatchResp.DispatchedJobID == "" { 5089 t.Fatalf("Bad response") 5090 } 5091 5092 state := s1.fsm.State() 5093 ws := memdb.NewWatchSet() 5094 out, err := state.JobByID(ws, tc.parameterizedJob.Namespace, dispatchResp.DispatchedJobID) 5095 if err != nil { 5096 t.Fatalf("err: %v", err) 5097 } 5098 if out == nil { 5099 t.Fatalf("expected job") 5100 } 5101 if out.CreateIndex != dispatchResp.JobCreateIndex { 5102 t.Fatalf("index mis-match") 5103 } 5104 if out.ParentID != tc.parameterizedJob.ID { 5105 t.Fatalf("bad parent ID") 5106 } 5107 if !out.Dispatched { 5108 t.Fatal("expected dispatched job") 5109 } 5110 if out.IsParameterized() { 5111 t.Fatal("dispatched job should not be parameterized") 5112 } 5113 if out.ParameterizedJob == nil { 5114 t.Fatal("parameter job config should exist") 5115 } 5116 5117 if tc.noEval { 5118 return 5119 } 5120 5121 // Lookup the evaluation 5122 eval, err := state.EvalByID(ws, dispatchResp.EvalID) 5123 if err != nil { 5124 t.Fatalf("err: %v", err) 5125 } 5126 5127 if eval == nil { 5128 t.Fatalf("expected eval") 5129 } 5130 if eval.CreateIndex != dispatchResp.EvalCreateIndex { 5131 t.Fatalf("index mis-match") 5132 } 5133 } else { 5134 if !tc.err { 5135 t.Fatalf("Got unexpected error: %v", dispatchErr) 5136 } else if !strings.Contains(dispatchErr.Error(), tc.errStr) { 5137 t.Fatalf("Expected err to include %q; got %v", tc.errStr, dispatchErr) 5138 } 5139 } 5140 }) 5141 } 5142 }