github.com/adityamillind98/nomad@v0.11.8/nomad/job_endpoint_test.go (about) 1 package nomad 2 3 import ( 4 "errors" 5 "fmt" 6 "reflect" 7 "strings" 8 "testing" 9 "time" 10 11 memdb "github.com/hashicorp/go-memdb" 12 msgpackrpc "github.com/hashicorp/net-rpc-msgpackrpc" 13 "github.com/kr/pretty" 14 "github.com/stretchr/testify/require" 15 16 "github.com/hashicorp/nomad/acl" 17 "github.com/hashicorp/nomad/command/agent/consul" 18 "github.com/hashicorp/nomad/helper" 19 "github.com/hashicorp/nomad/helper/uuid" 20 "github.com/hashicorp/nomad/nomad/mock" 21 "github.com/hashicorp/nomad/nomad/structs" 22 "github.com/hashicorp/nomad/testutil" 23 ) 24 25 func TestJobEndpoint_Register(t *testing.T) { 26 t.Parallel() 27 28 s1, cleanupS1 := TestServer(t, func(c *Config) { 29 c.NumSchedulers = 0 // Prevent automatic dequeue 30 }) 31 defer cleanupS1() 32 codec := rpcClient(t, s1) 33 testutil.WaitForLeader(t, s1.RPC) 34 35 // Create the register request 36 job := mock.Job() 37 req := &structs.JobRegisterRequest{ 38 Job: job, 39 WriteRequest: structs.WriteRequest{ 40 Region: "global", 41 Namespace: job.Namespace, 42 }, 43 } 44 45 // Fetch the response 46 var resp structs.JobRegisterResponse 47 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { 48 t.Fatalf("err: %v", err) 49 } 50 if resp.Index == 0 { 51 t.Fatalf("bad index: %d", resp.Index) 52 } 53 54 // Check for the node in the FSM 55 state := s1.fsm.State() 56 ws := memdb.NewWatchSet() 57 out, err := state.JobByID(ws, job.Namespace, job.ID) 58 if err != nil { 59 t.Fatalf("err: %v", err) 60 } 61 if out == nil { 62 t.Fatalf("expected job") 63 } 64 if out.CreateIndex != resp.JobModifyIndex { 65 t.Fatalf("index mis-match") 66 } 67 serviceName := out.TaskGroups[0].Tasks[0].Services[0].Name 68 expectedServiceName := "web-frontend" 69 if serviceName != expectedServiceName { 70 t.Fatalf("Expected Service Name: %s, Actual: %s", expectedServiceName, serviceName) 71 } 72 73 // Lookup the evaluation 74 eval, err := state.EvalByID(ws, resp.EvalID) 75 if err != nil { 76 t.Fatalf("err: %v", err) 77 } 78 if eval == nil { 79 t.Fatalf("expected eval") 80 } 81 if eval.CreateIndex != resp.EvalCreateIndex { 82 t.Fatalf("index mis-match") 83 } 84 85 if eval.Priority != job.Priority { 86 t.Fatalf("bad: %#v", eval) 87 } 88 if eval.Type != job.Type { 89 t.Fatalf("bad: %#v", eval) 90 } 91 if eval.TriggeredBy != structs.EvalTriggerJobRegister { 92 t.Fatalf("bad: %#v", eval) 93 } 94 if eval.JobID != job.ID { 95 t.Fatalf("bad: %#v", eval) 96 } 97 if eval.JobModifyIndex != resp.JobModifyIndex { 98 t.Fatalf("bad: %#v", eval) 99 } 100 if eval.Status != structs.EvalStatusPending { 101 t.Fatalf("bad: %#v", eval) 102 } 103 if eval.CreateTime == 0 { 104 t.Fatalf("eval CreateTime is unset: %#v", eval) 105 } 106 if eval.ModifyTime == 0 { 107 t.Fatalf("eval ModifyTime is unset: %#v", eval) 108 } 109 } 110 111 func TestJobEndpoint_Register_Connect(t *testing.T) { 112 t.Parallel() 113 require := require.New(t) 114 115 s1, cleanupS1 := TestServer(t, func(c *Config) { 116 c.NumSchedulers = 0 // Prevent automatic dequeue 117 }) 118 defer cleanupS1() 119 codec := rpcClient(t, s1) 120 testutil.WaitForLeader(t, s1.RPC) 121 122 // Create the register request 123 job := mock.Job() 124 job.TaskGroups[0].Networks = structs.Networks{{ 125 Mode: "bridge", 126 }} 127 job.TaskGroups[0].Services = []*structs.Service{{ 128 Name: "backend", 129 PortLabel: "8080", 130 Connect: &structs.ConsulConnect{ 131 SidecarService: &structs.ConsulSidecarService{}, 132 }}, 133 } 134 req := &structs.JobRegisterRequest{ 135 Job: job, 136 WriteRequest: structs.WriteRequest{ 137 Region: "global", 138 Namespace: job.Namespace, 139 }, 140 } 141 142 // Fetch the response 143 var resp structs.JobRegisterResponse 144 require.NoError(msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp)) 145 require.NotZero(resp.Index) 146 147 // Check for the node in the FSM 148 state := s1.fsm.State() 149 ws := memdb.NewWatchSet() 150 out, err := state.JobByID(ws, job.Namespace, job.ID) 151 require.NoError(err) 152 require.NotNil(out) 153 require.Equal(resp.JobModifyIndex, out.CreateIndex) 154 155 // Check that the sidecar task was injected 156 require.Len(out.TaskGroups[0].Tasks, 2) 157 sidecarTask := out.TaskGroups[0].Tasks[1] 158 require.Equal("connect-proxy-backend", sidecarTask.Name) 159 require.Equal("connect-proxy:backend", string(sidecarTask.Kind)) 160 require.Equal("connect-proxy-backend", out.TaskGroups[0].Networks[0].DynamicPorts[0].Label) 161 162 // Check that round tripping the job doesn't change the sidecarTask 163 out.Meta["test"] = "abc" 164 req.Job = out 165 require.NoError(msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp)) 166 require.NotZero(resp.Index) 167 // Check for the new node in the FSM 168 state = s1.fsm.State() 169 ws = memdb.NewWatchSet() 170 out, err = state.JobByID(ws, job.Namespace, job.ID) 171 require.NoError(err) 172 require.NotNil(out) 173 require.Equal(resp.JobModifyIndex, out.CreateIndex) 174 175 require.Len(out.TaskGroups[0].Tasks, 2) 176 require.Exactly(sidecarTask, out.TaskGroups[0].Tasks[1]) 177 } 178 179 func TestJobEndpoint_Register_ConnectExposeCheck(t *testing.T) { 180 t.Parallel() 181 r := require.New(t) 182 183 s1, cleanupS1 := TestServer(t, func(c *Config) { 184 c.NumSchedulers = 0 // Prevent automatic dequeue 185 }) 186 defer cleanupS1() 187 codec := rpcClient(t, s1) 188 testutil.WaitForLeader(t, s1.RPC) 189 190 // Setup the job we are going to register 191 job := mock.Job() 192 job.TaskGroups[0].Networks = structs.Networks{{ 193 Mode: "bridge", 194 DynamicPorts: []structs.Port{{ 195 Label: "hcPort", 196 To: -1, 197 }, { 198 Label: "v2Port", 199 To: -1, 200 }}, 201 }} 202 job.TaskGroups[0].Services = []*structs.Service{{ 203 Name: "backend", 204 PortLabel: "8080", 205 Checks: []*structs.ServiceCheck{{ 206 Name: "check1", 207 Type: "http", 208 Protocol: "http", 209 Path: "/health", 210 Expose: true, 211 PortLabel: "hcPort", 212 Interval: 1 * time.Second, 213 Timeout: 1 * time.Second, 214 }, { 215 Name: "check2", 216 Type: "script", 217 Command: "/bin/true", 218 Interval: 1 * time.Second, 219 Timeout: 1 * time.Second, 220 }, { 221 Name: "check3", 222 Type: "grpc", 223 Protocol: "grpc", 224 Path: "/v2/health", 225 Expose: true, 226 PortLabel: "v2Port", 227 Interval: 1 * time.Second, 228 Timeout: 1 * time.Second, 229 }}, 230 Connect: &structs.ConsulConnect{ 231 SidecarService: &structs.ConsulSidecarService{}}, 232 }} 233 234 // Create the register request 235 req := &structs.JobRegisterRequest{ 236 Job: job, 237 WriteRequest: structs.WriteRequest{ 238 Region: "global", 239 Namespace: job.Namespace, 240 }, 241 } 242 243 // Fetch the response 244 var resp structs.JobRegisterResponse 245 r.NoError(msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp)) 246 r.NotZero(resp.Index) 247 248 // Check for the node in the FSM 249 state := s1.fsm.State() 250 ws := memdb.NewWatchSet() 251 out, err := state.JobByID(ws, job.Namespace, job.ID) 252 r.NoError(err) 253 r.NotNil(out) 254 r.Equal(resp.JobModifyIndex, out.CreateIndex) 255 256 // Check that the new expose paths got created 257 r.Len(out.TaskGroups[0].Services[0].Connect.SidecarService.Proxy.Expose.Paths, 2) 258 httpPath := out.TaskGroups[0].Services[0].Connect.SidecarService.Proxy.Expose.Paths[0] 259 r.Equal(structs.ConsulExposePath{ 260 Path: "/health", 261 Protocol: "http", 262 LocalPathPort: 8080, 263 ListenerPort: "hcPort", 264 }, httpPath) 265 grpcPath := out.TaskGroups[0].Services[0].Connect.SidecarService.Proxy.Expose.Paths[1] 266 r.Equal(structs.ConsulExposePath{ 267 Path: "/v2/health", 268 Protocol: "grpc", 269 LocalPathPort: 8080, 270 ListenerPort: "v2Port", 271 }, grpcPath) 272 273 // make sure round tripping does not create duplicate expose paths 274 out.Meta["test"] = "abc" 275 req.Job = out 276 r.NoError(msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp)) 277 r.NotZero(resp.Index) 278 279 // Check for the new node in the FSM 280 state = s1.fsm.State() 281 ws = memdb.NewWatchSet() 282 out, err = state.JobByID(ws, job.Namespace, job.ID) 283 r.NoError(err) 284 r.NotNil(out) 285 r.Equal(resp.JobModifyIndex, out.CreateIndex) 286 287 // make sure we are not re-adding what has already been added 288 r.Len(out.TaskGroups[0].Services[0].Connect.SidecarService.Proxy.Expose.Paths, 2) 289 } 290 291 func TestJobEndpoint_Register_ConnectWithSidecarTask(t *testing.T) { 292 t.Parallel() 293 require := require.New(t) 294 295 s1, cleanupS1 := TestServer(t, func(c *Config) { 296 c.NumSchedulers = 0 // Prevent automatic dequeue 297 }) 298 defer cleanupS1() 299 codec := rpcClient(t, s1) 300 testutil.WaitForLeader(t, s1.RPC) 301 302 // Create the register request 303 job := mock.Job() 304 job.TaskGroups[0].Networks = structs.Networks{ 305 { 306 Mode: "bridge", 307 }, 308 } 309 job.TaskGroups[0].Services = []*structs.Service{ 310 { 311 Name: "backend", 312 PortLabel: "8080", 313 Connect: &structs.ConsulConnect{ 314 SidecarService: &structs.ConsulSidecarService{}, 315 SidecarTask: &structs.SidecarTask{ 316 Meta: map[string]string{ 317 "source": "test", 318 }, 319 Resources: &structs.Resources{ 320 CPU: 500, 321 }, 322 Config: map[string]interface{}{ 323 "labels": map[string]string{ 324 "foo": "bar", 325 }, 326 }, 327 }, 328 }, 329 }, 330 } 331 req := &structs.JobRegisterRequest{ 332 Job: job, 333 WriteRequest: structs.WriteRequest{ 334 Region: "global", 335 Namespace: job.Namespace, 336 }, 337 } 338 339 // Fetch the response 340 var resp structs.JobRegisterResponse 341 require.NoError(msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp)) 342 require.NotZero(resp.Index) 343 344 // Check for the node in the FSM 345 state := s1.fsm.State() 346 ws := memdb.NewWatchSet() 347 out, err := state.JobByID(ws, job.Namespace, job.ID) 348 require.NoError(err) 349 require.NotNil(out) 350 require.Equal(resp.JobModifyIndex, out.CreateIndex) 351 352 // Check that the sidecar task was injected 353 require.Len(out.TaskGroups[0].Tasks, 2) 354 sidecarTask := out.TaskGroups[0].Tasks[1] 355 require.Equal("connect-proxy-backend", sidecarTask.Name) 356 require.Equal("connect-proxy:backend", string(sidecarTask.Kind)) 357 require.Equal("connect-proxy-backend", out.TaskGroups[0].Networks[0].DynamicPorts[0].Label) 358 359 // Check that the correct fields were overridden from the sidecar_task stanza 360 require.Equal("test", sidecarTask.Meta["source"]) 361 require.Equal(500, sidecarTask.Resources.CPU) 362 require.Equal(connectSidecarResources().MemoryMB, sidecarTask.Resources.MemoryMB) 363 cfg := connectDriverConfig 364 cfg["labels"] = map[string]interface{}{ 365 "foo": "bar", 366 } 367 require.Equal(cfg, sidecarTask.Config) 368 369 // Check that round tripping the job doesn't change the sidecarTask 370 out.Meta["test"] = "abc" 371 req.Job = out 372 require.NoError(msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp)) 373 require.NotZero(resp.Index) 374 // Check for the new node in the FSM 375 state = s1.fsm.State() 376 ws = memdb.NewWatchSet() 377 out, err = state.JobByID(ws, job.Namespace, job.ID) 378 require.NoError(err) 379 require.NotNil(out) 380 require.Equal(resp.JobModifyIndex, out.CreateIndex) 381 382 require.Len(out.TaskGroups[0].Tasks, 2) 383 require.Exactly(sidecarTask, out.TaskGroups[0].Tasks[1]) 384 385 } 386 387 // TestJobEndpoint_Register_Connect_AllowUnauthenticatedFalse asserts that a job 388 // submission fails allow_unauthenticated is false, and either an invalid or no 389 // operator Consul token is provided. 390 func TestJobEndpoint_Register_Connect_AllowUnauthenticatedFalse(t *testing.T) { 391 t.Parallel() 392 393 s1, cleanupS1 := TestServer(t, func(c *Config) { 394 c.NumSchedulers = 0 // Prevent automatic dequeue 395 c.ConsulConfig.AllowUnauthenticated = helper.BoolToPtr(false) 396 }) 397 defer cleanupS1() 398 codec := rpcClient(t, s1) 399 testutil.WaitForLeader(t, s1.RPC) 400 401 // Create the register request 402 job := mock.Job() 403 job.TaskGroups[0].Networks = structs.Networks{ 404 { 405 Mode: "bridge", 406 }, 407 } 408 job.TaskGroups[0].Services = []*structs.Service{ 409 { 410 Name: "service1", // matches consul.ExamplePolicyID1 411 PortLabel: "8080", 412 Connect: &structs.ConsulConnect{ 413 SidecarService: &structs.ConsulSidecarService{}, 414 }, 415 }, 416 } 417 418 newRequest := func(job *structs.Job) *structs.JobRegisterRequest { 419 return &structs.JobRegisterRequest{ 420 Job: job, 421 WriteRequest: structs.WriteRequest{ 422 Region: "global", 423 Namespace: job.Namespace, 424 }, 425 } 426 } 427 428 noTokenOnJob := func(t *testing.T) { 429 fsmState := s1.State() 430 ws := memdb.NewWatchSet() 431 storedJob, err := fsmState.JobByID(ws, job.Namespace, job.ID) 432 require.NoError(t, err) 433 require.NotNil(t, storedJob) 434 require.Empty(t, storedJob.ConsulToken) 435 } 436 437 // Each variation of the provided Consul operator token 438 noOpToken := "" 439 unrecognizedOpToken := uuid.Generate() 440 unauthorizedOpToken := consul.ExampleOperatorTokenID3 441 authorizedOpToken := consul.ExampleOperatorTokenID1 442 443 t.Run("no token provided", func(t *testing.T) { 444 request := newRequest(job) 445 request.Job.ConsulToken = noOpToken 446 var response structs.JobRegisterResponse 447 err := msgpackrpc.CallWithCodec(codec, "Job.Register", request, &response) 448 require.EqualError(t, err, "operator token denied: missing consul token") 449 }) 450 451 t.Run("unknown token provided", func(t *testing.T) { 452 request := newRequest(job) 453 request.Job.ConsulToken = unrecognizedOpToken 454 var response structs.JobRegisterResponse 455 err := msgpackrpc.CallWithCodec(codec, "Job.Register", request, &response) 456 require.EqualError(t, err, "operator token denied: unable to validate operator consul token: no such token") 457 }) 458 459 t.Run("unauthorized token provided", func(t *testing.T) { 460 request := newRequest(job) 461 request.Job.ConsulToken = unauthorizedOpToken 462 var response structs.JobRegisterResponse 463 err := msgpackrpc.CallWithCodec(codec, "Job.Register", request, &response) 464 require.EqualError(t, err, "operator token denied: permission denied for \"service1\"") 465 }) 466 467 t.Run("authorized token provided", func(t *testing.T) { 468 request := newRequest(job) 469 request.Job.ConsulToken = authorizedOpToken 470 var response structs.JobRegisterResponse 471 err := msgpackrpc.CallWithCodec(codec, "Job.Register", request, &response) 472 require.NoError(t, err) 473 noTokenOnJob(t) 474 }) 475 } 476 477 func TestJobEndpoint_Register_ACL(t *testing.T) { 478 t.Parallel() 479 480 s1, root, cleanupS1 := TestACLServer(t, func(c *Config) { 481 c.NumSchedulers = 0 // Prevent automatic dequeue 482 }) 483 defer cleanupS1() 484 testutil.WaitForLeader(t, s1.RPC) 485 486 newVolumeJob := func(readonlyVolume bool) *structs.Job { 487 j := mock.Job() 488 tg := j.TaskGroups[0] 489 tg.Volumes = map[string]*structs.VolumeRequest{ 490 "ca-certs": { 491 Type: structs.VolumeTypeHost, 492 Source: "prod-ca-certs", 493 ReadOnly: readonlyVolume, 494 }, 495 "csi": { 496 Type: structs.VolumeTypeCSI, 497 Source: "prod-db", 498 }, 499 } 500 501 tg.Tasks[0].VolumeMounts = []*structs.VolumeMount{ 502 { 503 Volume: "ca-certs", 504 Destination: "/etc/ca-certificates", 505 // Task readonly does not effect acls 506 ReadOnly: true, 507 }, 508 } 509 510 return j 511 } 512 513 newCSIPluginJob := func() *structs.Job { 514 j := mock.Job() 515 t := j.TaskGroups[0].Tasks[0] 516 t.CSIPluginConfig = &structs.TaskCSIPluginConfig{ 517 ID: "foo", 518 Type: "node", 519 } 520 return j 521 } 522 523 submitJobPolicy := mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob, acl.NamespaceCapabilitySubmitJob}) 524 525 submitJobToken := mock.CreatePolicyAndToken(t, s1.State(), 1001, "test-submit-job", submitJobPolicy) 526 527 volumesPolicyReadWrite := mock.HostVolumePolicy("prod-*", "", []string{acl.HostVolumeCapabilityMountReadWrite}) 528 529 volumesPolicyCSIMount := mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityCSIMountVolume}) + 530 mock.PluginPolicy("read") 531 532 submitJobWithVolumesReadWriteToken := mock.CreatePolicyAndToken(t, s1.State(), 1002, "test-submit-volumes", submitJobPolicy+ 533 volumesPolicyReadWrite+ 534 volumesPolicyCSIMount) 535 536 volumesPolicyReadOnly := mock.HostVolumePolicy("prod-*", "", []string{acl.HostVolumeCapabilityMountReadOnly}) 537 538 submitJobWithVolumesReadOnlyToken := mock.CreatePolicyAndToken(t, s1.State(), 1003, "test-submit-volumes-readonly", submitJobPolicy+ 539 volumesPolicyReadOnly+ 540 volumesPolicyCSIMount) 541 542 pluginPolicy := mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityCSIRegisterPlugin}) 543 pluginToken := mock.CreatePolicyAndToken(t, s1.State(), 1005, "test-csi-register-plugin", submitJobPolicy+pluginPolicy) 544 545 cases := []struct { 546 Name string 547 Job *structs.Job 548 Token string 549 ErrExpected bool 550 }{ 551 { 552 Name: "without a token", 553 Job: mock.Job(), 554 Token: "", 555 ErrExpected: true, 556 }, 557 { 558 Name: "with a token", 559 Job: mock.Job(), 560 Token: root.SecretID, 561 ErrExpected: false, 562 }, 563 { 564 Name: "with a token that can submit a job, but not use a required volume", 565 Job: newVolumeJob(false), 566 Token: submitJobToken.SecretID, 567 ErrExpected: true, 568 }, 569 { 570 Name: "with a token that can submit a job, and use all required volumes", 571 Job: newVolumeJob(false), 572 Token: submitJobWithVolumesReadWriteToken.SecretID, 573 ErrExpected: false, 574 }, 575 { 576 Name: "with a token that can submit a job, but only has readonly access", 577 Job: newVolumeJob(false), 578 Token: submitJobWithVolumesReadOnlyToken.SecretID, 579 ErrExpected: true, 580 }, 581 { 582 Name: "with a token that can submit a job, and readonly volume access is enough", 583 Job: newVolumeJob(true), 584 Token: submitJobWithVolumesReadOnlyToken.SecretID, 585 ErrExpected: false, 586 }, 587 { 588 Name: "with a token that can submit a job, plugin rejected", 589 Job: newCSIPluginJob(), 590 Token: submitJobToken.SecretID, 591 ErrExpected: true, 592 }, 593 { 594 Name: "with a token that also has csi-register-plugin, accepted", 595 Job: newCSIPluginJob(), 596 Token: pluginToken.SecretID, 597 ErrExpected: false, 598 }, 599 } 600 601 for _, tt := range cases { 602 t.Run(tt.Name, func(t *testing.T) { 603 codec := rpcClient(t, s1) 604 req := &structs.JobRegisterRequest{ 605 Job: tt.Job, 606 WriteRequest: structs.WriteRequest{ 607 Region: "global", 608 Namespace: tt.Job.Namespace, 609 }, 610 } 611 req.AuthToken = tt.Token 612 613 // Try without a token, expect failure 614 var resp structs.JobRegisterResponse 615 err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp) 616 617 // If we expected an error, then the job should _not_ be registered. 618 if tt.ErrExpected { 619 require.Error(t, err, "expected error") 620 return 621 } 622 623 if !tt.ErrExpected { 624 require.NoError(t, err, "unexpected error") 625 } 626 627 require.NotEqual(t, 0, resp.Index) 628 629 state := s1.fsm.State() 630 ws := memdb.NewWatchSet() 631 out, err := state.JobByID(ws, tt.Job.Namespace, tt.Job.ID) 632 require.NoError(t, err) 633 require.NotNil(t, out) 634 require.Equal(t, tt.Job.TaskGroups, out.TaskGroups) 635 }) 636 } 637 } 638 639 func TestJobEndpoint_Register_InvalidNamespace(t *testing.T) { 640 t.Parallel() 641 642 s1, cleanupS1 := TestServer(t, func(c *Config) { 643 c.NumSchedulers = 0 // Prevent automatic dequeue 644 }) 645 defer cleanupS1() 646 codec := rpcClient(t, s1) 647 testutil.WaitForLeader(t, s1.RPC) 648 649 // Create the register request 650 job := mock.Job() 651 job.Namespace = "foo" 652 req := &structs.JobRegisterRequest{ 653 Job: job, 654 WriteRequest: structs.WriteRequest{ 655 Region: "global", 656 Namespace: job.Namespace, 657 }, 658 } 659 660 // Try without a token, expect failure 661 var resp structs.JobRegisterResponse 662 err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp) 663 if err == nil || !strings.Contains(err.Error(), "nonexistent namespace") { 664 t.Fatalf("expected namespace error: %v", err) 665 } 666 667 // Check for the job in the FSM 668 state := s1.fsm.State() 669 ws := memdb.NewWatchSet() 670 out, err := state.JobByID(ws, job.Namespace, job.ID) 671 if err != nil { 672 t.Fatalf("err: %v", err) 673 } 674 if out != nil { 675 t.Fatalf("expected no job") 676 } 677 } 678 679 func TestJobEndpoint_Register_Payload(t *testing.T) { 680 t.Parallel() 681 682 s1, cleanupS1 := TestServer(t, func(c *Config) { 683 c.NumSchedulers = 0 // Prevent automatic dequeue 684 }) 685 defer cleanupS1() 686 codec := rpcClient(t, s1) 687 testutil.WaitForLeader(t, s1.RPC) 688 689 // Create the register request with a job containing an invalid driver 690 // config 691 job := mock.Job() 692 job.Payload = []byte{0x1} 693 req := &structs.JobRegisterRequest{ 694 Job: job, 695 WriteRequest: structs.WriteRequest{ 696 Region: "global", 697 Namespace: job.Namespace, 698 }, 699 } 700 701 // Fetch the response 702 var resp structs.JobRegisterResponse 703 err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp) 704 if err == nil { 705 t.Fatalf("expected a validation error") 706 } 707 708 if !strings.Contains(err.Error(), "payload") { 709 t.Fatalf("expected a payload error but got: %v", err) 710 } 711 } 712 713 func TestJobEndpoint_Register_Existing(t *testing.T) { 714 t.Parallel() 715 716 s1, cleanupS1 := TestServer(t, func(c *Config) { 717 c.NumSchedulers = 0 // Prevent automatic dequeue 718 }) 719 defer cleanupS1() 720 codec := rpcClient(t, s1) 721 testutil.WaitForLeader(t, s1.RPC) 722 723 // Create the register request 724 job := mock.Job() 725 req := &structs.JobRegisterRequest{ 726 Job: job, 727 WriteRequest: structs.WriteRequest{ 728 Region: "global", 729 Namespace: job.Namespace, 730 }, 731 } 732 733 // Fetch the response 734 var resp structs.JobRegisterResponse 735 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { 736 t.Fatalf("err: %v", err) 737 } 738 if resp.Index == 0 { 739 t.Fatalf("bad index: %d", resp.Index) 740 } 741 742 // Update the job definition 743 job2 := mock.Job() 744 job2.Priority = 100 745 job2.ID = job.ID 746 req.Job = job2 747 748 // Attempt update 749 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { 750 t.Fatalf("err: %v", err) 751 } 752 if resp.Index == 0 { 753 t.Fatalf("bad index: %d", resp.Index) 754 } 755 756 // Check for the node in the FSM 757 state := s1.fsm.State() 758 ws := memdb.NewWatchSet() 759 out, err := state.JobByID(ws, job.Namespace, job.ID) 760 if err != nil { 761 t.Fatalf("err: %v", err) 762 } 763 if out == nil { 764 t.Fatalf("expected job") 765 } 766 if out.ModifyIndex != resp.JobModifyIndex { 767 t.Fatalf("index mis-match") 768 } 769 if out.Priority != 100 { 770 t.Fatalf("expected update") 771 } 772 if out.Version != 1 { 773 t.Fatalf("expected update") 774 } 775 776 // Lookup the evaluation 777 eval, err := state.EvalByID(ws, resp.EvalID) 778 if err != nil { 779 t.Fatalf("err: %v", err) 780 } 781 if eval == nil { 782 t.Fatalf("expected eval") 783 } 784 if eval.CreateIndex != resp.EvalCreateIndex { 785 t.Fatalf("index mis-match") 786 } 787 788 if eval.Priority != job2.Priority { 789 t.Fatalf("bad: %#v", eval) 790 } 791 if eval.Type != job2.Type { 792 t.Fatalf("bad: %#v", eval) 793 } 794 if eval.TriggeredBy != structs.EvalTriggerJobRegister { 795 t.Fatalf("bad: %#v", eval) 796 } 797 if eval.JobID != job2.ID { 798 t.Fatalf("bad: %#v", eval) 799 } 800 if eval.JobModifyIndex != resp.JobModifyIndex { 801 t.Fatalf("bad: %#v", eval) 802 } 803 if eval.Status != structs.EvalStatusPending { 804 t.Fatalf("bad: %#v", eval) 805 } 806 if eval.CreateTime == 0 { 807 t.Fatalf("eval CreateTime is unset: %#v", eval) 808 } 809 if eval.ModifyTime == 0 { 810 t.Fatalf("eval ModifyTime is unset: %#v", eval) 811 } 812 813 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { 814 t.Fatalf("err: %v", err) 815 } 816 if resp.Index == 0 { 817 t.Fatalf("bad index: %d", resp.Index) 818 } 819 820 // Check to ensure the job version didn't get bumped because we submitted 821 // the same job 822 state = s1.fsm.State() 823 ws = memdb.NewWatchSet() 824 out, err = state.JobByID(ws, job.Namespace, job.ID) 825 if err != nil { 826 t.Fatalf("err: %v", err) 827 } 828 if out == nil { 829 t.Fatalf("expected job") 830 } 831 if out.Version != 1 { 832 t.Fatalf("expected no update; got %v; diff %v", out.Version, pretty.Diff(job2, out)) 833 } 834 } 835 836 func TestJobEndpoint_Register_Periodic(t *testing.T) { 837 t.Parallel() 838 839 s1, cleanupS1 := TestServer(t, func(c *Config) { 840 c.NumSchedulers = 0 // Prevent automatic dequeue 841 }) 842 defer cleanupS1() 843 codec := rpcClient(t, s1) 844 testutil.WaitForLeader(t, s1.RPC) 845 846 // Create the register request for a periodic job. 847 job := mock.PeriodicJob() 848 req := &structs.JobRegisterRequest{ 849 Job: job, 850 WriteRequest: structs.WriteRequest{ 851 Region: "global", 852 Namespace: job.Namespace, 853 }, 854 } 855 856 // Fetch the response 857 var resp structs.JobRegisterResponse 858 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { 859 t.Fatalf("err: %v", err) 860 } 861 if resp.JobModifyIndex == 0 { 862 t.Fatalf("bad index: %d", resp.Index) 863 } 864 865 // Check for the node in the FSM 866 state := s1.fsm.State() 867 ws := memdb.NewWatchSet() 868 out, err := state.JobByID(ws, job.Namespace, job.ID) 869 if err != nil { 870 t.Fatalf("err: %v", err) 871 } 872 if out == nil { 873 t.Fatalf("expected job") 874 } 875 if out.CreateIndex != resp.JobModifyIndex { 876 t.Fatalf("index mis-match") 877 } 878 serviceName := out.TaskGroups[0].Tasks[0].Services[0].Name 879 expectedServiceName := "web-frontend" 880 if serviceName != expectedServiceName { 881 t.Fatalf("Expected Service Name: %s, Actual: %s", expectedServiceName, serviceName) 882 } 883 884 if resp.EvalID != "" { 885 t.Fatalf("Register created an eval for a periodic job") 886 } 887 } 888 889 func TestJobEndpoint_Register_ParameterizedJob(t *testing.T) { 890 t.Parallel() 891 892 s1, cleanupS1 := TestServer(t, func(c *Config) { 893 c.NumSchedulers = 0 // Prevent automatic dequeue 894 }) 895 defer cleanupS1() 896 codec := rpcClient(t, s1) 897 testutil.WaitForLeader(t, s1.RPC) 898 899 // Create the register request for a parameterized job. 900 job := mock.BatchJob() 901 job.ParameterizedJob = &structs.ParameterizedJobConfig{} 902 req := &structs.JobRegisterRequest{ 903 Job: job, 904 WriteRequest: structs.WriteRequest{ 905 Region: "global", 906 Namespace: job.Namespace, 907 }, 908 } 909 910 // Fetch the response 911 var resp structs.JobRegisterResponse 912 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { 913 t.Fatalf("err: %v", err) 914 } 915 if resp.JobModifyIndex == 0 { 916 t.Fatalf("bad index: %d", resp.Index) 917 } 918 919 // Check for the job in the FSM 920 state := s1.fsm.State() 921 ws := memdb.NewWatchSet() 922 out, err := state.JobByID(ws, job.Namespace, job.ID) 923 if err != nil { 924 t.Fatalf("err: %v", err) 925 } 926 if out == nil { 927 t.Fatalf("expected job") 928 } 929 if out.CreateIndex != resp.JobModifyIndex { 930 t.Fatalf("index mis-match") 931 } 932 if resp.EvalID != "" { 933 t.Fatalf("Register created an eval for a parameterized job") 934 } 935 } 936 937 func TestJobEndpoint_Register_Dispatched(t *testing.T) { 938 t.Parallel() 939 require := require.New(t) 940 941 s1, cleanupS1 := TestServer(t, func(c *Config) { 942 c.NumSchedulers = 0 // Prevent automatic dequeue 943 }) 944 defer cleanupS1() 945 codec := rpcClient(t, s1) 946 testutil.WaitForLeader(t, s1.RPC) 947 948 // Create the register request with a job with 'Dispatch' set to true 949 job := mock.Job() 950 job.Dispatched = true 951 req := &structs.JobRegisterRequest{ 952 Job: job, 953 WriteRequest: structs.WriteRequest{ 954 Region: "global", 955 Namespace: job.Namespace, 956 }, 957 } 958 959 // Fetch the response 960 var resp structs.JobRegisterResponse 961 err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp) 962 require.Error(err) 963 require.Contains(err.Error(), "job can't be submitted with 'Dispatched'") 964 } 965 966 func TestJobEndpoint_Register_EnforceIndex(t *testing.T) { 967 t.Parallel() 968 969 s1, cleanupS1 := TestServer(t, func(c *Config) { 970 c.NumSchedulers = 0 // Prevent automatic dequeue 971 }) 972 defer cleanupS1() 973 codec := rpcClient(t, s1) 974 testutil.WaitForLeader(t, s1.RPC) 975 976 // Create the register request and enforcing an incorrect index 977 job := mock.Job() 978 req := &structs.JobRegisterRequest{ 979 Job: job, 980 EnforceIndex: true, 981 JobModifyIndex: 100, // Not registered yet so not possible 982 WriteRequest: structs.WriteRequest{ 983 Region: "global", 984 Namespace: job.Namespace, 985 }, 986 } 987 988 // Fetch the response 989 var resp structs.JobRegisterResponse 990 err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp) 991 if err == nil || !strings.Contains(err.Error(), RegisterEnforceIndexErrPrefix) { 992 t.Fatalf("expected enforcement error") 993 } 994 995 // Create the register request and enforcing it is new 996 req = &structs.JobRegisterRequest{ 997 Job: job, 998 EnforceIndex: true, 999 JobModifyIndex: 0, 1000 WriteRequest: structs.WriteRequest{ 1001 Region: "global", 1002 Namespace: job.Namespace, 1003 }, 1004 } 1005 1006 // Fetch the response 1007 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { 1008 t.Fatalf("err: %v", err) 1009 } 1010 if resp.Index == 0 { 1011 t.Fatalf("bad index: %d", resp.Index) 1012 } 1013 1014 curIndex := resp.JobModifyIndex 1015 1016 // Check for the node in the FSM 1017 state := s1.fsm.State() 1018 ws := memdb.NewWatchSet() 1019 out, err := state.JobByID(ws, job.Namespace, job.ID) 1020 if err != nil { 1021 t.Fatalf("err: %v", err) 1022 } 1023 if out == nil { 1024 t.Fatalf("expected job") 1025 } 1026 if out.CreateIndex != resp.JobModifyIndex { 1027 t.Fatalf("index mis-match") 1028 } 1029 1030 // Reregister request and enforcing it be a new job 1031 req = &structs.JobRegisterRequest{ 1032 Job: job, 1033 EnforceIndex: true, 1034 JobModifyIndex: 0, 1035 WriteRequest: structs.WriteRequest{ 1036 Region: "global", 1037 Namespace: job.Namespace, 1038 }, 1039 } 1040 1041 // Fetch the response 1042 err = msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp) 1043 if err == nil || !strings.Contains(err.Error(), RegisterEnforceIndexErrPrefix) { 1044 t.Fatalf("expected enforcement error") 1045 } 1046 1047 // Reregister request and enforcing it be at an incorrect index 1048 req = &structs.JobRegisterRequest{ 1049 Job: job, 1050 EnforceIndex: true, 1051 JobModifyIndex: curIndex - 1, 1052 WriteRequest: structs.WriteRequest{ 1053 Region: "global", 1054 Namespace: job.Namespace, 1055 }, 1056 } 1057 1058 // Fetch the response 1059 err = msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp) 1060 if err == nil || !strings.Contains(err.Error(), RegisterEnforceIndexErrPrefix) { 1061 t.Fatalf("expected enforcement error") 1062 } 1063 1064 // Reregister request and enforcing it be at the correct index 1065 job.Priority = job.Priority + 1 1066 req = &structs.JobRegisterRequest{ 1067 Job: job, 1068 EnforceIndex: true, 1069 JobModifyIndex: curIndex, 1070 WriteRequest: structs.WriteRequest{ 1071 Region: "global", 1072 Namespace: job.Namespace, 1073 }, 1074 } 1075 1076 // Fetch the response 1077 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { 1078 t.Fatalf("err: %v", err) 1079 } 1080 if resp.Index == 0 { 1081 t.Fatalf("bad index: %d", resp.Index) 1082 } 1083 1084 out, err = state.JobByID(ws, job.Namespace, job.ID) 1085 if err != nil { 1086 t.Fatalf("err: %v", err) 1087 } 1088 if out == nil { 1089 t.Fatalf("expected job") 1090 } 1091 if out.Priority != job.Priority { 1092 t.Fatalf("priority mis-match") 1093 } 1094 } 1095 1096 // TestJobEndpoint_Register_Vault_Disabled asserts that submitting a job that 1097 // uses Vault when Vault is *disabled* results in an error. 1098 func TestJobEndpoint_Register_Vault_Disabled(t *testing.T) { 1099 t.Parallel() 1100 1101 s1, cleanupS1 := TestServer(t, func(c *Config) { 1102 c.NumSchedulers = 0 // Prevent automatic dequeue 1103 f := false 1104 c.VaultConfig.Enabled = &f 1105 }) 1106 defer cleanupS1() 1107 codec := rpcClient(t, s1) 1108 testutil.WaitForLeader(t, s1.RPC) 1109 1110 // Create the register request with a job asking for a vault policy 1111 job := mock.Job() 1112 job.TaskGroups[0].Tasks[0].Vault = &structs.Vault{ 1113 Policies: []string{"foo"}, 1114 ChangeMode: structs.VaultChangeModeRestart, 1115 } 1116 req := &structs.JobRegisterRequest{ 1117 Job: job, 1118 WriteRequest: structs.WriteRequest{ 1119 Region: "global", 1120 Namespace: job.Namespace, 1121 }, 1122 } 1123 1124 // Fetch the response 1125 var resp structs.JobRegisterResponse 1126 err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp) 1127 if err == nil || !strings.Contains(err.Error(), "Vault not enabled") { 1128 t.Fatalf("expected Vault not enabled error: %v", err) 1129 } 1130 } 1131 1132 // TestJobEndpoint_Register_Vault_AllowUnauthenticated asserts submitting a job 1133 // with a Vault policy but without a Vault token is *succeeds* if 1134 // allow_unauthenticated=true. 1135 func TestJobEndpoint_Register_Vault_AllowUnauthenticated(t *testing.T) { 1136 t.Parallel() 1137 1138 s1, cleanupS1 := TestServer(t, func(c *Config) { 1139 c.NumSchedulers = 0 // Prevent automatic dequeue 1140 }) 1141 defer cleanupS1() 1142 codec := rpcClient(t, s1) 1143 testutil.WaitForLeader(t, s1.RPC) 1144 1145 // Enable vault and allow authenticated 1146 tr := true 1147 s1.config.VaultConfig.Enabled = &tr 1148 s1.config.VaultConfig.AllowUnauthenticated = &tr 1149 1150 // Replace the Vault Client on the server 1151 s1.vault = &TestVaultClient{} 1152 1153 // Create the register request with a job asking for a vault policy 1154 job := mock.Job() 1155 job.TaskGroups[0].Tasks[0].Vault = &structs.Vault{ 1156 Policies: []string{"foo"}, 1157 ChangeMode: structs.VaultChangeModeRestart, 1158 } 1159 req := &structs.JobRegisterRequest{ 1160 Job: job, 1161 WriteRequest: structs.WriteRequest{ 1162 Region: "global", 1163 Namespace: job.Namespace, 1164 }, 1165 } 1166 1167 // Fetch the response 1168 var resp structs.JobRegisterResponse 1169 err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp) 1170 if err != nil { 1171 t.Fatalf("bad: %v", err) 1172 } 1173 1174 // Check for the job in the FSM 1175 state := s1.fsm.State() 1176 ws := memdb.NewWatchSet() 1177 out, err := state.JobByID(ws, job.Namespace, job.ID) 1178 if err != nil { 1179 t.Fatalf("err: %v", err) 1180 } 1181 if out == nil { 1182 t.Fatalf("expected job") 1183 } 1184 if out.CreateIndex != resp.JobModifyIndex { 1185 t.Fatalf("index mis-match") 1186 } 1187 } 1188 1189 // TestJobEndpoint_Register_Vault_OverrideConstraint asserts that job 1190 // submitters can specify their own Vault constraint to override the 1191 // automatically injected one. 1192 func TestJobEndpoint_Register_Vault_OverrideConstraint(t *testing.T) { 1193 t.Parallel() 1194 1195 s1, cleanupS1 := TestServer(t, func(c *Config) { 1196 c.NumSchedulers = 0 // Prevent automatic dequeue 1197 }) 1198 defer cleanupS1() 1199 codec := rpcClient(t, s1) 1200 testutil.WaitForLeader(t, s1.RPC) 1201 1202 // Enable vault and allow authenticated 1203 tr := true 1204 s1.config.VaultConfig.Enabled = &tr 1205 s1.config.VaultConfig.AllowUnauthenticated = &tr 1206 1207 // Replace the Vault Client on the server 1208 s1.vault = &TestVaultClient{} 1209 1210 // Create the register request with a job asking for a vault policy 1211 job := mock.Job() 1212 job.TaskGroups[0].Tasks[0].Vault = &structs.Vault{ 1213 Policies: []string{"foo"}, 1214 ChangeMode: structs.VaultChangeModeRestart, 1215 } 1216 job.TaskGroups[0].Tasks[0].Constraints = []*structs.Constraint{ 1217 { 1218 LTarget: "${attr.vault.version}", 1219 Operand: "is_set", 1220 }, 1221 } 1222 req := &structs.JobRegisterRequest{ 1223 Job: job, 1224 WriteRequest: structs.WriteRequest{ 1225 Region: "global", 1226 Namespace: job.Namespace, 1227 }, 1228 } 1229 1230 // Fetch the response 1231 var resp structs.JobRegisterResponse 1232 err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp) 1233 require.NoError(t, err) 1234 1235 // Check for the job in the FSM 1236 state := s1.fsm.State() 1237 ws := memdb.NewWatchSet() 1238 out, err := state.JobByID(ws, job.Namespace, job.ID) 1239 require.NoError(t, err) 1240 require.NotNil(t, out) 1241 require.Equal(t, resp.JobModifyIndex, out.CreateIndex) 1242 1243 // Assert constraint was not overridden by the server 1244 outConstraints := out.TaskGroups[0].Tasks[0].Constraints 1245 require.Len(t, outConstraints, 1) 1246 require.True(t, job.TaskGroups[0].Tasks[0].Constraints[0].Equals(outConstraints[0])) 1247 } 1248 1249 func TestJobEndpoint_Register_Vault_NoToken(t *testing.T) { 1250 t.Parallel() 1251 1252 s1, cleanupS1 := TestServer(t, func(c *Config) { 1253 c.NumSchedulers = 0 // Prevent automatic dequeue 1254 }) 1255 defer cleanupS1() 1256 codec := rpcClient(t, s1) 1257 testutil.WaitForLeader(t, s1.RPC) 1258 1259 // Enable vault 1260 tr, f := true, false 1261 s1.config.VaultConfig.Enabled = &tr 1262 s1.config.VaultConfig.AllowUnauthenticated = &f 1263 1264 // Replace the Vault Client on the server 1265 s1.vault = &TestVaultClient{} 1266 1267 // Create the register request with a job asking for a vault policy but 1268 // don't send a Vault token 1269 job := mock.Job() 1270 job.TaskGroups[0].Tasks[0].Vault = &structs.Vault{ 1271 Policies: []string{"foo"}, 1272 ChangeMode: structs.VaultChangeModeRestart, 1273 } 1274 req := &structs.JobRegisterRequest{ 1275 Job: job, 1276 WriteRequest: structs.WriteRequest{ 1277 Region: "global", 1278 Namespace: job.Namespace, 1279 }, 1280 } 1281 1282 // Fetch the response 1283 var resp structs.JobRegisterResponse 1284 err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp) 1285 if err == nil || !strings.Contains(err.Error(), "missing Vault Token") { 1286 t.Fatalf("expected Vault not enabled error: %v", err) 1287 } 1288 } 1289 1290 func TestJobEndpoint_Register_Vault_Policies(t *testing.T) { 1291 t.Parallel() 1292 1293 s1, cleanupS1 := TestServer(t, func(c *Config) { 1294 c.NumSchedulers = 0 // Prevent automatic dequeue 1295 }) 1296 defer cleanupS1() 1297 codec := rpcClient(t, s1) 1298 testutil.WaitForLeader(t, s1.RPC) 1299 1300 // Enable vault 1301 tr, f := true, false 1302 s1.config.VaultConfig.Enabled = &tr 1303 s1.config.VaultConfig.AllowUnauthenticated = &f 1304 1305 // Replace the Vault Client on the server 1306 tvc := &TestVaultClient{} 1307 s1.vault = tvc 1308 1309 // Add three tokens: one that allows the requesting policy, one that does 1310 // not and one that returns an error 1311 policy := "foo" 1312 1313 badToken := uuid.Generate() 1314 badPolicies := []string{"a", "b", "c"} 1315 tvc.SetLookupTokenAllowedPolicies(badToken, badPolicies) 1316 1317 goodToken := uuid.Generate() 1318 goodPolicies := []string{"foo", "bar", "baz"} 1319 tvc.SetLookupTokenAllowedPolicies(goodToken, goodPolicies) 1320 1321 rootToken := uuid.Generate() 1322 rootPolicies := []string{"root"} 1323 tvc.SetLookupTokenAllowedPolicies(rootToken, rootPolicies) 1324 1325 errToken := uuid.Generate() 1326 expectedErr := fmt.Errorf("return errors from vault") 1327 tvc.SetLookupTokenError(errToken, expectedErr) 1328 1329 // Create the register request with a job asking for a vault policy but 1330 // send the bad Vault token 1331 job := mock.Job() 1332 job.VaultToken = badToken 1333 job.TaskGroups[0].Tasks[0].Vault = &structs.Vault{ 1334 Policies: []string{policy}, 1335 ChangeMode: structs.VaultChangeModeRestart, 1336 } 1337 req := &structs.JobRegisterRequest{ 1338 Job: job, 1339 WriteRequest: structs.WriteRequest{ 1340 Region: "global", 1341 Namespace: job.Namespace, 1342 }, 1343 } 1344 1345 // Fetch the response 1346 var resp structs.JobRegisterResponse 1347 err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp) 1348 if err == nil || !strings.Contains(err.Error(), 1349 "doesn't allow access to the following policies: "+policy) { 1350 t.Fatalf("expected permission denied error: %v", err) 1351 } 1352 1353 // Use the err token 1354 job.VaultToken = errToken 1355 err = msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp) 1356 if err == nil || !strings.Contains(err.Error(), expectedErr.Error()) { 1357 t.Fatalf("expected permission denied error: %v", err) 1358 } 1359 1360 // Use the good token 1361 job.VaultToken = goodToken 1362 1363 // Fetch the response 1364 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { 1365 t.Fatalf("bad: %v", err) 1366 } 1367 1368 // Check for the job in the FSM 1369 state := s1.fsm.State() 1370 ws := memdb.NewWatchSet() 1371 out, err := state.JobByID(ws, job.Namespace, job.ID) 1372 if err != nil { 1373 t.Fatalf("err: %v", err) 1374 } 1375 if out == nil { 1376 t.Fatalf("expected job") 1377 } 1378 if out.CreateIndex != resp.JobModifyIndex { 1379 t.Fatalf("index mis-match") 1380 } 1381 if out.VaultToken != "" { 1382 t.Fatalf("vault token not cleared") 1383 } 1384 1385 // Check that an implicit constraint was created 1386 constraints := out.TaskGroups[0].Constraints 1387 if l := len(constraints); l != 1 { 1388 t.Fatalf("Unexpected number of tests: %v", l) 1389 } 1390 1391 if !constraints[0].Equal(vaultConstraint) { 1392 t.Fatalf("bad constraint; got %#v; want %#v", constraints[0], vaultConstraint) 1393 } 1394 1395 // Create the register request with another job asking for a vault policy but 1396 // send the root Vault token 1397 job2 := mock.Job() 1398 job2.VaultToken = rootToken 1399 job.TaskGroups[0].Tasks[0].Vault = &structs.Vault{ 1400 Policies: []string{policy}, 1401 ChangeMode: structs.VaultChangeModeRestart, 1402 } 1403 req = &structs.JobRegisterRequest{ 1404 Job: job2, 1405 WriteRequest: structs.WriteRequest{ 1406 Region: "global", 1407 Namespace: job.Namespace, 1408 }, 1409 } 1410 1411 // Fetch the response 1412 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { 1413 t.Fatalf("bad: %v", err) 1414 } 1415 1416 // Check for the job in the FSM 1417 out, err = state.JobByID(ws, job2.Namespace, job2.ID) 1418 if err != nil { 1419 t.Fatalf("err: %v", err) 1420 } 1421 if out == nil { 1422 t.Fatalf("expected job") 1423 } 1424 if out.CreateIndex != resp.JobModifyIndex { 1425 t.Fatalf("index mis-match") 1426 } 1427 if out.VaultToken != "" { 1428 t.Fatalf("vault token not cleared") 1429 } 1430 } 1431 1432 // TestJobEndpoint_Register_SemverConstraint asserts that semver ordering is 1433 // used when evaluating semver constraints. 1434 func TestJobEndpoint_Register_SemverConstraint(t *testing.T) { 1435 t.Parallel() 1436 1437 s1, cleanupS1 := TestServer(t, nil) 1438 defer cleanupS1() 1439 codec := rpcClient(t, s1) 1440 testutil.WaitForLeader(t, s1.RPC) 1441 state := s1.State() 1442 1443 // Create a job with a semver constraint 1444 job := mock.Job() 1445 job.Constraints = []*structs.Constraint{ 1446 { 1447 LTarget: "${attr.vault.version}", 1448 RTarget: ">= 0.6.1", 1449 Operand: structs.ConstraintSemver, 1450 }, 1451 } 1452 job.TaskGroups[0].Count = 1 1453 1454 // Insert 2 Nodes, 1 matching the constraint, 1 not 1455 node1 := mock.Node() 1456 node1.Attributes["vault.version"] = "1.3.0-beta1+ent" 1457 node1.ComputeClass() 1458 require.NoError(t, state.UpsertNode(1, node1)) 1459 1460 node2 := mock.Node() 1461 delete(node2.Attributes, "vault.version") 1462 node2.ComputeClass() 1463 require.NoError(t, state.UpsertNode(2, node2)) 1464 1465 // Create the register request 1466 req := &structs.JobRegisterRequest{ 1467 Job: job, 1468 WriteRequest: structs.WriteRequest{ 1469 Region: "global", 1470 Namespace: job.Namespace, 1471 }, 1472 } 1473 1474 // Fetch the response 1475 var resp structs.JobRegisterResponse 1476 require.NoError(t, msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp)) 1477 require.NotZero(t, resp.Index) 1478 1479 // Wait for placements 1480 allocReq := &structs.JobSpecificRequest{ 1481 JobID: job.ID, 1482 QueryOptions: structs.QueryOptions{ 1483 Region: "global", 1484 Namespace: structs.DefaultNamespace, 1485 }, 1486 } 1487 1488 testutil.WaitForResult(func() (bool, error) { 1489 resp := structs.JobAllocationsResponse{} 1490 err := msgpackrpc.CallWithCodec(codec, "Job.Allocations", allocReq, &resp) 1491 if err != nil { 1492 return false, err 1493 } 1494 if n := len(resp.Allocations); n != 1 { 1495 return false, fmt.Errorf("expected 1 alloc, found %d", n) 1496 } 1497 alloc := resp.Allocations[0] 1498 if alloc.NodeID != node1.ID { 1499 return false, fmt.Errorf("expected alloc to be one node=%q but found node=%q", 1500 node1.ID, alloc.NodeID) 1501 } 1502 return true, nil 1503 }, func(waitErr error) { 1504 evals, err := state.EvalsByJob(nil, structs.DefaultNamespace, job.ID) 1505 require.NoError(t, err) 1506 for i, e := range evals { 1507 t.Logf("%d Eval: %s", i, pretty.Sprint(e)) 1508 } 1509 1510 require.NoError(t, waitErr) 1511 }) 1512 } 1513 1514 func TestJobEndpoint_Revert(t *testing.T) { 1515 t.Parallel() 1516 1517 s1, cleanupS1 := TestServer(t, func(c *Config) { 1518 c.NumSchedulers = 0 // Prevent automatic dequeue 1519 }) 1520 defer cleanupS1() 1521 codec := rpcClient(t, s1) 1522 testutil.WaitForLeader(t, s1.RPC) 1523 1524 // Create the initial register request 1525 job := mock.Job() 1526 job.Priority = 100 1527 req := &structs.JobRegisterRequest{ 1528 Job: job, 1529 WriteRequest: structs.WriteRequest{ 1530 Region: "global", 1531 Namespace: job.Namespace, 1532 }, 1533 } 1534 1535 // Fetch the response 1536 var resp structs.JobRegisterResponse 1537 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { 1538 t.Fatalf("err: %v", err) 1539 } 1540 if resp.Index == 0 { 1541 t.Fatalf("bad index: %d", resp.Index) 1542 } 1543 1544 // Reregister again to get another version 1545 job2 := job.Copy() 1546 job2.Priority = 1 1547 req = &structs.JobRegisterRequest{ 1548 Job: job2, 1549 WriteRequest: structs.WriteRequest{ 1550 Region: "global", 1551 Namespace: job.Namespace, 1552 }, 1553 } 1554 1555 // Fetch the response 1556 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { 1557 t.Fatalf("err: %v", err) 1558 } 1559 if resp.Index == 0 { 1560 t.Fatalf("bad index: %d", resp.Index) 1561 } 1562 1563 // Create revert request and enforcing it be at an incorrect version 1564 revertReq := &structs.JobRevertRequest{ 1565 JobID: job.ID, 1566 JobVersion: 0, 1567 EnforcePriorVersion: helper.Uint64ToPtr(10), 1568 WriteRequest: structs.WriteRequest{ 1569 Region: "global", 1570 Namespace: job.Namespace, 1571 }, 1572 } 1573 1574 // Fetch the response 1575 err := msgpackrpc.CallWithCodec(codec, "Job.Revert", revertReq, &resp) 1576 if err == nil || !strings.Contains(err.Error(), "enforcing version 10") { 1577 t.Fatalf("expected enforcement error") 1578 } 1579 1580 // Create revert request and enforcing it be at the current version 1581 revertReq = &structs.JobRevertRequest{ 1582 JobID: job.ID, 1583 JobVersion: 1, 1584 WriteRequest: structs.WriteRequest{ 1585 Region: "global", 1586 Namespace: job.Namespace, 1587 }, 1588 } 1589 1590 // Fetch the response 1591 err = msgpackrpc.CallWithCodec(codec, "Job.Revert", revertReq, &resp) 1592 if err == nil || !strings.Contains(err.Error(), "current version") { 1593 t.Fatalf("expected current version err: %v", err) 1594 } 1595 1596 // Create revert request and enforcing it be at version 1 1597 revertReq = &structs.JobRevertRequest{ 1598 JobID: job.ID, 1599 JobVersion: 0, 1600 EnforcePriorVersion: helper.Uint64ToPtr(1), 1601 WriteRequest: structs.WriteRequest{ 1602 Region: "global", 1603 Namespace: job.Namespace, 1604 }, 1605 } 1606 1607 // Fetch the response 1608 if err := msgpackrpc.CallWithCodec(codec, "Job.Revert", revertReq, &resp); err != nil { 1609 t.Fatalf("err: %v", err) 1610 } 1611 if resp.Index == 0 { 1612 t.Fatalf("bad index: %d", resp.Index) 1613 } 1614 if resp.EvalID == "" || resp.EvalCreateIndex == 0 { 1615 t.Fatalf("bad created eval: %+v", resp) 1616 } 1617 if resp.JobModifyIndex == 0 { 1618 t.Fatalf("bad job modify index: %d", resp.JobModifyIndex) 1619 } 1620 1621 // Create revert request and don't enforce. We are at version 2 but it is 1622 // the same as version 0 1623 revertReq = &structs.JobRevertRequest{ 1624 JobID: job.ID, 1625 JobVersion: 0, 1626 WriteRequest: structs.WriteRequest{ 1627 Region: "global", 1628 Namespace: job.Namespace, 1629 }, 1630 } 1631 1632 // Fetch the response 1633 if err := msgpackrpc.CallWithCodec(codec, "Job.Revert", revertReq, &resp); err != nil { 1634 t.Fatalf("err: %v", err) 1635 } 1636 if resp.Index == 0 { 1637 t.Fatalf("bad index: %d", resp.Index) 1638 } 1639 if resp.EvalID == "" || resp.EvalCreateIndex == 0 { 1640 t.Fatalf("bad created eval: %+v", resp) 1641 } 1642 if resp.JobModifyIndex == 0 { 1643 t.Fatalf("bad job modify index: %d", resp.JobModifyIndex) 1644 } 1645 1646 // Check that the job is at the correct version and that the eval was 1647 // created 1648 state := s1.fsm.State() 1649 ws := memdb.NewWatchSet() 1650 out, err := state.JobByID(ws, job.Namespace, job.ID) 1651 if err != nil { 1652 t.Fatalf("err: %v", err) 1653 } 1654 if out == nil { 1655 t.Fatalf("expected job") 1656 } 1657 if out.Priority != job.Priority { 1658 t.Fatalf("priority mis-match") 1659 } 1660 if out.Version != 2 { 1661 t.Fatalf("got version %d; want %d", out.Version, 2) 1662 } 1663 1664 eout, err := state.EvalByID(ws, resp.EvalID) 1665 if err != nil { 1666 t.Fatalf("err: %v", err) 1667 } 1668 if eout == nil { 1669 t.Fatalf("expected eval") 1670 } 1671 if eout.JobID != job.ID { 1672 t.Fatalf("job id mis-match") 1673 } 1674 1675 versions, err := state.JobVersionsByID(ws, job.Namespace, job.ID) 1676 if err != nil { 1677 t.Fatalf("err: %v", err) 1678 } 1679 if len(versions) != 3 { 1680 t.Fatalf("got %d versions; want %d", len(versions), 3) 1681 } 1682 } 1683 1684 func TestJobEndpoint_Revert_Vault_NoToken(t *testing.T) { 1685 t.Parallel() 1686 1687 s1, cleanupS1 := TestServer(t, func(c *Config) { 1688 c.NumSchedulers = 0 // Prevent automatic dequeue 1689 }) 1690 defer cleanupS1() 1691 codec := rpcClient(t, s1) 1692 testutil.WaitForLeader(t, s1.RPC) 1693 1694 // Enable vault 1695 tr, f := true, false 1696 s1.config.VaultConfig.Enabled = &tr 1697 s1.config.VaultConfig.AllowUnauthenticated = &f 1698 1699 // Replace the Vault Client on the server 1700 tvc := &TestVaultClient{} 1701 s1.vault = tvc 1702 1703 // Add three tokens: one that allows the requesting policy, one that does 1704 // not and one that returns an error 1705 policy := "foo" 1706 1707 goodToken := uuid.Generate() 1708 goodPolicies := []string{"foo", "bar", "baz"} 1709 tvc.SetLookupTokenAllowedPolicies(goodToken, goodPolicies) 1710 1711 // Create the initial register request 1712 job := mock.Job() 1713 job.VaultToken = goodToken 1714 job.Priority = 100 1715 job.TaskGroups[0].Tasks[0].Vault = &structs.Vault{ 1716 Policies: []string{policy}, 1717 ChangeMode: structs.VaultChangeModeRestart, 1718 } 1719 req := &structs.JobRegisterRequest{ 1720 Job: job, 1721 WriteRequest: structs.WriteRequest{ 1722 Region: "global", 1723 Namespace: job.Namespace, 1724 }, 1725 } 1726 1727 // Fetch the response 1728 var resp structs.JobRegisterResponse 1729 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { 1730 t.Fatalf("err: %v", err) 1731 } 1732 1733 // Reregister again to get another version 1734 job2 := job.Copy() 1735 job2.Priority = 1 1736 req = &structs.JobRegisterRequest{ 1737 Job: job2, 1738 WriteRequest: structs.WriteRequest{ 1739 Region: "global", 1740 Namespace: job.Namespace, 1741 }, 1742 } 1743 1744 // Fetch the response 1745 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { 1746 t.Fatalf("err: %v", err) 1747 } 1748 1749 revertReq := &structs.JobRevertRequest{ 1750 JobID: job.ID, 1751 JobVersion: 1, 1752 WriteRequest: structs.WriteRequest{ 1753 Region: "global", 1754 Namespace: job.Namespace, 1755 }, 1756 } 1757 1758 // Fetch the response 1759 err := msgpackrpc.CallWithCodec(codec, "Job.Revert", revertReq, &resp) 1760 if err == nil || !strings.Contains(err.Error(), "current version") { 1761 t.Fatalf("expected current version err: %v", err) 1762 } 1763 1764 // Create revert request and enforcing it be at version 1 1765 revertReq = &structs.JobRevertRequest{ 1766 JobID: job.ID, 1767 JobVersion: 0, 1768 EnforcePriorVersion: helper.Uint64ToPtr(1), 1769 WriteRequest: structs.WriteRequest{ 1770 Region: "global", 1771 Namespace: job.Namespace, 1772 }, 1773 } 1774 1775 // Fetch the response 1776 err = msgpackrpc.CallWithCodec(codec, "Job.Revert", revertReq, &resp) 1777 if err == nil || !strings.Contains(err.Error(), "missing Vault Token") { 1778 t.Fatalf("expected Vault not enabled error: %v", err) 1779 } 1780 } 1781 1782 // TestJobEndpoint_Revert_Vault_Policies asserts that job revert uses the 1783 // revert request's Vault token when authorizing policies. 1784 func TestJobEndpoint_Revert_Vault_Policies(t *testing.T) { 1785 t.Parallel() 1786 1787 s1, cleanupS1 := TestServer(t, func(c *Config) { 1788 c.NumSchedulers = 0 // Prevent automatic dequeue 1789 }) 1790 defer cleanupS1() 1791 codec := rpcClient(t, s1) 1792 testutil.WaitForLeader(t, s1.RPC) 1793 1794 // Enable vault 1795 tr, f := true, false 1796 s1.config.VaultConfig.Enabled = &tr 1797 s1.config.VaultConfig.AllowUnauthenticated = &f 1798 1799 // Replace the Vault Client on the server 1800 tvc := &TestVaultClient{} 1801 s1.vault = tvc 1802 1803 // Add three tokens: one that allows the requesting policy, one that does 1804 // not and one that returns an error 1805 policy := "foo" 1806 1807 badToken := uuid.Generate() 1808 badPolicies := []string{"a", "b", "c"} 1809 tvc.SetLookupTokenAllowedPolicies(badToken, badPolicies) 1810 1811 registerGoodToken := uuid.Generate() 1812 goodPolicies := []string{"foo", "bar", "baz"} 1813 tvc.SetLookupTokenAllowedPolicies(registerGoodToken, goodPolicies) 1814 1815 revertGoodToken := uuid.Generate() 1816 revertGoodPolicies := []string{"foo", "bar_revert", "baz_revert"} 1817 tvc.SetLookupTokenAllowedPolicies(revertGoodToken, revertGoodPolicies) 1818 1819 rootToken := uuid.Generate() 1820 rootPolicies := []string{"root"} 1821 tvc.SetLookupTokenAllowedPolicies(rootToken, rootPolicies) 1822 1823 errToken := uuid.Generate() 1824 expectedErr := fmt.Errorf("return errors from vault") 1825 tvc.SetLookupTokenError(errToken, expectedErr) 1826 1827 // Create the initial register request 1828 job := mock.Job() 1829 job.VaultToken = registerGoodToken 1830 job.Priority = 100 1831 job.TaskGroups[0].Tasks[0].Vault = &structs.Vault{ 1832 Policies: []string{policy}, 1833 ChangeMode: structs.VaultChangeModeRestart, 1834 } 1835 req := &structs.JobRegisterRequest{ 1836 Job: job, 1837 WriteRequest: structs.WriteRequest{ 1838 Region: "global", 1839 Namespace: job.Namespace, 1840 }, 1841 } 1842 1843 // Fetch the response 1844 var resp structs.JobRegisterResponse 1845 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { 1846 t.Fatalf("err: %v", err) 1847 } 1848 1849 // Reregister again to get another version 1850 job2 := job.Copy() 1851 job2.Priority = 1 1852 req = &structs.JobRegisterRequest{ 1853 Job: job2, 1854 WriteRequest: structs.WriteRequest{ 1855 Region: "global", 1856 Namespace: job.Namespace, 1857 }, 1858 } 1859 1860 // Fetch the response 1861 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { 1862 t.Fatalf("err: %v", err) 1863 } 1864 1865 // Create the revert request with the bad Vault token 1866 revertReq := &structs.JobRevertRequest{ 1867 JobID: job.ID, 1868 JobVersion: 0, 1869 WriteRequest: structs.WriteRequest{ 1870 Region: "global", 1871 Namespace: job.Namespace, 1872 }, 1873 VaultToken: badToken, 1874 } 1875 1876 // Fetch the response 1877 err := msgpackrpc.CallWithCodec(codec, "Job.Revert", revertReq, &resp) 1878 if err == nil || !strings.Contains(err.Error(), 1879 "doesn't allow access to the following policies: "+policy) { 1880 t.Fatalf("expected permission denied error: %v", err) 1881 } 1882 1883 // Use the err token 1884 revertReq.VaultToken = errToken 1885 err = msgpackrpc.CallWithCodec(codec, "Job.Revert", revertReq, &resp) 1886 if err == nil || !strings.Contains(err.Error(), expectedErr.Error()) { 1887 t.Fatalf("expected permission denied error: %v", err) 1888 } 1889 1890 // Use a good token 1891 revertReq.VaultToken = revertGoodToken 1892 if err := msgpackrpc.CallWithCodec(codec, "Job.Revert", revertReq, &resp); err != nil { 1893 t.Fatalf("bad: %v", err) 1894 } 1895 } 1896 1897 func TestJobEndpoint_Revert_ACL(t *testing.T) { 1898 t.Parallel() 1899 require := require.New(t) 1900 1901 s1, root, cleanupS1 := TestACLServer(t, func(c *Config) { 1902 c.NumSchedulers = 0 // Prevent automatic dequeue 1903 }) 1904 1905 defer cleanupS1() 1906 codec := rpcClient(t, s1) 1907 state := s1.fsm.State() 1908 testutil.WaitForLeader(t, s1.RPC) 1909 1910 // Create the jobs 1911 job := mock.Job() 1912 err := state.UpsertJob(300, job) 1913 require.Nil(err) 1914 1915 job2 := job.Copy() 1916 job2.Priority = 1 1917 err = state.UpsertJob(400, job2) 1918 require.Nil(err) 1919 1920 // Create revert request and enforcing it be at the current version 1921 revertReq := &structs.JobRevertRequest{ 1922 JobID: job.ID, 1923 JobVersion: 0, 1924 WriteRequest: structs.WriteRequest{ 1925 Region: "global", 1926 Namespace: job.Namespace, 1927 }, 1928 } 1929 1930 // Attempt to fetch the response without a valid token 1931 var resp structs.JobRegisterResponse 1932 err = msgpackrpc.CallWithCodec(codec, "Job.Revert", revertReq, &resp) 1933 require.NotNil(err) 1934 require.Contains(err.Error(), "Permission denied") 1935 1936 // Attempt to fetch the response with an invalid token 1937 invalidToken := mock.CreatePolicyAndToken(t, state, 1001, "test-invalid", 1938 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob})) 1939 1940 revertReq.AuthToken = invalidToken.SecretID 1941 var invalidResp structs.JobRegisterResponse 1942 err = msgpackrpc.CallWithCodec(codec, "Job.Revert", revertReq, &invalidResp) 1943 require.NotNil(err) 1944 require.Contains(err.Error(), "Permission denied") 1945 1946 // Fetch the response with a valid management token 1947 revertReq.AuthToken = root.SecretID 1948 var validResp structs.JobRegisterResponse 1949 err = msgpackrpc.CallWithCodec(codec, "Job.Revert", revertReq, &validResp) 1950 require.Nil(err) 1951 1952 // Try with a valid non-management token 1953 validToken := mock.CreatePolicyAndToken(t, state, 1003, "test-valid", 1954 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilitySubmitJob})) 1955 1956 revertReq.AuthToken = validToken.SecretID 1957 var validResp2 structs.JobRegisterResponse 1958 err = msgpackrpc.CallWithCodec(codec, "Job.Revert", revertReq, &validResp2) 1959 require.Nil(err) 1960 } 1961 1962 func TestJobEndpoint_Stable(t *testing.T) { 1963 t.Parallel() 1964 1965 s1, cleanupS1 := TestServer(t, func(c *Config) { 1966 c.NumSchedulers = 0 // Prevent automatic dequeue 1967 }) 1968 defer cleanupS1() 1969 codec := rpcClient(t, s1) 1970 testutil.WaitForLeader(t, s1.RPC) 1971 1972 // Create the initial register request 1973 job := mock.Job() 1974 req := &structs.JobRegisterRequest{ 1975 Job: job, 1976 WriteRequest: structs.WriteRequest{ 1977 Region: "global", 1978 Namespace: job.Namespace, 1979 }, 1980 } 1981 1982 // Fetch the response 1983 var resp structs.JobRegisterResponse 1984 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { 1985 t.Fatalf("err: %v", err) 1986 } 1987 if resp.Index == 0 { 1988 t.Fatalf("bad index: %d", resp.Index) 1989 } 1990 1991 // Create stability request 1992 stableReq := &structs.JobStabilityRequest{ 1993 JobID: job.ID, 1994 JobVersion: 0, 1995 Stable: true, 1996 WriteRequest: structs.WriteRequest{ 1997 Region: "global", 1998 Namespace: job.Namespace, 1999 }, 2000 } 2001 2002 // Fetch the response 2003 var stableResp structs.JobStabilityResponse 2004 if err := msgpackrpc.CallWithCodec(codec, "Job.Stable", stableReq, &stableResp); err != nil { 2005 t.Fatalf("err: %v", err) 2006 } 2007 if stableResp.Index == 0 { 2008 t.Fatalf("bad index: %d", resp.Index) 2009 } 2010 2011 // Check that the job is marked stable 2012 state := s1.fsm.State() 2013 ws := memdb.NewWatchSet() 2014 out, err := state.JobByID(ws, job.Namespace, job.ID) 2015 if err != nil { 2016 t.Fatalf("err: %v", err) 2017 } 2018 if out == nil { 2019 t.Fatalf("expected job") 2020 } 2021 if !out.Stable { 2022 t.Fatalf("Job is not marked stable") 2023 } 2024 } 2025 2026 func TestJobEndpoint_Stable_ACL(t *testing.T) { 2027 t.Parallel() 2028 require := require.New(t) 2029 2030 s1, root, cleanupS1 := TestACLServer(t, func(c *Config) { 2031 c.NumSchedulers = 0 // Prevent automatic dequeue 2032 }) 2033 defer cleanupS1() 2034 codec := rpcClient(t, s1) 2035 state := s1.fsm.State() 2036 testutil.WaitForLeader(t, s1.RPC) 2037 2038 // Register the job 2039 job := mock.Job() 2040 err := state.UpsertJob(1000, job) 2041 require.Nil(err) 2042 2043 // Create stability request 2044 stableReq := &structs.JobStabilityRequest{ 2045 JobID: job.ID, 2046 JobVersion: 0, 2047 Stable: true, 2048 WriteRequest: structs.WriteRequest{ 2049 Region: "global", 2050 Namespace: job.Namespace, 2051 }, 2052 } 2053 2054 // Attempt to fetch the token without a token 2055 var stableResp structs.JobStabilityResponse 2056 err = msgpackrpc.CallWithCodec(codec, "Job.Stable", stableReq, &stableResp) 2057 require.NotNil(err) 2058 require.Contains("Permission denied", err.Error()) 2059 2060 // Expect failure for request with an invalid token 2061 invalidToken := mock.CreatePolicyAndToken(t, state, 1003, "test-invalid", 2062 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityListJobs})) 2063 2064 stableReq.AuthToken = invalidToken.SecretID 2065 var invalidStableResp structs.JobStabilityResponse 2066 err = msgpackrpc.CallWithCodec(codec, "Job.Stable", stableReq, &invalidStableResp) 2067 require.NotNil(err) 2068 require.Contains("Permission denied", err.Error()) 2069 2070 // Attempt to fetch with a management token 2071 stableReq.AuthToken = root.SecretID 2072 var validStableResp structs.JobStabilityResponse 2073 err = msgpackrpc.CallWithCodec(codec, "Job.Stable", stableReq, &validStableResp) 2074 require.Nil(err) 2075 2076 // Attempt to fetch with a valid token 2077 validToken := mock.CreatePolicyAndToken(t, state, 1005, "test-invalid", 2078 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilitySubmitJob})) 2079 2080 stableReq.AuthToken = validToken.SecretID 2081 var validStableResp2 structs.JobStabilityResponse 2082 err = msgpackrpc.CallWithCodec(codec, "Job.Stable", stableReq, &validStableResp2) 2083 require.Nil(err) 2084 2085 // Check that the job is marked stable 2086 ws := memdb.NewWatchSet() 2087 out, err := state.JobByID(ws, job.Namespace, job.ID) 2088 require.Nil(err) 2089 require.NotNil(job) 2090 require.Equal(true, out.Stable) 2091 } 2092 2093 func TestJobEndpoint_Evaluate(t *testing.T) { 2094 t.Parallel() 2095 2096 s1, cleanupS1 := TestServer(t, func(c *Config) { 2097 c.NumSchedulers = 0 // Prevent automatic dequeue 2098 }) 2099 defer cleanupS1() 2100 codec := rpcClient(t, s1) 2101 testutil.WaitForLeader(t, s1.RPC) 2102 2103 // Create the register request 2104 job := mock.Job() 2105 req := &structs.JobRegisterRequest{ 2106 Job: job, 2107 WriteRequest: structs.WriteRequest{ 2108 Region: "global", 2109 Namespace: job.Namespace, 2110 }, 2111 } 2112 2113 // Fetch the response 2114 var resp structs.JobRegisterResponse 2115 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { 2116 t.Fatalf("err: %v", err) 2117 } 2118 if resp.Index == 0 { 2119 t.Fatalf("bad index: %d", resp.Index) 2120 } 2121 2122 // Force a re-evaluation 2123 reEval := &structs.JobEvaluateRequest{ 2124 JobID: job.ID, 2125 WriteRequest: structs.WriteRequest{ 2126 Region: "global", 2127 Namespace: job.Namespace, 2128 }, 2129 } 2130 2131 // Fetch the response 2132 if err := msgpackrpc.CallWithCodec(codec, "Job.Evaluate", reEval, &resp); err != nil { 2133 t.Fatalf("err: %v", err) 2134 } 2135 if resp.Index == 0 { 2136 t.Fatalf("bad index: %d", resp.Index) 2137 } 2138 2139 // Lookup the evaluation 2140 state := s1.fsm.State() 2141 ws := memdb.NewWatchSet() 2142 eval, err := state.EvalByID(ws, resp.EvalID) 2143 if err != nil { 2144 t.Fatalf("err: %v", err) 2145 } 2146 if eval == nil { 2147 t.Fatalf("expected eval") 2148 } 2149 if eval.CreateIndex != resp.EvalCreateIndex { 2150 t.Fatalf("index mis-match") 2151 } 2152 2153 if eval.Priority != job.Priority { 2154 t.Fatalf("bad: %#v", eval) 2155 } 2156 if eval.Type != job.Type { 2157 t.Fatalf("bad: %#v", eval) 2158 } 2159 if eval.TriggeredBy != structs.EvalTriggerJobRegister { 2160 t.Fatalf("bad: %#v", eval) 2161 } 2162 if eval.JobID != job.ID { 2163 t.Fatalf("bad: %#v", eval) 2164 } 2165 if eval.JobModifyIndex != resp.JobModifyIndex { 2166 t.Fatalf("bad: %#v", eval) 2167 } 2168 if eval.Status != structs.EvalStatusPending { 2169 t.Fatalf("bad: %#v", eval) 2170 } 2171 if eval.CreateTime == 0 { 2172 t.Fatalf("eval CreateTime is unset: %#v", eval) 2173 } 2174 if eval.ModifyTime == 0 { 2175 t.Fatalf("eval ModifyTime is unset: %#v", eval) 2176 } 2177 } 2178 2179 func TestJobEndpoint_ForceRescheduleEvaluate(t *testing.T) { 2180 t.Parallel() 2181 require := require.New(t) 2182 2183 s1, cleanupS1 := TestServer(t, func(c *Config) { 2184 c.NumSchedulers = 0 // Prevent automatic dequeue 2185 }) 2186 defer cleanupS1() 2187 codec := rpcClient(t, s1) 2188 testutil.WaitForLeader(t, s1.RPC) 2189 2190 // Create the register request 2191 job := mock.Job() 2192 req := &structs.JobRegisterRequest{ 2193 Job: job, 2194 WriteRequest: structs.WriteRequest{ 2195 Region: "global", 2196 Namespace: job.Namespace, 2197 }, 2198 } 2199 2200 // Fetch the response 2201 var resp structs.JobRegisterResponse 2202 err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp) 2203 require.Nil(err) 2204 require.NotEqual(0, resp.Index) 2205 2206 state := s1.fsm.State() 2207 job, err = state.JobByID(nil, structs.DefaultNamespace, job.ID) 2208 require.Nil(err) 2209 2210 // Create a failed alloc 2211 alloc := mock.Alloc() 2212 alloc.Job = job 2213 alloc.JobID = job.ID 2214 alloc.TaskGroup = job.TaskGroups[0].Name 2215 alloc.Namespace = job.Namespace 2216 alloc.ClientStatus = structs.AllocClientStatusFailed 2217 err = s1.State().UpsertAllocs(resp.Index+1, []*structs.Allocation{alloc}) 2218 require.Nil(err) 2219 2220 // Force a re-evaluation 2221 reEval := &structs.JobEvaluateRequest{ 2222 JobID: job.ID, 2223 EvalOptions: structs.EvalOptions{ForceReschedule: true}, 2224 WriteRequest: structs.WriteRequest{ 2225 Region: "global", 2226 Namespace: job.Namespace, 2227 }, 2228 } 2229 2230 // Fetch the response 2231 err = msgpackrpc.CallWithCodec(codec, "Job.Evaluate", reEval, &resp) 2232 require.Nil(err) 2233 require.NotEqual(0, resp.Index) 2234 2235 // Lookup the evaluation 2236 ws := memdb.NewWatchSet() 2237 eval, err := state.EvalByID(ws, resp.EvalID) 2238 require.Nil(err) 2239 require.NotNil(eval) 2240 require.Equal(eval.CreateIndex, resp.EvalCreateIndex) 2241 require.Equal(eval.Priority, job.Priority) 2242 require.Equal(eval.Type, job.Type) 2243 require.Equal(eval.TriggeredBy, structs.EvalTriggerJobRegister) 2244 require.Equal(eval.JobID, job.ID) 2245 require.Equal(eval.JobModifyIndex, resp.JobModifyIndex) 2246 require.Equal(eval.Status, structs.EvalStatusPending) 2247 require.NotZero(eval.CreateTime) 2248 require.NotZero(eval.ModifyTime) 2249 2250 // Lookup the alloc, verify DesiredTransition ForceReschedule 2251 alloc, err = state.AllocByID(ws, alloc.ID) 2252 require.NotNil(alloc) 2253 require.Nil(err) 2254 require.True(*alloc.DesiredTransition.ForceReschedule) 2255 } 2256 2257 func TestJobEndpoint_Evaluate_ACL(t *testing.T) { 2258 t.Parallel() 2259 require := require.New(t) 2260 2261 s1, root, cleanupS1 := TestACLServer(t, func(c *Config) { 2262 c.NumSchedulers = 0 // Prevent automatic dequeue 2263 }) 2264 defer cleanupS1() 2265 codec := rpcClient(t, s1) 2266 testutil.WaitForLeader(t, s1.RPC) 2267 state := s1.fsm.State() 2268 2269 // Create the job 2270 job := mock.Job() 2271 err := state.UpsertJob(300, job) 2272 require.Nil(err) 2273 2274 // Force a re-evaluation 2275 reEval := &structs.JobEvaluateRequest{ 2276 JobID: job.ID, 2277 WriteRequest: structs.WriteRequest{ 2278 Region: "global", 2279 Namespace: job.Namespace, 2280 }, 2281 } 2282 2283 // Attempt to fetch the response without a token 2284 var resp structs.JobRegisterResponse 2285 err = msgpackrpc.CallWithCodec(codec, "Job.Evaluate", reEval, &resp) 2286 require.NotNil(err) 2287 require.Contains(err.Error(), "Permission denied") 2288 2289 // Attempt to fetch the response with an invalid token 2290 invalidToken := mock.CreatePolicyAndToken(t, state, 1003, "test-invalid", 2291 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityListJobs})) 2292 2293 reEval.AuthToken = invalidToken.SecretID 2294 var invalidResp structs.JobRegisterResponse 2295 err = msgpackrpc.CallWithCodec(codec, "Job.Evaluate", reEval, &invalidResp) 2296 require.NotNil(err) 2297 require.Contains(err.Error(), "Permission denied") 2298 2299 // Fetch the response with a valid management token 2300 reEval.AuthToken = root.SecretID 2301 var validResp structs.JobRegisterResponse 2302 err = msgpackrpc.CallWithCodec(codec, "Job.Evaluate", reEval, &validResp) 2303 require.Nil(err) 2304 2305 // Fetch the response with a valid token 2306 validToken := mock.CreatePolicyAndToken(t, state, 1005, "test-valid", 2307 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob})) 2308 2309 reEval.AuthToken = validToken.SecretID 2310 var validResp2 structs.JobRegisterResponse 2311 err = msgpackrpc.CallWithCodec(codec, "Job.Evaluate", reEval, &validResp2) 2312 require.Nil(err) 2313 2314 // Lookup the evaluation 2315 ws := memdb.NewWatchSet() 2316 eval, err := state.EvalByID(ws, validResp2.EvalID) 2317 require.Nil(err) 2318 require.NotNil(eval) 2319 2320 require.Equal(eval.CreateIndex, validResp2.EvalCreateIndex) 2321 require.Equal(eval.Priority, job.Priority) 2322 require.Equal(eval.Type, job.Type) 2323 require.Equal(eval.TriggeredBy, structs.EvalTriggerJobRegister) 2324 require.Equal(eval.JobID, job.ID) 2325 require.Equal(eval.JobModifyIndex, validResp2.JobModifyIndex) 2326 require.Equal(eval.Status, structs.EvalStatusPending) 2327 require.NotZero(eval.CreateTime) 2328 require.NotZero(eval.ModifyTime) 2329 } 2330 2331 func TestJobEndpoint_Evaluate_Periodic(t *testing.T) { 2332 t.Parallel() 2333 2334 s1, cleanupS1 := TestServer(t, func(c *Config) { 2335 c.NumSchedulers = 0 // Prevent automatic dequeue 2336 }) 2337 defer cleanupS1() 2338 codec := rpcClient(t, s1) 2339 testutil.WaitForLeader(t, s1.RPC) 2340 2341 // Create the register request 2342 job := mock.PeriodicJob() 2343 req := &structs.JobRegisterRequest{ 2344 Job: job, 2345 WriteRequest: structs.WriteRequest{ 2346 Region: "global", 2347 Namespace: job.Namespace, 2348 }, 2349 } 2350 2351 // Fetch the response 2352 var resp structs.JobRegisterResponse 2353 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { 2354 t.Fatalf("err: %v", err) 2355 } 2356 if resp.JobModifyIndex == 0 { 2357 t.Fatalf("bad index: %d", resp.Index) 2358 } 2359 2360 // Force a re-evaluation 2361 reEval := &structs.JobEvaluateRequest{ 2362 JobID: job.ID, 2363 WriteRequest: structs.WriteRequest{ 2364 Region: "global", 2365 Namespace: job.Namespace, 2366 }, 2367 } 2368 2369 // Fetch the response 2370 if err := msgpackrpc.CallWithCodec(codec, "Job.Evaluate", reEval, &resp); err == nil { 2371 t.Fatal("expect an err") 2372 } 2373 } 2374 2375 func TestJobEndpoint_Evaluate_ParameterizedJob(t *testing.T) { 2376 t.Parallel() 2377 2378 s1, cleanupS1 := TestServer(t, func(c *Config) { 2379 c.NumSchedulers = 0 // Prevent automatic dequeue 2380 }) 2381 defer cleanupS1() 2382 codec := rpcClient(t, s1) 2383 testutil.WaitForLeader(t, s1.RPC) 2384 2385 // Create the register request 2386 job := mock.BatchJob() 2387 job.ParameterizedJob = &structs.ParameterizedJobConfig{} 2388 req := &structs.JobRegisterRequest{ 2389 Job: job, 2390 WriteRequest: structs.WriteRequest{ 2391 Region: "global", 2392 Namespace: job.Namespace, 2393 }, 2394 } 2395 2396 // Fetch the response 2397 var resp structs.JobRegisterResponse 2398 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { 2399 t.Fatalf("err: %v", err) 2400 } 2401 if resp.JobModifyIndex == 0 { 2402 t.Fatalf("bad index: %d", resp.Index) 2403 } 2404 2405 // Force a re-evaluation 2406 reEval := &structs.JobEvaluateRequest{ 2407 JobID: job.ID, 2408 WriteRequest: structs.WriteRequest{ 2409 Region: "global", 2410 Namespace: job.Namespace, 2411 }, 2412 } 2413 2414 // Fetch the response 2415 if err := msgpackrpc.CallWithCodec(codec, "Job.Evaluate", reEval, &resp); err == nil { 2416 t.Fatal("expect an err") 2417 } 2418 } 2419 2420 func TestJobEndpoint_Deregister(t *testing.T) { 2421 t.Parallel() 2422 require := require.New(t) 2423 2424 s1, cleanupS1 := TestServer(t, func(c *Config) { 2425 c.NumSchedulers = 0 // Prevent automatic dequeue 2426 }) 2427 defer cleanupS1() 2428 codec := rpcClient(t, s1) 2429 testutil.WaitForLeader(t, s1.RPC) 2430 2431 // Create the register requests 2432 job := mock.Job() 2433 reg := &structs.JobRegisterRequest{ 2434 Job: job, 2435 WriteRequest: structs.WriteRequest{ 2436 Region: "global", 2437 Namespace: job.Namespace, 2438 }, 2439 } 2440 2441 // Fetch the response 2442 var resp structs.JobRegisterResponse 2443 require.Nil(msgpackrpc.CallWithCodec(codec, "Job.Register", reg, &resp)) 2444 2445 // Deregister but don't purge 2446 dereg := &structs.JobDeregisterRequest{ 2447 JobID: job.ID, 2448 Purge: false, 2449 WriteRequest: structs.WriteRequest{ 2450 Region: "global", 2451 Namespace: job.Namespace, 2452 }, 2453 } 2454 var resp2 structs.JobDeregisterResponse 2455 require.Nil(msgpackrpc.CallWithCodec(codec, "Job.Deregister", dereg, &resp2)) 2456 require.NotZero(resp2.Index) 2457 2458 // Check for the job in the FSM 2459 state := s1.fsm.State() 2460 out, err := state.JobByID(nil, job.Namespace, job.ID) 2461 require.Nil(err) 2462 require.NotNil(out) 2463 require.True(out.Stop) 2464 2465 // Lookup the evaluation 2466 eval, err := state.EvalByID(nil, resp2.EvalID) 2467 require.Nil(err) 2468 require.NotNil(eval) 2469 require.EqualValues(resp2.EvalCreateIndex, eval.CreateIndex) 2470 require.Equal(job.Priority, eval.Priority) 2471 require.Equal(job.Type, eval.Type) 2472 require.Equal(structs.EvalTriggerJobDeregister, eval.TriggeredBy) 2473 require.Equal(job.ID, eval.JobID) 2474 require.Equal(structs.EvalStatusPending, eval.Status) 2475 require.NotZero(eval.CreateTime) 2476 require.NotZero(eval.ModifyTime) 2477 2478 // Deregister and purge 2479 dereg2 := &structs.JobDeregisterRequest{ 2480 JobID: job.ID, 2481 Purge: true, 2482 WriteRequest: structs.WriteRequest{ 2483 Region: "global", 2484 Namespace: job.Namespace, 2485 }, 2486 } 2487 var resp3 structs.JobDeregisterResponse 2488 require.Nil(msgpackrpc.CallWithCodec(codec, "Job.Deregister", dereg2, &resp3)) 2489 require.NotZero(resp3.Index) 2490 2491 // Check for the job in the FSM 2492 out, err = state.JobByID(nil, job.Namespace, job.ID) 2493 require.Nil(err) 2494 require.Nil(out) 2495 2496 // Lookup the evaluation 2497 eval, err = state.EvalByID(nil, resp3.EvalID) 2498 require.Nil(err) 2499 require.NotNil(eval) 2500 2501 require.EqualValues(resp3.EvalCreateIndex, eval.CreateIndex) 2502 require.Equal(job.Priority, eval.Priority) 2503 require.Equal(job.Type, eval.Type) 2504 require.Equal(structs.EvalTriggerJobDeregister, eval.TriggeredBy) 2505 require.Equal(job.ID, eval.JobID) 2506 require.Equal(structs.EvalStatusPending, eval.Status) 2507 require.NotZero(eval.CreateTime) 2508 require.NotZero(eval.ModifyTime) 2509 } 2510 2511 func TestJobEndpoint_Deregister_ACL(t *testing.T) { 2512 t.Parallel() 2513 require := require.New(t) 2514 2515 s1, root, cleanupS1 := TestACLServer(t, func(c *Config) { 2516 c.NumSchedulers = 0 // Prevent automatic dequeue 2517 }) 2518 defer cleanupS1() 2519 codec := rpcClient(t, s1) 2520 testutil.WaitForLeader(t, s1.RPC) 2521 state := s1.fsm.State() 2522 2523 // Create and register a job 2524 job := mock.Job() 2525 err := state.UpsertJob(100, job) 2526 require.Nil(err) 2527 2528 // Deregister and purge 2529 req := &structs.JobDeregisterRequest{ 2530 JobID: job.ID, 2531 Purge: true, 2532 WriteRequest: structs.WriteRequest{ 2533 Region: "global", 2534 Namespace: job.Namespace, 2535 }, 2536 } 2537 2538 // Expect failure for request without a token 2539 var resp structs.JobDeregisterResponse 2540 err = msgpackrpc.CallWithCodec(codec, "Job.Deregister", req, &resp) 2541 require.NotNil(err) 2542 require.Contains(err.Error(), "Permission denied") 2543 2544 // Expect failure for request with an invalid token 2545 invalidToken := mock.CreatePolicyAndToken(t, state, 1003, "test-invalid", 2546 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityListJobs})) 2547 req.AuthToken = invalidToken.SecretID 2548 2549 var invalidResp structs.JobDeregisterResponse 2550 err = msgpackrpc.CallWithCodec(codec, "Job.Deregister", req, &invalidResp) 2551 require.NotNil(err) 2552 require.Contains(err.Error(), "Permission denied") 2553 2554 // Expect success with a valid management token 2555 req.AuthToken = root.SecretID 2556 2557 var validResp structs.JobDeregisterResponse 2558 err = msgpackrpc.CallWithCodec(codec, "Job.Deregister", req, &validResp) 2559 require.Nil(err) 2560 require.NotEqual(validResp.Index, 0) 2561 2562 // Expect success with a valid token 2563 validToken := mock.CreatePolicyAndToken(t, state, 1005, "test-valid", 2564 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilitySubmitJob})) 2565 req.AuthToken = validToken.SecretID 2566 2567 // Check for the job in the FSM 2568 out, err := state.JobByID(nil, job.Namespace, job.ID) 2569 require.Nil(err) 2570 require.Nil(out) 2571 2572 // Lookup the evaluation 2573 eval, err := state.EvalByID(nil, validResp.EvalID) 2574 require.Nil(err) 2575 require.NotNil(eval, nil) 2576 2577 require.Equal(eval.CreateIndex, validResp.EvalCreateIndex) 2578 require.Equal(eval.Priority, structs.JobDefaultPriority) 2579 require.Equal(eval.Type, structs.JobTypeService) 2580 require.Equal(eval.TriggeredBy, structs.EvalTriggerJobDeregister) 2581 require.Equal(eval.JobID, job.ID) 2582 require.Equal(eval.JobModifyIndex, validResp.JobModifyIndex) 2583 require.Equal(eval.Status, structs.EvalStatusPending) 2584 require.NotZero(eval.CreateTime) 2585 require.NotZero(eval.ModifyTime) 2586 2587 // Deregistration is not idempotent, produces a new eval after the job is 2588 // deregistered. TODO(langmartin) make it idempotent. 2589 var validResp2 structs.JobDeregisterResponse 2590 err = msgpackrpc.CallWithCodec(codec, "Job.Deregister", req, &validResp2) 2591 require.NoError(err) 2592 require.NotEqual("", validResp2.EvalID) 2593 require.NotEqual(validResp.EvalID, validResp2.EvalID) 2594 } 2595 2596 func TestJobEndpoint_Deregister_Nonexistent(t *testing.T) { 2597 t.Parallel() 2598 2599 s1, cleanupS1 := TestServer(t, func(c *Config) { 2600 c.NumSchedulers = 0 // Prevent automatic dequeue 2601 }) 2602 defer cleanupS1() 2603 codec := rpcClient(t, s1) 2604 testutil.WaitForLeader(t, s1.RPC) 2605 2606 // Deregister 2607 jobID := "foo" 2608 dereg := &structs.JobDeregisterRequest{ 2609 JobID: jobID, 2610 WriteRequest: structs.WriteRequest{ 2611 Region: "global", 2612 Namespace: structs.DefaultNamespace, 2613 }, 2614 } 2615 var resp2 structs.JobDeregisterResponse 2616 if err := msgpackrpc.CallWithCodec(codec, "Job.Deregister", dereg, &resp2); err != nil { 2617 t.Fatalf("err: %v", err) 2618 } 2619 if resp2.JobModifyIndex == 0 { 2620 t.Fatalf("bad index: %d", resp2.Index) 2621 } 2622 2623 // Lookup the evaluation 2624 state := s1.fsm.State() 2625 ws := memdb.NewWatchSet() 2626 eval, err := state.EvalByID(ws, resp2.EvalID) 2627 if err != nil { 2628 t.Fatalf("err: %v", err) 2629 } 2630 if eval == nil { 2631 t.Fatalf("expected eval") 2632 } 2633 if eval.CreateIndex != resp2.EvalCreateIndex { 2634 t.Fatalf("index mis-match") 2635 } 2636 2637 if eval.Priority != structs.JobDefaultPriority { 2638 t.Fatalf("bad: %#v", eval) 2639 } 2640 if eval.Type != structs.JobTypeService { 2641 t.Fatalf("bad: %#v", eval) 2642 } 2643 if eval.TriggeredBy != structs.EvalTriggerJobDeregister { 2644 t.Fatalf("bad: %#v", eval) 2645 } 2646 if eval.JobID != jobID { 2647 t.Fatalf("bad: %#v", eval) 2648 } 2649 if eval.JobModifyIndex != resp2.JobModifyIndex { 2650 t.Fatalf("bad: %#v", eval) 2651 } 2652 if eval.Status != structs.EvalStatusPending { 2653 t.Fatalf("bad: %#v", eval) 2654 } 2655 if eval.CreateTime == 0 { 2656 t.Fatalf("eval CreateTime is unset: %#v", eval) 2657 } 2658 if eval.ModifyTime == 0 { 2659 t.Fatalf("eval ModifyTime is unset: %#v", eval) 2660 } 2661 } 2662 2663 func TestJobEndpoint_Deregister_Periodic(t *testing.T) { 2664 t.Parallel() 2665 2666 s1, cleanupS1 := TestServer(t, func(c *Config) { 2667 c.NumSchedulers = 0 // Prevent automatic dequeue 2668 }) 2669 defer cleanupS1() 2670 codec := rpcClient(t, s1) 2671 testutil.WaitForLeader(t, s1.RPC) 2672 2673 // Create the register request 2674 job := mock.PeriodicJob() 2675 reg := &structs.JobRegisterRequest{ 2676 Job: job, 2677 WriteRequest: structs.WriteRequest{ 2678 Region: "global", 2679 Namespace: job.Namespace, 2680 }, 2681 } 2682 2683 // Fetch the response 2684 var resp structs.JobRegisterResponse 2685 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", reg, &resp); err != nil { 2686 t.Fatalf("err: %v", err) 2687 } 2688 2689 // Deregister 2690 dereg := &structs.JobDeregisterRequest{ 2691 JobID: job.ID, 2692 Purge: true, 2693 WriteRequest: structs.WriteRequest{ 2694 Region: "global", 2695 Namespace: job.Namespace, 2696 }, 2697 } 2698 var resp2 structs.JobDeregisterResponse 2699 if err := msgpackrpc.CallWithCodec(codec, "Job.Deregister", dereg, &resp2); err != nil { 2700 t.Fatalf("err: %v", err) 2701 } 2702 if resp2.JobModifyIndex == 0 { 2703 t.Fatalf("bad index: %d", resp2.Index) 2704 } 2705 2706 // Check for the node in the FSM 2707 state := s1.fsm.State() 2708 ws := memdb.NewWatchSet() 2709 out, err := state.JobByID(ws, job.Namespace, job.ID) 2710 if err != nil { 2711 t.Fatalf("err: %v", err) 2712 } 2713 if out != nil { 2714 t.Fatalf("unexpected job") 2715 } 2716 2717 if resp.EvalID != "" { 2718 t.Fatalf("Deregister created an eval for a periodic job") 2719 } 2720 } 2721 2722 func TestJobEndpoint_Deregister_ParameterizedJob(t *testing.T) { 2723 t.Parallel() 2724 2725 s1, cleanupS1 := TestServer(t, func(c *Config) { 2726 c.NumSchedulers = 0 // Prevent automatic dequeue 2727 }) 2728 defer cleanupS1() 2729 codec := rpcClient(t, s1) 2730 testutil.WaitForLeader(t, s1.RPC) 2731 2732 // Create the register request 2733 job := mock.BatchJob() 2734 job.ParameterizedJob = &structs.ParameterizedJobConfig{} 2735 reg := &structs.JobRegisterRequest{ 2736 Job: job, 2737 WriteRequest: structs.WriteRequest{ 2738 Region: "global", 2739 Namespace: job.Namespace, 2740 }, 2741 } 2742 2743 // Fetch the response 2744 var resp structs.JobRegisterResponse 2745 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", reg, &resp); err != nil { 2746 t.Fatalf("err: %v", err) 2747 } 2748 2749 // Deregister 2750 dereg := &structs.JobDeregisterRequest{ 2751 JobID: job.ID, 2752 Purge: true, 2753 WriteRequest: structs.WriteRequest{ 2754 Region: "global", 2755 Namespace: job.Namespace, 2756 }, 2757 } 2758 var resp2 structs.JobDeregisterResponse 2759 if err := msgpackrpc.CallWithCodec(codec, "Job.Deregister", dereg, &resp2); err != nil { 2760 t.Fatalf("err: %v", err) 2761 } 2762 if resp2.JobModifyIndex == 0 { 2763 t.Fatalf("bad index: %d", resp2.Index) 2764 } 2765 2766 // Check for the node in the FSM 2767 state := s1.fsm.State() 2768 ws := memdb.NewWatchSet() 2769 out, err := state.JobByID(ws, job.Namespace, job.ID) 2770 if err != nil { 2771 t.Fatalf("err: %v", err) 2772 } 2773 if out != nil { 2774 t.Fatalf("unexpected job") 2775 } 2776 2777 if resp.EvalID != "" { 2778 t.Fatalf("Deregister created an eval for a parameterized job") 2779 } 2780 } 2781 2782 func TestJobEndpoint_BatchDeregister(t *testing.T) { 2783 t.Parallel() 2784 require := require.New(t) 2785 2786 s1, cleanupS1 := TestServer(t, func(c *Config) { 2787 c.NumSchedulers = 0 // Prevent automatic dequeue 2788 }) 2789 defer cleanupS1() 2790 codec := rpcClient(t, s1) 2791 testutil.WaitForLeader(t, s1.RPC) 2792 2793 // Create the register requests 2794 job := mock.Job() 2795 reg := &structs.JobRegisterRequest{ 2796 Job: job, 2797 WriteRequest: structs.WriteRequest{ 2798 Region: "global", 2799 Namespace: job.Namespace, 2800 }, 2801 } 2802 2803 // Fetch the response 2804 var resp structs.JobRegisterResponse 2805 require.Nil(msgpackrpc.CallWithCodec(codec, "Job.Register", reg, &resp)) 2806 2807 job2 := mock.Job() 2808 job2.Priority = 1 2809 reg2 := &structs.JobRegisterRequest{ 2810 Job: job2, 2811 WriteRequest: structs.WriteRequest{ 2812 Region: "global", 2813 Namespace: job2.Namespace, 2814 }, 2815 } 2816 2817 // Fetch the response 2818 require.Nil(msgpackrpc.CallWithCodec(codec, "Job.Register", reg2, &resp)) 2819 2820 // Deregister 2821 dereg := &structs.JobBatchDeregisterRequest{ 2822 Jobs: map[structs.NamespacedID]*structs.JobDeregisterOptions{ 2823 { 2824 ID: job.ID, 2825 Namespace: job.Namespace, 2826 }: {}, 2827 { 2828 ID: job2.ID, 2829 Namespace: job2.Namespace, 2830 }: { 2831 Purge: true, 2832 }, 2833 }, 2834 WriteRequest: structs.WriteRequest{ 2835 Region: "global", 2836 Namespace: job.Namespace, 2837 }, 2838 } 2839 var resp2 structs.JobBatchDeregisterResponse 2840 require.Nil(msgpackrpc.CallWithCodec(codec, "Job.BatchDeregister", dereg, &resp2)) 2841 require.NotZero(resp2.Index) 2842 2843 // Check for the job in the FSM 2844 state := s1.fsm.State() 2845 out, err := state.JobByID(nil, job.Namespace, job.ID) 2846 require.Nil(err) 2847 require.NotNil(out) 2848 require.True(out.Stop) 2849 2850 out, err = state.JobByID(nil, job2.Namespace, job2.ID) 2851 require.Nil(err) 2852 require.Nil(out) 2853 2854 // Lookup the evaluation 2855 for jobNS, eval := range resp2.JobEvals { 2856 expectedJob := job 2857 if jobNS.ID != job.ID { 2858 expectedJob = job2 2859 } 2860 2861 eval, err := state.EvalByID(nil, eval) 2862 require.Nil(err) 2863 require.NotNil(eval) 2864 require.EqualValues(resp2.Index, eval.CreateIndex) 2865 require.Equal(expectedJob.Priority, eval.Priority) 2866 require.Equal(expectedJob.Type, eval.Type) 2867 require.Equal(structs.EvalTriggerJobDeregister, eval.TriggeredBy) 2868 require.Equal(expectedJob.ID, eval.JobID) 2869 require.Equal(structs.EvalStatusPending, eval.Status) 2870 require.NotZero(eval.CreateTime) 2871 require.NotZero(eval.ModifyTime) 2872 } 2873 } 2874 2875 func TestJobEndpoint_BatchDeregister_ACL(t *testing.T) { 2876 t.Parallel() 2877 require := require.New(t) 2878 2879 s1, root, cleanupS1 := TestACLServer(t, func(c *Config) { 2880 c.NumSchedulers = 0 // Prevent automatic dequeue 2881 }) 2882 defer cleanupS1() 2883 codec := rpcClient(t, s1) 2884 testutil.WaitForLeader(t, s1.RPC) 2885 state := s1.fsm.State() 2886 2887 // Create and register a job 2888 job, job2 := mock.Job(), mock.Job() 2889 require.Nil(state.UpsertJob(100, job)) 2890 require.Nil(state.UpsertJob(101, job2)) 2891 2892 // Deregister 2893 req := &structs.JobBatchDeregisterRequest{ 2894 Jobs: map[structs.NamespacedID]*structs.JobDeregisterOptions{ 2895 { 2896 ID: job.ID, 2897 Namespace: job.Namespace, 2898 }: {}, 2899 { 2900 ID: job2.ID, 2901 Namespace: job2.Namespace, 2902 }: {}, 2903 }, 2904 WriteRequest: structs.WriteRequest{ 2905 Region: "global", 2906 }, 2907 } 2908 2909 // Expect failure for request without a token 2910 var resp structs.JobBatchDeregisterResponse 2911 err := msgpackrpc.CallWithCodec(codec, "Job.BatchDeregister", req, &resp) 2912 require.NotNil(err) 2913 require.True(structs.IsErrPermissionDenied(err)) 2914 2915 // Expect failure for request with an invalid token 2916 invalidToken := mock.CreatePolicyAndToken(t, state, 1003, "test-invalid", 2917 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityListJobs})) 2918 req.AuthToken = invalidToken.SecretID 2919 2920 var invalidResp structs.JobDeregisterResponse 2921 err = msgpackrpc.CallWithCodec(codec, "Job.BatchDeregister", req, &invalidResp) 2922 require.NotNil(err) 2923 require.True(structs.IsErrPermissionDenied(err)) 2924 2925 // Expect success with a valid management token 2926 req.AuthToken = root.SecretID 2927 2928 var validResp structs.JobDeregisterResponse 2929 err = msgpackrpc.CallWithCodec(codec, "Job.BatchDeregister", req, &validResp) 2930 require.Nil(err) 2931 require.NotEqual(validResp.Index, 0) 2932 2933 // Expect success with a valid token 2934 validToken := mock.CreatePolicyAndToken(t, state, 1005, "test-valid", 2935 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilitySubmitJob})) 2936 req.AuthToken = validToken.SecretID 2937 2938 var validResp2 structs.JobDeregisterResponse 2939 err = msgpackrpc.CallWithCodec(codec, "Job.BatchDeregister", req, &validResp2) 2940 require.Nil(err) 2941 require.NotEqual(validResp2.Index, 0) 2942 } 2943 2944 func TestJobEndpoint_GetJob(t *testing.T) { 2945 t.Parallel() 2946 2947 s1, cleanupS1 := TestServer(t, nil) 2948 defer cleanupS1() 2949 codec := rpcClient(t, s1) 2950 testutil.WaitForLeader(t, s1.RPC) 2951 2952 // Create the register request 2953 job := mock.Job() 2954 reg := &structs.JobRegisterRequest{ 2955 Job: job, 2956 WriteRequest: structs.WriteRequest{ 2957 Region: "global", 2958 Namespace: job.Namespace, 2959 }, 2960 } 2961 2962 // Fetch the response 2963 var resp structs.JobRegisterResponse 2964 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", reg, &resp); err != nil { 2965 t.Fatalf("err: %v", err) 2966 } 2967 job.CreateIndex = resp.JobModifyIndex 2968 job.ModifyIndex = resp.JobModifyIndex 2969 job.JobModifyIndex = resp.JobModifyIndex 2970 2971 // Lookup the job 2972 get := &structs.JobSpecificRequest{ 2973 JobID: job.ID, 2974 QueryOptions: structs.QueryOptions{ 2975 Region: "global", 2976 Namespace: job.Namespace, 2977 }, 2978 } 2979 var resp2 structs.SingleJobResponse 2980 if err := msgpackrpc.CallWithCodec(codec, "Job.GetJob", get, &resp2); err != nil { 2981 t.Fatalf("err: %v", err) 2982 } 2983 if resp2.Index != resp.JobModifyIndex { 2984 t.Fatalf("Bad index: %d %d", resp2.Index, resp.Index) 2985 } 2986 2987 // Make a copy of the origin job and change the service name so that we can 2988 // do a deep equal with the response from the GET JOB Api 2989 j := job 2990 j.TaskGroups[0].Tasks[0].Services[0].Name = "web-frontend" 2991 for tgix, tg := range j.TaskGroups { 2992 for tidx, t := range tg.Tasks { 2993 for sidx, service := range t.Services { 2994 for cidx, check := range service.Checks { 2995 check.Name = resp2.Job.TaskGroups[tgix].Tasks[tidx].Services[sidx].Checks[cidx].Name 2996 } 2997 } 2998 } 2999 } 3000 3001 // Clear the submit times 3002 j.SubmitTime = 0 3003 resp2.Job.SubmitTime = 0 3004 3005 if !reflect.DeepEqual(j, resp2.Job) { 3006 t.Fatalf("bad: %#v %#v", job, resp2.Job) 3007 } 3008 3009 // Lookup non-existing job 3010 get.JobID = "foobarbaz" 3011 if err := msgpackrpc.CallWithCodec(codec, "Job.GetJob", get, &resp2); err != nil { 3012 t.Fatalf("err: %v", err) 3013 } 3014 if resp2.Index != resp.JobModifyIndex { 3015 t.Fatalf("Bad index: %d %d", resp2.Index, resp.Index) 3016 } 3017 if resp2.Job != nil { 3018 t.Fatalf("unexpected job") 3019 } 3020 } 3021 3022 func TestJobEndpoint_GetJob_ACL(t *testing.T) { 3023 t.Parallel() 3024 require := require.New(t) 3025 3026 s1, root, cleanupS1 := TestACLServer(t, nil) 3027 defer cleanupS1() 3028 codec := rpcClient(t, s1) 3029 testutil.WaitForLeader(t, s1.RPC) 3030 state := s1.fsm.State() 3031 3032 // Create the job 3033 job := mock.Job() 3034 err := state.UpsertJob(1000, job) 3035 require.Nil(err) 3036 3037 // Lookup the job 3038 get := &structs.JobSpecificRequest{ 3039 JobID: job.ID, 3040 QueryOptions: structs.QueryOptions{ 3041 Region: "global", 3042 Namespace: job.Namespace, 3043 }, 3044 } 3045 3046 // Looking up the job without a token should fail 3047 var resp structs.SingleJobResponse 3048 err = msgpackrpc.CallWithCodec(codec, "Job.GetJob", get, &resp) 3049 require.NotNil(err) 3050 require.Contains(err.Error(), "Permission denied") 3051 3052 // Expect failure for request with an invalid token 3053 invalidToken := mock.CreatePolicyAndToken(t, state, 1003, "test-invalid", 3054 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityListJobs})) 3055 3056 get.AuthToken = invalidToken.SecretID 3057 var invalidResp structs.SingleJobResponse 3058 err = msgpackrpc.CallWithCodec(codec, "Job.GetJob", get, &invalidResp) 3059 require.NotNil(err) 3060 require.Contains(err.Error(), "Permission denied") 3061 3062 // Looking up the job with a management token should succeed 3063 get.AuthToken = root.SecretID 3064 var validResp structs.SingleJobResponse 3065 err = msgpackrpc.CallWithCodec(codec, "Job.GetJob", get, &validResp) 3066 require.Nil(err) 3067 require.Equal(job.ID, validResp.Job.ID) 3068 3069 // Looking up the job with a valid token should succeed 3070 validToken := mock.CreatePolicyAndToken(t, state, 1005, "test-valid", 3071 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob})) 3072 3073 get.AuthToken = validToken.SecretID 3074 var validResp2 structs.SingleJobResponse 3075 err = msgpackrpc.CallWithCodec(codec, "Job.GetJob", get, &validResp2) 3076 require.Nil(err) 3077 require.Equal(job.ID, validResp2.Job.ID) 3078 } 3079 3080 func TestJobEndpoint_GetJob_Blocking(t *testing.T) { 3081 t.Parallel() 3082 3083 s1, cleanupS1 := TestServer(t, nil) 3084 defer cleanupS1() 3085 state := s1.fsm.State() 3086 codec := rpcClient(t, s1) 3087 testutil.WaitForLeader(t, s1.RPC) 3088 3089 // Create the jobs 3090 job1 := mock.Job() 3091 job2 := mock.Job() 3092 3093 // Upsert a job we are not interested in first. 3094 time.AfterFunc(100*time.Millisecond, func() { 3095 if err := state.UpsertJob(100, job1); err != nil { 3096 t.Fatalf("err: %v", err) 3097 } 3098 }) 3099 3100 // Upsert another job later which should trigger the watch. 3101 time.AfterFunc(200*time.Millisecond, func() { 3102 if err := state.UpsertJob(200, job2); err != nil { 3103 t.Fatalf("err: %v", err) 3104 } 3105 }) 3106 3107 req := &structs.JobSpecificRequest{ 3108 JobID: job2.ID, 3109 QueryOptions: structs.QueryOptions{ 3110 Region: "global", 3111 Namespace: job2.Namespace, 3112 MinQueryIndex: 150, 3113 }, 3114 } 3115 start := time.Now() 3116 var resp structs.SingleJobResponse 3117 if err := msgpackrpc.CallWithCodec(codec, "Job.GetJob", req, &resp); err != nil { 3118 t.Fatalf("err: %v", err) 3119 } 3120 3121 if elapsed := time.Since(start); elapsed < 200*time.Millisecond { 3122 t.Fatalf("should block (returned in %s) %#v", elapsed, resp) 3123 } 3124 if resp.Index != 200 { 3125 t.Fatalf("Bad index: %d %d", resp.Index, 200) 3126 } 3127 if resp.Job == nil || resp.Job.ID != job2.ID { 3128 t.Fatalf("bad: %#v", resp.Job) 3129 } 3130 3131 // Job delete fires watches 3132 time.AfterFunc(100*time.Millisecond, func() { 3133 if err := state.DeleteJob(300, job2.Namespace, job2.ID); err != nil { 3134 t.Fatalf("err: %v", err) 3135 } 3136 }) 3137 3138 req.QueryOptions.MinQueryIndex = 250 3139 start = time.Now() 3140 3141 var resp2 structs.SingleJobResponse 3142 if err := msgpackrpc.CallWithCodec(codec, "Job.GetJob", req, &resp2); err != nil { 3143 t.Fatalf("err: %v", err) 3144 } 3145 3146 if elapsed := time.Since(start); elapsed < 100*time.Millisecond { 3147 t.Fatalf("should block (returned in %s) %#v", elapsed, resp2) 3148 } 3149 if resp2.Index != 300 { 3150 t.Fatalf("Bad index: %d %d", resp2.Index, 300) 3151 } 3152 if resp2.Job != nil { 3153 t.Fatalf("bad: %#v", resp2.Job) 3154 } 3155 } 3156 3157 func TestJobEndpoint_GetJobVersions(t *testing.T) { 3158 t.Parallel() 3159 3160 s1, cleanupS1 := TestServer(t, nil) 3161 defer cleanupS1() 3162 codec := rpcClient(t, s1) 3163 testutil.WaitForLeader(t, s1.RPC) 3164 3165 // Create the register request 3166 job := mock.Job() 3167 job.Priority = 88 3168 reg := &structs.JobRegisterRequest{ 3169 Job: job, 3170 WriteRequest: structs.WriteRequest{ 3171 Region: "global", 3172 Namespace: job.Namespace, 3173 }, 3174 } 3175 3176 // Fetch the response 3177 var resp structs.JobRegisterResponse 3178 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", reg, &resp); err != nil { 3179 t.Fatalf("err: %v", err) 3180 } 3181 3182 // Register the job again to create another version 3183 job.Priority = 100 3184 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", reg, &resp); err != nil { 3185 t.Fatalf("err: %v", err) 3186 } 3187 3188 // Lookup the job 3189 get := &structs.JobVersionsRequest{ 3190 JobID: job.ID, 3191 QueryOptions: structs.QueryOptions{ 3192 Region: "global", 3193 Namespace: job.Namespace, 3194 }, 3195 } 3196 var versionsResp structs.JobVersionsResponse 3197 if err := msgpackrpc.CallWithCodec(codec, "Job.GetJobVersions", get, &versionsResp); err != nil { 3198 t.Fatalf("err: %v", err) 3199 } 3200 if versionsResp.Index != resp.JobModifyIndex { 3201 t.Fatalf("Bad index: %d %d", versionsResp.Index, resp.Index) 3202 } 3203 3204 // Make sure there are two job versions 3205 versions := versionsResp.Versions 3206 if l := len(versions); l != 2 { 3207 t.Fatalf("Got %d versions; want 2", l) 3208 } 3209 3210 if v := versions[0]; v.Priority != 100 || v.ID != job.ID || v.Version != 1 { 3211 t.Fatalf("bad: %+v", v) 3212 } 3213 if v := versions[1]; v.Priority != 88 || v.ID != job.ID || v.Version != 0 { 3214 t.Fatalf("bad: %+v", v) 3215 } 3216 3217 // Lookup non-existing job 3218 get.JobID = "foobarbaz" 3219 if err := msgpackrpc.CallWithCodec(codec, "Job.GetJobVersions", get, &versionsResp); err != nil { 3220 t.Fatalf("err: %v", err) 3221 } 3222 if versionsResp.Index != resp.JobModifyIndex { 3223 t.Fatalf("Bad index: %d %d", versionsResp.Index, resp.Index) 3224 } 3225 if l := len(versionsResp.Versions); l != 0 { 3226 t.Fatalf("unexpected versions: %d", l) 3227 } 3228 } 3229 3230 func TestJobEndpoint_GetJobVersions_ACL(t *testing.T) { 3231 t.Parallel() 3232 require := require.New(t) 3233 3234 s1, root, cleanupS1 := TestACLServer(t, nil) 3235 defer cleanupS1() 3236 codec := rpcClient(t, s1) 3237 testutil.WaitForLeader(t, s1.RPC) 3238 state := s1.fsm.State() 3239 3240 // Create two versions of a job with different priorities 3241 job := mock.Job() 3242 job.Priority = 88 3243 err := state.UpsertJob(10, job) 3244 require.Nil(err) 3245 3246 job.Priority = 100 3247 err = state.UpsertJob(100, job) 3248 require.Nil(err) 3249 3250 // Lookup the job 3251 get := &structs.JobVersionsRequest{ 3252 JobID: job.ID, 3253 QueryOptions: structs.QueryOptions{ 3254 Region: "global", 3255 Namespace: job.Namespace, 3256 }, 3257 } 3258 3259 // Attempt to fetch without a token should fail 3260 var resp structs.JobVersionsResponse 3261 err = msgpackrpc.CallWithCodec(codec, "Job.GetJobVersions", get, &resp) 3262 require.NotNil(err) 3263 require.Contains(err.Error(), "Permission denied") 3264 3265 // Expect failure for request with an invalid token 3266 invalidToken := mock.CreatePolicyAndToken(t, state, 1003, "test-invalid", 3267 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityListJobs})) 3268 3269 get.AuthToken = invalidToken.SecretID 3270 var invalidResp structs.JobVersionsResponse 3271 err = msgpackrpc.CallWithCodec(codec, "Job.GetJobVersions", get, &invalidResp) 3272 require.NotNil(err) 3273 require.Contains(err.Error(), "Permission denied") 3274 3275 // Expect success for request with a valid management token 3276 get.AuthToken = root.SecretID 3277 var validResp structs.JobVersionsResponse 3278 err = msgpackrpc.CallWithCodec(codec, "Job.GetJobVersions", get, &validResp) 3279 require.Nil(err) 3280 3281 // Expect success for request with a valid token 3282 validToken := mock.CreatePolicyAndToken(t, state, 1005, "test-valid", 3283 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob})) 3284 3285 get.AuthToken = validToken.SecretID 3286 var validResp2 structs.JobVersionsResponse 3287 err = msgpackrpc.CallWithCodec(codec, "Job.GetJobVersions", get, &validResp2) 3288 require.Nil(err) 3289 3290 // Make sure there are two job versions 3291 versions := validResp2.Versions 3292 require.Equal(2, len(versions)) 3293 require.Equal(versions[0].ID, job.ID) 3294 require.Equal(versions[1].ID, job.ID) 3295 } 3296 3297 func TestJobEndpoint_GetJobVersions_Diff(t *testing.T) { 3298 t.Parallel() 3299 3300 s1, cleanupS1 := TestServer(t, nil) 3301 defer cleanupS1() 3302 codec := rpcClient(t, s1) 3303 testutil.WaitForLeader(t, s1.RPC) 3304 3305 // Create the register request 3306 job := mock.Job() 3307 job.Priority = 88 3308 reg := &structs.JobRegisterRequest{ 3309 Job: job, 3310 WriteRequest: structs.WriteRequest{ 3311 Region: "global", 3312 Namespace: job.Namespace, 3313 }, 3314 } 3315 3316 // Fetch the response 3317 var resp structs.JobRegisterResponse 3318 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", reg, &resp); err != nil { 3319 t.Fatalf("err: %v", err) 3320 } 3321 3322 // Register the job again to create another version 3323 job.Priority = 90 3324 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", reg, &resp); err != nil { 3325 t.Fatalf("err: %v", err) 3326 } 3327 3328 // Register the job again to create another version 3329 job.Priority = 100 3330 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", reg, &resp); err != nil { 3331 t.Fatalf("err: %v", err) 3332 } 3333 3334 // Lookup the job 3335 get := &structs.JobVersionsRequest{ 3336 JobID: job.ID, 3337 Diffs: true, 3338 QueryOptions: structs.QueryOptions{ 3339 Region: "global", 3340 Namespace: job.Namespace, 3341 }, 3342 } 3343 var versionsResp structs.JobVersionsResponse 3344 if err := msgpackrpc.CallWithCodec(codec, "Job.GetJobVersions", get, &versionsResp); err != nil { 3345 t.Fatalf("err: %v", err) 3346 } 3347 if versionsResp.Index != resp.JobModifyIndex { 3348 t.Fatalf("Bad index: %d %d", versionsResp.Index, resp.Index) 3349 } 3350 3351 // Make sure there are two job versions 3352 versions := versionsResp.Versions 3353 if l := len(versions); l != 3 { 3354 t.Fatalf("Got %d versions; want 3", l) 3355 } 3356 3357 if v := versions[0]; v.Priority != 100 || v.ID != job.ID || v.Version != 2 { 3358 t.Fatalf("bad: %+v", v) 3359 } 3360 if v := versions[1]; v.Priority != 90 || v.ID != job.ID || v.Version != 1 { 3361 t.Fatalf("bad: %+v", v) 3362 } 3363 if v := versions[2]; v.Priority != 88 || v.ID != job.ID || v.Version != 0 { 3364 t.Fatalf("bad: %+v", v) 3365 } 3366 3367 // Ensure we got diffs 3368 diffs := versionsResp.Diffs 3369 if l := len(diffs); l != 2 { 3370 t.Fatalf("Got %d diffs; want 2", l) 3371 } 3372 d1 := diffs[0] 3373 if len(d1.Fields) != 1 { 3374 t.Fatalf("Got too many diffs: %#v", d1) 3375 } 3376 if d1.Fields[0].Name != "Priority" { 3377 t.Fatalf("Got wrong field: %#v", d1) 3378 } 3379 if d1.Fields[0].Old != "90" && d1.Fields[0].New != "100" { 3380 t.Fatalf("Got wrong field values: %#v", d1) 3381 } 3382 d2 := diffs[1] 3383 if len(d2.Fields) != 1 { 3384 t.Fatalf("Got too many diffs: %#v", d2) 3385 } 3386 if d2.Fields[0].Name != "Priority" { 3387 t.Fatalf("Got wrong field: %#v", d2) 3388 } 3389 if d2.Fields[0].Old != "88" && d1.Fields[0].New != "90" { 3390 t.Fatalf("Got wrong field values: %#v", d2) 3391 } 3392 } 3393 3394 func TestJobEndpoint_GetJobVersions_Blocking(t *testing.T) { 3395 t.Parallel() 3396 3397 s1, cleanupS1 := TestServer(t, nil) 3398 defer cleanupS1() 3399 state := s1.fsm.State() 3400 codec := rpcClient(t, s1) 3401 testutil.WaitForLeader(t, s1.RPC) 3402 3403 // Create the jobs 3404 job1 := mock.Job() 3405 job2 := mock.Job() 3406 job3 := mock.Job() 3407 job3.ID = job2.ID 3408 job3.Priority = 1 3409 3410 // Upsert a job we are not interested in first. 3411 time.AfterFunc(100*time.Millisecond, func() { 3412 if err := state.UpsertJob(100, job1); err != nil { 3413 t.Fatalf("err: %v", err) 3414 } 3415 }) 3416 3417 // Upsert another job later which should trigger the watch. 3418 time.AfterFunc(200*time.Millisecond, func() { 3419 if err := state.UpsertJob(200, job2); err != nil { 3420 t.Fatalf("err: %v", err) 3421 } 3422 }) 3423 3424 req := &structs.JobVersionsRequest{ 3425 JobID: job2.ID, 3426 QueryOptions: structs.QueryOptions{ 3427 Region: "global", 3428 Namespace: job2.Namespace, 3429 MinQueryIndex: 150, 3430 }, 3431 } 3432 start := time.Now() 3433 var resp structs.JobVersionsResponse 3434 if err := msgpackrpc.CallWithCodec(codec, "Job.GetJobVersions", req, &resp); err != nil { 3435 t.Fatalf("err: %v", err) 3436 } 3437 3438 if elapsed := time.Since(start); elapsed < 200*time.Millisecond { 3439 t.Fatalf("should block (returned in %s) %#v", elapsed, resp) 3440 } 3441 if resp.Index != 200 { 3442 t.Fatalf("Bad index: %d %d", resp.Index, 200) 3443 } 3444 if len(resp.Versions) != 1 || resp.Versions[0].ID != job2.ID { 3445 t.Fatalf("bad: %#v", resp.Versions) 3446 } 3447 3448 // Upsert the job again which should trigger the watch. 3449 time.AfterFunc(100*time.Millisecond, func() { 3450 if err := state.UpsertJob(300, job3); err != nil { 3451 t.Fatalf("err: %v", err) 3452 } 3453 }) 3454 3455 req2 := &structs.JobVersionsRequest{ 3456 JobID: job3.ID, 3457 QueryOptions: structs.QueryOptions{ 3458 Region: "global", 3459 Namespace: job3.Namespace, 3460 MinQueryIndex: 250, 3461 }, 3462 } 3463 var resp2 structs.JobVersionsResponse 3464 start = time.Now() 3465 if err := msgpackrpc.CallWithCodec(codec, "Job.GetJobVersions", req2, &resp2); err != nil { 3466 t.Fatalf("err: %v", err) 3467 } 3468 3469 if elapsed := time.Since(start); elapsed < 100*time.Millisecond { 3470 t.Fatalf("should block (returned in %s) %#v", elapsed, resp) 3471 } 3472 if resp2.Index != 300 { 3473 t.Fatalf("Bad index: %d %d", resp.Index, 300) 3474 } 3475 if len(resp2.Versions) != 2 { 3476 t.Fatalf("bad: %#v", resp2.Versions) 3477 } 3478 } 3479 3480 func TestJobEndpoint_GetJobSummary(t *testing.T) { 3481 t.Parallel() 3482 3483 s1, cleanupS1 := TestServer(t, func(c *Config) { 3484 c.NumSchedulers = 0 // Prevent automatic dequeue 3485 }) 3486 3487 defer cleanupS1() 3488 codec := rpcClient(t, s1) 3489 testutil.WaitForLeader(t, s1.RPC) 3490 3491 // Create the register request 3492 job := mock.Job() 3493 reg := &structs.JobRegisterRequest{ 3494 Job: job, 3495 WriteRequest: structs.WriteRequest{ 3496 Region: "global", 3497 Namespace: job.Namespace, 3498 }, 3499 } 3500 3501 // Fetch the response 3502 var resp structs.JobRegisterResponse 3503 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", reg, &resp); err != nil { 3504 t.Fatalf("err: %v", err) 3505 } 3506 job.CreateIndex = resp.JobModifyIndex 3507 job.ModifyIndex = resp.JobModifyIndex 3508 job.JobModifyIndex = resp.JobModifyIndex 3509 3510 // Lookup the job summary 3511 get := &structs.JobSummaryRequest{ 3512 JobID: job.ID, 3513 QueryOptions: structs.QueryOptions{ 3514 Region: "global", 3515 Namespace: job.Namespace, 3516 }, 3517 } 3518 var resp2 structs.JobSummaryResponse 3519 if err := msgpackrpc.CallWithCodec(codec, "Job.Summary", get, &resp2); err != nil { 3520 t.Fatalf("err: %v", err) 3521 } 3522 if resp2.Index != resp.JobModifyIndex { 3523 t.Fatalf("Bad index: %d %d", resp2.Index, resp.Index) 3524 } 3525 3526 expectedJobSummary := structs.JobSummary{ 3527 JobID: job.ID, 3528 Namespace: job.Namespace, 3529 Summary: map[string]structs.TaskGroupSummary{ 3530 "web": {}, 3531 }, 3532 Children: new(structs.JobChildrenSummary), 3533 CreateIndex: job.CreateIndex, 3534 ModifyIndex: job.CreateIndex, 3535 } 3536 3537 if !reflect.DeepEqual(resp2.JobSummary, &expectedJobSummary) { 3538 t.Fatalf("expected: %v, actual: %v", expectedJobSummary, resp2.JobSummary) 3539 } 3540 } 3541 3542 func TestJobEndpoint_Summary_ACL(t *testing.T) { 3543 t.Parallel() 3544 require := require.New(t) 3545 3546 s1, root, cleanupS1 := TestACLServer(t, func(c *Config) { 3547 c.NumSchedulers = 0 // Prevent automatic dequeue 3548 }) 3549 defer cleanupS1() 3550 codec := rpcClient(t, s1) 3551 testutil.WaitForLeader(t, s1.RPC) 3552 3553 // Create the job 3554 job := mock.Job() 3555 reg := &structs.JobRegisterRequest{ 3556 Job: job, 3557 WriteRequest: structs.WriteRequest{ 3558 Region: "global", 3559 Namespace: job.Namespace, 3560 }, 3561 } 3562 reg.AuthToken = root.SecretID 3563 3564 var err error 3565 3566 // Register the job with a valid token 3567 var regResp structs.JobRegisterResponse 3568 err = msgpackrpc.CallWithCodec(codec, "Job.Register", reg, ®Resp) 3569 require.Nil(err) 3570 3571 job.CreateIndex = regResp.JobModifyIndex 3572 job.ModifyIndex = regResp.JobModifyIndex 3573 job.JobModifyIndex = regResp.JobModifyIndex 3574 3575 req := &structs.JobSummaryRequest{ 3576 JobID: job.ID, 3577 QueryOptions: structs.QueryOptions{ 3578 Region: "global", 3579 Namespace: job.Namespace, 3580 }, 3581 } 3582 3583 // Expect failure for request without a token 3584 var resp structs.JobSummaryResponse 3585 err = msgpackrpc.CallWithCodec(codec, "Job.Summary", req, &resp) 3586 require.NotNil(err) 3587 3588 expectedJobSummary := &structs.JobSummary{ 3589 JobID: job.ID, 3590 Namespace: job.Namespace, 3591 Summary: map[string]structs.TaskGroupSummary{ 3592 "web": {}, 3593 }, 3594 Children: new(structs.JobChildrenSummary), 3595 CreateIndex: job.CreateIndex, 3596 ModifyIndex: job.ModifyIndex, 3597 } 3598 3599 // Expect success when using a management token 3600 req.AuthToken = root.SecretID 3601 var mgmtResp structs.JobSummaryResponse 3602 err = msgpackrpc.CallWithCodec(codec, "Job.Summary", req, &mgmtResp) 3603 require.Nil(err) 3604 require.Equal(expectedJobSummary, mgmtResp.JobSummary) 3605 3606 // Create the namespace policy and tokens 3607 state := s1.fsm.State() 3608 3609 // Expect failure for request with an invalid token 3610 invalidToken := mock.CreatePolicyAndToken(t, state, 1003, "test-invalid", 3611 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityListJobs})) 3612 3613 req.AuthToken = invalidToken.SecretID 3614 var invalidResp structs.JobSummaryResponse 3615 err = msgpackrpc.CallWithCodec(codec, "Job.Summary", req, &invalidResp) 3616 require.NotNil(err) 3617 3618 // Try with a valid token 3619 validToken := mock.CreatePolicyAndToken(t, state, 1001, "test-valid", 3620 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob})) 3621 3622 req.AuthToken = validToken.SecretID 3623 var authResp structs.JobSummaryResponse 3624 err = msgpackrpc.CallWithCodec(codec, "Job.Summary", req, &authResp) 3625 require.Nil(err) 3626 require.Equal(expectedJobSummary, authResp.JobSummary) 3627 } 3628 3629 func TestJobEndpoint_GetJobSummary_Blocking(t *testing.T) { 3630 t.Parallel() 3631 3632 s1, cleanupS1 := TestServer(t, nil) 3633 defer cleanupS1() 3634 state := s1.fsm.State() 3635 codec := rpcClient(t, s1) 3636 testutil.WaitForLeader(t, s1.RPC) 3637 3638 // Create a job and insert it 3639 job1 := mock.Job() 3640 time.AfterFunc(200*time.Millisecond, func() { 3641 if err := state.UpsertJob(100, job1); err != nil { 3642 t.Fatalf("err: %v", err) 3643 } 3644 }) 3645 3646 // Ensure the job summary request gets fired 3647 req := &structs.JobSummaryRequest{ 3648 JobID: job1.ID, 3649 QueryOptions: structs.QueryOptions{ 3650 Region: "global", 3651 Namespace: job1.Namespace, 3652 MinQueryIndex: 50, 3653 }, 3654 } 3655 var resp structs.JobSummaryResponse 3656 start := time.Now() 3657 if err := msgpackrpc.CallWithCodec(codec, "Job.Summary", req, &resp); err != nil { 3658 t.Fatalf("err: %v", err) 3659 } 3660 if elapsed := time.Since(start); elapsed < 200*time.Millisecond { 3661 t.Fatalf("should block (returned in %s) %#v", elapsed, resp) 3662 } 3663 3664 // Upsert an allocation for the job which should trigger the watch. 3665 time.AfterFunc(200*time.Millisecond, func() { 3666 alloc := mock.Alloc() 3667 alloc.JobID = job1.ID 3668 alloc.Job = job1 3669 if err := state.UpsertAllocs(200, []*structs.Allocation{alloc}); err != nil { 3670 t.Fatalf("err: %v", err) 3671 } 3672 }) 3673 req = &structs.JobSummaryRequest{ 3674 JobID: job1.ID, 3675 QueryOptions: structs.QueryOptions{ 3676 Region: "global", 3677 Namespace: job1.Namespace, 3678 MinQueryIndex: 199, 3679 }, 3680 } 3681 start = time.Now() 3682 var resp1 structs.JobSummaryResponse 3683 if err := msgpackrpc.CallWithCodec(codec, "Job.Summary", req, &resp1); err != nil { 3684 t.Fatalf("err: %v", err) 3685 } 3686 3687 if elapsed := time.Since(start); elapsed < 200*time.Millisecond { 3688 t.Fatalf("should block (returned in %s) %#v", elapsed, resp) 3689 } 3690 if resp1.Index != 200 { 3691 t.Fatalf("Bad index: %d %d", resp.Index, 200) 3692 } 3693 if resp1.JobSummary == nil { 3694 t.Fatalf("bad: %#v", resp) 3695 } 3696 3697 // Job delete fires watches 3698 time.AfterFunc(100*time.Millisecond, func() { 3699 if err := state.DeleteJob(300, job1.Namespace, job1.ID); err != nil { 3700 t.Fatalf("err: %v", err) 3701 } 3702 }) 3703 3704 req.QueryOptions.MinQueryIndex = 250 3705 start = time.Now() 3706 3707 var resp2 structs.SingleJobResponse 3708 if err := msgpackrpc.CallWithCodec(codec, "Job.Summary", req, &resp2); err != nil { 3709 t.Fatalf("err: %v", err) 3710 } 3711 3712 if elapsed := time.Since(start); elapsed < 100*time.Millisecond { 3713 t.Fatalf("should block (returned in %s) %#v", elapsed, resp2) 3714 } 3715 if resp2.Index != 300 { 3716 t.Fatalf("Bad index: %d %d", resp2.Index, 300) 3717 } 3718 if resp2.Job != nil { 3719 t.Fatalf("bad: %#v", resp2.Job) 3720 } 3721 } 3722 3723 func TestJobEndpoint_ListJobs(t *testing.T) { 3724 t.Parallel() 3725 3726 s1, cleanupS1 := TestServer(t, nil) 3727 defer cleanupS1() 3728 codec := rpcClient(t, s1) 3729 testutil.WaitForLeader(t, s1.RPC) 3730 3731 // Create the register request 3732 job := mock.Job() 3733 state := s1.fsm.State() 3734 err := state.UpsertJob(1000, job) 3735 if err != nil { 3736 t.Fatalf("err: %v", err) 3737 } 3738 3739 // Lookup the jobs 3740 get := &structs.JobListRequest{ 3741 QueryOptions: structs.QueryOptions{ 3742 Region: "global", 3743 Namespace: job.Namespace, 3744 }, 3745 } 3746 var resp2 structs.JobListResponse 3747 if err := msgpackrpc.CallWithCodec(codec, "Job.List", get, &resp2); err != nil { 3748 t.Fatalf("err: %v", err) 3749 } 3750 if resp2.Index != 1000 { 3751 t.Fatalf("Bad index: %d %d", resp2.Index, 1000) 3752 } 3753 3754 if len(resp2.Jobs) != 1 { 3755 t.Fatalf("bad: %#v", resp2.Jobs) 3756 } 3757 if resp2.Jobs[0].ID != job.ID { 3758 t.Fatalf("bad: %#v", resp2.Jobs[0]) 3759 } 3760 3761 // Lookup the jobs by prefix 3762 get = &structs.JobListRequest{ 3763 QueryOptions: structs.QueryOptions{ 3764 Region: "global", 3765 Namespace: job.Namespace, 3766 Prefix: resp2.Jobs[0].ID[:4], 3767 }, 3768 } 3769 var resp3 structs.JobListResponse 3770 if err := msgpackrpc.CallWithCodec(codec, "Job.List", get, &resp3); err != nil { 3771 t.Fatalf("err: %v", err) 3772 } 3773 if resp3.Index != 1000 { 3774 t.Fatalf("Bad index: %d %d", resp3.Index, 1000) 3775 } 3776 3777 if len(resp3.Jobs) != 1 { 3778 t.Fatalf("bad: %#v", resp3.Jobs) 3779 } 3780 if resp3.Jobs[0].ID != job.ID { 3781 t.Fatalf("bad: %#v", resp3.Jobs[0]) 3782 } 3783 } 3784 3785 func TestJobEndpoint_ListJobs_WithACL(t *testing.T) { 3786 t.Parallel() 3787 require := require.New(t) 3788 3789 s1, root, cleanupS1 := TestACLServer(t, func(c *Config) { 3790 c.NumSchedulers = 0 // Prevent automatic dequeue 3791 }) 3792 defer cleanupS1() 3793 codec := rpcClient(t, s1) 3794 testutil.WaitForLeader(t, s1.RPC) 3795 state := s1.fsm.State() 3796 3797 var err error 3798 3799 // Create the register request 3800 job := mock.Job() 3801 err = state.UpsertJob(1000, job) 3802 require.Nil(err) 3803 3804 req := &structs.JobListRequest{ 3805 QueryOptions: structs.QueryOptions{ 3806 Region: "global", 3807 Namespace: job.Namespace, 3808 }, 3809 } 3810 3811 // Expect failure for request without a token 3812 var resp structs.JobListResponse 3813 err = msgpackrpc.CallWithCodec(codec, "Job.List", req, &resp) 3814 require.NotNil(err) 3815 3816 // Expect success for request with a management token 3817 var mgmtResp structs.JobListResponse 3818 req.AuthToken = root.SecretID 3819 err = msgpackrpc.CallWithCodec(codec, "Job.List", req, &mgmtResp) 3820 require.Nil(err) 3821 require.Equal(1, len(mgmtResp.Jobs)) 3822 require.Equal(job.ID, mgmtResp.Jobs[0].ID) 3823 3824 // Expect failure for request with a token that has incorrect permissions 3825 invalidToken := mock.CreatePolicyAndToken(t, state, 1003, "test-invalid", 3826 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob})) 3827 3828 req.AuthToken = invalidToken.SecretID 3829 var invalidResp structs.JobListResponse 3830 err = msgpackrpc.CallWithCodec(codec, "Job.List", req, &invalidResp) 3831 require.NotNil(err) 3832 3833 // Try with a valid token with correct permissions 3834 validToken := mock.CreatePolicyAndToken(t, state, 1001, "test-valid", 3835 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityListJobs})) 3836 var validResp structs.JobListResponse 3837 req.AuthToken = validToken.SecretID 3838 3839 err = msgpackrpc.CallWithCodec(codec, "Job.List", req, &validResp) 3840 require.Nil(err) 3841 require.Equal(1, len(validResp.Jobs)) 3842 require.Equal(job.ID, validResp.Jobs[0].ID) 3843 } 3844 3845 func TestJobEndpoint_ListJobs_Blocking(t *testing.T) { 3846 t.Parallel() 3847 3848 s1, cleanupS1 := TestServer(t, nil) 3849 defer cleanupS1() 3850 state := s1.fsm.State() 3851 codec := rpcClient(t, s1) 3852 testutil.WaitForLeader(t, s1.RPC) 3853 3854 // Create the job 3855 job := mock.Job() 3856 3857 // Upsert job triggers watches 3858 time.AfterFunc(100*time.Millisecond, func() { 3859 if err := state.UpsertJob(100, job); err != nil { 3860 t.Fatalf("err: %v", err) 3861 } 3862 }) 3863 3864 req := &structs.JobListRequest{ 3865 QueryOptions: structs.QueryOptions{ 3866 Region: "global", 3867 Namespace: job.Namespace, 3868 MinQueryIndex: 50, 3869 }, 3870 } 3871 start := time.Now() 3872 var resp structs.JobListResponse 3873 if err := msgpackrpc.CallWithCodec(codec, "Job.List", req, &resp); err != nil { 3874 t.Fatalf("err: %v", err) 3875 } 3876 3877 if elapsed := time.Since(start); elapsed < 100*time.Millisecond { 3878 t.Fatalf("should block (returned in %s) %#v", elapsed, resp) 3879 } 3880 if resp.Index != 100 { 3881 t.Fatalf("Bad index: %d %d", resp.Index, 100) 3882 } 3883 if len(resp.Jobs) != 1 || resp.Jobs[0].ID != job.ID { 3884 t.Fatalf("bad: %#v", resp) 3885 } 3886 3887 // Job deletion triggers watches 3888 time.AfterFunc(100*time.Millisecond, func() { 3889 if err := state.DeleteJob(200, job.Namespace, job.ID); err != nil { 3890 t.Fatalf("err: %v", err) 3891 } 3892 }) 3893 3894 req.MinQueryIndex = 150 3895 start = time.Now() 3896 var resp2 structs.JobListResponse 3897 if err := msgpackrpc.CallWithCodec(codec, "Job.List", req, &resp2); err != nil { 3898 t.Fatalf("err: %v", err) 3899 } 3900 3901 if elapsed := time.Since(start); elapsed < 100*time.Millisecond { 3902 t.Fatalf("should block (returned in %s) %#v", elapsed, resp2) 3903 } 3904 if resp2.Index != 200 { 3905 t.Fatalf("Bad index: %d %d", resp2.Index, 200) 3906 } 3907 if len(resp2.Jobs) != 0 { 3908 t.Fatalf("bad: %#v", resp2) 3909 } 3910 } 3911 3912 func TestJobEndpoint_Allocations(t *testing.T) { 3913 t.Parallel() 3914 3915 s1, cleanupS1 := TestServer(t, nil) 3916 defer cleanupS1() 3917 codec := rpcClient(t, s1) 3918 testutil.WaitForLeader(t, s1.RPC) 3919 3920 // Create the register request 3921 alloc1 := mock.Alloc() 3922 alloc2 := mock.Alloc() 3923 alloc2.JobID = alloc1.JobID 3924 state := s1.fsm.State() 3925 state.UpsertJobSummary(998, mock.JobSummary(alloc1.JobID)) 3926 state.UpsertJobSummary(999, mock.JobSummary(alloc2.JobID)) 3927 err := state.UpsertAllocs(1000, 3928 []*structs.Allocation{alloc1, alloc2}) 3929 if err != nil { 3930 t.Fatalf("err: %v", err) 3931 } 3932 3933 // Lookup the jobs 3934 get := &structs.JobSpecificRequest{ 3935 JobID: alloc1.JobID, 3936 QueryOptions: structs.QueryOptions{ 3937 Region: "global", 3938 Namespace: alloc1.Job.Namespace, 3939 }, 3940 } 3941 var resp2 structs.JobAllocationsResponse 3942 if err := msgpackrpc.CallWithCodec(codec, "Job.Allocations", get, &resp2); err != nil { 3943 t.Fatalf("err: %v", err) 3944 } 3945 if resp2.Index != 1000 { 3946 t.Fatalf("Bad index: %d %d", resp2.Index, 1000) 3947 } 3948 3949 if len(resp2.Allocations) != 2 { 3950 t.Fatalf("bad: %#v", resp2.Allocations) 3951 } 3952 } 3953 3954 func TestJobEndpoint_Allocations_ACL(t *testing.T) { 3955 t.Parallel() 3956 require := require.New(t) 3957 3958 s1, root, cleanupS1 := TestACLServer(t, nil) 3959 defer cleanupS1() 3960 codec := rpcClient(t, s1) 3961 testutil.WaitForLeader(t, s1.RPC) 3962 state := s1.fsm.State() 3963 3964 // Create allocations for a job 3965 alloc1 := mock.Alloc() 3966 alloc2 := mock.Alloc() 3967 alloc2.JobID = alloc1.JobID 3968 state.UpsertJobSummary(998, mock.JobSummary(alloc1.JobID)) 3969 state.UpsertJobSummary(999, mock.JobSummary(alloc2.JobID)) 3970 err := state.UpsertAllocs(1000, 3971 []*structs.Allocation{alloc1, alloc2}) 3972 require.Nil(err) 3973 3974 // Look up allocations for that job 3975 get := &structs.JobSpecificRequest{ 3976 JobID: alloc1.JobID, 3977 QueryOptions: structs.QueryOptions{ 3978 Region: "global", 3979 Namespace: alloc1.Job.Namespace, 3980 }, 3981 } 3982 3983 // Attempt to fetch the response without a token should fail 3984 var resp structs.JobAllocationsResponse 3985 err = msgpackrpc.CallWithCodec(codec, "Job.Allocations", get, &resp) 3986 require.NotNil(err) 3987 require.Contains(err.Error(), "Permission denied") 3988 3989 // Attempt to fetch the response with an invalid token should fail 3990 invalidToken := mock.CreatePolicyAndToken(t, state, 1001, "test-invalid", 3991 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityListJobs})) 3992 3993 get.AuthToken = invalidToken.SecretID 3994 var invalidResp structs.JobAllocationsResponse 3995 err = msgpackrpc.CallWithCodec(codec, "Job.Allocations", get, &invalidResp) 3996 require.NotNil(err) 3997 require.Contains(err.Error(), "Permission denied") 3998 3999 // Attempt to fetch the response with valid management token should succeed 4000 get.AuthToken = root.SecretID 4001 var validResp structs.JobAllocationsResponse 4002 err = msgpackrpc.CallWithCodec(codec, "Job.Allocations", get, &validResp) 4003 require.Nil(err) 4004 4005 // Attempt to fetch the response with valid management token should succeed 4006 validToken := mock.CreatePolicyAndToken(t, state, 1005, "test-valid", 4007 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob})) 4008 4009 get.AuthToken = validToken.SecretID 4010 var validResp2 structs.JobAllocationsResponse 4011 err = msgpackrpc.CallWithCodec(codec, "Job.Allocations", get, &validResp2) 4012 require.Nil(err) 4013 4014 require.Equal(2, len(validResp2.Allocations)) 4015 } 4016 4017 func TestJobEndpoint_Allocations_Blocking(t *testing.T) { 4018 t.Parallel() 4019 4020 s1, cleanupS1 := TestServer(t, nil) 4021 defer cleanupS1() 4022 codec := rpcClient(t, s1) 4023 testutil.WaitForLeader(t, s1.RPC) 4024 4025 // Create the register request 4026 alloc1 := mock.Alloc() 4027 alloc2 := mock.Alloc() 4028 alloc2.JobID = "job1" 4029 state := s1.fsm.State() 4030 4031 // First upsert an unrelated alloc 4032 time.AfterFunc(100*time.Millisecond, func() { 4033 state.UpsertJobSummary(99, mock.JobSummary(alloc1.JobID)) 4034 err := state.UpsertAllocs(100, []*structs.Allocation{alloc1}) 4035 if err != nil { 4036 t.Fatalf("err: %v", err) 4037 } 4038 }) 4039 4040 // Upsert an alloc for the job we are interested in later 4041 time.AfterFunc(200*time.Millisecond, func() { 4042 state.UpsertJobSummary(199, mock.JobSummary(alloc2.JobID)) 4043 err := state.UpsertAllocs(200, []*structs.Allocation{alloc2}) 4044 if err != nil { 4045 t.Fatalf("err: %v", err) 4046 } 4047 }) 4048 4049 // Lookup the jobs 4050 get := &structs.JobSpecificRequest{ 4051 JobID: "job1", 4052 QueryOptions: structs.QueryOptions{ 4053 Region: "global", 4054 Namespace: alloc1.Job.Namespace, 4055 MinQueryIndex: 150, 4056 }, 4057 } 4058 var resp structs.JobAllocationsResponse 4059 start := time.Now() 4060 if err := msgpackrpc.CallWithCodec(codec, "Job.Allocations", get, &resp); err != nil { 4061 t.Fatalf("err: %v", err) 4062 } 4063 4064 if elapsed := time.Since(start); elapsed < 200*time.Millisecond { 4065 t.Fatalf("should block (returned in %s) %#v", elapsed, resp) 4066 } 4067 if resp.Index != 200 { 4068 t.Fatalf("Bad index: %d %d", resp.Index, 200) 4069 } 4070 if len(resp.Allocations) != 1 || resp.Allocations[0].JobID != "job1" { 4071 t.Fatalf("bad: %#v", resp.Allocations) 4072 } 4073 } 4074 4075 // TestJobEndpoint_Allocations_NoJobID asserts not setting a JobID in the 4076 // request returns an error. 4077 func TestJobEndpoint_Allocations_NoJobID(t *testing.T) { 4078 t.Parallel() 4079 4080 s1, cleanupS1 := TestServer(t, nil) 4081 defer cleanupS1() 4082 codec := rpcClient(t, s1) 4083 testutil.WaitForLeader(t, s1.RPC) 4084 4085 get := &structs.JobSpecificRequest{ 4086 JobID: "", 4087 QueryOptions: structs.QueryOptions{ 4088 Region: "global", 4089 Namespace: structs.DefaultNamespace, 4090 }, 4091 } 4092 4093 var resp structs.JobAllocationsResponse 4094 err := msgpackrpc.CallWithCodec(codec, "Job.Allocations", get, &resp) 4095 require.Error(t, err) 4096 require.Contains(t, err.Error(), "missing job ID") 4097 } 4098 4099 func TestJobEndpoint_Evaluations(t *testing.T) { 4100 t.Parallel() 4101 4102 s1, cleanupS1 := TestServer(t, nil) 4103 defer cleanupS1() 4104 codec := rpcClient(t, s1) 4105 testutil.WaitForLeader(t, s1.RPC) 4106 4107 // Create the register request 4108 eval1 := mock.Eval() 4109 eval2 := mock.Eval() 4110 eval2.JobID = eval1.JobID 4111 state := s1.fsm.State() 4112 err := state.UpsertEvals(1000, 4113 []*structs.Evaluation{eval1, eval2}) 4114 if err != nil { 4115 t.Fatalf("err: %v", err) 4116 } 4117 4118 // Lookup the jobs 4119 get := &structs.JobSpecificRequest{ 4120 JobID: eval1.JobID, 4121 QueryOptions: structs.QueryOptions{ 4122 Region: "global", 4123 Namespace: eval1.Namespace, 4124 }, 4125 } 4126 var resp2 structs.JobEvaluationsResponse 4127 if err := msgpackrpc.CallWithCodec(codec, "Job.Evaluations", get, &resp2); err != nil { 4128 t.Fatalf("err: %v", err) 4129 } 4130 if resp2.Index != 1000 { 4131 t.Fatalf("Bad index: %d %d", resp2.Index, 1000) 4132 } 4133 4134 if len(resp2.Evaluations) != 2 { 4135 t.Fatalf("bad: %#v", resp2.Evaluations) 4136 } 4137 } 4138 4139 func TestJobEndpoint_Evaluations_ACL(t *testing.T) { 4140 t.Parallel() 4141 require := require.New(t) 4142 4143 s1, root, cleanupS1 := TestACLServer(t, nil) 4144 defer cleanupS1() 4145 codec := rpcClient(t, s1) 4146 testutil.WaitForLeader(t, s1.RPC) 4147 state := s1.fsm.State() 4148 4149 // Create evaluations for the same job 4150 eval1 := mock.Eval() 4151 eval2 := mock.Eval() 4152 eval2.JobID = eval1.JobID 4153 err := state.UpsertEvals(1000, 4154 []*structs.Evaluation{eval1, eval2}) 4155 require.Nil(err) 4156 4157 // Lookup the jobs 4158 get := &structs.JobSpecificRequest{ 4159 JobID: eval1.JobID, 4160 QueryOptions: structs.QueryOptions{ 4161 Region: "global", 4162 Namespace: eval1.Namespace, 4163 }, 4164 } 4165 4166 // Attempt to fetch without providing a token 4167 var resp structs.JobEvaluationsResponse 4168 err = msgpackrpc.CallWithCodec(codec, "Job.Evaluations", get, &resp) 4169 require.NotNil(err) 4170 require.Contains(err.Error(), "Permission denied") 4171 4172 // Attempt to fetch the response with an invalid token 4173 invalidToken := mock.CreatePolicyAndToken(t, state, 1001, "test-invalid", 4174 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityListJobs})) 4175 4176 get.AuthToken = invalidToken.SecretID 4177 var invalidResp structs.JobEvaluationsResponse 4178 err = msgpackrpc.CallWithCodec(codec, "Job.Evaluations", get, &invalidResp) 4179 require.NotNil(err) 4180 require.Contains(err.Error(), "Permission denied") 4181 4182 // Attempt to fetch with valid management token should succeed 4183 get.AuthToken = root.SecretID 4184 var validResp structs.JobEvaluationsResponse 4185 err = msgpackrpc.CallWithCodec(codec, "Job.Evaluations", get, &validResp) 4186 require.Nil(err) 4187 require.Equal(2, len(validResp.Evaluations)) 4188 4189 // Attempt to fetch with valid token should succeed 4190 validToken := mock.CreatePolicyAndToken(t, state, 1003, "test-valid", 4191 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob})) 4192 4193 get.AuthToken = validToken.SecretID 4194 var validResp2 structs.JobEvaluationsResponse 4195 err = msgpackrpc.CallWithCodec(codec, "Job.Evaluations", get, &validResp2) 4196 require.Nil(err) 4197 require.Equal(2, len(validResp2.Evaluations)) 4198 } 4199 4200 func TestJobEndpoint_Evaluations_Blocking(t *testing.T) { 4201 t.Parallel() 4202 4203 s1, cleanupS1 := TestServer(t, nil) 4204 defer cleanupS1() 4205 codec := rpcClient(t, s1) 4206 testutil.WaitForLeader(t, s1.RPC) 4207 4208 // Create the register request 4209 eval1 := mock.Eval() 4210 eval2 := mock.Eval() 4211 eval2.JobID = "job1" 4212 state := s1.fsm.State() 4213 4214 // First upsert an unrelated eval 4215 time.AfterFunc(100*time.Millisecond, func() { 4216 err := state.UpsertEvals(100, []*structs.Evaluation{eval1}) 4217 if err != nil { 4218 t.Fatalf("err: %v", err) 4219 } 4220 }) 4221 4222 // Upsert an eval for the job we are interested in later 4223 time.AfterFunc(200*time.Millisecond, func() { 4224 err := state.UpsertEvals(200, []*structs.Evaluation{eval2}) 4225 if err != nil { 4226 t.Fatalf("err: %v", err) 4227 } 4228 }) 4229 4230 // Lookup the jobs 4231 get := &structs.JobSpecificRequest{ 4232 JobID: "job1", 4233 QueryOptions: structs.QueryOptions{ 4234 Region: "global", 4235 Namespace: eval1.Namespace, 4236 MinQueryIndex: 150, 4237 }, 4238 } 4239 var resp structs.JobEvaluationsResponse 4240 start := time.Now() 4241 if err := msgpackrpc.CallWithCodec(codec, "Job.Evaluations", get, &resp); err != nil { 4242 t.Fatalf("err: %v", err) 4243 } 4244 4245 if elapsed := time.Since(start); elapsed < 200*time.Millisecond { 4246 t.Fatalf("should block (returned in %s) %#v", elapsed, resp) 4247 } 4248 if resp.Index != 200 { 4249 t.Fatalf("Bad index: %d %d", resp.Index, 200) 4250 } 4251 if len(resp.Evaluations) != 1 || resp.Evaluations[0].JobID != "job1" { 4252 t.Fatalf("bad: %#v", resp.Evaluations) 4253 } 4254 } 4255 4256 func TestJobEndpoint_Deployments(t *testing.T) { 4257 t.Parallel() 4258 4259 s1, cleanupS1 := TestServer(t, nil) 4260 defer cleanupS1() 4261 codec := rpcClient(t, s1) 4262 testutil.WaitForLeader(t, s1.RPC) 4263 state := s1.fsm.State() 4264 require := require.New(t) 4265 4266 // Create the register request 4267 j := mock.Job() 4268 d1 := mock.Deployment() 4269 d2 := mock.Deployment() 4270 d1.JobID = j.ID 4271 d2.JobID = j.ID 4272 require.Nil(state.UpsertJob(1000, j), "UpsertJob") 4273 d1.JobCreateIndex = j.CreateIndex 4274 d2.JobCreateIndex = j.CreateIndex 4275 4276 require.Nil(state.UpsertDeployment(1001, d1), "UpsertDeployment") 4277 require.Nil(state.UpsertDeployment(1002, d2), "UpsertDeployment") 4278 4279 // Lookup the jobs 4280 get := &structs.JobSpecificRequest{ 4281 JobID: j.ID, 4282 QueryOptions: structs.QueryOptions{ 4283 Region: "global", 4284 Namespace: j.Namespace, 4285 }, 4286 } 4287 var resp structs.DeploymentListResponse 4288 require.Nil(msgpackrpc.CallWithCodec(codec, "Job.Deployments", get, &resp), "RPC") 4289 require.EqualValues(1002, resp.Index, "response index") 4290 require.Len(resp.Deployments, 2, "deployments for job") 4291 } 4292 4293 func TestJobEndpoint_Deployments_ACL(t *testing.T) { 4294 t.Parallel() 4295 require := require.New(t) 4296 4297 s1, root, cleanupS1 := TestACLServer(t, nil) 4298 defer cleanupS1() 4299 codec := rpcClient(t, s1) 4300 testutil.WaitForLeader(t, s1.RPC) 4301 state := s1.fsm.State() 4302 4303 // Create a job and corresponding deployments 4304 j := mock.Job() 4305 d1 := mock.Deployment() 4306 d2 := mock.Deployment() 4307 d1.JobID = j.ID 4308 d2.JobID = j.ID 4309 require.Nil(state.UpsertJob(1000, j), "UpsertJob") 4310 d1.JobCreateIndex = j.CreateIndex 4311 d2.JobCreateIndex = j.CreateIndex 4312 require.Nil(state.UpsertDeployment(1001, d1), "UpsertDeployment") 4313 require.Nil(state.UpsertDeployment(1002, d2), "UpsertDeployment") 4314 4315 // Lookup the jobs 4316 get := &structs.JobSpecificRequest{ 4317 JobID: j.ID, 4318 QueryOptions: structs.QueryOptions{ 4319 Region: "global", 4320 Namespace: j.Namespace, 4321 }, 4322 } 4323 // Lookup with no token should fail 4324 var resp structs.DeploymentListResponse 4325 err := msgpackrpc.CallWithCodec(codec, "Job.Deployments", get, &resp) 4326 require.NotNil(err) 4327 require.Contains(err.Error(), "Permission denied") 4328 4329 // Attempt to fetch the response with an invalid token 4330 invalidToken := mock.CreatePolicyAndToken(t, state, 1001, "test-invalid", 4331 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityListJobs})) 4332 4333 get.AuthToken = invalidToken.SecretID 4334 var invalidResp structs.DeploymentListResponse 4335 err = msgpackrpc.CallWithCodec(codec, "Job.Deployments", get, &invalidResp) 4336 require.NotNil(err) 4337 require.Contains(err.Error(), "Permission denied") 4338 4339 // Lookup with valid management token should succeed 4340 get.AuthToken = root.SecretID 4341 var validResp structs.DeploymentListResponse 4342 require.Nil(msgpackrpc.CallWithCodec(codec, "Job.Deployments", get, &validResp), "RPC") 4343 require.EqualValues(1002, validResp.Index, "response index") 4344 require.Len(validResp.Deployments, 2, "deployments for job") 4345 4346 // Lookup with valid token should succeed 4347 validToken := mock.CreatePolicyAndToken(t, state, 1005, "test-valid", 4348 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob})) 4349 4350 get.AuthToken = validToken.SecretID 4351 var validResp2 structs.DeploymentListResponse 4352 require.Nil(msgpackrpc.CallWithCodec(codec, "Job.Deployments", get, &validResp2), "RPC") 4353 require.EqualValues(1002, validResp2.Index, "response index") 4354 require.Len(validResp2.Deployments, 2, "deployments for job") 4355 } 4356 4357 func TestJobEndpoint_Deployments_Blocking(t *testing.T) { 4358 t.Parallel() 4359 4360 s1, cleanupS1 := TestServer(t, nil) 4361 defer cleanupS1() 4362 codec := rpcClient(t, s1) 4363 testutil.WaitForLeader(t, s1.RPC) 4364 state := s1.fsm.State() 4365 require := require.New(t) 4366 4367 // Create the register request 4368 j := mock.Job() 4369 d1 := mock.Deployment() 4370 d2 := mock.Deployment() 4371 d2.JobID = j.ID 4372 require.Nil(state.UpsertJob(50, j), "UpsertJob") 4373 d2.JobCreateIndex = j.CreateIndex 4374 // First upsert an unrelated eval 4375 time.AfterFunc(100*time.Millisecond, func() { 4376 require.Nil(state.UpsertDeployment(100, d1), "UpsertDeployment") 4377 }) 4378 4379 // Upsert an eval for the job we are interested in later 4380 time.AfterFunc(200*time.Millisecond, func() { 4381 require.Nil(state.UpsertDeployment(200, d2), "UpsertDeployment") 4382 }) 4383 4384 // Lookup the jobs 4385 get := &structs.JobSpecificRequest{ 4386 JobID: d2.JobID, 4387 QueryOptions: structs.QueryOptions{ 4388 Region: "global", 4389 Namespace: d2.Namespace, 4390 MinQueryIndex: 150, 4391 }, 4392 } 4393 var resp structs.DeploymentListResponse 4394 start := time.Now() 4395 require.Nil(msgpackrpc.CallWithCodec(codec, "Job.Deployments", get, &resp), "RPC") 4396 require.EqualValues(200, resp.Index, "response index") 4397 require.Len(resp.Deployments, 1, "deployments for job") 4398 require.Equal(d2.ID, resp.Deployments[0].ID, "returned deployment") 4399 if elapsed := time.Since(start); elapsed < 200*time.Millisecond { 4400 t.Fatalf("should block (returned in %s) %#v", elapsed, resp) 4401 } 4402 } 4403 4404 func TestJobEndpoint_LatestDeployment(t *testing.T) { 4405 t.Parallel() 4406 4407 s1, cleanupS1 := TestServer(t, nil) 4408 defer cleanupS1() 4409 codec := rpcClient(t, s1) 4410 testutil.WaitForLeader(t, s1.RPC) 4411 state := s1.fsm.State() 4412 require := require.New(t) 4413 4414 // Create the register request 4415 j := mock.Job() 4416 d1 := mock.Deployment() 4417 d2 := mock.Deployment() 4418 d1.JobID = j.ID 4419 d2.JobID = j.ID 4420 d2.CreateIndex = d1.CreateIndex + 100 4421 d2.ModifyIndex = d2.CreateIndex + 100 4422 require.Nil(state.UpsertJob(1000, j), "UpsertJob") 4423 d1.JobCreateIndex = j.CreateIndex 4424 d2.JobCreateIndex = j.CreateIndex 4425 require.Nil(state.UpsertDeployment(1001, d1), "UpsertDeployment") 4426 require.Nil(state.UpsertDeployment(1002, d2), "UpsertDeployment") 4427 4428 // Lookup the jobs 4429 get := &structs.JobSpecificRequest{ 4430 JobID: j.ID, 4431 QueryOptions: structs.QueryOptions{ 4432 Region: "global", 4433 Namespace: j.Namespace, 4434 }, 4435 } 4436 var resp structs.SingleDeploymentResponse 4437 require.Nil(msgpackrpc.CallWithCodec(codec, "Job.LatestDeployment", get, &resp), "RPC") 4438 require.EqualValues(1002, resp.Index, "response index") 4439 require.NotNil(resp.Deployment, "want a deployment") 4440 require.Equal(d2.ID, resp.Deployment.ID, "latest deployment for job") 4441 } 4442 4443 func TestJobEndpoint_LatestDeployment_ACL(t *testing.T) { 4444 t.Parallel() 4445 require := require.New(t) 4446 4447 s1, root, cleanupS1 := TestACLServer(t, nil) 4448 defer cleanupS1() 4449 codec := rpcClient(t, s1) 4450 testutil.WaitForLeader(t, s1.RPC) 4451 state := s1.fsm.State() 4452 4453 // Create a job and deployments 4454 j := mock.Job() 4455 d1 := mock.Deployment() 4456 d2 := mock.Deployment() 4457 d1.JobID = j.ID 4458 d2.JobID = j.ID 4459 d2.CreateIndex = d1.CreateIndex + 100 4460 d2.ModifyIndex = d2.CreateIndex + 100 4461 require.Nil(state.UpsertJob(1000, j), "UpsertJob") 4462 d1.JobCreateIndex = j.CreateIndex 4463 d2.JobCreateIndex = j.CreateIndex 4464 require.Nil(state.UpsertDeployment(1001, d1), "UpsertDeployment") 4465 require.Nil(state.UpsertDeployment(1002, d2), "UpsertDeployment") 4466 4467 // Lookup the jobs 4468 get := &structs.JobSpecificRequest{ 4469 JobID: j.ID, 4470 QueryOptions: structs.QueryOptions{ 4471 Region: "global", 4472 Namespace: j.Namespace, 4473 }, 4474 } 4475 4476 // Attempt to fetch the response without a token should fail 4477 var resp structs.SingleDeploymentResponse 4478 err := msgpackrpc.CallWithCodec(codec, "Job.LatestDeployment", get, &resp) 4479 require.NotNil(err) 4480 require.Contains(err.Error(), "Permission denied") 4481 4482 // Attempt to fetch the response with an invalid token should fail 4483 invalidToken := mock.CreatePolicyAndToken(t, state, 1001, "test-invalid", 4484 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityListJobs})) 4485 4486 get.AuthToken = invalidToken.SecretID 4487 var invalidResp structs.SingleDeploymentResponse 4488 err = msgpackrpc.CallWithCodec(codec, "Job.LatestDeployment", get, &invalidResp) 4489 require.NotNil(err) 4490 require.Contains(err.Error(), "Permission denied") 4491 4492 // Fetching latest deployment with a valid management token should succeed 4493 get.AuthToken = root.SecretID 4494 var validResp structs.SingleDeploymentResponse 4495 require.Nil(msgpackrpc.CallWithCodec(codec, "Job.LatestDeployment", get, &validResp), "RPC") 4496 require.EqualValues(1002, validResp.Index, "response index") 4497 require.NotNil(validResp.Deployment, "want a deployment") 4498 require.Equal(d2.ID, validResp.Deployment.ID, "latest deployment for job") 4499 4500 // Fetching latest deployment with a valid token should succeed 4501 validToken := mock.CreatePolicyAndToken(t, state, 1004, "test-valid", 4502 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob})) 4503 4504 get.AuthToken = validToken.SecretID 4505 var validResp2 structs.SingleDeploymentResponse 4506 require.Nil(msgpackrpc.CallWithCodec(codec, "Job.LatestDeployment", get, &validResp2), "RPC") 4507 require.EqualValues(1002, validResp2.Index, "response index") 4508 require.NotNil(validResp2.Deployment, "want a deployment") 4509 require.Equal(d2.ID, validResp2.Deployment.ID, "latest deployment for job") 4510 } 4511 4512 func TestJobEndpoint_LatestDeployment_Blocking(t *testing.T) { 4513 t.Parallel() 4514 4515 s1, cleanupS1 := TestServer(t, nil) 4516 defer cleanupS1() 4517 codec := rpcClient(t, s1) 4518 testutil.WaitForLeader(t, s1.RPC) 4519 state := s1.fsm.State() 4520 require := require.New(t) 4521 4522 // Create the register request 4523 j := mock.Job() 4524 d1 := mock.Deployment() 4525 d2 := mock.Deployment() 4526 d2.JobID = j.ID 4527 require.Nil(state.UpsertJob(50, j), "UpsertJob") 4528 d2.JobCreateIndex = j.CreateIndex 4529 4530 // First upsert an unrelated eval 4531 time.AfterFunc(100*time.Millisecond, func() { 4532 require.Nil(state.UpsertDeployment(100, d1), "UpsertDeployment") 4533 }) 4534 4535 // Upsert an eval for the job we are interested in later 4536 time.AfterFunc(200*time.Millisecond, func() { 4537 require.Nil(state.UpsertDeployment(200, d2), "UpsertDeployment") 4538 }) 4539 4540 // Lookup the jobs 4541 get := &structs.JobSpecificRequest{ 4542 JobID: d2.JobID, 4543 QueryOptions: structs.QueryOptions{ 4544 Region: "global", 4545 Namespace: d2.Namespace, 4546 MinQueryIndex: 150, 4547 }, 4548 } 4549 var resp structs.SingleDeploymentResponse 4550 start := time.Now() 4551 require.Nil(msgpackrpc.CallWithCodec(codec, "Job.LatestDeployment", get, &resp), "RPC") 4552 require.EqualValues(200, resp.Index, "response index") 4553 require.NotNil(resp.Deployment, "deployment for job") 4554 require.Equal(d2.ID, resp.Deployment.ID, "returned deployment") 4555 if elapsed := time.Since(start); elapsed < 200*time.Millisecond { 4556 t.Fatalf("should block (returned in %s) %#v", elapsed, resp) 4557 } 4558 } 4559 4560 func TestJobEndpoint_Plan_ACL(t *testing.T) { 4561 t.Parallel() 4562 4563 s1, root, cleanupS1 := TestACLServer(t, func(c *Config) { 4564 c.NumSchedulers = 0 // Prevent automatic dequeue 4565 }) 4566 defer cleanupS1() 4567 codec := rpcClient(t, s1) 4568 testutil.WaitForLeader(t, s1.RPC) 4569 4570 // Create a plan request 4571 job := mock.Job() 4572 planReq := &structs.JobPlanRequest{ 4573 Job: job, 4574 Diff: true, 4575 WriteRequest: structs.WriteRequest{ 4576 Region: "global", 4577 Namespace: job.Namespace, 4578 }, 4579 } 4580 4581 // Try without a token, expect failure 4582 var planResp structs.JobPlanResponse 4583 if err := msgpackrpc.CallWithCodec(codec, "Job.Plan", planReq, &planResp); err == nil { 4584 t.Fatalf("expected error") 4585 } 4586 4587 // Try with a token 4588 planReq.AuthToken = root.SecretID 4589 if err := msgpackrpc.CallWithCodec(codec, "Job.Plan", planReq, &planResp); err != nil { 4590 t.Fatalf("err: %v", err) 4591 } 4592 } 4593 4594 func TestJobEndpoint_Plan_WithDiff(t *testing.T) { 4595 t.Parallel() 4596 4597 s1, cleanupS1 := TestServer(t, func(c *Config) { 4598 c.NumSchedulers = 0 // Prevent automatic dequeue 4599 }) 4600 defer cleanupS1() 4601 codec := rpcClient(t, s1) 4602 testutil.WaitForLeader(t, s1.RPC) 4603 4604 // Create the register request 4605 job := mock.Job() 4606 req := &structs.JobRegisterRequest{ 4607 Job: job, 4608 WriteRequest: structs.WriteRequest{ 4609 Region: "global", 4610 Namespace: job.Namespace, 4611 }, 4612 } 4613 4614 // Fetch the response 4615 var resp structs.JobRegisterResponse 4616 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { 4617 t.Fatalf("err: %v", err) 4618 } 4619 if resp.Index == 0 { 4620 t.Fatalf("bad index: %d", resp.Index) 4621 } 4622 4623 // Create a plan request 4624 planReq := &structs.JobPlanRequest{ 4625 Job: job, 4626 Diff: true, 4627 WriteRequest: structs.WriteRequest{ 4628 Region: "global", 4629 Namespace: job.Namespace, 4630 }, 4631 } 4632 4633 // Fetch the response 4634 var planResp structs.JobPlanResponse 4635 if err := msgpackrpc.CallWithCodec(codec, "Job.Plan", planReq, &planResp); err != nil { 4636 t.Fatalf("err: %v", err) 4637 } 4638 4639 // Check the response 4640 if planResp.JobModifyIndex == 0 { 4641 t.Fatalf("bad cas: %d", planResp.JobModifyIndex) 4642 } 4643 if planResp.Annotations == nil { 4644 t.Fatalf("no annotations") 4645 } 4646 if planResp.Diff == nil { 4647 t.Fatalf("no diff") 4648 } 4649 if len(planResp.FailedTGAllocs) == 0 { 4650 t.Fatalf("no failed task group alloc metrics") 4651 } 4652 } 4653 4654 func TestJobEndpoint_Plan_NoDiff(t *testing.T) { 4655 t.Parallel() 4656 4657 s1, cleanupS1 := TestServer(t, func(c *Config) { 4658 c.NumSchedulers = 0 // Prevent automatic dequeue 4659 }) 4660 defer cleanupS1() 4661 codec := rpcClient(t, s1) 4662 testutil.WaitForLeader(t, s1.RPC) 4663 4664 // Create the register request 4665 job := mock.Job() 4666 req := &structs.JobRegisterRequest{ 4667 Job: job, 4668 WriteRequest: structs.WriteRequest{ 4669 Region: "global", 4670 Namespace: job.Namespace, 4671 }, 4672 } 4673 4674 // Fetch the response 4675 var resp structs.JobRegisterResponse 4676 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { 4677 t.Fatalf("err: %v", err) 4678 } 4679 if resp.Index == 0 { 4680 t.Fatalf("bad index: %d", resp.Index) 4681 } 4682 4683 // Create a plan request 4684 planReq := &structs.JobPlanRequest{ 4685 Job: job, 4686 Diff: false, 4687 WriteRequest: structs.WriteRequest{ 4688 Region: "global", 4689 Namespace: job.Namespace, 4690 }, 4691 } 4692 4693 // Fetch the response 4694 var planResp structs.JobPlanResponse 4695 if err := msgpackrpc.CallWithCodec(codec, "Job.Plan", planReq, &planResp); err != nil { 4696 t.Fatalf("err: %v", err) 4697 } 4698 4699 // Check the response 4700 if planResp.JobModifyIndex == 0 { 4701 t.Fatalf("bad cas: %d", planResp.JobModifyIndex) 4702 } 4703 if planResp.Annotations == nil { 4704 t.Fatalf("no annotations") 4705 } 4706 if planResp.Diff != nil { 4707 t.Fatalf("got diff") 4708 } 4709 if len(planResp.FailedTGAllocs) == 0 { 4710 t.Fatalf("no failed task group alloc metrics") 4711 } 4712 } 4713 4714 func TestJobEndpoint_ImplicitConstraints_Vault(t *testing.T) { 4715 t.Parallel() 4716 4717 s1, cleanupS1 := TestServer(t, func(c *Config) { 4718 c.NumSchedulers = 0 // Prevent automatic dequeue 4719 }) 4720 defer cleanupS1() 4721 codec := rpcClient(t, s1) 4722 testutil.WaitForLeader(t, s1.RPC) 4723 4724 // Enable vault 4725 tr, f := true, false 4726 s1.config.VaultConfig.Enabled = &tr 4727 s1.config.VaultConfig.AllowUnauthenticated = &f 4728 4729 // Replace the Vault Client on the server 4730 tvc := &TestVaultClient{} 4731 s1.vault = tvc 4732 4733 policy := "foo" 4734 goodToken := uuid.Generate() 4735 goodPolicies := []string{"foo", "bar", "baz"} 4736 tvc.SetLookupTokenAllowedPolicies(goodToken, goodPolicies) 4737 4738 // Create the register request with a job asking for a vault policy 4739 job := mock.Job() 4740 job.VaultToken = goodToken 4741 job.TaskGroups[0].Tasks[0].Vault = &structs.Vault{ 4742 Policies: []string{policy}, 4743 ChangeMode: structs.VaultChangeModeRestart, 4744 } 4745 req := &structs.JobRegisterRequest{ 4746 Job: job, 4747 WriteRequest: structs.WriteRequest{ 4748 Region: "global", 4749 Namespace: job.Namespace, 4750 }, 4751 } 4752 4753 // Fetch the response 4754 var resp structs.JobRegisterResponse 4755 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { 4756 t.Fatalf("bad: %v", err) 4757 } 4758 4759 // Check for the job in the FSM 4760 state := s1.fsm.State() 4761 ws := memdb.NewWatchSet() 4762 out, err := state.JobByID(ws, job.Namespace, job.ID) 4763 if err != nil { 4764 t.Fatalf("err: %v", err) 4765 } 4766 if out == nil { 4767 t.Fatalf("expected job") 4768 } 4769 if out.CreateIndex != resp.JobModifyIndex { 4770 t.Fatalf("index mis-match") 4771 } 4772 4773 // Check that there is an implicit vault constraint 4774 constraints := out.TaskGroups[0].Constraints 4775 if len(constraints) != 1 { 4776 t.Fatalf("Expected an implicit constraint") 4777 } 4778 4779 if !constraints[0].Equal(vaultConstraint) { 4780 t.Fatalf("Expected implicit vault constraint") 4781 } 4782 } 4783 4784 func TestJobEndpoint_ValidateJob_ConsulConnect(t *testing.T) { 4785 t.Parallel() 4786 4787 s1, cleanupS1 := TestServer(t, nil) 4788 defer cleanupS1() 4789 codec := rpcClient(t, s1) 4790 testutil.WaitForLeader(t, s1.RPC) 4791 4792 validateJob := func(j *structs.Job) error { 4793 req := &structs.JobRegisterRequest{ 4794 Job: j, 4795 WriteRequest: structs.WriteRequest{ 4796 Region: "global", 4797 Namespace: j.Namespace, 4798 }, 4799 } 4800 var resp structs.JobValidateResponse 4801 if err := msgpackrpc.CallWithCodec(codec, "Job.Validate", req, &resp); err != nil { 4802 return err 4803 } 4804 4805 if resp.Error != "" { 4806 return errors.New(resp.Error) 4807 } 4808 4809 if len(resp.ValidationErrors) != 0 { 4810 return errors.New(strings.Join(resp.ValidationErrors, ",")) 4811 } 4812 4813 if resp.Warnings != "" { 4814 return errors.New(resp.Warnings) 4815 } 4816 4817 return nil 4818 } 4819 4820 tgServices := []*structs.Service{ 4821 { 4822 Name: "count-api", 4823 PortLabel: "9001", 4824 Connect: &structs.ConsulConnect{ 4825 SidecarService: &structs.ConsulSidecarService{}, 4826 }, 4827 }, 4828 } 4829 4830 t.Run("plain job", func(t *testing.T) { 4831 j := mock.Job() 4832 require.NoError(t, validateJob(j)) 4833 }) 4834 t.Run("valid consul connect", func(t *testing.T) { 4835 j := mock.Job() 4836 4837 tg := j.TaskGroups[0] 4838 tg.Services = tgServices 4839 tg.Networks = structs.Networks{ 4840 {Mode: "bridge"}, 4841 } 4842 4843 err := validateJob(j) 4844 require.NoError(t, err) 4845 }) 4846 4847 t.Run("consul connect but missing network", func(t *testing.T) { 4848 j := mock.Job() 4849 4850 tg := j.TaskGroups[0] 4851 tg.Services = tgServices 4852 tg.Networks = nil 4853 4854 err := validateJob(j) 4855 require.Error(t, err) 4856 require.Contains(t, err.Error(), `Consul Connect sidecars require exactly 1 network`) 4857 }) 4858 4859 t.Run("consul connect but non bridge network", func(t *testing.T) { 4860 j := mock.Job() 4861 4862 tg := j.TaskGroups[0] 4863 tg.Services = tgServices 4864 4865 tg.Networks = structs.Networks{ 4866 {Mode: "host"}, 4867 } 4868 4869 err := validateJob(j) 4870 require.Error(t, err) 4871 require.Contains(t, err.Error(), `Consul Connect sidecar requires bridge network, found "host" in group "web"`) 4872 }) 4873 4874 } 4875 4876 func TestJobEndpoint_ImplicitConstraints_Signals(t *testing.T) { 4877 t.Parallel() 4878 4879 s1, cleanupS1 := TestServer(t, func(c *Config) { 4880 c.NumSchedulers = 0 // Prevent automatic dequeue 4881 }) 4882 defer cleanupS1() 4883 codec := rpcClient(t, s1) 4884 testutil.WaitForLeader(t, s1.RPC) 4885 4886 // Create the register request with a job asking for a template that sends a 4887 // signal 4888 job := mock.Job() 4889 signal1 := "SIGUSR1" 4890 signal2 := "SIGHUP" 4891 job.TaskGroups[0].Tasks[0].Templates = []*structs.Template{ 4892 { 4893 SourcePath: "foo", 4894 DestPath: "bar", 4895 ChangeMode: structs.TemplateChangeModeSignal, 4896 ChangeSignal: signal1, 4897 }, 4898 { 4899 SourcePath: "foo", 4900 DestPath: "baz", 4901 ChangeMode: structs.TemplateChangeModeSignal, 4902 ChangeSignal: signal2, 4903 }, 4904 } 4905 req := &structs.JobRegisterRequest{ 4906 Job: job, 4907 WriteRequest: structs.WriteRequest{ 4908 Region: "global", 4909 Namespace: job.Namespace, 4910 }, 4911 } 4912 4913 // Fetch the response 4914 var resp structs.JobRegisterResponse 4915 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { 4916 t.Fatalf("bad: %v", err) 4917 } 4918 4919 // Check for the job in the FSM 4920 state := s1.fsm.State() 4921 ws := memdb.NewWatchSet() 4922 out, err := state.JobByID(ws, job.Namespace, job.ID) 4923 if err != nil { 4924 t.Fatalf("err: %v", err) 4925 } 4926 if out == nil { 4927 t.Fatalf("expected job") 4928 } 4929 if out.CreateIndex != resp.JobModifyIndex { 4930 t.Fatalf("index mis-match") 4931 } 4932 4933 // Check that there is an implicit signal constraint 4934 constraints := out.TaskGroups[0].Constraints 4935 if len(constraints) != 1 { 4936 t.Fatalf("Expected an implicit constraint") 4937 } 4938 4939 sigConstraint := getSignalConstraint([]string{signal1, signal2}) 4940 if !strings.HasPrefix(sigConstraint.RTarget, "SIGHUP") { 4941 t.Fatalf("signals not sorted: %v", sigConstraint.RTarget) 4942 } 4943 4944 if !constraints[0].Equal(sigConstraint) { 4945 t.Fatalf("Expected implicit vault constraint") 4946 } 4947 } 4948 4949 func TestJobEndpoint_ValidateJobUpdate(t *testing.T) { 4950 t.Parallel() 4951 require := require.New(t) 4952 old := mock.Job() 4953 new := mock.Job() 4954 4955 if err := validateJobUpdate(old, new); err != nil { 4956 t.Errorf("expected update to be valid but got: %v", err) 4957 } 4958 4959 new.Type = "batch" 4960 if err := validateJobUpdate(old, new); err == nil { 4961 t.Errorf("expected err when setting new job to a different type") 4962 } else { 4963 t.Log(err) 4964 } 4965 4966 new = mock.Job() 4967 new.Periodic = &structs.PeriodicConfig{Enabled: true} 4968 if err := validateJobUpdate(old, new); err == nil { 4969 t.Errorf("expected err when setting new job to periodic") 4970 } else { 4971 t.Log(err) 4972 } 4973 4974 new = mock.Job() 4975 new.ParameterizedJob = &structs.ParameterizedJobConfig{} 4976 if err := validateJobUpdate(old, new); err == nil { 4977 t.Errorf("expected err when setting new job to parameterized") 4978 } else { 4979 t.Log(err) 4980 } 4981 4982 new = mock.Job() 4983 new.Dispatched = true 4984 require.Error(validateJobUpdate(old, new), 4985 "expected err when setting new job to dispatched") 4986 require.Error(validateJobUpdate(nil, new), 4987 "expected err when setting new job to dispatched") 4988 require.Error(validateJobUpdate(new, old), 4989 "expected err when setting dispatched to false") 4990 require.NoError(validateJobUpdate(nil, old)) 4991 } 4992 4993 func TestJobEndpoint_ValidateJobUpdate_ACL(t *testing.T) { 4994 t.Parallel() 4995 require := require.New(t) 4996 4997 s1, root, cleanupS1 := TestACLServer(t, func(c *Config) { 4998 c.NumSchedulers = 0 // Prevent automatic dequeue 4999 }) 5000 defer cleanupS1() 5001 codec := rpcClient(t, s1) 5002 testutil.WaitForLeader(t, s1.RPC) 5003 5004 job := mock.Job() 5005 5006 req := &structs.JobValidateRequest{ 5007 Job: job, 5008 WriteRequest: structs.WriteRequest{ 5009 Region: "global", 5010 Namespace: job.Namespace, 5011 }, 5012 } 5013 5014 // Attempt to update without providing a valid token 5015 var resp structs.JobValidateResponse 5016 err := msgpackrpc.CallWithCodec(codec, "Job.Validate", req, &resp) 5017 require.NotNil(err) 5018 5019 // Update with a valid token 5020 req.AuthToken = root.SecretID 5021 var validResp structs.JobValidateResponse 5022 err = msgpackrpc.CallWithCodec(codec, "Job.Validate", req, &validResp) 5023 require.Nil(err) 5024 5025 require.Equal("", validResp.Error) 5026 require.Equal("", validResp.Warnings) 5027 } 5028 5029 func TestJobEndpoint_Dispatch_ACL(t *testing.T) { 5030 t.Parallel() 5031 require := require.New(t) 5032 5033 s1, root, cleanupS1 := TestACLServer(t, func(c *Config) { 5034 c.NumSchedulers = 0 // Prevent automatic dequeue 5035 }) 5036 defer cleanupS1() 5037 codec := rpcClient(t, s1) 5038 testutil.WaitForLeader(t, s1.RPC) 5039 state := s1.fsm.State() 5040 5041 // Create a parameterized job 5042 job := mock.BatchJob() 5043 job.ParameterizedJob = &structs.ParameterizedJobConfig{} 5044 err := state.UpsertJob(400, job) 5045 require.Nil(err) 5046 5047 req := &structs.JobDispatchRequest{ 5048 JobID: job.ID, 5049 WriteRequest: structs.WriteRequest{ 5050 Region: "global", 5051 Namespace: job.Namespace, 5052 }, 5053 } 5054 5055 // Attempt to fetch the response without a token should fail 5056 var resp structs.JobDispatchResponse 5057 err = msgpackrpc.CallWithCodec(codec, "Job.Dispatch", req, &resp) 5058 require.NotNil(err) 5059 require.Contains(err.Error(), "Permission denied") 5060 5061 // Attempt to fetch the response with an invalid token should fail 5062 invalidToken := mock.CreatePolicyAndToken(t, state, 1001, "test-invalid", 5063 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityListJobs})) 5064 req.AuthToken = invalidToken.SecretID 5065 5066 var invalidResp structs.JobDispatchResponse 5067 err = msgpackrpc.CallWithCodec(codec, "Job.Dispatch", req, &invalidResp) 5068 require.NotNil(err) 5069 require.Contains(err.Error(), "Permission denied") 5070 5071 // Dispatch with a valid management token should succeed 5072 req.AuthToken = root.SecretID 5073 5074 var validResp structs.JobDispatchResponse 5075 err = msgpackrpc.CallWithCodec(codec, "Job.Dispatch", req, &validResp) 5076 require.Nil(err) 5077 require.NotNil(validResp.EvalID) 5078 require.NotNil(validResp.DispatchedJobID) 5079 require.NotEqual(validResp.DispatchedJobID, "") 5080 5081 // Dispatch with a valid token should succeed 5082 validToken := mock.CreatePolicyAndToken(t, state, 1003, "test-valid", 5083 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityDispatchJob})) 5084 req.AuthToken = validToken.SecretID 5085 5086 var validResp2 structs.JobDispatchResponse 5087 err = msgpackrpc.CallWithCodec(codec, "Job.Dispatch", req, &validResp2) 5088 require.Nil(err) 5089 require.NotNil(validResp2.EvalID) 5090 require.NotNil(validResp2.DispatchedJobID) 5091 require.NotEqual(validResp2.DispatchedJobID, "") 5092 5093 ws := memdb.NewWatchSet() 5094 out, err := state.JobByID(ws, job.Namespace, validResp2.DispatchedJobID) 5095 require.Nil(err) 5096 require.NotNil(out) 5097 require.Equal(out.ParentID, job.ID) 5098 5099 // Look up the evaluation 5100 eval, err := state.EvalByID(ws, validResp2.EvalID) 5101 require.Nil(err) 5102 require.NotNil(eval) 5103 require.Equal(eval.CreateIndex, validResp2.EvalCreateIndex) 5104 } 5105 5106 func TestJobEndpoint_Dispatch(t *testing.T) { 5107 t.Parallel() 5108 5109 // No requirements 5110 d1 := mock.BatchJob() 5111 d1.ParameterizedJob = &structs.ParameterizedJobConfig{} 5112 5113 // Require input data 5114 d2 := mock.BatchJob() 5115 d2.ParameterizedJob = &structs.ParameterizedJobConfig{ 5116 Payload: structs.DispatchPayloadRequired, 5117 } 5118 5119 // Disallow input data 5120 d3 := mock.BatchJob() 5121 d3.ParameterizedJob = &structs.ParameterizedJobConfig{ 5122 Payload: structs.DispatchPayloadForbidden, 5123 } 5124 5125 // Require meta 5126 d4 := mock.BatchJob() 5127 d4.ParameterizedJob = &structs.ParameterizedJobConfig{ 5128 MetaRequired: []string{"foo", "bar"}, 5129 } 5130 5131 // Optional meta 5132 d5 := mock.BatchJob() 5133 d5.ParameterizedJob = &structs.ParameterizedJobConfig{ 5134 MetaOptional: []string{"foo", "bar"}, 5135 } 5136 5137 // Periodic dispatch job 5138 d6 := mock.PeriodicJob() 5139 d6.ParameterizedJob = &structs.ParameterizedJobConfig{} 5140 5141 d7 := mock.BatchJob() 5142 d7.ParameterizedJob = &structs.ParameterizedJobConfig{} 5143 d7.Stop = true 5144 5145 reqNoInputNoMeta := &structs.JobDispatchRequest{} 5146 reqInputDataNoMeta := &structs.JobDispatchRequest{ 5147 Payload: []byte("hello world"), 5148 } 5149 reqNoInputDataMeta := &structs.JobDispatchRequest{ 5150 Meta: map[string]string{ 5151 "foo": "f1", 5152 "bar": "f2", 5153 }, 5154 } 5155 reqInputDataMeta := &structs.JobDispatchRequest{ 5156 Payload: []byte("hello world"), 5157 Meta: map[string]string{ 5158 "foo": "f1", 5159 "bar": "f2", 5160 }, 5161 } 5162 reqBadMeta := &structs.JobDispatchRequest{ 5163 Payload: []byte("hello world"), 5164 Meta: map[string]string{ 5165 "foo": "f1", 5166 "bar": "f2", 5167 "baz": "f3", 5168 }, 5169 } 5170 reqInputDataTooLarge := &structs.JobDispatchRequest{ 5171 Payload: make([]byte, DispatchPayloadSizeLimit+100), 5172 } 5173 5174 type testCase struct { 5175 name string 5176 parameterizedJob *structs.Job 5177 dispatchReq *structs.JobDispatchRequest 5178 noEval bool 5179 err bool 5180 errStr string 5181 } 5182 cases := []testCase{ 5183 { 5184 name: "optional input data w/ data", 5185 parameterizedJob: d1, 5186 dispatchReq: reqInputDataNoMeta, 5187 err: false, 5188 }, 5189 { 5190 name: "optional input data w/o data", 5191 parameterizedJob: d1, 5192 dispatchReq: reqNoInputNoMeta, 5193 err: false, 5194 }, 5195 { 5196 name: "require input data w/ data", 5197 parameterizedJob: d2, 5198 dispatchReq: reqInputDataNoMeta, 5199 err: false, 5200 }, 5201 { 5202 name: "require input data w/o data", 5203 parameterizedJob: d2, 5204 dispatchReq: reqNoInputNoMeta, 5205 err: true, 5206 errStr: "not provided but required", 5207 }, 5208 { 5209 name: "disallow input data w/o data", 5210 parameterizedJob: d3, 5211 dispatchReq: reqNoInputNoMeta, 5212 err: false, 5213 }, 5214 { 5215 name: "disallow input data w/ data", 5216 parameterizedJob: d3, 5217 dispatchReq: reqInputDataNoMeta, 5218 err: true, 5219 errStr: "provided but forbidden", 5220 }, 5221 { 5222 name: "require meta w/ meta", 5223 parameterizedJob: d4, 5224 dispatchReq: reqInputDataMeta, 5225 err: false, 5226 }, 5227 { 5228 name: "require meta w/o meta", 5229 parameterizedJob: d4, 5230 dispatchReq: reqNoInputNoMeta, 5231 err: true, 5232 errStr: "did not provide required meta keys", 5233 }, 5234 { 5235 name: "optional meta w/ meta", 5236 parameterizedJob: d5, 5237 dispatchReq: reqNoInputDataMeta, 5238 err: false, 5239 }, 5240 { 5241 name: "optional meta w/o meta", 5242 parameterizedJob: d5, 5243 dispatchReq: reqNoInputNoMeta, 5244 err: false, 5245 }, 5246 { 5247 name: "optional meta w/ bad meta", 5248 parameterizedJob: d5, 5249 dispatchReq: reqBadMeta, 5250 err: true, 5251 errStr: "unpermitted metadata keys", 5252 }, 5253 { 5254 name: "optional input w/ too big of input", 5255 parameterizedJob: d1, 5256 dispatchReq: reqInputDataTooLarge, 5257 err: true, 5258 errStr: "Payload exceeds maximum size", 5259 }, 5260 { 5261 name: "periodic job dispatched, ensure no eval", 5262 parameterizedJob: d6, 5263 dispatchReq: reqNoInputNoMeta, 5264 noEval: true, 5265 }, 5266 { 5267 name: "periodic job stopped, ensure error", 5268 parameterizedJob: d7, 5269 dispatchReq: reqNoInputNoMeta, 5270 err: true, 5271 errStr: "stopped", 5272 }, 5273 } 5274 5275 for _, tc := range cases { 5276 t.Run(tc.name, func(t *testing.T) { 5277 s1, cleanupS1 := TestServer(t, func(c *Config) { 5278 c.NumSchedulers = 0 // Prevent automatic dequeue 5279 }) 5280 defer cleanupS1() 5281 codec := rpcClient(t, s1) 5282 testutil.WaitForLeader(t, s1.RPC) 5283 5284 // Create the register request 5285 regReq := &structs.JobRegisterRequest{ 5286 Job: tc.parameterizedJob, 5287 WriteRequest: structs.WriteRequest{ 5288 Region: "global", 5289 Namespace: tc.parameterizedJob.Namespace, 5290 }, 5291 } 5292 5293 // Fetch the response 5294 var regResp structs.JobRegisterResponse 5295 if err := msgpackrpc.CallWithCodec(codec, "Job.Register", regReq, ®Resp); err != nil { 5296 t.Fatalf("err: %v", err) 5297 } 5298 5299 // Now try to dispatch 5300 tc.dispatchReq.JobID = tc.parameterizedJob.ID 5301 tc.dispatchReq.WriteRequest = structs.WriteRequest{ 5302 Region: "global", 5303 Namespace: tc.parameterizedJob.Namespace, 5304 } 5305 5306 var dispatchResp structs.JobDispatchResponse 5307 dispatchErr := msgpackrpc.CallWithCodec(codec, "Job.Dispatch", tc.dispatchReq, &dispatchResp) 5308 5309 if dispatchErr == nil { 5310 if tc.err { 5311 t.Fatalf("Expected error: %v", dispatchErr) 5312 } 5313 5314 // Check that we got an eval and job id back 5315 switch dispatchResp.EvalID { 5316 case "": 5317 if !tc.noEval { 5318 t.Fatalf("Bad response") 5319 } 5320 default: 5321 if tc.noEval { 5322 t.Fatalf("Got eval %q", dispatchResp.EvalID) 5323 } 5324 } 5325 5326 if dispatchResp.DispatchedJobID == "" { 5327 t.Fatalf("Bad response") 5328 } 5329 5330 state := s1.fsm.State() 5331 ws := memdb.NewWatchSet() 5332 out, err := state.JobByID(ws, tc.parameterizedJob.Namespace, dispatchResp.DispatchedJobID) 5333 if err != nil { 5334 t.Fatalf("err: %v", err) 5335 } 5336 if out == nil { 5337 t.Fatalf("expected job") 5338 } 5339 if out.CreateIndex != dispatchResp.JobCreateIndex { 5340 t.Fatalf("index mis-match") 5341 } 5342 if out.ParentID != tc.parameterizedJob.ID { 5343 t.Fatalf("bad parent ID") 5344 } 5345 if !out.Dispatched { 5346 t.Fatal("expected dispatched job") 5347 } 5348 if out.IsParameterized() { 5349 t.Fatal("dispatched job should not be parameterized") 5350 } 5351 if out.ParameterizedJob == nil { 5352 t.Fatal("parameter job config should exist") 5353 } 5354 5355 if tc.noEval { 5356 return 5357 } 5358 5359 // Lookup the evaluation 5360 eval, err := state.EvalByID(ws, dispatchResp.EvalID) 5361 if err != nil { 5362 t.Fatalf("err: %v", err) 5363 } 5364 5365 if eval == nil { 5366 t.Fatalf("expected eval") 5367 } 5368 if eval.CreateIndex != dispatchResp.EvalCreateIndex { 5369 t.Fatalf("index mis-match") 5370 } 5371 } else { 5372 if !tc.err { 5373 t.Fatalf("Got unexpected error: %v", dispatchErr) 5374 } else if !strings.Contains(dispatchErr.Error(), tc.errStr) { 5375 t.Fatalf("Expected err to include %q; got %v", tc.errStr, dispatchErr) 5376 } 5377 } 5378 }) 5379 } 5380 } 5381 5382 func TestJobEndpoint_Scale(t *testing.T) { 5383 t.Parallel() 5384 require := require.New(t) 5385 5386 s1, cleanupS1 := TestServer(t, nil) 5387 defer cleanupS1() 5388 codec := rpcClient(t, s1) 5389 testutil.WaitForLeader(t, s1.RPC) 5390 state := s1.fsm.State() 5391 5392 job := mock.Job() 5393 count := job.TaskGroups[0].Count 5394 err := state.UpsertJob(1000, job) 5395 require.Nil(err) 5396 5397 scale := &structs.JobScaleRequest{ 5398 JobID: job.ID, 5399 Target: map[string]string{ 5400 structs.ScalingTargetGroup: job.TaskGroups[0].Name, 5401 }, 5402 Count: helper.Int64ToPtr(int64(count + 1)), 5403 Message: "because of the load", 5404 Meta: map[string]interface{}{ 5405 "metrics": map[string]string{ 5406 "1": "a", 5407 "2": "b", 5408 }, 5409 "other": "value", 5410 }, 5411 PolicyOverride: false, 5412 WriteRequest: structs.WriteRequest{ 5413 Region: "global", 5414 Namespace: job.Namespace, 5415 }, 5416 } 5417 var resp structs.JobRegisterResponse 5418 err = msgpackrpc.CallWithCodec(codec, "Job.Scale", scale, &resp) 5419 require.NoError(err) 5420 require.NotEmpty(resp.EvalID) 5421 require.Greater(resp.EvalCreateIndex, resp.JobModifyIndex) 5422 } 5423 5424 func TestJobEndpoint_Scale_ACL(t *testing.T) { 5425 t.Parallel() 5426 require := require.New(t) 5427 5428 s1, root, cleanupS1 := TestACLServer(t, nil) 5429 defer cleanupS1() 5430 codec := rpcClient(t, s1) 5431 testutil.WaitForLeader(t, s1.RPC) 5432 state := s1.fsm.State() 5433 5434 job := mock.Job() 5435 err := state.UpsertJob(1000, job) 5436 require.Nil(err) 5437 5438 scale := &structs.JobScaleRequest{ 5439 JobID: job.ID, 5440 Target: map[string]string{ 5441 structs.ScalingTargetGroup: job.TaskGroups[0].Name, 5442 }, 5443 Message: "because of the load", 5444 WriteRequest: structs.WriteRequest{ 5445 Region: "global", 5446 Namespace: job.Namespace, 5447 }, 5448 } 5449 5450 // Scale without a token should fail 5451 var resp structs.JobRegisterResponse 5452 err = msgpackrpc.CallWithCodec(codec, "Job.Scale", scale, &resp) 5453 require.NotNil(err) 5454 require.Contains(err.Error(), "Permission denied") 5455 5456 // Expect failure for request with an invalid token 5457 invalidToken := mock.CreatePolicyAndToken(t, state, 1003, "test-invalid", 5458 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityListJobs})) 5459 scale.AuthToken = invalidToken.SecretID 5460 var invalidResp structs.JobRegisterResponse 5461 require.NotNil(err) 5462 err = msgpackrpc.CallWithCodec(codec, "Job.Scale", scale, &invalidResp) 5463 require.Contains(err.Error(), "Permission denied") 5464 5465 type testCase struct { 5466 authToken string 5467 name string 5468 } 5469 cases := []testCase{ 5470 { 5471 name: "mgmt token should succeed", 5472 authToken: root.SecretID, 5473 }, 5474 { 5475 name: "write disposition should succeed", 5476 authToken: mock.CreatePolicyAndToken(t, state, 1005, "test-valid-write", 5477 mock.NamespacePolicy(structs.DefaultNamespace, "write", nil)). 5478 SecretID, 5479 }, 5480 { 5481 name: "autoscaler disposition should succeed", 5482 authToken: mock.CreatePolicyAndToken(t, state, 1005, "test-valid-autoscaler", 5483 mock.NamespacePolicy(structs.DefaultNamespace, "scale", nil)). 5484 SecretID, 5485 }, 5486 { 5487 name: "submit-job capability should succeed", 5488 authToken: mock.CreatePolicyAndToken(t, state, 1005, "test-valid-submit-job", 5489 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilitySubmitJob})).SecretID, 5490 }, 5491 { 5492 name: "scale-job capability should succeed", 5493 authToken: mock.CreatePolicyAndToken(t, state, 1005, "test-valid-scale-job", 5494 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityScaleJob})). 5495 SecretID, 5496 }, 5497 } 5498 5499 for _, tc := range cases { 5500 scale.AuthToken = tc.authToken 5501 var resp structs.JobRegisterResponse 5502 err = msgpackrpc.CallWithCodec(codec, "Job.Scale", scale, &resp) 5503 require.NoError(err, tc.name) 5504 require.NotNil(resp.EvalID) 5505 } 5506 5507 } 5508 5509 func TestJobEndpoint_Scale_Invalid(t *testing.T) { 5510 t.Parallel() 5511 require := require.New(t) 5512 5513 s1, cleanupS1 := TestServer(t, nil) 5514 defer cleanupS1() 5515 codec := rpcClient(t, s1) 5516 testutil.WaitForLeader(t, s1.RPC) 5517 state := s1.fsm.State() 5518 5519 job := mock.Job() 5520 count := job.TaskGroups[0].Count 5521 5522 // check before job registration 5523 scale := &structs.JobScaleRequest{ 5524 JobID: job.ID, 5525 Target: map[string]string{ 5526 structs.ScalingTargetGroup: job.TaskGroups[0].Name, 5527 }, 5528 Count: helper.Int64ToPtr(int64(count) + 1), 5529 Message: "this should fail", 5530 Meta: map[string]interface{}{ 5531 "metrics": map[string]string{ 5532 "1": "a", 5533 "2": "b", 5534 }, 5535 "other": "value", 5536 }, 5537 PolicyOverride: false, 5538 WriteRequest: structs.WriteRequest{ 5539 Region: "global", 5540 Namespace: job.Namespace, 5541 }, 5542 } 5543 var resp structs.JobRegisterResponse 5544 err := msgpackrpc.CallWithCodec(codec, "Job.Scale", scale, &resp) 5545 require.Error(err) 5546 require.Contains(err.Error(), "not found") 5547 5548 // register the job 5549 err = state.UpsertJob(1000, job) 5550 require.Nil(err) 5551 5552 scale.Count = helper.Int64ToPtr(10) 5553 scale.Message = "error message" 5554 scale.Error = true 5555 err = msgpackrpc.CallWithCodec(codec, "Job.Scale", scale, &resp) 5556 require.Error(err) 5557 require.Contains(err.Error(), "should not contain count if error is true") 5558 } 5559 5560 func TestJobEndpoint_Scale_NoEval(t *testing.T) { 5561 t.Parallel() 5562 require := require.New(t) 5563 5564 s1, cleanupS1 := TestServer(t, nil) 5565 defer cleanupS1() 5566 codec := rpcClient(t, s1) 5567 testutil.WaitForLeader(t, s1.RPC) 5568 state := s1.fsm.State() 5569 5570 job := mock.Job() 5571 groupName := job.TaskGroups[0].Name 5572 var resp structs.JobRegisterResponse 5573 err := msgpackrpc.CallWithCodec(codec, "Job.Register", &structs.JobRegisterRequest{ 5574 Job: job, 5575 WriteRequest: structs.WriteRequest{ 5576 Region: "global", 5577 Namespace: job.Namespace, 5578 }, 5579 }, &resp) 5580 jobCreateIndex := resp.Index 5581 require.NoError(err) 5582 5583 scale := &structs.JobScaleRequest{ 5584 JobID: job.ID, 5585 Target: map[string]string{ 5586 structs.ScalingTargetGroup: groupName, 5587 }, 5588 Count: nil, // no count => no eval 5589 Message: "something informative", 5590 Meta: map[string]interface{}{ 5591 "metrics": map[string]string{ 5592 "1": "a", 5593 "2": "b", 5594 }, 5595 "other": "value", 5596 }, 5597 PolicyOverride: false, 5598 WriteRequest: structs.WriteRequest{ 5599 Region: "global", 5600 Namespace: job.Namespace, 5601 }, 5602 } 5603 err = msgpackrpc.CallWithCodec(codec, "Job.Scale", scale, &resp) 5604 require.NoError(err) 5605 require.Empty(resp.EvalID) 5606 require.Empty(resp.EvalCreateIndex) 5607 5608 jobEvents, eventsIndex, err := state.ScalingEventsByJob(nil, job.Namespace, job.ID) 5609 require.NoError(err) 5610 require.NotNil(jobEvents) 5611 require.Len(jobEvents, 1) 5612 require.Contains(jobEvents, groupName) 5613 groupEvents := jobEvents[groupName] 5614 require.Len(groupEvents, 1) 5615 event := groupEvents[0] 5616 require.Nil(event.EvalID) 5617 require.Greater(eventsIndex, jobCreateIndex) 5618 } 5619 5620 func TestJobEndpoint_InvalidCount(t *testing.T) { 5621 t.Parallel() 5622 require := require.New(t) 5623 5624 s1, cleanupS1 := TestServer(t, nil) 5625 defer cleanupS1() 5626 codec := rpcClient(t, s1) 5627 testutil.WaitForLeader(t, s1.RPC) 5628 state := s1.fsm.State() 5629 5630 job := mock.Job() 5631 err := state.UpsertJob(1000, job) 5632 require.Nil(err) 5633 5634 scale := &structs.JobScaleRequest{ 5635 JobID: job.ID, 5636 Target: map[string]string{ 5637 structs.ScalingTargetGroup: job.TaskGroups[0].Name, 5638 }, 5639 Count: helper.Int64ToPtr(int64(-1)), 5640 WriteRequest: structs.WriteRequest{ 5641 Region: "global", 5642 Namespace: job.Namespace, 5643 }, 5644 } 5645 var resp structs.JobRegisterResponse 5646 err = msgpackrpc.CallWithCodec(codec, "Job.Scale", scale, &resp) 5647 require.Error(err) 5648 } 5649 5650 func TestJobEndpoint_GetScaleStatus(t *testing.T) { 5651 t.Parallel() 5652 require := require.New(t) 5653 5654 s1, cleanupS1 := TestServer(t, nil) 5655 defer cleanupS1() 5656 codec := rpcClient(t, s1) 5657 testutil.WaitForLeader(t, s1.RPC) 5658 state := s1.fsm.State() 5659 5660 jobV1 := mock.Job() 5661 5662 // check before registration 5663 // Fetch the scaling status 5664 get := &structs.JobScaleStatusRequest{ 5665 JobID: jobV1.ID, 5666 QueryOptions: structs.QueryOptions{ 5667 Region: "global", 5668 Namespace: jobV1.Namespace, 5669 }, 5670 } 5671 var resp2 structs.JobScaleStatusResponse 5672 require.NoError(msgpackrpc.CallWithCodec(codec, "Job.ScaleStatus", get, &resp2)) 5673 require.Nil(resp2.JobScaleStatus) 5674 5675 // stopped (previous version) 5676 require.NoError(state.UpsertJob(1000, jobV1), "UpsertJob") 5677 a0 := mock.Alloc() 5678 a0.Job = jobV1 5679 a0.Namespace = jobV1.Namespace 5680 a0.JobID = jobV1.ID 5681 a0.ClientStatus = structs.AllocClientStatusComplete 5682 require.NoError(state.UpsertAllocs(1010, []*structs.Allocation{a0}), "UpsertAllocs") 5683 5684 jobV2 := jobV1.Copy() 5685 require.NoError(state.UpsertJob(1100, jobV2), "UpsertJob") 5686 a1 := mock.Alloc() 5687 a1.Job = jobV2 5688 a1.Namespace = jobV2.Namespace 5689 a1.JobID = jobV2.ID 5690 a1.ClientStatus = structs.AllocClientStatusRunning 5691 // healthy 5692 a1.DeploymentStatus = &structs.AllocDeploymentStatus{ 5693 Healthy: helper.BoolToPtr(true), 5694 } 5695 a2 := mock.Alloc() 5696 a2.Job = jobV2 5697 a2.Namespace = jobV2.Namespace 5698 a2.JobID = jobV2.ID 5699 a2.ClientStatus = structs.AllocClientStatusPending 5700 // unhealthy 5701 a2.DeploymentStatus = &structs.AllocDeploymentStatus{ 5702 Healthy: helper.BoolToPtr(false), 5703 } 5704 a3 := mock.Alloc() 5705 a3.Job = jobV2 5706 a3.Namespace = jobV2.Namespace 5707 a3.JobID = jobV2.ID 5708 a3.ClientStatus = structs.AllocClientStatusRunning 5709 // canary 5710 a3.DeploymentStatus = &structs.AllocDeploymentStatus{ 5711 Healthy: helper.BoolToPtr(true), 5712 Canary: true, 5713 } 5714 // no health 5715 a4 := mock.Alloc() 5716 a4.Job = jobV2 5717 a4.Namespace = jobV2.Namespace 5718 a4.JobID = jobV2.ID 5719 a4.ClientStatus = structs.AllocClientStatusRunning 5720 // upsert allocations 5721 require.NoError(state.UpsertAllocs(1110, []*structs.Allocation{a1, a2, a3, a4}), "UpsertAllocs") 5722 5723 event := &structs.ScalingEvent{ 5724 Time: time.Now().Unix(), 5725 Count: helper.Int64ToPtr(5), 5726 Message: "message", 5727 Error: false, 5728 Meta: map[string]interface{}{ 5729 "a": "b", 5730 }, 5731 EvalID: nil, 5732 } 5733 5734 require.NoError(state.UpsertScalingEvent(1003, &structs.ScalingEventRequest{ 5735 Namespace: jobV2.Namespace, 5736 JobID: jobV2.ID, 5737 TaskGroup: jobV2.TaskGroups[0].Name, 5738 ScalingEvent: event, 5739 }), "UpsertScalingEvent") 5740 5741 // check after job registration 5742 require.NoError(msgpackrpc.CallWithCodec(codec, "Job.ScaleStatus", get, &resp2)) 5743 require.NotNil(resp2.JobScaleStatus) 5744 5745 expectedStatus := structs.JobScaleStatus{ 5746 JobID: jobV2.ID, 5747 JobCreateIndex: jobV2.CreateIndex, 5748 JobModifyIndex: a1.CreateIndex, 5749 JobStopped: jobV2.Stop, 5750 TaskGroups: map[string]*structs.TaskGroupScaleStatus{ 5751 jobV2.TaskGroups[0].Name: { 5752 Desired: jobV2.TaskGroups[0].Count, 5753 Placed: 3, 5754 Running: 2, 5755 Healthy: 1, 5756 Unhealthy: 1, 5757 Events: []*structs.ScalingEvent{event}, 5758 }, 5759 }, 5760 } 5761 5762 require.True(reflect.DeepEqual(*resp2.JobScaleStatus, expectedStatus)) 5763 } 5764 5765 func TestJobEndpoint_GetScaleStatus_ACL(t *testing.T) { 5766 t.Parallel() 5767 require := require.New(t) 5768 5769 s1, root, cleanupS1 := TestACLServer(t, nil) 5770 defer cleanupS1() 5771 codec := rpcClient(t, s1) 5772 testutil.WaitForLeader(t, s1.RPC) 5773 state := s1.fsm.State() 5774 5775 // Create the job 5776 job := mock.Job() 5777 err := state.UpsertJob(1000, job) 5778 require.Nil(err) 5779 5780 // Get the job scale status 5781 get := &structs.JobScaleStatusRequest{ 5782 JobID: job.ID, 5783 QueryOptions: structs.QueryOptions{ 5784 Region: "global", 5785 Namespace: job.Namespace, 5786 }, 5787 } 5788 5789 // Get without a token should fail 5790 var resp structs.JobScaleStatusResponse 5791 err = msgpackrpc.CallWithCodec(codec, "Job.ScaleStatus", get, &resp) 5792 require.NotNil(err) 5793 require.Contains(err.Error(), "Permission denied") 5794 5795 // Expect failure for request with an invalid token 5796 invalidToken := mock.CreatePolicyAndToken(t, state, 1003, "test-invalid", 5797 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityListJobs})) 5798 get.AuthToken = invalidToken.SecretID 5799 require.NotNil(err) 5800 err = msgpackrpc.CallWithCodec(codec, "Job.ScaleStatus", get, &resp) 5801 require.Contains(err.Error(), "Permission denied") 5802 5803 type testCase struct { 5804 authToken string 5805 name string 5806 } 5807 cases := []testCase{ 5808 { 5809 name: "mgmt token should succeed", 5810 authToken: root.SecretID, 5811 }, 5812 { 5813 name: "read disposition should succeed", 5814 authToken: mock.CreatePolicyAndToken(t, state, 1005, "test-valid-read", 5815 mock.NamespacePolicy(structs.DefaultNamespace, "read", nil)). 5816 SecretID, 5817 }, 5818 { 5819 name: "write disposition should succeed", 5820 authToken: mock.CreatePolicyAndToken(t, state, 1005, "test-valid-write", 5821 mock.NamespacePolicy(structs.DefaultNamespace, "write", nil)). 5822 SecretID, 5823 }, 5824 { 5825 name: "autoscaler disposition should succeed", 5826 authToken: mock.CreatePolicyAndToken(t, state, 1005, "test-valid-autoscaler", 5827 mock.NamespacePolicy(structs.DefaultNamespace, "scale", nil)). 5828 SecretID, 5829 }, 5830 { 5831 name: "read-job capability should succeed", 5832 authToken: mock.CreatePolicyAndToken(t, state, 1005, "test-valid-read-job", 5833 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob})).SecretID, 5834 }, 5835 { 5836 name: "read-job-scaling capability should succeed", 5837 authToken: mock.CreatePolicyAndToken(t, state, 1005, "test-valid-read-job-scaling", 5838 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJobScaling})). 5839 SecretID, 5840 }, 5841 } 5842 5843 for _, tc := range cases { 5844 get.AuthToken = tc.authToken 5845 var validResp structs.JobScaleStatusResponse 5846 err = msgpackrpc.CallWithCodec(codec, "Job.ScaleStatus", get, &validResp) 5847 require.NoError(err, tc.name) 5848 require.NotNil(validResp.JobScaleStatus) 5849 } 5850 }