github.com/manicqin/nomad@v0.9.5/nomad/job_endpoint.go (about) 1 package nomad 2 3 import ( 4 "context" 5 "fmt" 6 "sort" 7 "strings" 8 "time" 9 10 metrics "github.com/armon/go-metrics" 11 log "github.com/hashicorp/go-hclog" 12 memdb "github.com/hashicorp/go-memdb" 13 multierror "github.com/hashicorp/go-multierror" 14 15 "github.com/golang/snappy" 16 "github.com/hashicorp/consul/lib" 17 "github.com/hashicorp/nomad/acl" 18 "github.com/hashicorp/nomad/helper" 19 "github.com/hashicorp/nomad/helper/uuid" 20 "github.com/hashicorp/nomad/nomad/state" 21 "github.com/hashicorp/nomad/nomad/structs" 22 "github.com/hashicorp/nomad/scheduler" 23 ) 24 25 const ( 26 // RegisterEnforceIndexErrPrefix is the prefix to use in errors caused by 27 // enforcing the job modify index during registers. 28 RegisterEnforceIndexErrPrefix = "Enforcing job modify index" 29 30 // DispatchPayloadSizeLimit is the maximum size of the uncompressed input 31 // data payload. 32 DispatchPayloadSizeLimit = 16 * 1024 33 ) 34 35 var ( 36 37 // allowRescheduleTransition is the transition that allows failed 38 // allocations to be force rescheduled. We create a one off 39 // variable to avoid creating a new object for every request. 40 allowForceRescheduleTransition = &structs.DesiredTransition{ 41 ForceReschedule: helper.BoolToPtr(true), 42 } 43 ) 44 45 // Job endpoint is used for job interactions 46 type Job struct { 47 srv *Server 48 logger log.Logger 49 50 // builtin admission controllers 51 mutators []jobMutator 52 validators []jobValidator 53 } 54 55 // NewJobEndpoints creates a new job endpoint with builtin admission controllers 56 func NewJobEndpoints(s *Server) *Job { 57 return &Job{ 58 srv: s, 59 logger: s.logger.Named("job"), 60 mutators: []jobMutator{ 61 jobCanonicalizer{}, 62 jobConnectHook{}, 63 jobImpliedConstraints{}, 64 }, 65 validators: []jobValidator{ 66 jobConnectHook{}, 67 jobValidate{}, 68 }, 69 } 70 } 71 72 // Register is used to upsert a job for scheduling 73 func (j *Job) Register(args *structs.JobRegisterRequest, reply *structs.JobRegisterResponse) error { 74 if done, err := j.srv.forward("Job.Register", args, args, reply); done { 75 return err 76 } 77 defer metrics.MeasureSince([]string{"nomad", "job", "register"}, time.Now()) 78 79 // Validate the arguments 80 if args.Job == nil { 81 return fmt.Errorf("missing job for registration") 82 } 83 84 // defensive check; http layer and RPC requester should ensure namespaces are set consistently 85 if args.RequestNamespace() != args.Job.Namespace { 86 return fmt.Errorf("mismatched request namespace in request: %q, %q", args.RequestNamespace(), args.Job.Namespace) 87 } 88 89 // Run admission controllers 90 job, warnings, err := j.admissionControllers(args.Job) 91 if err != nil { 92 return err 93 } 94 args.Job = job 95 96 // Set the warning message 97 reply.Warnings = structs.MergeMultierrorWarnings(warnings...) 98 99 // Check job submission permissions 100 if aclObj, err := j.srv.ResolveToken(args.AuthToken); err != nil { 101 return err 102 } else if aclObj != nil { 103 if !aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilitySubmitJob) { 104 return structs.ErrPermissionDenied 105 } 106 107 // Validate Volume Permsissions 108 for _, tg := range args.Job.TaskGroups { 109 for _, vol := range tg.Volumes { 110 if vol.Type != structs.VolumeTypeHost { 111 return structs.ErrPermissionDenied 112 } 113 114 // If a volume is readonly, then we allow access if the user has ReadOnly 115 // or ReadWrite access to the volume. Otherwise we only allow access if 116 // they have ReadWrite access. 117 if vol.ReadOnly { 118 if !aclObj.AllowHostVolumeOperation(vol.Source, acl.HostVolumeCapabilityMountReadOnly) && 119 !aclObj.AllowHostVolumeOperation(vol.Source, acl.HostVolumeCapabilityMountReadWrite) { 120 return structs.ErrPermissionDenied 121 } 122 } else { 123 if !aclObj.AllowHostVolumeOperation(vol.Source, acl.HostVolumeCapabilityMountReadWrite) { 124 return structs.ErrPermissionDenied 125 } 126 } 127 } 128 129 for _, t := range tg.Tasks { 130 for _, vm := range t.VolumeMounts { 131 vol := tg.Volumes[vm.Volume] 132 if vm.PropagationMode == structs.VolumeMountPropagationBidirectional && 133 !aclObj.AllowHostVolumeOperation(vol.Source, acl.HostVolumeCapabilityMountReadWrite) { 134 return structs.ErrPermissionDenied 135 } 136 } 137 } 138 } 139 140 // Check if override is set and we do not have permissions 141 if args.PolicyOverride { 142 if !aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilitySentinelOverride) { 143 j.logger.Warn("policy override attempted without permissions for job", "job", args.Job.ID) 144 return structs.ErrPermissionDenied 145 } 146 j.logger.Warn("policy override set for job", "job", args.Job.ID) 147 } 148 } 149 150 // Lookup the job 151 snap, err := j.srv.State().Snapshot() 152 if err != nil { 153 return err 154 } 155 ws := memdb.NewWatchSet() 156 existingJob, err := snap.JobByID(ws, args.RequestNamespace(), args.Job.ID) 157 if err != nil { 158 return err 159 } 160 161 // If EnforceIndex set, check it before trying to apply 162 if args.EnforceIndex { 163 jmi := args.JobModifyIndex 164 if existingJob != nil { 165 if jmi == 0 { 166 return fmt.Errorf("%s 0: job already exists", RegisterEnforceIndexErrPrefix) 167 } else if jmi != existingJob.JobModifyIndex { 168 return fmt.Errorf("%s %d: job exists with conflicting job modify index: %d", 169 RegisterEnforceIndexErrPrefix, jmi, existingJob.JobModifyIndex) 170 } 171 } else if jmi != 0 { 172 return fmt.Errorf("%s %d: job does not exist", RegisterEnforceIndexErrPrefix, jmi) 173 } 174 } 175 176 // Validate job transitions if its an update 177 if err := validateJobUpdate(existingJob, args.Job); err != nil { 178 return err 179 } 180 181 // Ensure that the job has permissions for the requested Vault tokens 182 policies := args.Job.VaultPolicies() 183 if len(policies) != 0 { 184 vconf := j.srv.config.VaultConfig 185 if !vconf.IsEnabled() { 186 return fmt.Errorf("Vault not enabled and Vault policies requested") 187 } 188 189 // Have to check if the user has permissions 190 if !vconf.AllowsUnauthenticated() { 191 if args.Job.VaultToken == "" { 192 return fmt.Errorf("Vault policies requested but missing Vault Token") 193 } 194 195 vault := j.srv.vault 196 s, err := vault.LookupToken(context.Background(), args.Job.VaultToken) 197 if err != nil { 198 return err 199 } 200 201 allowedPolicies, err := PoliciesFrom(s) 202 if err != nil { 203 return err 204 } 205 206 // If we are given a root token it can access all policies 207 if !lib.StrContains(allowedPolicies, "root") { 208 flatPolicies := structs.VaultPoliciesSet(policies) 209 subset, offending := helper.SliceStringIsSubset(allowedPolicies, flatPolicies) 210 if !subset { 211 return fmt.Errorf("Passed Vault Token doesn't allow access to the following policies: %s", 212 strings.Join(offending, ", ")) 213 } 214 } 215 } 216 } 217 218 // Enforce Sentinel policies 219 policyWarnings, err := j.enforceSubmitJob(args.PolicyOverride, args.Job) 220 if err != nil { 221 return err 222 } 223 if policyWarnings != nil { 224 warnings = append(warnings, policyWarnings) 225 reply.Warnings = structs.MergeMultierrorWarnings(warnings...) 226 } 227 228 // Clear the Vault token 229 args.Job.VaultToken = "" 230 231 // Check if the job has changed at all 232 if existingJob == nil || existingJob.SpecChanged(args.Job) { 233 // Set the submit time 234 args.Job.SetSubmitTime() 235 236 // Commit this update via Raft 237 fsmErr, index, err := j.srv.raftApply(structs.JobRegisterRequestType, args) 238 if err, ok := fsmErr.(error); ok && err != nil { 239 j.logger.Error("registering job failed", "error", err, "fsm", true) 240 return err 241 } 242 if err != nil { 243 j.logger.Error("registering job failed", "error", err, "raft", true) 244 return err 245 } 246 247 // Populate the reply with job information 248 reply.JobModifyIndex = index 249 } else { 250 reply.JobModifyIndex = existingJob.JobModifyIndex 251 } 252 253 // If the job is periodic or parameterized, we don't create an eval. 254 if args.Job.IsPeriodic() || args.Job.IsParameterized() { 255 return nil 256 } 257 258 // Create a new evaluation 259 now := time.Now().UTC().UnixNano() 260 eval := &structs.Evaluation{ 261 ID: uuid.Generate(), 262 Namespace: args.RequestNamespace(), 263 Priority: args.Job.Priority, 264 Type: args.Job.Type, 265 TriggeredBy: structs.EvalTriggerJobRegister, 266 JobID: args.Job.ID, 267 JobModifyIndex: reply.JobModifyIndex, 268 Status: structs.EvalStatusPending, 269 CreateTime: now, 270 ModifyTime: now, 271 } 272 update := &structs.EvalUpdateRequest{ 273 Evals: []*structs.Evaluation{eval}, 274 WriteRequest: structs.WriteRequest{Region: args.Region}, 275 } 276 277 // Commit this evaluation via Raft 278 // XXX: There is a risk of partial failure where the JobRegister succeeds 279 // but that the EvalUpdate does not. 280 _, evalIndex, err := j.srv.raftApply(structs.EvalUpdateRequestType, update) 281 if err != nil { 282 j.logger.Error("eval create failed", "error", err, "method", "register") 283 return err 284 } 285 286 // Populate the reply with eval information 287 reply.EvalID = eval.ID 288 reply.EvalCreateIndex = evalIndex 289 reply.Index = evalIndex 290 return nil 291 } 292 293 // getSignalConstraint builds a suitable constraint based on the required 294 // signals 295 func getSignalConstraint(signals []string) *structs.Constraint { 296 sort.Strings(signals) 297 return &structs.Constraint{ 298 Operand: structs.ConstraintSetContains, 299 LTarget: "${attr.os.signals}", 300 RTarget: strings.Join(signals, ","), 301 } 302 } 303 304 // Summary retrieves the summary of a job 305 func (j *Job) Summary(args *structs.JobSummaryRequest, 306 reply *structs.JobSummaryResponse) error { 307 308 if done, err := j.srv.forward("Job.Summary", args, args, reply); done { 309 return err 310 } 311 defer metrics.MeasureSince([]string{"nomad", "job_summary", "get_job_summary"}, time.Now()) 312 313 // Check for read-job permissions 314 if aclObj, err := j.srv.ResolveToken(args.AuthToken); err != nil { 315 return err 316 } else if aclObj != nil && !aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilityReadJob) { 317 return structs.ErrPermissionDenied 318 } 319 320 // Setup the blocking query 321 opts := blockingOptions{ 322 queryOpts: &args.QueryOptions, 323 queryMeta: &reply.QueryMeta, 324 run: func(ws memdb.WatchSet, state *state.StateStore) error { 325 // Look for job summary 326 out, err := state.JobSummaryByID(ws, args.RequestNamespace(), args.JobID) 327 if err != nil { 328 return err 329 } 330 331 // Setup the output 332 reply.JobSummary = out 333 if out != nil { 334 reply.Index = out.ModifyIndex 335 } else { 336 // Use the last index that affected the job_summary table 337 index, err := state.Index("job_summary") 338 if err != nil { 339 return err 340 } 341 reply.Index = index 342 } 343 344 // Set the query response 345 j.srv.setQueryMeta(&reply.QueryMeta) 346 return nil 347 }} 348 return j.srv.blockingRPC(&opts) 349 } 350 351 // Validate validates a job 352 func (j *Job) Validate(args *structs.JobValidateRequest, reply *structs.JobValidateResponse) error { 353 defer metrics.MeasureSince([]string{"nomad", "job", "validate"}, time.Now()) 354 355 // defensive check; http layer and RPC requester should ensure namespaces are set consistently 356 if args.RequestNamespace() != args.Job.Namespace { 357 return fmt.Errorf("mismatched request namespace in request: %q, %q", args.RequestNamespace(), args.Job.Namespace) 358 } 359 360 job, mutateWarnings, err := j.admissionMutators(args.Job) 361 if err != nil { 362 return err 363 } 364 args.Job = job 365 366 // Check for read-job permissions 367 if aclObj, err := j.srv.ResolveToken(args.AuthToken); err != nil { 368 return err 369 } else if aclObj != nil && !aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilityReadJob) { 370 return structs.ErrPermissionDenied 371 } 372 373 // Validate the job and capture any warnings 374 validateWarnings, err := j.admissionValidators(args.Job) 375 if err != nil { 376 if merr, ok := err.(*multierror.Error); ok { 377 for _, err := range merr.Errors { 378 reply.ValidationErrors = append(reply.ValidationErrors, err.Error()) 379 } 380 reply.Error = merr.Error() 381 } else { 382 reply.ValidationErrors = append(reply.ValidationErrors, err.Error()) 383 reply.Error = err.Error() 384 } 385 } 386 387 validateWarnings = append(validateWarnings, mutateWarnings...) 388 389 // Set the warning message 390 reply.Warnings = structs.MergeMultierrorWarnings(validateWarnings...) 391 reply.DriverConfigValidated = true 392 return nil 393 } 394 395 // Revert is used to revert the job to a prior version 396 func (j *Job) Revert(args *structs.JobRevertRequest, reply *structs.JobRegisterResponse) error { 397 if done, err := j.srv.forward("Job.Revert", args, args, reply); done { 398 return err 399 } 400 defer metrics.MeasureSince([]string{"nomad", "job", "revert"}, time.Now()) 401 402 // Check for submit-job permissions 403 if aclObj, err := j.srv.ResolveToken(args.AuthToken); err != nil { 404 return err 405 } else if aclObj != nil && !aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilitySubmitJob) { 406 return structs.ErrPermissionDenied 407 } 408 409 // Validate the arguments 410 if args.JobID == "" { 411 return fmt.Errorf("missing job ID for revert") 412 } 413 414 // Lookup the job by version 415 snap, err := j.srv.fsm.State().Snapshot() 416 if err != nil { 417 return err 418 } 419 420 ws := memdb.NewWatchSet() 421 cur, err := snap.JobByID(ws, args.RequestNamespace(), args.JobID) 422 if err != nil { 423 return err 424 } 425 if cur == nil { 426 return fmt.Errorf("job %q not found", args.JobID) 427 } 428 if args.JobVersion == cur.Version { 429 return fmt.Errorf("can't revert to current version") 430 } 431 432 jobV, err := snap.JobByIDAndVersion(ws, args.RequestNamespace(), args.JobID, args.JobVersion) 433 if err != nil { 434 return err 435 } 436 if jobV == nil { 437 return fmt.Errorf("job %q in namespace %q at version %d not found", args.JobID, args.RequestNamespace(), args.JobVersion) 438 } 439 440 // Build the register request 441 revJob := jobV.Copy() 442 // Use Vault Token from revert request to perform registration of reverted job. 443 revJob.VaultToken = args.VaultToken 444 reg := &structs.JobRegisterRequest{ 445 Job: revJob, 446 WriteRequest: args.WriteRequest, 447 } 448 449 // If the request is enforcing the existing version do a check. 450 if args.EnforcePriorVersion != nil { 451 if cur.Version != *args.EnforcePriorVersion { 452 return fmt.Errorf("Current job has version %d; enforcing version %d", cur.Version, *args.EnforcePriorVersion) 453 } 454 455 reg.EnforceIndex = true 456 reg.JobModifyIndex = cur.JobModifyIndex 457 } 458 459 // Register the version. 460 return j.Register(reg, reply) 461 } 462 463 // Stable is used to mark the job version as stable 464 func (j *Job) Stable(args *structs.JobStabilityRequest, reply *structs.JobStabilityResponse) error { 465 if done, err := j.srv.forward("Job.Stable", args, args, reply); done { 466 return err 467 } 468 defer metrics.MeasureSince([]string{"nomad", "job", "stable"}, time.Now()) 469 470 // Check for read-job permissions 471 if aclObj, err := j.srv.ResolveToken(args.AuthToken); err != nil { 472 return err 473 } else if aclObj != nil && !aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilitySubmitJob) { 474 return structs.ErrPermissionDenied 475 } 476 477 // Validate the arguments 478 if args.JobID == "" { 479 return fmt.Errorf("missing job ID for marking job as stable") 480 } 481 482 // Lookup the job by version 483 snap, err := j.srv.fsm.State().Snapshot() 484 if err != nil { 485 return err 486 } 487 488 ws := memdb.NewWatchSet() 489 jobV, err := snap.JobByIDAndVersion(ws, args.RequestNamespace(), args.JobID, args.JobVersion) 490 if err != nil { 491 return err 492 } 493 if jobV == nil { 494 return fmt.Errorf("job %q in namespace %q at version %d not found", args.JobID, args.RequestNamespace(), args.JobVersion) 495 } 496 497 // Commit this stability request via Raft 498 _, modifyIndex, err := j.srv.raftApply(structs.JobStabilityRequestType, args) 499 if err != nil { 500 j.logger.Error("submitting job stability request failed", "error", err) 501 return err 502 } 503 504 // Setup the reply 505 reply.Index = modifyIndex 506 return nil 507 } 508 509 // Evaluate is used to force a job for re-evaluation 510 func (j *Job) Evaluate(args *structs.JobEvaluateRequest, reply *structs.JobRegisterResponse) error { 511 if done, err := j.srv.forward("Job.Evaluate", args, args, reply); done { 512 return err 513 } 514 defer metrics.MeasureSince([]string{"nomad", "job", "evaluate"}, time.Now()) 515 516 // Check for read-job permissions 517 if aclObj, err := j.srv.ResolveToken(args.AuthToken); err != nil { 518 return err 519 } else if aclObj != nil && !aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilityReadJob) { 520 return structs.ErrPermissionDenied 521 } 522 523 // Validate the arguments 524 if args.JobID == "" { 525 return fmt.Errorf("missing job ID for evaluation") 526 } 527 528 // Lookup the job 529 snap, err := j.srv.fsm.State().Snapshot() 530 if err != nil { 531 return err 532 } 533 ws := memdb.NewWatchSet() 534 job, err := snap.JobByID(ws, args.RequestNamespace(), args.JobID) 535 if err != nil { 536 return err 537 } 538 if job == nil { 539 return fmt.Errorf("job not found") 540 } 541 542 if job.IsPeriodic() { 543 return fmt.Errorf("can't evaluate periodic job") 544 } else if job.IsParameterized() { 545 return fmt.Errorf("can't evaluate parameterized job") 546 } 547 548 forceRescheduleAllocs := make(map[string]*structs.DesiredTransition) 549 550 if args.EvalOptions.ForceReschedule { 551 // Find any failed allocs that could be force rescheduled 552 allocs, err := snap.AllocsByJob(ws, args.RequestNamespace(), args.JobID, false) 553 if err != nil { 554 return err 555 } 556 557 for _, alloc := range allocs { 558 taskGroup := job.LookupTaskGroup(alloc.TaskGroup) 559 // Forcing rescheduling is only allowed if task group has rescheduling enabled 560 if taskGroup == nil || !taskGroup.ReschedulePolicy.Enabled() { 561 continue 562 } 563 564 if alloc.NextAllocation == "" && alloc.ClientStatus == structs.AllocClientStatusFailed && !alloc.DesiredTransition.ShouldForceReschedule() { 565 forceRescheduleAllocs[alloc.ID] = allowForceRescheduleTransition 566 } 567 } 568 } 569 570 // Create a new evaluation 571 now := time.Now().UTC().UnixNano() 572 eval := &structs.Evaluation{ 573 ID: uuid.Generate(), 574 Namespace: args.RequestNamespace(), 575 Priority: job.Priority, 576 Type: job.Type, 577 TriggeredBy: structs.EvalTriggerJobRegister, 578 JobID: job.ID, 579 JobModifyIndex: job.ModifyIndex, 580 Status: structs.EvalStatusPending, 581 CreateTime: now, 582 ModifyTime: now, 583 } 584 585 // Create a AllocUpdateDesiredTransitionRequest request with the eval and any forced rescheduled allocs 586 updateTransitionReq := &structs.AllocUpdateDesiredTransitionRequest{ 587 Allocs: forceRescheduleAllocs, 588 Evals: []*structs.Evaluation{eval}, 589 } 590 _, evalIndex, err := j.srv.raftApply(structs.AllocUpdateDesiredTransitionRequestType, updateTransitionReq) 591 592 if err != nil { 593 j.logger.Error("eval create failed", "error", err, "method", "evaluate") 594 return err 595 } 596 597 // Setup the reply 598 reply.EvalID = eval.ID 599 reply.EvalCreateIndex = evalIndex 600 reply.JobModifyIndex = job.ModifyIndex 601 reply.Index = evalIndex 602 return nil 603 } 604 605 // Deregister is used to remove a job the cluster. 606 func (j *Job) Deregister(args *structs.JobDeregisterRequest, reply *structs.JobDeregisterResponse) error { 607 if done, err := j.srv.forward("Job.Deregister", args, args, reply); done { 608 return err 609 } 610 defer metrics.MeasureSince([]string{"nomad", "job", "deregister"}, time.Now()) 611 612 // Check for submit-job permissions 613 if aclObj, err := j.srv.ResolveToken(args.AuthToken); err != nil { 614 return err 615 } else if aclObj != nil && !aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilitySubmitJob) { 616 return structs.ErrPermissionDenied 617 } 618 619 // Validate the arguments 620 if args.JobID == "" { 621 return fmt.Errorf("missing job ID for deregistering") 622 } 623 624 // Lookup the job 625 snap, err := j.srv.fsm.State().Snapshot() 626 if err != nil { 627 return err 628 } 629 ws := memdb.NewWatchSet() 630 job, err := snap.JobByID(ws, args.RequestNamespace(), args.JobID) 631 if err != nil { 632 return err 633 } 634 635 // Commit this update via Raft 636 _, index, err := j.srv.raftApply(structs.JobDeregisterRequestType, args) 637 if err != nil { 638 j.logger.Error("deregister failed", "error", err) 639 return err 640 } 641 642 // Populate the reply with job information 643 reply.JobModifyIndex = index 644 645 // If the job is periodic or parameterized, we don't create an eval. 646 if job != nil && (job.IsPeriodic() || job.IsParameterized()) { 647 return nil 648 } 649 650 // Create a new evaluation 651 // XXX: The job priority / type is strange for this, since it's not a high 652 // priority even if the job was. 653 now := time.Now().UTC().UnixNano() 654 eval := &structs.Evaluation{ 655 ID: uuid.Generate(), 656 Namespace: args.RequestNamespace(), 657 Priority: structs.JobDefaultPriority, 658 Type: structs.JobTypeService, 659 TriggeredBy: structs.EvalTriggerJobDeregister, 660 JobID: args.JobID, 661 JobModifyIndex: index, 662 Status: structs.EvalStatusPending, 663 CreateTime: now, 664 ModifyTime: now, 665 } 666 update := &structs.EvalUpdateRequest{ 667 Evals: []*structs.Evaluation{eval}, 668 WriteRequest: structs.WriteRequest{Region: args.Region}, 669 } 670 671 // Commit this evaluation via Raft 672 _, evalIndex, err := j.srv.raftApply(structs.EvalUpdateRequestType, update) 673 if err != nil { 674 j.logger.Error("eval create failed", "error", err, "method", "deregister") 675 return err 676 } 677 678 // Populate the reply with eval information 679 reply.EvalID = eval.ID 680 reply.EvalCreateIndex = evalIndex 681 reply.Index = evalIndex 682 return nil 683 } 684 685 // BatchDeregister is used to remove a set of jobs from the cluster. 686 func (j *Job) BatchDeregister(args *structs.JobBatchDeregisterRequest, reply *structs.JobBatchDeregisterResponse) error { 687 if done, err := j.srv.forward("Job.BatchDeregister", args, args, reply); done { 688 return err 689 } 690 defer metrics.MeasureSince([]string{"nomad", "job", "batch_deregister"}, time.Now()) 691 692 // Resolve the ACL token 693 aclObj, err := j.srv.ResolveToken(args.AuthToken) 694 if err != nil { 695 return err 696 } 697 698 // Validate the arguments 699 if len(args.Jobs) == 0 { 700 return fmt.Errorf("given no jobs to deregister") 701 } 702 if len(args.Evals) != 0 { 703 return fmt.Errorf("evaluations should not be populated") 704 } 705 706 // Loop through checking for permissions 707 for jobNS := range args.Jobs { 708 // Check for submit-job permissions 709 if aclObj != nil && !aclObj.AllowNsOp(jobNS.Namespace, acl.NamespaceCapabilitySubmitJob) { 710 return structs.ErrPermissionDenied 711 } 712 } 713 714 // Grab a snapshot 715 snap, err := j.srv.fsm.State().Snapshot() 716 if err != nil { 717 return err 718 } 719 720 // Loop through to create evals 721 for jobNS, options := range args.Jobs { 722 if options == nil { 723 return fmt.Errorf("no deregister options provided for %v", jobNS) 724 } 725 726 job, err := snap.JobByID(nil, jobNS.Namespace, jobNS.ID) 727 if err != nil { 728 return err 729 } 730 731 // If the job is periodic or parameterized, we don't create an eval. 732 if job != nil && (job.IsPeriodic() || job.IsParameterized()) { 733 continue 734 } 735 736 priority := structs.JobDefaultPriority 737 jtype := structs.JobTypeService 738 if job != nil { 739 priority = job.Priority 740 jtype = job.Type 741 } 742 743 // Create a new evaluation 744 now := time.Now().UTC().UnixNano() 745 eval := &structs.Evaluation{ 746 ID: uuid.Generate(), 747 Namespace: jobNS.Namespace, 748 Priority: priority, 749 Type: jtype, 750 TriggeredBy: structs.EvalTriggerJobDeregister, 751 JobID: jobNS.ID, 752 Status: structs.EvalStatusPending, 753 CreateTime: now, 754 ModifyTime: now, 755 } 756 args.Evals = append(args.Evals, eval) 757 } 758 759 // Commit this update via Raft 760 _, index, err := j.srv.raftApply(structs.JobBatchDeregisterRequestType, args) 761 if err != nil { 762 j.logger.Error("batch deregister failed", "error", err) 763 return err 764 } 765 766 reply.Index = index 767 return nil 768 } 769 770 // GetJob is used to request information about a specific job 771 func (j *Job) GetJob(args *structs.JobSpecificRequest, 772 reply *structs.SingleJobResponse) error { 773 if done, err := j.srv.forward("Job.GetJob", args, args, reply); done { 774 return err 775 } 776 defer metrics.MeasureSince([]string{"nomad", "job", "get_job"}, time.Now()) 777 778 // Check for read-job permissions 779 if aclObj, err := j.srv.ResolveToken(args.AuthToken); err != nil { 780 return err 781 } else if aclObj != nil && !aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilityReadJob) { 782 return structs.ErrPermissionDenied 783 } 784 785 // Setup the blocking query 786 opts := blockingOptions{ 787 queryOpts: &args.QueryOptions, 788 queryMeta: &reply.QueryMeta, 789 run: func(ws memdb.WatchSet, state *state.StateStore) error { 790 // Look for the job 791 out, err := state.JobByID(ws, args.RequestNamespace(), args.JobID) 792 if err != nil { 793 return err 794 } 795 796 // Setup the output 797 reply.Job = out 798 if out != nil { 799 reply.Index = out.ModifyIndex 800 } else { 801 // Use the last index that affected the nodes table 802 index, err := state.Index("jobs") 803 if err != nil { 804 return err 805 } 806 reply.Index = index 807 } 808 809 // Set the query response 810 j.srv.setQueryMeta(&reply.QueryMeta) 811 return nil 812 }} 813 return j.srv.blockingRPC(&opts) 814 } 815 816 // GetJobVersions is used to retrieve all tracked versions of a job. 817 func (j *Job) GetJobVersions(args *structs.JobVersionsRequest, 818 reply *structs.JobVersionsResponse) error { 819 if done, err := j.srv.forward("Job.GetJobVersions", args, args, reply); done { 820 return err 821 } 822 defer metrics.MeasureSince([]string{"nomad", "job", "get_job_versions"}, time.Now()) 823 824 // Check for read-job permissions 825 if aclObj, err := j.srv.ResolveToken(args.AuthToken); err != nil { 826 return err 827 } else if aclObj != nil && !aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilityReadJob) { 828 return structs.ErrPermissionDenied 829 } 830 831 // Setup the blocking query 832 opts := blockingOptions{ 833 queryOpts: &args.QueryOptions, 834 queryMeta: &reply.QueryMeta, 835 run: func(ws memdb.WatchSet, state *state.StateStore) error { 836 // Look for the job 837 out, err := state.JobVersionsByID(ws, args.RequestNamespace(), args.JobID) 838 if err != nil { 839 return err 840 } 841 842 // Setup the output 843 reply.Versions = out 844 if len(out) != 0 { 845 reply.Index = out[0].ModifyIndex 846 847 // Compute the diffs 848 if args.Diffs { 849 for i := 0; i < len(out)-1; i++ { 850 old, new := out[i+1], out[i] 851 d, err := old.Diff(new, true) 852 if err != nil { 853 return fmt.Errorf("failed to create job diff: %v", err) 854 } 855 reply.Diffs = append(reply.Diffs, d) 856 } 857 } 858 } else { 859 // Use the last index that affected the nodes table 860 index, err := state.Index("job_version") 861 if err != nil { 862 return err 863 } 864 reply.Index = index 865 } 866 867 // Set the query response 868 j.srv.setQueryMeta(&reply.QueryMeta) 869 return nil 870 }} 871 return j.srv.blockingRPC(&opts) 872 } 873 874 // List is used to list the jobs registered in the system 875 func (j *Job) List(args *structs.JobListRequest, 876 reply *structs.JobListResponse) error { 877 if done, err := j.srv.forward("Job.List", args, args, reply); done { 878 return err 879 } 880 defer metrics.MeasureSince([]string{"nomad", "job", "list"}, time.Now()) 881 882 // Check for list-job permissions 883 if aclObj, err := j.srv.ResolveToken(args.AuthToken); err != nil { 884 return err 885 } else if aclObj != nil && !aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilityListJobs) { 886 return structs.ErrPermissionDenied 887 } 888 889 // Setup the blocking query 890 opts := blockingOptions{ 891 queryOpts: &args.QueryOptions, 892 queryMeta: &reply.QueryMeta, 893 run: func(ws memdb.WatchSet, state *state.StateStore) error { 894 // Capture all the jobs 895 var err error 896 var iter memdb.ResultIterator 897 if prefix := args.QueryOptions.Prefix; prefix != "" { 898 iter, err = state.JobsByIDPrefix(ws, args.RequestNamespace(), prefix) 899 } else { 900 iter, err = state.JobsByNamespace(ws, args.RequestNamespace()) 901 } 902 if err != nil { 903 return err 904 } 905 906 var jobs []*structs.JobListStub 907 for { 908 raw := iter.Next() 909 if raw == nil { 910 break 911 } 912 job := raw.(*structs.Job) 913 summary, err := state.JobSummaryByID(ws, args.RequestNamespace(), job.ID) 914 if err != nil { 915 return fmt.Errorf("unable to look up summary for job: %v", job.ID) 916 } 917 jobs = append(jobs, job.Stub(summary)) 918 } 919 reply.Jobs = jobs 920 921 // Use the last index that affected the jobs table or summary 922 jindex, err := state.Index("jobs") 923 if err != nil { 924 return err 925 } 926 sindex, err := state.Index("job_summary") 927 if err != nil { 928 return err 929 } 930 reply.Index = helper.Uint64Max(jindex, sindex) 931 932 // Set the query response 933 j.srv.setQueryMeta(&reply.QueryMeta) 934 return nil 935 }} 936 return j.srv.blockingRPC(&opts) 937 } 938 939 // Allocations is used to list the allocations for a job 940 func (j *Job) Allocations(args *structs.JobSpecificRequest, 941 reply *structs.JobAllocationsResponse) error { 942 if done, err := j.srv.forward("Job.Allocations", args, args, reply); done { 943 return err 944 } 945 defer metrics.MeasureSince([]string{"nomad", "job", "allocations"}, time.Now()) 946 947 // Check for read-job permissions 948 if aclObj, err := j.srv.ResolveToken(args.AuthToken); err != nil { 949 return err 950 } else if aclObj != nil && !aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilityReadJob) { 951 return structs.ErrPermissionDenied 952 } 953 954 // Ensure JobID is set otherwise everything works and never returns 955 // allocations which can hide bugs in request code. 956 if args.JobID == "" { 957 return fmt.Errorf("missing job ID") 958 } 959 960 // Setup the blocking query 961 opts := blockingOptions{ 962 queryOpts: &args.QueryOptions, 963 queryMeta: &reply.QueryMeta, 964 run: func(ws memdb.WatchSet, state *state.StateStore) error { 965 // Capture the allocations 966 allocs, err := state.AllocsByJob(ws, args.RequestNamespace(), args.JobID, args.All) 967 if err != nil { 968 return err 969 } 970 971 // Convert to stubs 972 if len(allocs) > 0 { 973 reply.Allocations = make([]*structs.AllocListStub, 0, len(allocs)) 974 for _, alloc := range allocs { 975 reply.Allocations = append(reply.Allocations, alloc.Stub()) 976 } 977 } 978 979 // Use the last index that affected the allocs table 980 index, err := state.Index("allocs") 981 if err != nil { 982 return err 983 } 984 reply.Index = index 985 986 // Set the query response 987 j.srv.setQueryMeta(&reply.QueryMeta) 988 return nil 989 990 }} 991 return j.srv.blockingRPC(&opts) 992 } 993 994 // Evaluations is used to list the evaluations for a job 995 func (j *Job) Evaluations(args *structs.JobSpecificRequest, 996 reply *structs.JobEvaluationsResponse) error { 997 if done, err := j.srv.forward("Job.Evaluations", args, args, reply); done { 998 return err 999 } 1000 defer metrics.MeasureSince([]string{"nomad", "job", "evaluations"}, time.Now()) 1001 1002 // Check for read-job permissions 1003 if aclObj, err := j.srv.ResolveToken(args.AuthToken); err != nil { 1004 return err 1005 } else if aclObj != nil && !aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilityReadJob) { 1006 return structs.ErrPermissionDenied 1007 } 1008 1009 // Setup the blocking query 1010 opts := blockingOptions{ 1011 queryOpts: &args.QueryOptions, 1012 queryMeta: &reply.QueryMeta, 1013 run: func(ws memdb.WatchSet, state *state.StateStore) error { 1014 // Capture the evals 1015 var err error 1016 reply.Evaluations, err = state.EvalsByJob(ws, args.RequestNamespace(), args.JobID) 1017 if err != nil { 1018 return err 1019 } 1020 1021 // Use the last index that affected the evals table 1022 index, err := state.Index("evals") 1023 if err != nil { 1024 return err 1025 } 1026 reply.Index = index 1027 1028 // Set the query response 1029 j.srv.setQueryMeta(&reply.QueryMeta) 1030 return nil 1031 }} 1032 1033 return j.srv.blockingRPC(&opts) 1034 } 1035 1036 // Deployments is used to list the deployments for a job 1037 func (j *Job) Deployments(args *structs.JobSpecificRequest, 1038 reply *structs.DeploymentListResponse) error { 1039 if done, err := j.srv.forward("Job.Deployments", args, args, reply); done { 1040 return err 1041 } 1042 defer metrics.MeasureSince([]string{"nomad", "job", "deployments"}, time.Now()) 1043 1044 // Check for read-job permissions 1045 if aclObj, err := j.srv.ResolveToken(args.AuthToken); err != nil { 1046 return err 1047 } else if aclObj != nil && !aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilityReadJob) { 1048 return structs.ErrPermissionDenied 1049 } 1050 1051 // Setup the blocking query 1052 opts := blockingOptions{ 1053 queryOpts: &args.QueryOptions, 1054 queryMeta: &reply.QueryMeta, 1055 run: func(ws memdb.WatchSet, state *state.StateStore) error { 1056 // Capture the deployments 1057 deploys, err := state.DeploymentsByJobID(ws, args.RequestNamespace(), args.JobID, args.All) 1058 if err != nil { 1059 return err 1060 } 1061 1062 // Use the last index that affected the deployment table 1063 index, err := state.Index("deployment") 1064 if err != nil { 1065 return err 1066 } 1067 reply.Index = index 1068 reply.Deployments = deploys 1069 1070 // Set the query response 1071 j.srv.setQueryMeta(&reply.QueryMeta) 1072 return nil 1073 1074 }} 1075 return j.srv.blockingRPC(&opts) 1076 } 1077 1078 // LatestDeployment is used to retrieve the latest deployment for a job 1079 func (j *Job) LatestDeployment(args *structs.JobSpecificRequest, 1080 reply *structs.SingleDeploymentResponse) error { 1081 if done, err := j.srv.forward("Job.LatestDeployment", args, args, reply); done { 1082 return err 1083 } 1084 defer metrics.MeasureSince([]string{"nomad", "job", "latest_deployment"}, time.Now()) 1085 1086 // Check for read-job permissions 1087 if aclObj, err := j.srv.ResolveToken(args.AuthToken); err != nil { 1088 return err 1089 } else if aclObj != nil && !aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilityReadJob) { 1090 return structs.ErrPermissionDenied 1091 } 1092 1093 // Setup the blocking query 1094 opts := blockingOptions{ 1095 queryOpts: &args.QueryOptions, 1096 queryMeta: &reply.QueryMeta, 1097 run: func(ws memdb.WatchSet, state *state.StateStore) error { 1098 // Capture the deployments 1099 deploys, err := state.DeploymentsByJobID(ws, args.RequestNamespace(), args.JobID, args.All) 1100 if err != nil { 1101 return err 1102 } 1103 1104 // Use the last index that affected the deployment table 1105 index, err := state.Index("deployment") 1106 if err != nil { 1107 return err 1108 } 1109 reply.Index = index 1110 if len(deploys) > 0 { 1111 sort.Slice(deploys, func(i, j int) bool { 1112 return deploys[i].CreateIndex > deploys[j].CreateIndex 1113 }) 1114 reply.Deployment = deploys[0] 1115 } 1116 1117 // Set the query response 1118 j.srv.setQueryMeta(&reply.QueryMeta) 1119 return nil 1120 1121 }} 1122 return j.srv.blockingRPC(&opts) 1123 } 1124 1125 // Plan is used to cause a dry-run evaluation of the Job and return the results 1126 // with a potential diff containing annotations. 1127 func (j *Job) Plan(args *structs.JobPlanRequest, reply *structs.JobPlanResponse) error { 1128 if done, err := j.srv.forward("Job.Plan", args, args, reply); done { 1129 return err 1130 } 1131 defer metrics.MeasureSince([]string{"nomad", "job", "plan"}, time.Now()) 1132 1133 // Validate the arguments 1134 if args.Job == nil { 1135 return fmt.Errorf("Job required for plan") 1136 } 1137 1138 // Run admission controllers 1139 job, warnings, err := j.admissionControllers(args.Job) 1140 if err != nil { 1141 return err 1142 } 1143 args.Job = job 1144 1145 // Set the warning message 1146 reply.Warnings = structs.MergeMultierrorWarnings(warnings...) 1147 1148 // Check job submission permissions, which we assume is the same for plan 1149 if aclObj, err := j.srv.ResolveToken(args.AuthToken); err != nil { 1150 return err 1151 } else if aclObj != nil { 1152 if !aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilitySubmitJob) { 1153 return structs.ErrPermissionDenied 1154 } 1155 // Check if override is set and we do not have permissions 1156 if args.PolicyOverride { 1157 if !aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilitySentinelOverride) { 1158 return structs.ErrPermissionDenied 1159 } 1160 } 1161 } 1162 1163 // Enforce Sentinel policies 1164 policyWarnings, err := j.enforceSubmitJob(args.PolicyOverride, args.Job) 1165 if err != nil { 1166 return err 1167 } 1168 if policyWarnings != nil { 1169 warnings = append(warnings, policyWarnings) 1170 reply.Warnings = structs.MergeMultierrorWarnings(warnings...) 1171 } 1172 1173 // Acquire a snapshot of the state 1174 snap, err := j.srv.fsm.State().Snapshot() 1175 if err != nil { 1176 return err 1177 } 1178 1179 // Get the original job 1180 ws := memdb.NewWatchSet() 1181 oldJob, err := snap.JobByID(ws, args.RequestNamespace(), args.Job.ID) 1182 if err != nil { 1183 return err 1184 } 1185 1186 var index uint64 1187 var updatedIndex uint64 1188 1189 if oldJob != nil { 1190 index = oldJob.JobModifyIndex 1191 1192 // We want to reuse deployments where possible, so only insert the job if 1193 // it has changed or the job didn't exist 1194 if oldJob.SpecChanged(args.Job) { 1195 // Insert the updated Job into the snapshot 1196 updatedIndex = oldJob.JobModifyIndex + 1 1197 snap.UpsertJob(updatedIndex, args.Job) 1198 } 1199 } else if oldJob == nil { 1200 // Insert the updated Job into the snapshot 1201 snap.UpsertJob(100, args.Job) 1202 } 1203 1204 // Create an eval and mark it as requiring annotations and insert that as well 1205 now := time.Now().UTC().UnixNano() 1206 eval := &structs.Evaluation{ 1207 ID: uuid.Generate(), 1208 Namespace: args.RequestNamespace(), 1209 Priority: args.Job.Priority, 1210 Type: args.Job.Type, 1211 TriggeredBy: structs.EvalTriggerJobRegister, 1212 JobID: args.Job.ID, 1213 JobModifyIndex: updatedIndex, 1214 Status: structs.EvalStatusPending, 1215 AnnotatePlan: true, 1216 // Timestamps are added for consistency but this eval is never persisted 1217 CreateTime: now, 1218 ModifyTime: now, 1219 } 1220 1221 snap.UpsertEvals(100, []*structs.Evaluation{eval}) 1222 1223 // Create an in-memory Planner that returns no errors and stores the 1224 // submitted plan and created evals. 1225 planner := &scheduler.Harness{ 1226 State: &snap.StateStore, 1227 } 1228 1229 // Create the scheduler and run it 1230 sched, err := scheduler.NewScheduler(eval.Type, j.logger, snap, planner) 1231 if err != nil { 1232 return err 1233 } 1234 1235 if err := sched.Process(eval); err != nil { 1236 return err 1237 } 1238 1239 // Annotate and store the diff 1240 if plans := len(planner.Plans); plans != 1 { 1241 return fmt.Errorf("scheduler resulted in an unexpected number of plans: %v", plans) 1242 } 1243 annotations := planner.Plans[0].Annotations 1244 if args.Diff { 1245 jobDiff, err := oldJob.Diff(args.Job, true) 1246 if err != nil { 1247 return fmt.Errorf("failed to create job diff: %v", err) 1248 } 1249 1250 if err := scheduler.Annotate(jobDiff, annotations); err != nil { 1251 return fmt.Errorf("failed to annotate job diff: %v", err) 1252 } 1253 reply.Diff = jobDiff 1254 } 1255 1256 // Grab the failures 1257 if len(planner.Evals) != 1 { 1258 return fmt.Errorf("scheduler resulted in an unexpected number of eval updates: %v", planner.Evals) 1259 } 1260 updatedEval := planner.Evals[0] 1261 1262 // If it is a periodic job calculate the next launch 1263 if args.Job.IsPeriodic() && args.Job.Periodic.Enabled { 1264 reply.NextPeriodicLaunch, err = args.Job.Periodic.Next(time.Now().In(args.Job.Periodic.GetLocation())) 1265 if err != nil { 1266 return fmt.Errorf("Failed to parse cron expression: %v", err) 1267 } 1268 } 1269 1270 reply.FailedTGAllocs = updatedEval.FailedTGAllocs 1271 reply.JobModifyIndex = index 1272 reply.Annotations = annotations 1273 reply.CreatedEvals = planner.CreateEvals 1274 reply.Index = index 1275 return nil 1276 } 1277 1278 // validateJobUpdate ensures updates to a job are valid. 1279 func validateJobUpdate(old, new *structs.Job) error { 1280 // Validate Dispatch not set on new Jobs 1281 if old == nil { 1282 if new.Dispatched { 1283 return fmt.Errorf("job can't be submitted with 'Dispatched' set") 1284 } 1285 return nil 1286 } 1287 1288 // Type transitions are disallowed 1289 if old.Type != new.Type { 1290 return fmt.Errorf("cannot update job from type %q to %q", old.Type, new.Type) 1291 } 1292 1293 // Transitioning to/from periodic is disallowed 1294 if old.IsPeriodic() && !new.IsPeriodic() { 1295 return fmt.Errorf("cannot update periodic job to being non-periodic") 1296 } 1297 if new.IsPeriodic() && !old.IsPeriodic() { 1298 return fmt.Errorf("cannot update non-periodic job to being periodic") 1299 } 1300 1301 // Transitioning to/from parameterized is disallowed 1302 if old.IsParameterized() && !new.IsParameterized() { 1303 return fmt.Errorf("cannot update non-parameterized job to being parameterized") 1304 } 1305 if new.IsParameterized() && !old.IsParameterized() { 1306 return fmt.Errorf("cannot update parameterized job to being non-parameterized") 1307 } 1308 1309 if old.Dispatched != new.Dispatched { 1310 return fmt.Errorf("field 'Dispatched' is read-only") 1311 } 1312 1313 return nil 1314 } 1315 1316 // Dispatch a parameterized job. 1317 func (j *Job) Dispatch(args *structs.JobDispatchRequest, reply *structs.JobDispatchResponse) error { 1318 if done, err := j.srv.forward("Job.Dispatch", args, args, reply); done { 1319 return err 1320 } 1321 defer metrics.MeasureSince([]string{"nomad", "job", "dispatch"}, time.Now()) 1322 1323 // Check for submit-job permissions 1324 if aclObj, err := j.srv.ResolveToken(args.AuthToken); err != nil { 1325 return err 1326 } else if aclObj != nil && !aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilityDispatchJob) { 1327 return structs.ErrPermissionDenied 1328 } 1329 1330 // Lookup the parameterized job 1331 if args.JobID == "" { 1332 return fmt.Errorf("missing parameterized job ID") 1333 } 1334 1335 snap, err := j.srv.fsm.State().Snapshot() 1336 if err != nil { 1337 return err 1338 } 1339 ws := memdb.NewWatchSet() 1340 parameterizedJob, err := snap.JobByID(ws, args.RequestNamespace(), args.JobID) 1341 if err != nil { 1342 return err 1343 } 1344 if parameterizedJob == nil { 1345 return fmt.Errorf("parameterized job not found") 1346 } 1347 1348 if !parameterizedJob.IsParameterized() { 1349 return fmt.Errorf("Specified job %q is not a parameterized job", args.JobID) 1350 } 1351 1352 if parameterizedJob.Stop { 1353 return fmt.Errorf("Specified job %q is stopped", args.JobID) 1354 } 1355 1356 // Validate the arguments 1357 if err := validateDispatchRequest(args, parameterizedJob); err != nil { 1358 return err 1359 } 1360 1361 // Derive the child job and commit it via Raft 1362 dispatchJob := parameterizedJob.Copy() 1363 dispatchJob.ID = structs.DispatchedID(parameterizedJob.ID, time.Now()) 1364 dispatchJob.ParentID = parameterizedJob.ID 1365 dispatchJob.Name = dispatchJob.ID 1366 dispatchJob.SetSubmitTime() 1367 dispatchJob.Dispatched = true 1368 1369 // Merge in the meta data 1370 for k, v := range args.Meta { 1371 if dispatchJob.Meta == nil { 1372 dispatchJob.Meta = make(map[string]string, len(args.Meta)) 1373 } 1374 dispatchJob.Meta[k] = v 1375 } 1376 1377 // Compress the payload 1378 dispatchJob.Payload = snappy.Encode(nil, args.Payload) 1379 1380 regReq := &structs.JobRegisterRequest{ 1381 Job: dispatchJob, 1382 WriteRequest: args.WriteRequest, 1383 } 1384 1385 // Commit this update via Raft 1386 fsmErr, jobCreateIndex, err := j.srv.raftApply(structs.JobRegisterRequestType, regReq) 1387 if err, ok := fsmErr.(error); ok && err != nil { 1388 j.logger.Error("dispatched job register failed", "error", err, "fsm", true) 1389 return err 1390 } 1391 if err != nil { 1392 j.logger.Error("dispatched job register failed", "error", err, "raft", true) 1393 return err 1394 } 1395 1396 reply.JobCreateIndex = jobCreateIndex 1397 reply.DispatchedJobID = dispatchJob.ID 1398 reply.Index = jobCreateIndex 1399 1400 // If the job is periodic, we don't create an eval. 1401 if !dispatchJob.IsPeriodic() { 1402 // Create a new evaluation 1403 now := time.Now().UTC().UnixNano() 1404 eval := &structs.Evaluation{ 1405 ID: uuid.Generate(), 1406 Namespace: args.RequestNamespace(), 1407 Priority: dispatchJob.Priority, 1408 Type: dispatchJob.Type, 1409 TriggeredBy: structs.EvalTriggerJobRegister, 1410 JobID: dispatchJob.ID, 1411 JobModifyIndex: jobCreateIndex, 1412 Status: structs.EvalStatusPending, 1413 CreateTime: now, 1414 ModifyTime: now, 1415 } 1416 update := &structs.EvalUpdateRequest{ 1417 Evals: []*structs.Evaluation{eval}, 1418 WriteRequest: structs.WriteRequest{Region: args.Region}, 1419 } 1420 1421 // Commit this evaluation via Raft 1422 _, evalIndex, err := j.srv.raftApply(structs.EvalUpdateRequestType, update) 1423 if err != nil { 1424 j.logger.Error("eval create failed", "error", err, "method", "dispatch") 1425 return err 1426 } 1427 1428 // Setup the reply 1429 reply.EvalID = eval.ID 1430 reply.EvalCreateIndex = evalIndex 1431 reply.Index = evalIndex 1432 } 1433 1434 return nil 1435 } 1436 1437 // validateDispatchRequest returns whether the request is valid given the 1438 // parameterized job. 1439 func validateDispatchRequest(req *structs.JobDispatchRequest, job *structs.Job) error { 1440 // Check the payload constraint is met 1441 hasInputData := len(req.Payload) != 0 1442 if job.ParameterizedJob.Payload == structs.DispatchPayloadRequired && !hasInputData { 1443 return fmt.Errorf("Payload is not provided but required by parameterized job") 1444 } else if job.ParameterizedJob.Payload == structs.DispatchPayloadForbidden && hasInputData { 1445 return fmt.Errorf("Payload provided but forbidden by parameterized job") 1446 } 1447 1448 // Check the payload doesn't exceed the size limit 1449 if l := len(req.Payload); l > DispatchPayloadSizeLimit { 1450 return fmt.Errorf("Payload exceeds maximum size; %d > %d", l, DispatchPayloadSizeLimit) 1451 } 1452 1453 // Check if the metadata is a set 1454 keys := make(map[string]struct{}, len(req.Meta)) 1455 for k := range keys { 1456 if _, ok := keys[k]; ok { 1457 return fmt.Errorf("Duplicate key %q in passed metadata", k) 1458 } 1459 keys[k] = struct{}{} 1460 } 1461 1462 required := helper.SliceStringToSet(job.ParameterizedJob.MetaRequired) 1463 optional := helper.SliceStringToSet(job.ParameterizedJob.MetaOptional) 1464 1465 // Check the metadata key constraints are met 1466 unpermitted := make(map[string]struct{}) 1467 for k := range req.Meta { 1468 _, req := required[k] 1469 _, opt := optional[k] 1470 if !req && !opt { 1471 unpermitted[k] = struct{}{} 1472 } 1473 } 1474 1475 if len(unpermitted) != 0 { 1476 flat := make([]string, 0, len(unpermitted)) 1477 for k := range unpermitted { 1478 flat = append(flat, k) 1479 } 1480 1481 return fmt.Errorf("Dispatch request included unpermitted metadata keys: %v", flat) 1482 } 1483 1484 missing := make(map[string]struct{}) 1485 for _, k := range job.ParameterizedJob.MetaRequired { 1486 if _, ok := req.Meta[k]; !ok { 1487 missing[k] = struct{}{} 1488 } 1489 } 1490 1491 if len(missing) != 0 { 1492 flat := make([]string, 0, len(missing)) 1493 for k := range missing { 1494 flat = append(flat, k) 1495 } 1496 1497 return fmt.Errorf("Dispatch did not provide required meta keys: %v", flat) 1498 } 1499 1500 return nil 1501 }