github.com/hspak/nomad@v0.7.2-0.20180309000617-bc4ae22a39a5/nomad/job_endpoint.go (about) 1 package nomad 2 3 import ( 4 "context" 5 "fmt" 6 "sort" 7 "strings" 8 "time" 9 10 "github.com/armon/go-metrics" 11 "github.com/golang/snappy" 12 "github.com/hashicorp/consul/lib" 13 memdb "github.com/hashicorp/go-memdb" 14 "github.com/hashicorp/go-multierror" 15 "github.com/hashicorp/nomad/acl" 16 "github.com/hashicorp/nomad/client/driver" 17 "github.com/hashicorp/nomad/helper" 18 "github.com/hashicorp/nomad/helper/uuid" 19 "github.com/hashicorp/nomad/nomad/state" 20 "github.com/hashicorp/nomad/nomad/structs" 21 "github.com/hashicorp/nomad/scheduler" 22 ) 23 24 const ( 25 // RegisterEnforceIndexErrPrefix is the prefix to use in errors caused by 26 // enforcing the job modify index during registers. 27 RegisterEnforceIndexErrPrefix = "Enforcing job modify index" 28 29 // DispatchPayloadSizeLimit is the maximum size of the uncompressed input 30 // data payload. 31 DispatchPayloadSizeLimit = 16 * 1024 32 ) 33 34 var ( 35 // vaultConstraint is the implicit constraint added to jobs requesting a 36 // Vault token 37 vaultConstraint = &structs.Constraint{ 38 LTarget: "${attr.vault.version}", 39 RTarget: ">= 0.6.1", 40 Operand: structs.ConstraintVersion, 41 } 42 ) 43 44 // Job endpoint is used for job interactions 45 type Job struct { 46 srv *Server 47 } 48 49 // Register is used to upsert a job for scheduling 50 func (j *Job) Register(args *structs.JobRegisterRequest, reply *structs.JobRegisterResponse) error { 51 if done, err := j.srv.forward("Job.Register", args, args, reply); done { 52 return err 53 } 54 defer metrics.MeasureSince([]string{"nomad", "job", "register"}, time.Now()) 55 56 // Validate the arguments 57 if args.Job == nil { 58 return fmt.Errorf("missing job for registration") 59 } 60 61 // Initialize the job fields (sets defaults and any necessary init work). 62 canonicalizeWarnings := args.Job.Canonicalize() 63 64 // Add implicit constraints 65 setImplicitConstraints(args.Job) 66 67 // Validate the job and capture any warnings 68 err, warnings := validateJob(args.Job) 69 if err != nil { 70 return err 71 } 72 73 // Set the warning message 74 reply.Warnings = structs.MergeMultierrorWarnings(warnings, canonicalizeWarnings) 75 76 // Check job submission permissions 77 if aclObj, err := j.srv.ResolveToken(args.AuthToken); err != nil { 78 return err 79 } else if aclObj != nil { 80 if !aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilitySubmitJob) { 81 return structs.ErrPermissionDenied 82 } 83 // Check if override is set and we do not have permissions 84 if args.PolicyOverride { 85 if !aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilitySentinelOverride) { 86 j.srv.logger.Printf("[WARN] nomad.job: policy override attempted without permissions for Job %q", args.Job.ID) 87 return structs.ErrPermissionDenied 88 } 89 j.srv.logger.Printf("[WARN] nomad.job: policy override set for Job %q", args.Job.ID) 90 } 91 } 92 93 // Lookup the job 94 snap, err := j.srv.State().Snapshot() 95 if err != nil { 96 return err 97 } 98 ws := memdb.NewWatchSet() 99 existingJob, err := snap.JobByID(ws, args.RequestNamespace(), args.Job.ID) 100 if err != nil { 101 return err 102 } 103 104 // If EnforceIndex set, check it before trying to apply 105 if args.EnforceIndex { 106 jmi := args.JobModifyIndex 107 if existingJob != nil { 108 if jmi == 0 { 109 return fmt.Errorf("%s 0: job already exists", RegisterEnforceIndexErrPrefix) 110 } else if jmi != existingJob.JobModifyIndex { 111 return fmt.Errorf("%s %d: job exists with conflicting job modify index: %d", 112 RegisterEnforceIndexErrPrefix, jmi, existingJob.JobModifyIndex) 113 } 114 } else if jmi != 0 { 115 return fmt.Errorf("%s %d: job does not exist", RegisterEnforceIndexErrPrefix, jmi) 116 } 117 } 118 119 // Validate job transitions if its an update 120 if existingJob != nil { 121 if err := validateJobUpdate(existingJob, args.Job); err != nil { 122 return err 123 } 124 } 125 126 // Ensure that the job has permissions for the requested Vault tokens 127 policies := args.Job.VaultPolicies() 128 if len(policies) != 0 { 129 vconf := j.srv.config.VaultConfig 130 if !vconf.IsEnabled() { 131 return fmt.Errorf("Vault not enabled and Vault policies requested") 132 } 133 134 // Have to check if the user has permissions 135 if !vconf.AllowsUnauthenticated() { 136 if args.Job.VaultToken == "" { 137 return fmt.Errorf("Vault policies requested but missing Vault Token") 138 } 139 140 vault := j.srv.vault 141 s, err := vault.LookupToken(context.Background(), args.Job.VaultToken) 142 if err != nil { 143 return err 144 } 145 146 allowedPolicies, err := PoliciesFrom(s) 147 if err != nil { 148 return err 149 } 150 151 // If we are given a root token it can access all policies 152 if !lib.StrContains(allowedPolicies, "root") { 153 flatPolicies := structs.VaultPoliciesSet(policies) 154 subset, offending := helper.SliceStringIsSubset(allowedPolicies, flatPolicies) 155 if !subset { 156 return fmt.Errorf("Passed Vault Token doesn't allow access to the following policies: %s", 157 strings.Join(offending, ", ")) 158 } 159 } 160 } 161 } 162 163 // Enforce Sentinel policies 164 policyWarnings, err := j.enforceSubmitJob(args.PolicyOverride, args.Job) 165 if err != nil { 166 return err 167 } 168 if policyWarnings != nil { 169 reply.Warnings = structs.MergeMultierrorWarnings(warnings, 170 canonicalizeWarnings, policyWarnings) 171 } 172 173 // Clear the Vault token 174 args.Job.VaultToken = "" 175 176 // Check if the job has changed at all 177 if existingJob == nil || existingJob.SpecChanged(args.Job) { 178 // Set the submit time 179 args.Job.SetSubmitTime() 180 181 // Commit this update via Raft 182 fsmErr, index, err := j.srv.raftApply(structs.JobRegisterRequestType, args) 183 if err, ok := fsmErr.(error); ok && err != nil { 184 j.srv.logger.Printf("[ERR] nomad.job: Register failed: %v", err) 185 return err 186 } 187 if err != nil { 188 j.srv.logger.Printf("[ERR] nomad.job: Register failed: %v", err) 189 return err 190 } 191 192 // Populate the reply with job information 193 reply.JobModifyIndex = index 194 } else { 195 reply.JobModifyIndex = existingJob.JobModifyIndex 196 } 197 198 // If the job is periodic or parameterized, we don't create an eval. 199 if args.Job.IsPeriodic() || args.Job.IsParameterized() { 200 return nil 201 } 202 203 // Create a new evaluation 204 eval := &structs.Evaluation{ 205 ID: uuid.Generate(), 206 Namespace: args.RequestNamespace(), 207 Priority: args.Job.Priority, 208 Type: args.Job.Type, 209 TriggeredBy: structs.EvalTriggerJobRegister, 210 JobID: args.Job.ID, 211 JobModifyIndex: reply.JobModifyIndex, 212 Status: structs.EvalStatusPending, 213 } 214 update := &structs.EvalUpdateRequest{ 215 Evals: []*structs.Evaluation{eval}, 216 WriteRequest: structs.WriteRequest{Region: args.Region}, 217 } 218 219 // Commit this evaluation via Raft 220 // XXX: There is a risk of partial failure where the JobRegister succeeds 221 // but that the EvalUpdate does not. 222 _, evalIndex, err := j.srv.raftApply(structs.EvalUpdateRequestType, update) 223 if err != nil { 224 j.srv.logger.Printf("[ERR] nomad.job: Eval create failed: %v", err) 225 return err 226 } 227 228 // Populate the reply with eval information 229 reply.EvalID = eval.ID 230 reply.EvalCreateIndex = evalIndex 231 reply.Index = evalIndex 232 return nil 233 } 234 235 // setImplicitConstraints adds implicit constraints to the job based on the 236 // features it is requesting. 237 func setImplicitConstraints(j *structs.Job) { 238 // Get the required Vault Policies 239 policies := j.VaultPolicies() 240 241 // Get the required signals 242 signals := j.RequiredSignals() 243 244 // Hot path 245 if len(signals) == 0 && len(policies) == 0 { 246 return 247 } 248 249 // Add Vault constraints 250 for _, tg := range j.TaskGroups { 251 _, ok := policies[tg.Name] 252 if !ok { 253 // Not requesting Vault 254 continue 255 } 256 257 found := false 258 for _, c := range tg.Constraints { 259 if c.Equal(vaultConstraint) { 260 found = true 261 break 262 } 263 } 264 265 if !found { 266 tg.Constraints = append(tg.Constraints, vaultConstraint) 267 } 268 } 269 270 // Add signal constraints 271 for _, tg := range j.TaskGroups { 272 tgSignals, ok := signals[tg.Name] 273 if !ok { 274 // Not requesting Vault 275 continue 276 } 277 278 // Flatten the signals 279 required := helper.MapStringStringSliceValueSet(tgSignals) 280 sigConstraint := getSignalConstraint(required) 281 282 found := false 283 for _, c := range tg.Constraints { 284 if c.Equal(sigConstraint) { 285 found = true 286 break 287 } 288 } 289 290 if !found { 291 tg.Constraints = append(tg.Constraints, sigConstraint) 292 } 293 } 294 } 295 296 // getSignalConstraint builds a suitable constraint based on the required 297 // signals 298 func getSignalConstraint(signals []string) *structs.Constraint { 299 return &structs.Constraint{ 300 Operand: structs.ConstraintSetContains, 301 LTarget: "${attr.os.signals}", 302 RTarget: strings.Join(signals, ","), 303 } 304 } 305 306 // Summary retrieves the summary of a job 307 func (j *Job) Summary(args *structs.JobSummaryRequest, 308 reply *structs.JobSummaryResponse) error { 309 310 if done, err := j.srv.forward("Job.Summary", args, args, reply); done { 311 return err 312 } 313 defer metrics.MeasureSince([]string{"nomad", "job_summary", "get_job_summary"}, time.Now()) 314 315 // Check for read-job permissions 316 if aclObj, err := j.srv.ResolveToken(args.AuthToken); err != nil { 317 return err 318 } else if aclObj != nil && !aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilityReadJob) { 319 return structs.ErrPermissionDenied 320 } 321 322 // Setup the blocking query 323 opts := blockingOptions{ 324 queryOpts: &args.QueryOptions, 325 queryMeta: &reply.QueryMeta, 326 run: func(ws memdb.WatchSet, state *state.StateStore) error { 327 // Look for job summary 328 out, err := state.JobSummaryByID(ws, args.RequestNamespace(), args.JobID) 329 if err != nil { 330 return err 331 } 332 333 // Setup the output 334 reply.JobSummary = out 335 if out != nil { 336 reply.Index = out.ModifyIndex 337 } else { 338 // Use the last index that affected the job_summary table 339 index, err := state.Index("job_summary") 340 if err != nil { 341 return err 342 } 343 reply.Index = index 344 } 345 346 // Set the query response 347 j.srv.setQueryMeta(&reply.QueryMeta) 348 return nil 349 }} 350 return j.srv.blockingRPC(&opts) 351 } 352 353 // Validate validates a job 354 func (j *Job) Validate(args *structs.JobValidateRequest, reply *structs.JobValidateResponse) error { 355 defer metrics.MeasureSince([]string{"nomad", "job", "validate"}, time.Now()) 356 357 // Check for read-job permissions 358 if aclObj, err := j.srv.ResolveToken(args.AuthToken); err != nil { 359 return err 360 } else if aclObj != nil && !aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilityReadJob) { 361 return structs.ErrPermissionDenied 362 } 363 364 // Initialize the job fields (sets defaults and any necessary init work). 365 canonicalizeWarnings := args.Job.Canonicalize() 366 367 // Add implicit constraints 368 setImplicitConstraints(args.Job) 369 370 // Validate the job and capture any warnings 371 err, warnings := validateJob(args.Job) 372 if err != nil { 373 if merr, ok := err.(*multierror.Error); ok { 374 for _, err := range merr.Errors { 375 reply.ValidationErrors = append(reply.ValidationErrors, err.Error()) 376 } 377 reply.Error = merr.Error() 378 } else { 379 reply.ValidationErrors = append(reply.ValidationErrors, err.Error()) 380 reply.Error = err.Error() 381 } 382 } 383 384 // Set the warning message 385 reply.Warnings = structs.MergeMultierrorWarnings(warnings, canonicalizeWarnings) 386 reply.DriverConfigValidated = true 387 return nil 388 } 389 390 // Revert is used to revert the job to a prior version 391 func (j *Job) Revert(args *structs.JobRevertRequest, reply *structs.JobRegisterResponse) error { 392 if done, err := j.srv.forward("Job.Revert", args, args, reply); done { 393 return err 394 } 395 defer metrics.MeasureSince([]string{"nomad", "job", "revert"}, time.Now()) 396 397 // Check for submit-job permissions 398 if aclObj, err := j.srv.ResolveToken(args.AuthToken); err != nil { 399 return err 400 } else if aclObj != nil && !aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilitySubmitJob) { 401 return structs.ErrPermissionDenied 402 } 403 404 // Validate the arguments 405 if args.JobID == "" { 406 return fmt.Errorf("missing job ID for revert") 407 } 408 409 // Lookup the job by version 410 snap, err := j.srv.fsm.State().Snapshot() 411 if err != nil { 412 return err 413 } 414 415 ws := memdb.NewWatchSet() 416 cur, err := snap.JobByID(ws, args.RequestNamespace(), args.JobID) 417 if err != nil { 418 return err 419 } 420 if cur == nil { 421 return fmt.Errorf("job %q not found", args.JobID) 422 } 423 if args.JobVersion == cur.Version { 424 return fmt.Errorf("can't revert to current version") 425 } 426 427 jobV, err := snap.JobByIDAndVersion(ws, args.RequestNamespace(), args.JobID, args.JobVersion) 428 if err != nil { 429 return err 430 } 431 if jobV == nil { 432 return fmt.Errorf("job %q in namespace %q at version %d not found", args.JobID, args.RequestNamespace(), args.JobVersion) 433 } 434 435 // Build the register request 436 reg := &structs.JobRegisterRequest{ 437 Job: jobV.Copy(), 438 WriteRequest: args.WriteRequest, 439 } 440 441 // If the request is enforcing the existing version do a check. 442 if args.EnforcePriorVersion != nil { 443 if cur.Version != *args.EnforcePriorVersion { 444 return fmt.Errorf("Current job has version %d; enforcing version %d", cur.Version, *args.EnforcePriorVersion) 445 } 446 447 reg.EnforceIndex = true 448 reg.JobModifyIndex = cur.JobModifyIndex 449 } 450 451 // Register the version. 452 return j.Register(reg, reply) 453 } 454 455 // Stable is used to mark the job version as stable 456 func (j *Job) Stable(args *structs.JobStabilityRequest, reply *structs.JobStabilityResponse) error { 457 if done, err := j.srv.forward("Job.Stable", args, args, reply); done { 458 return err 459 } 460 defer metrics.MeasureSince([]string{"nomad", "job", "stable"}, time.Now()) 461 462 // Check for read-job permissions 463 if aclObj, err := j.srv.ResolveToken(args.AuthToken); err != nil { 464 return err 465 } else if aclObj != nil && !aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilitySubmitJob) { 466 return structs.ErrPermissionDenied 467 } 468 469 // Validate the arguments 470 if args.JobID == "" { 471 return fmt.Errorf("missing job ID for marking job as stable") 472 } 473 474 // Lookup the job by version 475 snap, err := j.srv.fsm.State().Snapshot() 476 if err != nil { 477 return err 478 } 479 480 ws := memdb.NewWatchSet() 481 jobV, err := snap.JobByIDAndVersion(ws, args.RequestNamespace(), args.JobID, args.JobVersion) 482 if err != nil { 483 return err 484 } 485 if jobV == nil { 486 return fmt.Errorf("job %q in namespace %q at version %d not found", args.JobID, args.RequestNamespace(), args.JobVersion) 487 } 488 489 // Commit this stability request via Raft 490 _, modifyIndex, err := j.srv.raftApply(structs.JobStabilityRequestType, args) 491 if err != nil { 492 j.srv.logger.Printf("[ERR] nomad.job: Job stability request failed: %v", err) 493 return err 494 } 495 496 // Setup the reply 497 reply.Index = modifyIndex 498 return nil 499 } 500 501 // Evaluate is used to force a job for re-evaluation 502 func (j *Job) Evaluate(args *structs.JobEvaluateRequest, reply *structs.JobRegisterResponse) error { 503 if done, err := j.srv.forward("Job.Evaluate", args, args, reply); done { 504 return err 505 } 506 defer metrics.MeasureSince([]string{"nomad", "job", "evaluate"}, time.Now()) 507 508 // Check for read-job permissions 509 if aclObj, err := j.srv.ResolveToken(args.AuthToken); err != nil { 510 return err 511 } else if aclObj != nil && !aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilityReadJob) { 512 return structs.ErrPermissionDenied 513 } 514 515 // Validate the arguments 516 if args.JobID == "" { 517 return fmt.Errorf("missing job ID for evaluation") 518 } 519 520 // Lookup the job 521 snap, err := j.srv.fsm.State().Snapshot() 522 if err != nil { 523 return err 524 } 525 ws := memdb.NewWatchSet() 526 job, err := snap.JobByID(ws, args.RequestNamespace(), args.JobID) 527 if err != nil { 528 return err 529 } 530 if job == nil { 531 return fmt.Errorf("job not found") 532 } 533 534 if job.IsPeriodic() { 535 return fmt.Errorf("can't evaluate periodic job") 536 } else if job.IsParameterized() { 537 return fmt.Errorf("can't evaluate parameterized job") 538 } 539 540 // Create a new evaluation 541 eval := &structs.Evaluation{ 542 ID: uuid.Generate(), 543 Namespace: args.RequestNamespace(), 544 Priority: job.Priority, 545 Type: job.Type, 546 TriggeredBy: structs.EvalTriggerJobRegister, 547 JobID: job.ID, 548 JobModifyIndex: job.ModifyIndex, 549 Status: structs.EvalStatusPending, 550 } 551 update := &structs.EvalUpdateRequest{ 552 Evals: []*structs.Evaluation{eval}, 553 WriteRequest: structs.WriteRequest{Region: args.Region}, 554 } 555 556 // Commit this evaluation via Raft 557 _, evalIndex, err := j.srv.raftApply(structs.EvalUpdateRequestType, update) 558 if err != nil { 559 j.srv.logger.Printf("[ERR] nomad.job: Eval create failed: %v", err) 560 return err 561 } 562 563 // Setup the reply 564 reply.EvalID = eval.ID 565 reply.EvalCreateIndex = evalIndex 566 reply.JobModifyIndex = job.ModifyIndex 567 reply.Index = evalIndex 568 return nil 569 } 570 571 // Deregister is used to remove a job the cluster. 572 func (j *Job) Deregister(args *structs.JobDeregisterRequest, reply *structs.JobDeregisterResponse) error { 573 if done, err := j.srv.forward("Job.Deregister", args, args, reply); done { 574 return err 575 } 576 defer metrics.MeasureSince([]string{"nomad", "job", "deregister"}, time.Now()) 577 578 // Check for submit-job permissions 579 if aclObj, err := j.srv.ResolveToken(args.AuthToken); err != nil { 580 return err 581 } else if aclObj != nil && !aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilitySubmitJob) { 582 return structs.ErrPermissionDenied 583 } 584 585 // Validate the arguments 586 if args.JobID == "" { 587 return fmt.Errorf("missing job ID for deregistering") 588 } 589 590 // Lookup the job 591 snap, err := j.srv.fsm.State().Snapshot() 592 if err != nil { 593 return err 594 } 595 ws := memdb.NewWatchSet() 596 job, err := snap.JobByID(ws, args.RequestNamespace(), args.JobID) 597 if err != nil { 598 return err 599 } 600 601 // Commit this update via Raft 602 _, index, err := j.srv.raftApply(structs.JobDeregisterRequestType, args) 603 if err != nil { 604 j.srv.logger.Printf("[ERR] nomad.job: Deregister failed: %v", err) 605 return err 606 } 607 608 // Populate the reply with job information 609 reply.JobModifyIndex = index 610 611 // If the job is periodic or parameterized, we don't create an eval. 612 if job != nil && (job.IsPeriodic() || job.IsParameterized()) { 613 return nil 614 } 615 616 // Create a new evaluation 617 // XXX: The job priority / type is strange for this, since it's not a high 618 // priority even if the job was. The scheduler itself also doesn't matter, 619 // since all should be able to handle deregistration in the same way. 620 eval := &structs.Evaluation{ 621 ID: uuid.Generate(), 622 Namespace: args.RequestNamespace(), 623 Priority: structs.JobDefaultPriority, 624 Type: structs.JobTypeService, 625 TriggeredBy: structs.EvalTriggerJobDeregister, 626 JobID: args.JobID, 627 JobModifyIndex: index, 628 Status: structs.EvalStatusPending, 629 } 630 update := &structs.EvalUpdateRequest{ 631 Evals: []*structs.Evaluation{eval}, 632 WriteRequest: structs.WriteRequest{Region: args.Region}, 633 } 634 635 // Commit this evaluation via Raft 636 _, evalIndex, err := j.srv.raftApply(structs.EvalUpdateRequestType, update) 637 if err != nil { 638 j.srv.logger.Printf("[ERR] nomad.job: Eval create failed: %v", err) 639 return err 640 } 641 642 // Populate the reply with eval information 643 reply.EvalID = eval.ID 644 reply.EvalCreateIndex = evalIndex 645 reply.Index = evalIndex 646 return nil 647 } 648 649 // GetJob is used to request information about a specific job 650 func (j *Job) GetJob(args *structs.JobSpecificRequest, 651 reply *structs.SingleJobResponse) error { 652 if done, err := j.srv.forward("Job.GetJob", args, args, reply); done { 653 return err 654 } 655 defer metrics.MeasureSince([]string{"nomad", "job", "get_job"}, time.Now()) 656 657 // Check for read-job permissions 658 if aclObj, err := j.srv.ResolveToken(args.AuthToken); err != nil { 659 return err 660 } else if aclObj != nil && !aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilityReadJob) { 661 return structs.ErrPermissionDenied 662 } 663 664 // Setup the blocking query 665 opts := blockingOptions{ 666 queryOpts: &args.QueryOptions, 667 queryMeta: &reply.QueryMeta, 668 run: func(ws memdb.WatchSet, state *state.StateStore) error { 669 // Look for the job 670 out, err := state.JobByID(ws, args.RequestNamespace(), args.JobID) 671 if err != nil { 672 return err 673 } 674 675 // Setup the output 676 reply.Job = out 677 if out != nil { 678 reply.Index = out.ModifyIndex 679 } else { 680 // Use the last index that affected the nodes table 681 index, err := state.Index("jobs") 682 if err != nil { 683 return err 684 } 685 reply.Index = index 686 } 687 688 // Set the query response 689 j.srv.setQueryMeta(&reply.QueryMeta) 690 return nil 691 }} 692 return j.srv.blockingRPC(&opts) 693 } 694 695 // GetJobVersions is used to retrieve all tracked versions of a job. 696 func (j *Job) GetJobVersions(args *structs.JobVersionsRequest, 697 reply *structs.JobVersionsResponse) error { 698 if done, err := j.srv.forward("Job.GetJobVersions", args, args, reply); done { 699 return err 700 } 701 defer metrics.MeasureSince([]string{"nomad", "job", "get_job_versions"}, time.Now()) 702 703 // Check for read-job permissions 704 if aclObj, err := j.srv.ResolveToken(args.AuthToken); err != nil { 705 return err 706 } else if aclObj != nil && !aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilityReadJob) { 707 return structs.ErrPermissionDenied 708 } 709 710 // Setup the blocking query 711 opts := blockingOptions{ 712 queryOpts: &args.QueryOptions, 713 queryMeta: &reply.QueryMeta, 714 run: func(ws memdb.WatchSet, state *state.StateStore) error { 715 // Look for the job 716 out, err := state.JobVersionsByID(ws, args.RequestNamespace(), args.JobID) 717 if err != nil { 718 return err 719 } 720 721 // Setup the output 722 reply.Versions = out 723 if len(out) != 0 { 724 reply.Index = out[0].ModifyIndex 725 726 // Compute the diffs 727 if args.Diffs { 728 for i := 0; i < len(out)-1; i++ { 729 old, new := out[i+1], out[i] 730 d, err := old.Diff(new, true) 731 if err != nil { 732 return fmt.Errorf("failed to create job diff: %v", err) 733 } 734 reply.Diffs = append(reply.Diffs, d) 735 } 736 } 737 } else { 738 // Use the last index that affected the nodes table 739 index, err := state.Index("job_version") 740 if err != nil { 741 return err 742 } 743 reply.Index = index 744 } 745 746 // Set the query response 747 j.srv.setQueryMeta(&reply.QueryMeta) 748 return nil 749 }} 750 return j.srv.blockingRPC(&opts) 751 } 752 753 // List is used to list the jobs registered in the system 754 func (j *Job) List(args *structs.JobListRequest, 755 reply *structs.JobListResponse) error { 756 if done, err := j.srv.forward("Job.List", args, args, reply); done { 757 return err 758 } 759 defer metrics.MeasureSince([]string{"nomad", "job", "list"}, time.Now()) 760 761 // Check for list-job permissions 762 if aclObj, err := j.srv.ResolveToken(args.AuthToken); err != nil { 763 return err 764 } else if aclObj != nil && !aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilityListJobs) { 765 return structs.ErrPermissionDenied 766 } 767 768 // Setup the blocking query 769 opts := blockingOptions{ 770 queryOpts: &args.QueryOptions, 771 queryMeta: &reply.QueryMeta, 772 run: func(ws memdb.WatchSet, state *state.StateStore) error { 773 // Capture all the jobs 774 var err error 775 var iter memdb.ResultIterator 776 if prefix := args.QueryOptions.Prefix; prefix != "" { 777 iter, err = state.JobsByIDPrefix(ws, args.RequestNamespace(), prefix) 778 } else { 779 iter, err = state.JobsByNamespace(ws, args.RequestNamespace()) 780 } 781 if err != nil { 782 return err 783 } 784 785 var jobs []*structs.JobListStub 786 for { 787 raw := iter.Next() 788 if raw == nil { 789 break 790 } 791 job := raw.(*structs.Job) 792 summary, err := state.JobSummaryByID(ws, args.RequestNamespace(), job.ID) 793 if err != nil { 794 return fmt.Errorf("unable to look up summary for job: %v", job.ID) 795 } 796 jobs = append(jobs, job.Stub(summary)) 797 } 798 reply.Jobs = jobs 799 800 // Use the last index that affected the jobs table 801 index, err := state.Index("jobs") 802 if err != nil { 803 return err 804 } 805 reply.Index = index 806 807 // Set the query response 808 j.srv.setQueryMeta(&reply.QueryMeta) 809 return nil 810 }} 811 return j.srv.blockingRPC(&opts) 812 } 813 814 // Allocations is used to list the allocations for a job 815 func (j *Job) Allocations(args *structs.JobSpecificRequest, 816 reply *structs.JobAllocationsResponse) error { 817 if done, err := j.srv.forward("Job.Allocations", args, args, reply); done { 818 return err 819 } 820 defer metrics.MeasureSince([]string{"nomad", "job", "allocations"}, time.Now()) 821 822 // Check for read-job permissions 823 if aclObj, err := j.srv.ResolveToken(args.AuthToken); err != nil { 824 return err 825 } else if aclObj != nil && !aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilityReadJob) { 826 return structs.ErrPermissionDenied 827 } 828 829 // Setup the blocking query 830 opts := blockingOptions{ 831 queryOpts: &args.QueryOptions, 832 queryMeta: &reply.QueryMeta, 833 run: func(ws memdb.WatchSet, state *state.StateStore) error { 834 // Capture the allocations 835 allocs, err := state.AllocsByJob(ws, args.RequestNamespace(), args.JobID, args.AllAllocs) 836 if err != nil { 837 return err 838 } 839 840 // Convert to stubs 841 if len(allocs) > 0 { 842 reply.Allocations = make([]*structs.AllocListStub, 0, len(allocs)) 843 for _, alloc := range allocs { 844 reply.Allocations = append(reply.Allocations, alloc.Stub()) 845 } 846 } 847 848 // Use the last index that affected the allocs table 849 index, err := state.Index("allocs") 850 if err != nil { 851 return err 852 } 853 reply.Index = index 854 855 // Set the query response 856 j.srv.setQueryMeta(&reply.QueryMeta) 857 return nil 858 859 }} 860 return j.srv.blockingRPC(&opts) 861 } 862 863 // Evaluations is used to list the evaluations for a job 864 func (j *Job) Evaluations(args *structs.JobSpecificRequest, 865 reply *structs.JobEvaluationsResponse) error { 866 if done, err := j.srv.forward("Job.Evaluations", args, args, reply); done { 867 return err 868 } 869 defer metrics.MeasureSince([]string{"nomad", "job", "evaluations"}, time.Now()) 870 871 // Check for read-job permissions 872 if aclObj, err := j.srv.ResolveToken(args.AuthToken); err != nil { 873 return err 874 } else if aclObj != nil && !aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilityReadJob) { 875 return structs.ErrPermissionDenied 876 } 877 878 // Setup the blocking query 879 opts := blockingOptions{ 880 queryOpts: &args.QueryOptions, 881 queryMeta: &reply.QueryMeta, 882 run: func(ws memdb.WatchSet, state *state.StateStore) error { 883 // Capture the evals 884 var err error 885 reply.Evaluations, err = state.EvalsByJob(ws, args.RequestNamespace(), args.JobID) 886 if err != nil { 887 return err 888 } 889 890 // Use the last index that affected the evals table 891 index, err := state.Index("evals") 892 if err != nil { 893 return err 894 } 895 reply.Index = index 896 897 // Set the query response 898 j.srv.setQueryMeta(&reply.QueryMeta) 899 return nil 900 }} 901 902 return j.srv.blockingRPC(&opts) 903 } 904 905 // Deployments is used to list the deployments for a job 906 func (j *Job) Deployments(args *structs.JobSpecificRequest, 907 reply *structs.DeploymentListResponse) error { 908 if done, err := j.srv.forward("Job.Deployments", args, args, reply); done { 909 return err 910 } 911 defer metrics.MeasureSince([]string{"nomad", "job", "deployments"}, time.Now()) 912 913 // Check for read-job permissions 914 if aclObj, err := j.srv.ResolveToken(args.AuthToken); err != nil { 915 return err 916 } else if aclObj != nil && !aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilityReadJob) { 917 return structs.ErrPermissionDenied 918 } 919 920 // Setup the blocking query 921 opts := blockingOptions{ 922 queryOpts: &args.QueryOptions, 923 queryMeta: &reply.QueryMeta, 924 run: func(ws memdb.WatchSet, state *state.StateStore) error { 925 // Capture the deployments 926 deploys, err := state.DeploymentsByJobID(ws, args.RequestNamespace(), args.JobID) 927 if err != nil { 928 return err 929 } 930 931 // Use the last index that affected the deployment table 932 index, err := state.Index("deployment") 933 if err != nil { 934 return err 935 } 936 reply.Index = index 937 reply.Deployments = deploys 938 939 // Set the query response 940 j.srv.setQueryMeta(&reply.QueryMeta) 941 return nil 942 943 }} 944 return j.srv.blockingRPC(&opts) 945 } 946 947 // LatestDeployment is used to retrieve the latest deployment for a job 948 func (j *Job) LatestDeployment(args *structs.JobSpecificRequest, 949 reply *structs.SingleDeploymentResponse) error { 950 if done, err := j.srv.forward("Job.LatestDeployment", args, args, reply); done { 951 return err 952 } 953 defer metrics.MeasureSince([]string{"nomad", "job", "latest_deployment"}, time.Now()) 954 955 // Check for read-job permissions 956 if aclObj, err := j.srv.ResolveToken(args.AuthToken); err != nil { 957 return err 958 } else if aclObj != nil && !aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilityReadJob) { 959 return structs.ErrPermissionDenied 960 } 961 962 // Setup the blocking query 963 opts := blockingOptions{ 964 queryOpts: &args.QueryOptions, 965 queryMeta: &reply.QueryMeta, 966 run: func(ws memdb.WatchSet, state *state.StateStore) error { 967 // Capture the deployments 968 deploys, err := state.DeploymentsByJobID(ws, args.RequestNamespace(), args.JobID) 969 if err != nil { 970 return err 971 } 972 973 // Use the last index that affected the deployment table 974 index, err := state.Index("deployment") 975 if err != nil { 976 return err 977 } 978 reply.Index = index 979 if len(deploys) > 0 { 980 sort.Slice(deploys, func(i, j int) bool { 981 return deploys[i].CreateIndex > deploys[j].CreateIndex 982 }) 983 reply.Deployment = deploys[0] 984 } 985 986 // Set the query response 987 j.srv.setQueryMeta(&reply.QueryMeta) 988 return nil 989 990 }} 991 return j.srv.blockingRPC(&opts) 992 } 993 994 // Plan is used to cause a dry-run evaluation of the Job and return the results 995 // with a potential diff containing annotations. 996 func (j *Job) Plan(args *structs.JobPlanRequest, reply *structs.JobPlanResponse) error { 997 if done, err := j.srv.forward("Job.Plan", args, args, reply); done { 998 return err 999 } 1000 defer metrics.MeasureSince([]string{"nomad", "job", "plan"}, time.Now()) 1001 1002 // Validate the arguments 1003 if args.Job == nil { 1004 return fmt.Errorf("Job required for plan") 1005 } 1006 1007 // Initialize the job fields (sets defaults and any necessary init work). 1008 canonicalizeWarnings := args.Job.Canonicalize() 1009 1010 // Add implicit constraints 1011 setImplicitConstraints(args.Job) 1012 1013 // Validate the job and capture any warnings 1014 err, warnings := validateJob(args.Job) 1015 if err != nil { 1016 return err 1017 } 1018 1019 // Set the warning message 1020 reply.Warnings = structs.MergeMultierrorWarnings(warnings, canonicalizeWarnings) 1021 1022 // Check job submission permissions, which we assume is the same for plan 1023 if aclObj, err := j.srv.ResolveToken(args.AuthToken); err != nil { 1024 return err 1025 } else if aclObj != nil { 1026 if !aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilitySubmitJob) { 1027 return structs.ErrPermissionDenied 1028 } 1029 // Check if override is set and we do not have permissions 1030 if args.PolicyOverride { 1031 if !aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilitySentinelOverride) { 1032 return structs.ErrPermissionDenied 1033 } 1034 } 1035 } 1036 1037 // Enforce Sentinel policies 1038 policyWarnings, err := j.enforceSubmitJob(args.PolicyOverride, args.Job) 1039 if err != nil { 1040 return err 1041 } 1042 if policyWarnings != nil { 1043 reply.Warnings = structs.MergeMultierrorWarnings(warnings, 1044 canonicalizeWarnings, policyWarnings) 1045 } 1046 1047 // Acquire a snapshot of the state 1048 snap, err := j.srv.fsm.State().Snapshot() 1049 if err != nil { 1050 return err 1051 } 1052 1053 // Get the original job 1054 ws := memdb.NewWatchSet() 1055 oldJob, err := snap.JobByID(ws, args.RequestNamespace(), args.Job.ID) 1056 if err != nil { 1057 return err 1058 } 1059 1060 var index uint64 1061 var updatedIndex uint64 1062 1063 if oldJob != nil { 1064 index = oldJob.JobModifyIndex 1065 1066 // We want to reuse deployments where possible, so only insert the job if 1067 // it has changed or the job didn't exist 1068 if oldJob.SpecChanged(args.Job) { 1069 // Insert the updated Job into the snapshot 1070 updatedIndex = oldJob.JobModifyIndex + 1 1071 snap.UpsertJob(updatedIndex, args.Job) 1072 } 1073 } else if oldJob == nil { 1074 // Insert the updated Job into the snapshot 1075 snap.UpsertJob(100, args.Job) 1076 } 1077 1078 // Create an eval and mark it as requiring annotations and insert that as well 1079 eval := &structs.Evaluation{ 1080 ID: uuid.Generate(), 1081 Namespace: args.RequestNamespace(), 1082 Priority: args.Job.Priority, 1083 Type: args.Job.Type, 1084 TriggeredBy: structs.EvalTriggerJobRegister, 1085 JobID: args.Job.ID, 1086 JobModifyIndex: updatedIndex, 1087 Status: structs.EvalStatusPending, 1088 AnnotatePlan: true, 1089 } 1090 1091 snap.UpsertEvals(100, []*structs.Evaluation{eval}) 1092 1093 // Create an in-memory Planner that returns no errors and stores the 1094 // submitted plan and created evals. 1095 planner := &scheduler.Harness{ 1096 State: &snap.StateStore, 1097 } 1098 1099 // Create the scheduler and run it 1100 sched, err := scheduler.NewScheduler(eval.Type, j.srv.logger, snap, planner) 1101 if err != nil { 1102 return err 1103 } 1104 1105 if err := sched.Process(eval); err != nil { 1106 return err 1107 } 1108 1109 // Annotate and store the diff 1110 if plans := len(planner.Plans); plans != 1 { 1111 return fmt.Errorf("scheduler resulted in an unexpected number of plans: %v", plans) 1112 } 1113 annotations := planner.Plans[0].Annotations 1114 if args.Diff { 1115 jobDiff, err := oldJob.Diff(args.Job, true) 1116 if err != nil { 1117 return fmt.Errorf("failed to create job diff: %v", err) 1118 } 1119 1120 if err := scheduler.Annotate(jobDiff, annotations); err != nil { 1121 return fmt.Errorf("failed to annotate job diff: %v", err) 1122 } 1123 reply.Diff = jobDiff 1124 } 1125 1126 // Grab the failures 1127 if len(planner.Evals) != 1 { 1128 return fmt.Errorf("scheduler resulted in an unexpected number of eval updates: %v", planner.Evals) 1129 } 1130 updatedEval := planner.Evals[0] 1131 1132 // If it is a periodic job calculate the next launch 1133 if args.Job.IsPeriodic() && args.Job.Periodic.Enabled { 1134 reply.NextPeriodicLaunch = args.Job.Periodic.Next(time.Now().In(args.Job.Periodic.GetLocation())) 1135 } 1136 1137 reply.FailedTGAllocs = updatedEval.FailedTGAllocs 1138 reply.JobModifyIndex = index 1139 reply.Annotations = annotations 1140 reply.CreatedEvals = planner.CreateEvals 1141 reply.Index = index 1142 return nil 1143 } 1144 1145 // validateJob validates a Job and task drivers and returns an error if there is 1146 // a validation problem or if the Job is of a type a user is not allowed to 1147 // submit. 1148 func validateJob(job *structs.Job) (invalid, warnings error) { 1149 validationErrors := new(multierror.Error) 1150 if err := job.Validate(); err != nil { 1151 multierror.Append(validationErrors, err) 1152 } 1153 1154 // Get any warnings 1155 warnings = job.Warnings() 1156 1157 // Get the signals required 1158 signals := job.RequiredSignals() 1159 1160 // Validate the driver configurations. 1161 for _, tg := range job.TaskGroups { 1162 // Get the signals for the task group 1163 tgSignals, tgOk := signals[tg.Name] 1164 1165 for _, task := range tg.Tasks { 1166 d, err := driver.NewDriver( 1167 task.Driver, 1168 driver.NewEmptyDriverContext(), 1169 ) 1170 if err != nil { 1171 msg := "failed to create driver for task %q in group %q for validation: %v" 1172 multierror.Append(validationErrors, fmt.Errorf(msg, tg.Name, task.Name, err)) 1173 continue 1174 } 1175 1176 if err := d.Validate(task.Config); err != nil { 1177 formatted := fmt.Errorf("group %q -> task %q -> config: %v", tg.Name, task.Name, err) 1178 multierror.Append(validationErrors, formatted) 1179 } 1180 1181 // The task group didn't have any task that required signals 1182 if !tgOk { 1183 continue 1184 } 1185 1186 // This task requires signals. Ensure the driver is capable 1187 if required, ok := tgSignals[task.Name]; ok { 1188 abilities := d.Abilities() 1189 if !abilities.SendSignals { 1190 formatted := fmt.Errorf("group %q -> task %q: driver %q doesn't support sending signals. Requested signals are %v", 1191 tg.Name, task.Name, task.Driver, strings.Join(required, ", ")) 1192 multierror.Append(validationErrors, formatted) 1193 } 1194 } 1195 } 1196 } 1197 1198 if job.Type == structs.JobTypeCore { 1199 multierror.Append(validationErrors, fmt.Errorf("job type cannot be core")) 1200 } 1201 1202 if len(job.Payload) != 0 { 1203 multierror.Append(validationErrors, fmt.Errorf("job can't be submitted with a payload, only dispatched")) 1204 } 1205 1206 return validationErrors.ErrorOrNil(), warnings 1207 } 1208 1209 // validateJobUpdate ensures updates to a job are valid. 1210 func validateJobUpdate(old, new *structs.Job) error { 1211 // Type transitions are disallowed 1212 if old.Type != new.Type { 1213 return fmt.Errorf("cannot update job from type %q to %q", old.Type, new.Type) 1214 } 1215 1216 // Transitioning to/from periodic is disallowed 1217 if old.IsPeriodic() && !new.IsPeriodic() { 1218 return fmt.Errorf("cannot update periodic job to being non-periodic") 1219 } 1220 if new.IsPeriodic() && !old.IsPeriodic() { 1221 return fmt.Errorf("cannot update non-periodic job to being periodic") 1222 } 1223 1224 // Transitioning to/from parameterized is disallowed 1225 if old.IsParameterized() && !new.IsParameterized() { 1226 return fmt.Errorf("cannot update non-parameterized job to being parameterized") 1227 } 1228 if new.IsParameterized() && !old.IsParameterized() { 1229 return fmt.Errorf("cannot update parameterized job to being non-parameterized") 1230 } 1231 1232 return nil 1233 } 1234 1235 // Dispatch a parameterized job. 1236 func (j *Job) Dispatch(args *structs.JobDispatchRequest, reply *structs.JobDispatchResponse) error { 1237 if done, err := j.srv.forward("Job.Dispatch", args, args, reply); done { 1238 return err 1239 } 1240 defer metrics.MeasureSince([]string{"nomad", "job", "dispatch"}, time.Now()) 1241 1242 // Check for submit-job permissions 1243 if aclObj, err := j.srv.ResolveToken(args.AuthToken); err != nil { 1244 return err 1245 } else if aclObj != nil && !aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilityDispatchJob) { 1246 return structs.ErrPermissionDenied 1247 } 1248 1249 // Lookup the parameterized job 1250 if args.JobID == "" { 1251 return fmt.Errorf("missing parameterized job ID") 1252 } 1253 1254 snap, err := j.srv.fsm.State().Snapshot() 1255 if err != nil { 1256 return err 1257 } 1258 ws := memdb.NewWatchSet() 1259 parameterizedJob, err := snap.JobByID(ws, args.RequestNamespace(), args.JobID) 1260 if err != nil { 1261 return err 1262 } 1263 if parameterizedJob == nil { 1264 return fmt.Errorf("parameterized job not found") 1265 } 1266 1267 if !parameterizedJob.IsParameterized() { 1268 return fmt.Errorf("Specified job %q is not a parameterized job", args.JobID) 1269 } 1270 1271 if parameterizedJob.Stop { 1272 return fmt.Errorf("Specified job %q is stopped", args.JobID) 1273 } 1274 1275 // Validate the arguments 1276 if err := validateDispatchRequest(args, parameterizedJob); err != nil { 1277 return err 1278 } 1279 1280 // Derive the child job and commit it via Raft 1281 dispatchJob := parameterizedJob.Copy() 1282 dispatchJob.ParameterizedJob = nil 1283 dispatchJob.ID = structs.DispatchedID(parameterizedJob.ID, time.Now()) 1284 dispatchJob.ParentID = parameterizedJob.ID 1285 dispatchJob.Name = dispatchJob.ID 1286 dispatchJob.SetSubmitTime() 1287 1288 // Merge in the meta data 1289 for k, v := range args.Meta { 1290 if dispatchJob.Meta == nil { 1291 dispatchJob.Meta = make(map[string]string, len(args.Meta)) 1292 } 1293 dispatchJob.Meta[k] = v 1294 } 1295 1296 // Compress the payload 1297 dispatchJob.Payload = snappy.Encode(nil, args.Payload) 1298 1299 regReq := &structs.JobRegisterRequest{ 1300 Job: dispatchJob, 1301 WriteRequest: args.WriteRequest, 1302 } 1303 1304 // Commit this update via Raft 1305 fsmErr, jobCreateIndex, err := j.srv.raftApply(structs.JobRegisterRequestType, regReq) 1306 if err, ok := fsmErr.(error); ok && err != nil { 1307 j.srv.logger.Printf("[ERR] nomad.job: Dispatched job register failed: %v", err) 1308 return err 1309 } 1310 if err != nil { 1311 j.srv.logger.Printf("[ERR] nomad.job: Dispatched job register failed: %v", err) 1312 return err 1313 } 1314 1315 reply.JobCreateIndex = jobCreateIndex 1316 reply.DispatchedJobID = dispatchJob.ID 1317 reply.Index = jobCreateIndex 1318 1319 // If the job is periodic, we don't create an eval. 1320 if !dispatchJob.IsPeriodic() { 1321 // Create a new evaluation 1322 eval := &structs.Evaluation{ 1323 ID: uuid.Generate(), 1324 Namespace: args.RequestNamespace(), 1325 Priority: dispatchJob.Priority, 1326 Type: dispatchJob.Type, 1327 TriggeredBy: structs.EvalTriggerJobRegister, 1328 JobID: dispatchJob.ID, 1329 JobModifyIndex: jobCreateIndex, 1330 Status: structs.EvalStatusPending, 1331 } 1332 update := &structs.EvalUpdateRequest{ 1333 Evals: []*structs.Evaluation{eval}, 1334 WriteRequest: structs.WriteRequest{Region: args.Region}, 1335 } 1336 1337 // Commit this evaluation via Raft 1338 _, evalIndex, err := j.srv.raftApply(structs.EvalUpdateRequestType, update) 1339 if err != nil { 1340 j.srv.logger.Printf("[ERR] nomad.job: Eval create failed: %v", err) 1341 return err 1342 } 1343 1344 // Setup the reply 1345 reply.EvalID = eval.ID 1346 reply.EvalCreateIndex = evalIndex 1347 reply.Index = evalIndex 1348 } 1349 1350 return nil 1351 } 1352 1353 // validateDispatchRequest returns whether the request is valid given the 1354 // parameterized job. 1355 func validateDispatchRequest(req *structs.JobDispatchRequest, job *structs.Job) error { 1356 // Check the payload constraint is met 1357 hasInputData := len(req.Payload) != 0 1358 if job.ParameterizedJob.Payload == structs.DispatchPayloadRequired && !hasInputData { 1359 return fmt.Errorf("Payload is not provided but required by parameterized job") 1360 } else if job.ParameterizedJob.Payload == structs.DispatchPayloadForbidden && hasInputData { 1361 return fmt.Errorf("Payload provided but forbidden by parameterized job") 1362 } 1363 1364 // Check the payload doesn't exceed the size limit 1365 if l := len(req.Payload); l > DispatchPayloadSizeLimit { 1366 return fmt.Errorf("Payload exceeds maximum size; %d > %d", l, DispatchPayloadSizeLimit) 1367 } 1368 1369 // Check if the metadata is a set 1370 keys := make(map[string]struct{}, len(req.Meta)) 1371 for k := range keys { 1372 if _, ok := keys[k]; ok { 1373 return fmt.Errorf("Duplicate key %q in passed metadata", k) 1374 } 1375 keys[k] = struct{}{} 1376 } 1377 1378 required := helper.SliceStringToSet(job.ParameterizedJob.MetaRequired) 1379 optional := helper.SliceStringToSet(job.ParameterizedJob.MetaOptional) 1380 1381 // Check the metadata key constraints are met 1382 unpermitted := make(map[string]struct{}) 1383 for k := range req.Meta { 1384 _, req := required[k] 1385 _, opt := optional[k] 1386 if !req && !opt { 1387 unpermitted[k] = struct{}{} 1388 } 1389 } 1390 1391 if len(unpermitted) != 0 { 1392 flat := make([]string, 0, len(unpermitted)) 1393 for k := range unpermitted { 1394 flat = append(flat, k) 1395 } 1396 1397 return fmt.Errorf("Dispatch request included unpermitted metadata keys: %v", flat) 1398 } 1399 1400 missing := make(map[string]struct{}) 1401 for _, k := range job.ParameterizedJob.MetaRequired { 1402 if _, ok := req.Meta[k]; !ok { 1403 missing[k] = struct{}{} 1404 } 1405 } 1406 1407 if len(missing) != 0 { 1408 flat := make([]string, 0, len(missing)) 1409 for k := range missing { 1410 flat = append(flat, k) 1411 } 1412 1413 return fmt.Errorf("Dispatch did not provide required meta keys: %v", flat) 1414 } 1415 1416 return nil 1417 }