github.com/hspak/nomad@v0.7.2-0.20180309000617-bc4ae22a39a5/nomad/job_endpoint.go (about)

     1  package nomad
     2  
     3  import (
     4  	"context"
     5  	"fmt"
     6  	"sort"
     7  	"strings"
     8  	"time"
     9  
    10  	"github.com/armon/go-metrics"
    11  	"github.com/golang/snappy"
    12  	"github.com/hashicorp/consul/lib"
    13  	memdb "github.com/hashicorp/go-memdb"
    14  	"github.com/hashicorp/go-multierror"
    15  	"github.com/hashicorp/nomad/acl"
    16  	"github.com/hashicorp/nomad/client/driver"
    17  	"github.com/hashicorp/nomad/helper"
    18  	"github.com/hashicorp/nomad/helper/uuid"
    19  	"github.com/hashicorp/nomad/nomad/state"
    20  	"github.com/hashicorp/nomad/nomad/structs"
    21  	"github.com/hashicorp/nomad/scheduler"
    22  )
    23  
    24  const (
    25  	// RegisterEnforceIndexErrPrefix is the prefix to use in errors caused by
    26  	// enforcing the job modify index during registers.
    27  	RegisterEnforceIndexErrPrefix = "Enforcing job modify index"
    28  
    29  	// DispatchPayloadSizeLimit is the maximum size of the uncompressed input
    30  	// data payload.
    31  	DispatchPayloadSizeLimit = 16 * 1024
    32  )
    33  
    34  var (
    35  	// vaultConstraint is the implicit constraint added to jobs requesting a
    36  	// Vault token
    37  	vaultConstraint = &structs.Constraint{
    38  		LTarget: "${attr.vault.version}",
    39  		RTarget: ">= 0.6.1",
    40  		Operand: structs.ConstraintVersion,
    41  	}
    42  )
    43  
    44  // Job endpoint is used for job interactions
    45  type Job struct {
    46  	srv *Server
    47  }
    48  
    49  // Register is used to upsert a job for scheduling
    50  func (j *Job) Register(args *structs.JobRegisterRequest, reply *structs.JobRegisterResponse) error {
    51  	if done, err := j.srv.forward("Job.Register", args, args, reply); done {
    52  		return err
    53  	}
    54  	defer metrics.MeasureSince([]string{"nomad", "job", "register"}, time.Now())
    55  
    56  	// Validate the arguments
    57  	if args.Job == nil {
    58  		return fmt.Errorf("missing job for registration")
    59  	}
    60  
    61  	// Initialize the job fields (sets defaults and any necessary init work).
    62  	canonicalizeWarnings := args.Job.Canonicalize()
    63  
    64  	// Add implicit constraints
    65  	setImplicitConstraints(args.Job)
    66  
    67  	// Validate the job and capture any warnings
    68  	err, warnings := validateJob(args.Job)
    69  	if err != nil {
    70  		return err
    71  	}
    72  
    73  	// Set the warning message
    74  	reply.Warnings = structs.MergeMultierrorWarnings(warnings, canonicalizeWarnings)
    75  
    76  	// Check job submission permissions
    77  	if aclObj, err := j.srv.ResolveToken(args.AuthToken); err != nil {
    78  		return err
    79  	} else if aclObj != nil {
    80  		if !aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilitySubmitJob) {
    81  			return structs.ErrPermissionDenied
    82  		}
    83  		// Check if override is set and we do not have permissions
    84  		if args.PolicyOverride {
    85  			if !aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilitySentinelOverride) {
    86  				j.srv.logger.Printf("[WARN] nomad.job: policy override attempted without permissions for Job %q", args.Job.ID)
    87  				return structs.ErrPermissionDenied
    88  			}
    89  			j.srv.logger.Printf("[WARN] nomad.job: policy override set for Job %q", args.Job.ID)
    90  		}
    91  	}
    92  
    93  	// Lookup the job
    94  	snap, err := j.srv.State().Snapshot()
    95  	if err != nil {
    96  		return err
    97  	}
    98  	ws := memdb.NewWatchSet()
    99  	existingJob, err := snap.JobByID(ws, args.RequestNamespace(), args.Job.ID)
   100  	if err != nil {
   101  		return err
   102  	}
   103  
   104  	// If EnforceIndex set, check it before trying to apply
   105  	if args.EnforceIndex {
   106  		jmi := args.JobModifyIndex
   107  		if existingJob != nil {
   108  			if jmi == 0 {
   109  				return fmt.Errorf("%s 0: job already exists", RegisterEnforceIndexErrPrefix)
   110  			} else if jmi != existingJob.JobModifyIndex {
   111  				return fmt.Errorf("%s %d: job exists with conflicting job modify index: %d",
   112  					RegisterEnforceIndexErrPrefix, jmi, existingJob.JobModifyIndex)
   113  			}
   114  		} else if jmi != 0 {
   115  			return fmt.Errorf("%s %d: job does not exist", RegisterEnforceIndexErrPrefix, jmi)
   116  		}
   117  	}
   118  
   119  	// Validate job transitions if its an update
   120  	if existingJob != nil {
   121  		if err := validateJobUpdate(existingJob, args.Job); err != nil {
   122  			return err
   123  		}
   124  	}
   125  
   126  	// Ensure that the job has permissions for the requested Vault tokens
   127  	policies := args.Job.VaultPolicies()
   128  	if len(policies) != 0 {
   129  		vconf := j.srv.config.VaultConfig
   130  		if !vconf.IsEnabled() {
   131  			return fmt.Errorf("Vault not enabled and Vault policies requested")
   132  		}
   133  
   134  		// Have to check if the user has permissions
   135  		if !vconf.AllowsUnauthenticated() {
   136  			if args.Job.VaultToken == "" {
   137  				return fmt.Errorf("Vault policies requested but missing Vault Token")
   138  			}
   139  
   140  			vault := j.srv.vault
   141  			s, err := vault.LookupToken(context.Background(), args.Job.VaultToken)
   142  			if err != nil {
   143  				return err
   144  			}
   145  
   146  			allowedPolicies, err := PoliciesFrom(s)
   147  			if err != nil {
   148  				return err
   149  			}
   150  
   151  			// If we are given a root token it can access all policies
   152  			if !lib.StrContains(allowedPolicies, "root") {
   153  				flatPolicies := structs.VaultPoliciesSet(policies)
   154  				subset, offending := helper.SliceStringIsSubset(allowedPolicies, flatPolicies)
   155  				if !subset {
   156  					return fmt.Errorf("Passed Vault Token doesn't allow access to the following policies: %s",
   157  						strings.Join(offending, ", "))
   158  				}
   159  			}
   160  		}
   161  	}
   162  
   163  	// Enforce Sentinel policies
   164  	policyWarnings, err := j.enforceSubmitJob(args.PolicyOverride, args.Job)
   165  	if err != nil {
   166  		return err
   167  	}
   168  	if policyWarnings != nil {
   169  		reply.Warnings = structs.MergeMultierrorWarnings(warnings,
   170  			canonicalizeWarnings, policyWarnings)
   171  	}
   172  
   173  	// Clear the Vault token
   174  	args.Job.VaultToken = ""
   175  
   176  	// Check if the job has changed at all
   177  	if existingJob == nil || existingJob.SpecChanged(args.Job) {
   178  		// Set the submit time
   179  		args.Job.SetSubmitTime()
   180  
   181  		// Commit this update via Raft
   182  		fsmErr, index, err := j.srv.raftApply(structs.JobRegisterRequestType, args)
   183  		if err, ok := fsmErr.(error); ok && err != nil {
   184  			j.srv.logger.Printf("[ERR] nomad.job: Register failed: %v", err)
   185  			return err
   186  		}
   187  		if err != nil {
   188  			j.srv.logger.Printf("[ERR] nomad.job: Register failed: %v", err)
   189  			return err
   190  		}
   191  
   192  		// Populate the reply with job information
   193  		reply.JobModifyIndex = index
   194  	} else {
   195  		reply.JobModifyIndex = existingJob.JobModifyIndex
   196  	}
   197  
   198  	// If the job is periodic or parameterized, we don't create an eval.
   199  	if args.Job.IsPeriodic() || args.Job.IsParameterized() {
   200  		return nil
   201  	}
   202  
   203  	// Create a new evaluation
   204  	eval := &structs.Evaluation{
   205  		ID:             uuid.Generate(),
   206  		Namespace:      args.RequestNamespace(),
   207  		Priority:       args.Job.Priority,
   208  		Type:           args.Job.Type,
   209  		TriggeredBy:    structs.EvalTriggerJobRegister,
   210  		JobID:          args.Job.ID,
   211  		JobModifyIndex: reply.JobModifyIndex,
   212  		Status:         structs.EvalStatusPending,
   213  	}
   214  	update := &structs.EvalUpdateRequest{
   215  		Evals:        []*structs.Evaluation{eval},
   216  		WriteRequest: structs.WriteRequest{Region: args.Region},
   217  	}
   218  
   219  	// Commit this evaluation via Raft
   220  	// XXX: There is a risk of partial failure where the JobRegister succeeds
   221  	// but that the EvalUpdate does not.
   222  	_, evalIndex, err := j.srv.raftApply(structs.EvalUpdateRequestType, update)
   223  	if err != nil {
   224  		j.srv.logger.Printf("[ERR] nomad.job: Eval create failed: %v", err)
   225  		return err
   226  	}
   227  
   228  	// Populate the reply with eval information
   229  	reply.EvalID = eval.ID
   230  	reply.EvalCreateIndex = evalIndex
   231  	reply.Index = evalIndex
   232  	return nil
   233  }
   234  
   235  // setImplicitConstraints adds implicit constraints to the job based on the
   236  // features it is requesting.
   237  func setImplicitConstraints(j *structs.Job) {
   238  	// Get the required Vault Policies
   239  	policies := j.VaultPolicies()
   240  
   241  	// Get the required signals
   242  	signals := j.RequiredSignals()
   243  
   244  	// Hot path
   245  	if len(signals) == 0 && len(policies) == 0 {
   246  		return
   247  	}
   248  
   249  	// Add Vault constraints
   250  	for _, tg := range j.TaskGroups {
   251  		_, ok := policies[tg.Name]
   252  		if !ok {
   253  			// Not requesting Vault
   254  			continue
   255  		}
   256  
   257  		found := false
   258  		for _, c := range tg.Constraints {
   259  			if c.Equal(vaultConstraint) {
   260  				found = true
   261  				break
   262  			}
   263  		}
   264  
   265  		if !found {
   266  			tg.Constraints = append(tg.Constraints, vaultConstraint)
   267  		}
   268  	}
   269  
   270  	// Add signal constraints
   271  	for _, tg := range j.TaskGroups {
   272  		tgSignals, ok := signals[tg.Name]
   273  		if !ok {
   274  			// Not requesting Vault
   275  			continue
   276  		}
   277  
   278  		// Flatten the signals
   279  		required := helper.MapStringStringSliceValueSet(tgSignals)
   280  		sigConstraint := getSignalConstraint(required)
   281  
   282  		found := false
   283  		for _, c := range tg.Constraints {
   284  			if c.Equal(sigConstraint) {
   285  				found = true
   286  				break
   287  			}
   288  		}
   289  
   290  		if !found {
   291  			tg.Constraints = append(tg.Constraints, sigConstraint)
   292  		}
   293  	}
   294  }
   295  
   296  // getSignalConstraint builds a suitable constraint based on the required
   297  // signals
   298  func getSignalConstraint(signals []string) *structs.Constraint {
   299  	return &structs.Constraint{
   300  		Operand: structs.ConstraintSetContains,
   301  		LTarget: "${attr.os.signals}",
   302  		RTarget: strings.Join(signals, ","),
   303  	}
   304  }
   305  
   306  // Summary retrieves the summary of a job
   307  func (j *Job) Summary(args *structs.JobSummaryRequest,
   308  	reply *structs.JobSummaryResponse) error {
   309  
   310  	if done, err := j.srv.forward("Job.Summary", args, args, reply); done {
   311  		return err
   312  	}
   313  	defer metrics.MeasureSince([]string{"nomad", "job_summary", "get_job_summary"}, time.Now())
   314  
   315  	// Check for read-job permissions
   316  	if aclObj, err := j.srv.ResolveToken(args.AuthToken); err != nil {
   317  		return err
   318  	} else if aclObj != nil && !aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilityReadJob) {
   319  		return structs.ErrPermissionDenied
   320  	}
   321  
   322  	// Setup the blocking query
   323  	opts := blockingOptions{
   324  		queryOpts: &args.QueryOptions,
   325  		queryMeta: &reply.QueryMeta,
   326  		run: func(ws memdb.WatchSet, state *state.StateStore) error {
   327  			// Look for job summary
   328  			out, err := state.JobSummaryByID(ws, args.RequestNamespace(), args.JobID)
   329  			if err != nil {
   330  				return err
   331  			}
   332  
   333  			// Setup the output
   334  			reply.JobSummary = out
   335  			if out != nil {
   336  				reply.Index = out.ModifyIndex
   337  			} else {
   338  				// Use the last index that affected the job_summary table
   339  				index, err := state.Index("job_summary")
   340  				if err != nil {
   341  					return err
   342  				}
   343  				reply.Index = index
   344  			}
   345  
   346  			// Set the query response
   347  			j.srv.setQueryMeta(&reply.QueryMeta)
   348  			return nil
   349  		}}
   350  	return j.srv.blockingRPC(&opts)
   351  }
   352  
   353  // Validate validates a job
   354  func (j *Job) Validate(args *structs.JobValidateRequest, reply *structs.JobValidateResponse) error {
   355  	defer metrics.MeasureSince([]string{"nomad", "job", "validate"}, time.Now())
   356  
   357  	// Check for read-job permissions
   358  	if aclObj, err := j.srv.ResolveToken(args.AuthToken); err != nil {
   359  		return err
   360  	} else if aclObj != nil && !aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilityReadJob) {
   361  		return structs.ErrPermissionDenied
   362  	}
   363  
   364  	// Initialize the job fields (sets defaults and any necessary init work).
   365  	canonicalizeWarnings := args.Job.Canonicalize()
   366  
   367  	// Add implicit constraints
   368  	setImplicitConstraints(args.Job)
   369  
   370  	// Validate the job and capture any warnings
   371  	err, warnings := validateJob(args.Job)
   372  	if err != nil {
   373  		if merr, ok := err.(*multierror.Error); ok {
   374  			for _, err := range merr.Errors {
   375  				reply.ValidationErrors = append(reply.ValidationErrors, err.Error())
   376  			}
   377  			reply.Error = merr.Error()
   378  		} else {
   379  			reply.ValidationErrors = append(reply.ValidationErrors, err.Error())
   380  			reply.Error = err.Error()
   381  		}
   382  	}
   383  
   384  	// Set the warning message
   385  	reply.Warnings = structs.MergeMultierrorWarnings(warnings, canonicalizeWarnings)
   386  	reply.DriverConfigValidated = true
   387  	return nil
   388  }
   389  
   390  // Revert is used to revert the job to a prior version
   391  func (j *Job) Revert(args *structs.JobRevertRequest, reply *structs.JobRegisterResponse) error {
   392  	if done, err := j.srv.forward("Job.Revert", args, args, reply); done {
   393  		return err
   394  	}
   395  	defer metrics.MeasureSince([]string{"nomad", "job", "revert"}, time.Now())
   396  
   397  	// Check for submit-job permissions
   398  	if aclObj, err := j.srv.ResolveToken(args.AuthToken); err != nil {
   399  		return err
   400  	} else if aclObj != nil && !aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilitySubmitJob) {
   401  		return structs.ErrPermissionDenied
   402  	}
   403  
   404  	// Validate the arguments
   405  	if args.JobID == "" {
   406  		return fmt.Errorf("missing job ID for revert")
   407  	}
   408  
   409  	// Lookup the job by version
   410  	snap, err := j.srv.fsm.State().Snapshot()
   411  	if err != nil {
   412  		return err
   413  	}
   414  
   415  	ws := memdb.NewWatchSet()
   416  	cur, err := snap.JobByID(ws, args.RequestNamespace(), args.JobID)
   417  	if err != nil {
   418  		return err
   419  	}
   420  	if cur == nil {
   421  		return fmt.Errorf("job %q not found", args.JobID)
   422  	}
   423  	if args.JobVersion == cur.Version {
   424  		return fmt.Errorf("can't revert to current version")
   425  	}
   426  
   427  	jobV, err := snap.JobByIDAndVersion(ws, args.RequestNamespace(), args.JobID, args.JobVersion)
   428  	if err != nil {
   429  		return err
   430  	}
   431  	if jobV == nil {
   432  		return fmt.Errorf("job %q in namespace %q at version %d not found", args.JobID, args.RequestNamespace(), args.JobVersion)
   433  	}
   434  
   435  	// Build the register request
   436  	reg := &structs.JobRegisterRequest{
   437  		Job:          jobV.Copy(),
   438  		WriteRequest: args.WriteRequest,
   439  	}
   440  
   441  	// If the request is enforcing the existing version do a check.
   442  	if args.EnforcePriorVersion != nil {
   443  		if cur.Version != *args.EnforcePriorVersion {
   444  			return fmt.Errorf("Current job has version %d; enforcing version %d", cur.Version, *args.EnforcePriorVersion)
   445  		}
   446  
   447  		reg.EnforceIndex = true
   448  		reg.JobModifyIndex = cur.JobModifyIndex
   449  	}
   450  
   451  	// Register the version.
   452  	return j.Register(reg, reply)
   453  }
   454  
   455  // Stable is used to mark the job version as stable
   456  func (j *Job) Stable(args *structs.JobStabilityRequest, reply *structs.JobStabilityResponse) error {
   457  	if done, err := j.srv.forward("Job.Stable", args, args, reply); done {
   458  		return err
   459  	}
   460  	defer metrics.MeasureSince([]string{"nomad", "job", "stable"}, time.Now())
   461  
   462  	// Check for read-job permissions
   463  	if aclObj, err := j.srv.ResolveToken(args.AuthToken); err != nil {
   464  		return err
   465  	} else if aclObj != nil && !aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilitySubmitJob) {
   466  		return structs.ErrPermissionDenied
   467  	}
   468  
   469  	// Validate the arguments
   470  	if args.JobID == "" {
   471  		return fmt.Errorf("missing job ID for marking job as stable")
   472  	}
   473  
   474  	// Lookup the job by version
   475  	snap, err := j.srv.fsm.State().Snapshot()
   476  	if err != nil {
   477  		return err
   478  	}
   479  
   480  	ws := memdb.NewWatchSet()
   481  	jobV, err := snap.JobByIDAndVersion(ws, args.RequestNamespace(), args.JobID, args.JobVersion)
   482  	if err != nil {
   483  		return err
   484  	}
   485  	if jobV == nil {
   486  		return fmt.Errorf("job %q in namespace %q at version %d not found", args.JobID, args.RequestNamespace(), args.JobVersion)
   487  	}
   488  
   489  	// Commit this stability request via Raft
   490  	_, modifyIndex, err := j.srv.raftApply(structs.JobStabilityRequestType, args)
   491  	if err != nil {
   492  		j.srv.logger.Printf("[ERR] nomad.job: Job stability request failed: %v", err)
   493  		return err
   494  	}
   495  
   496  	// Setup the reply
   497  	reply.Index = modifyIndex
   498  	return nil
   499  }
   500  
   501  // Evaluate is used to force a job for re-evaluation
   502  func (j *Job) Evaluate(args *structs.JobEvaluateRequest, reply *structs.JobRegisterResponse) error {
   503  	if done, err := j.srv.forward("Job.Evaluate", args, args, reply); done {
   504  		return err
   505  	}
   506  	defer metrics.MeasureSince([]string{"nomad", "job", "evaluate"}, time.Now())
   507  
   508  	// Check for read-job permissions
   509  	if aclObj, err := j.srv.ResolveToken(args.AuthToken); err != nil {
   510  		return err
   511  	} else if aclObj != nil && !aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilityReadJob) {
   512  		return structs.ErrPermissionDenied
   513  	}
   514  
   515  	// Validate the arguments
   516  	if args.JobID == "" {
   517  		return fmt.Errorf("missing job ID for evaluation")
   518  	}
   519  
   520  	// Lookup the job
   521  	snap, err := j.srv.fsm.State().Snapshot()
   522  	if err != nil {
   523  		return err
   524  	}
   525  	ws := memdb.NewWatchSet()
   526  	job, err := snap.JobByID(ws, args.RequestNamespace(), args.JobID)
   527  	if err != nil {
   528  		return err
   529  	}
   530  	if job == nil {
   531  		return fmt.Errorf("job not found")
   532  	}
   533  
   534  	if job.IsPeriodic() {
   535  		return fmt.Errorf("can't evaluate periodic job")
   536  	} else if job.IsParameterized() {
   537  		return fmt.Errorf("can't evaluate parameterized job")
   538  	}
   539  
   540  	// Create a new evaluation
   541  	eval := &structs.Evaluation{
   542  		ID:             uuid.Generate(),
   543  		Namespace:      args.RequestNamespace(),
   544  		Priority:       job.Priority,
   545  		Type:           job.Type,
   546  		TriggeredBy:    structs.EvalTriggerJobRegister,
   547  		JobID:          job.ID,
   548  		JobModifyIndex: job.ModifyIndex,
   549  		Status:         structs.EvalStatusPending,
   550  	}
   551  	update := &structs.EvalUpdateRequest{
   552  		Evals:        []*structs.Evaluation{eval},
   553  		WriteRequest: structs.WriteRequest{Region: args.Region},
   554  	}
   555  
   556  	// Commit this evaluation via Raft
   557  	_, evalIndex, err := j.srv.raftApply(structs.EvalUpdateRequestType, update)
   558  	if err != nil {
   559  		j.srv.logger.Printf("[ERR] nomad.job: Eval create failed: %v", err)
   560  		return err
   561  	}
   562  
   563  	// Setup the reply
   564  	reply.EvalID = eval.ID
   565  	reply.EvalCreateIndex = evalIndex
   566  	reply.JobModifyIndex = job.ModifyIndex
   567  	reply.Index = evalIndex
   568  	return nil
   569  }
   570  
   571  // Deregister is used to remove a job the cluster.
   572  func (j *Job) Deregister(args *structs.JobDeregisterRequest, reply *structs.JobDeregisterResponse) error {
   573  	if done, err := j.srv.forward("Job.Deregister", args, args, reply); done {
   574  		return err
   575  	}
   576  	defer metrics.MeasureSince([]string{"nomad", "job", "deregister"}, time.Now())
   577  
   578  	// Check for submit-job permissions
   579  	if aclObj, err := j.srv.ResolveToken(args.AuthToken); err != nil {
   580  		return err
   581  	} else if aclObj != nil && !aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilitySubmitJob) {
   582  		return structs.ErrPermissionDenied
   583  	}
   584  
   585  	// Validate the arguments
   586  	if args.JobID == "" {
   587  		return fmt.Errorf("missing job ID for deregistering")
   588  	}
   589  
   590  	// Lookup the job
   591  	snap, err := j.srv.fsm.State().Snapshot()
   592  	if err != nil {
   593  		return err
   594  	}
   595  	ws := memdb.NewWatchSet()
   596  	job, err := snap.JobByID(ws, args.RequestNamespace(), args.JobID)
   597  	if err != nil {
   598  		return err
   599  	}
   600  
   601  	// Commit this update via Raft
   602  	_, index, err := j.srv.raftApply(structs.JobDeregisterRequestType, args)
   603  	if err != nil {
   604  		j.srv.logger.Printf("[ERR] nomad.job: Deregister failed: %v", err)
   605  		return err
   606  	}
   607  
   608  	// Populate the reply with job information
   609  	reply.JobModifyIndex = index
   610  
   611  	// If the job is periodic or parameterized, we don't create an eval.
   612  	if job != nil && (job.IsPeriodic() || job.IsParameterized()) {
   613  		return nil
   614  	}
   615  
   616  	// Create a new evaluation
   617  	// XXX: The job priority / type is strange for this, since it's not a high
   618  	// priority even if the job was. The scheduler itself also doesn't matter,
   619  	// since all should be able to handle deregistration in the same way.
   620  	eval := &structs.Evaluation{
   621  		ID:             uuid.Generate(),
   622  		Namespace:      args.RequestNamespace(),
   623  		Priority:       structs.JobDefaultPriority,
   624  		Type:           structs.JobTypeService,
   625  		TriggeredBy:    structs.EvalTriggerJobDeregister,
   626  		JobID:          args.JobID,
   627  		JobModifyIndex: index,
   628  		Status:         structs.EvalStatusPending,
   629  	}
   630  	update := &structs.EvalUpdateRequest{
   631  		Evals:        []*structs.Evaluation{eval},
   632  		WriteRequest: structs.WriteRequest{Region: args.Region},
   633  	}
   634  
   635  	// Commit this evaluation via Raft
   636  	_, evalIndex, err := j.srv.raftApply(structs.EvalUpdateRequestType, update)
   637  	if err != nil {
   638  		j.srv.logger.Printf("[ERR] nomad.job: Eval create failed: %v", err)
   639  		return err
   640  	}
   641  
   642  	// Populate the reply with eval information
   643  	reply.EvalID = eval.ID
   644  	reply.EvalCreateIndex = evalIndex
   645  	reply.Index = evalIndex
   646  	return nil
   647  }
   648  
   649  // GetJob is used to request information about a specific job
   650  func (j *Job) GetJob(args *structs.JobSpecificRequest,
   651  	reply *structs.SingleJobResponse) error {
   652  	if done, err := j.srv.forward("Job.GetJob", args, args, reply); done {
   653  		return err
   654  	}
   655  	defer metrics.MeasureSince([]string{"nomad", "job", "get_job"}, time.Now())
   656  
   657  	// Check for read-job permissions
   658  	if aclObj, err := j.srv.ResolveToken(args.AuthToken); err != nil {
   659  		return err
   660  	} else if aclObj != nil && !aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilityReadJob) {
   661  		return structs.ErrPermissionDenied
   662  	}
   663  
   664  	// Setup the blocking query
   665  	opts := blockingOptions{
   666  		queryOpts: &args.QueryOptions,
   667  		queryMeta: &reply.QueryMeta,
   668  		run: func(ws memdb.WatchSet, state *state.StateStore) error {
   669  			// Look for the job
   670  			out, err := state.JobByID(ws, args.RequestNamespace(), args.JobID)
   671  			if err != nil {
   672  				return err
   673  			}
   674  
   675  			// Setup the output
   676  			reply.Job = out
   677  			if out != nil {
   678  				reply.Index = out.ModifyIndex
   679  			} else {
   680  				// Use the last index that affected the nodes table
   681  				index, err := state.Index("jobs")
   682  				if err != nil {
   683  					return err
   684  				}
   685  				reply.Index = index
   686  			}
   687  
   688  			// Set the query response
   689  			j.srv.setQueryMeta(&reply.QueryMeta)
   690  			return nil
   691  		}}
   692  	return j.srv.blockingRPC(&opts)
   693  }
   694  
   695  // GetJobVersions is used to retrieve all tracked versions of a job.
   696  func (j *Job) GetJobVersions(args *structs.JobVersionsRequest,
   697  	reply *structs.JobVersionsResponse) error {
   698  	if done, err := j.srv.forward("Job.GetJobVersions", args, args, reply); done {
   699  		return err
   700  	}
   701  	defer metrics.MeasureSince([]string{"nomad", "job", "get_job_versions"}, time.Now())
   702  
   703  	// Check for read-job permissions
   704  	if aclObj, err := j.srv.ResolveToken(args.AuthToken); err != nil {
   705  		return err
   706  	} else if aclObj != nil && !aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilityReadJob) {
   707  		return structs.ErrPermissionDenied
   708  	}
   709  
   710  	// Setup the blocking query
   711  	opts := blockingOptions{
   712  		queryOpts: &args.QueryOptions,
   713  		queryMeta: &reply.QueryMeta,
   714  		run: func(ws memdb.WatchSet, state *state.StateStore) error {
   715  			// Look for the job
   716  			out, err := state.JobVersionsByID(ws, args.RequestNamespace(), args.JobID)
   717  			if err != nil {
   718  				return err
   719  			}
   720  
   721  			// Setup the output
   722  			reply.Versions = out
   723  			if len(out) != 0 {
   724  				reply.Index = out[0].ModifyIndex
   725  
   726  				// Compute the diffs
   727  				if args.Diffs {
   728  					for i := 0; i < len(out)-1; i++ {
   729  						old, new := out[i+1], out[i]
   730  						d, err := old.Diff(new, true)
   731  						if err != nil {
   732  							return fmt.Errorf("failed to create job diff: %v", err)
   733  						}
   734  						reply.Diffs = append(reply.Diffs, d)
   735  					}
   736  				}
   737  			} else {
   738  				// Use the last index that affected the nodes table
   739  				index, err := state.Index("job_version")
   740  				if err != nil {
   741  					return err
   742  				}
   743  				reply.Index = index
   744  			}
   745  
   746  			// Set the query response
   747  			j.srv.setQueryMeta(&reply.QueryMeta)
   748  			return nil
   749  		}}
   750  	return j.srv.blockingRPC(&opts)
   751  }
   752  
   753  // List is used to list the jobs registered in the system
   754  func (j *Job) List(args *structs.JobListRequest,
   755  	reply *structs.JobListResponse) error {
   756  	if done, err := j.srv.forward("Job.List", args, args, reply); done {
   757  		return err
   758  	}
   759  	defer metrics.MeasureSince([]string{"nomad", "job", "list"}, time.Now())
   760  
   761  	// Check for list-job permissions
   762  	if aclObj, err := j.srv.ResolveToken(args.AuthToken); err != nil {
   763  		return err
   764  	} else if aclObj != nil && !aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilityListJobs) {
   765  		return structs.ErrPermissionDenied
   766  	}
   767  
   768  	// Setup the blocking query
   769  	opts := blockingOptions{
   770  		queryOpts: &args.QueryOptions,
   771  		queryMeta: &reply.QueryMeta,
   772  		run: func(ws memdb.WatchSet, state *state.StateStore) error {
   773  			// Capture all the jobs
   774  			var err error
   775  			var iter memdb.ResultIterator
   776  			if prefix := args.QueryOptions.Prefix; prefix != "" {
   777  				iter, err = state.JobsByIDPrefix(ws, args.RequestNamespace(), prefix)
   778  			} else {
   779  				iter, err = state.JobsByNamespace(ws, args.RequestNamespace())
   780  			}
   781  			if err != nil {
   782  				return err
   783  			}
   784  
   785  			var jobs []*structs.JobListStub
   786  			for {
   787  				raw := iter.Next()
   788  				if raw == nil {
   789  					break
   790  				}
   791  				job := raw.(*structs.Job)
   792  				summary, err := state.JobSummaryByID(ws, args.RequestNamespace(), job.ID)
   793  				if err != nil {
   794  					return fmt.Errorf("unable to look up summary for job: %v", job.ID)
   795  				}
   796  				jobs = append(jobs, job.Stub(summary))
   797  			}
   798  			reply.Jobs = jobs
   799  
   800  			// Use the last index that affected the jobs table
   801  			index, err := state.Index("jobs")
   802  			if err != nil {
   803  				return err
   804  			}
   805  			reply.Index = index
   806  
   807  			// Set the query response
   808  			j.srv.setQueryMeta(&reply.QueryMeta)
   809  			return nil
   810  		}}
   811  	return j.srv.blockingRPC(&opts)
   812  }
   813  
   814  // Allocations is used to list the allocations for a job
   815  func (j *Job) Allocations(args *structs.JobSpecificRequest,
   816  	reply *structs.JobAllocationsResponse) error {
   817  	if done, err := j.srv.forward("Job.Allocations", args, args, reply); done {
   818  		return err
   819  	}
   820  	defer metrics.MeasureSince([]string{"nomad", "job", "allocations"}, time.Now())
   821  
   822  	// Check for read-job permissions
   823  	if aclObj, err := j.srv.ResolveToken(args.AuthToken); err != nil {
   824  		return err
   825  	} else if aclObj != nil && !aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilityReadJob) {
   826  		return structs.ErrPermissionDenied
   827  	}
   828  
   829  	// Setup the blocking query
   830  	opts := blockingOptions{
   831  		queryOpts: &args.QueryOptions,
   832  		queryMeta: &reply.QueryMeta,
   833  		run: func(ws memdb.WatchSet, state *state.StateStore) error {
   834  			// Capture the allocations
   835  			allocs, err := state.AllocsByJob(ws, args.RequestNamespace(), args.JobID, args.AllAllocs)
   836  			if err != nil {
   837  				return err
   838  			}
   839  
   840  			// Convert to stubs
   841  			if len(allocs) > 0 {
   842  				reply.Allocations = make([]*structs.AllocListStub, 0, len(allocs))
   843  				for _, alloc := range allocs {
   844  					reply.Allocations = append(reply.Allocations, alloc.Stub())
   845  				}
   846  			}
   847  
   848  			// Use the last index that affected the allocs table
   849  			index, err := state.Index("allocs")
   850  			if err != nil {
   851  				return err
   852  			}
   853  			reply.Index = index
   854  
   855  			// Set the query response
   856  			j.srv.setQueryMeta(&reply.QueryMeta)
   857  			return nil
   858  
   859  		}}
   860  	return j.srv.blockingRPC(&opts)
   861  }
   862  
   863  // Evaluations is used to list the evaluations for a job
   864  func (j *Job) Evaluations(args *structs.JobSpecificRequest,
   865  	reply *structs.JobEvaluationsResponse) error {
   866  	if done, err := j.srv.forward("Job.Evaluations", args, args, reply); done {
   867  		return err
   868  	}
   869  	defer metrics.MeasureSince([]string{"nomad", "job", "evaluations"}, time.Now())
   870  
   871  	// Check for read-job permissions
   872  	if aclObj, err := j.srv.ResolveToken(args.AuthToken); err != nil {
   873  		return err
   874  	} else if aclObj != nil && !aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilityReadJob) {
   875  		return structs.ErrPermissionDenied
   876  	}
   877  
   878  	// Setup the blocking query
   879  	opts := blockingOptions{
   880  		queryOpts: &args.QueryOptions,
   881  		queryMeta: &reply.QueryMeta,
   882  		run: func(ws memdb.WatchSet, state *state.StateStore) error {
   883  			// Capture the evals
   884  			var err error
   885  			reply.Evaluations, err = state.EvalsByJob(ws, args.RequestNamespace(), args.JobID)
   886  			if err != nil {
   887  				return err
   888  			}
   889  
   890  			// Use the last index that affected the evals table
   891  			index, err := state.Index("evals")
   892  			if err != nil {
   893  				return err
   894  			}
   895  			reply.Index = index
   896  
   897  			// Set the query response
   898  			j.srv.setQueryMeta(&reply.QueryMeta)
   899  			return nil
   900  		}}
   901  
   902  	return j.srv.blockingRPC(&opts)
   903  }
   904  
   905  // Deployments is used to list the deployments for a job
   906  func (j *Job) Deployments(args *structs.JobSpecificRequest,
   907  	reply *structs.DeploymentListResponse) error {
   908  	if done, err := j.srv.forward("Job.Deployments", args, args, reply); done {
   909  		return err
   910  	}
   911  	defer metrics.MeasureSince([]string{"nomad", "job", "deployments"}, time.Now())
   912  
   913  	// Check for read-job permissions
   914  	if aclObj, err := j.srv.ResolveToken(args.AuthToken); err != nil {
   915  		return err
   916  	} else if aclObj != nil && !aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilityReadJob) {
   917  		return structs.ErrPermissionDenied
   918  	}
   919  
   920  	// Setup the blocking query
   921  	opts := blockingOptions{
   922  		queryOpts: &args.QueryOptions,
   923  		queryMeta: &reply.QueryMeta,
   924  		run: func(ws memdb.WatchSet, state *state.StateStore) error {
   925  			// Capture the deployments
   926  			deploys, err := state.DeploymentsByJobID(ws, args.RequestNamespace(), args.JobID)
   927  			if err != nil {
   928  				return err
   929  			}
   930  
   931  			// Use the last index that affected the deployment table
   932  			index, err := state.Index("deployment")
   933  			if err != nil {
   934  				return err
   935  			}
   936  			reply.Index = index
   937  			reply.Deployments = deploys
   938  
   939  			// Set the query response
   940  			j.srv.setQueryMeta(&reply.QueryMeta)
   941  			return nil
   942  
   943  		}}
   944  	return j.srv.blockingRPC(&opts)
   945  }
   946  
   947  // LatestDeployment is used to retrieve the latest deployment for a job
   948  func (j *Job) LatestDeployment(args *structs.JobSpecificRequest,
   949  	reply *structs.SingleDeploymentResponse) error {
   950  	if done, err := j.srv.forward("Job.LatestDeployment", args, args, reply); done {
   951  		return err
   952  	}
   953  	defer metrics.MeasureSince([]string{"nomad", "job", "latest_deployment"}, time.Now())
   954  
   955  	// Check for read-job permissions
   956  	if aclObj, err := j.srv.ResolveToken(args.AuthToken); err != nil {
   957  		return err
   958  	} else if aclObj != nil && !aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilityReadJob) {
   959  		return structs.ErrPermissionDenied
   960  	}
   961  
   962  	// Setup the blocking query
   963  	opts := blockingOptions{
   964  		queryOpts: &args.QueryOptions,
   965  		queryMeta: &reply.QueryMeta,
   966  		run: func(ws memdb.WatchSet, state *state.StateStore) error {
   967  			// Capture the deployments
   968  			deploys, err := state.DeploymentsByJobID(ws, args.RequestNamespace(), args.JobID)
   969  			if err != nil {
   970  				return err
   971  			}
   972  
   973  			// Use the last index that affected the deployment table
   974  			index, err := state.Index("deployment")
   975  			if err != nil {
   976  				return err
   977  			}
   978  			reply.Index = index
   979  			if len(deploys) > 0 {
   980  				sort.Slice(deploys, func(i, j int) bool {
   981  					return deploys[i].CreateIndex > deploys[j].CreateIndex
   982  				})
   983  				reply.Deployment = deploys[0]
   984  			}
   985  
   986  			// Set the query response
   987  			j.srv.setQueryMeta(&reply.QueryMeta)
   988  			return nil
   989  
   990  		}}
   991  	return j.srv.blockingRPC(&opts)
   992  }
   993  
   994  // Plan is used to cause a dry-run evaluation of the Job and return the results
   995  // with a potential diff containing annotations.
   996  func (j *Job) Plan(args *structs.JobPlanRequest, reply *structs.JobPlanResponse) error {
   997  	if done, err := j.srv.forward("Job.Plan", args, args, reply); done {
   998  		return err
   999  	}
  1000  	defer metrics.MeasureSince([]string{"nomad", "job", "plan"}, time.Now())
  1001  
  1002  	// Validate the arguments
  1003  	if args.Job == nil {
  1004  		return fmt.Errorf("Job required for plan")
  1005  	}
  1006  
  1007  	// Initialize the job fields (sets defaults and any necessary init work).
  1008  	canonicalizeWarnings := args.Job.Canonicalize()
  1009  
  1010  	// Add implicit constraints
  1011  	setImplicitConstraints(args.Job)
  1012  
  1013  	// Validate the job and capture any warnings
  1014  	err, warnings := validateJob(args.Job)
  1015  	if err != nil {
  1016  		return err
  1017  	}
  1018  
  1019  	// Set the warning message
  1020  	reply.Warnings = structs.MergeMultierrorWarnings(warnings, canonicalizeWarnings)
  1021  
  1022  	// Check job submission permissions, which we assume is the same for plan
  1023  	if aclObj, err := j.srv.ResolveToken(args.AuthToken); err != nil {
  1024  		return err
  1025  	} else if aclObj != nil {
  1026  		if !aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilitySubmitJob) {
  1027  			return structs.ErrPermissionDenied
  1028  		}
  1029  		// Check if override is set and we do not have permissions
  1030  		if args.PolicyOverride {
  1031  			if !aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilitySentinelOverride) {
  1032  				return structs.ErrPermissionDenied
  1033  			}
  1034  		}
  1035  	}
  1036  
  1037  	// Enforce Sentinel policies
  1038  	policyWarnings, err := j.enforceSubmitJob(args.PolicyOverride, args.Job)
  1039  	if err != nil {
  1040  		return err
  1041  	}
  1042  	if policyWarnings != nil {
  1043  		reply.Warnings = structs.MergeMultierrorWarnings(warnings,
  1044  			canonicalizeWarnings, policyWarnings)
  1045  	}
  1046  
  1047  	// Acquire a snapshot of the state
  1048  	snap, err := j.srv.fsm.State().Snapshot()
  1049  	if err != nil {
  1050  		return err
  1051  	}
  1052  
  1053  	// Get the original job
  1054  	ws := memdb.NewWatchSet()
  1055  	oldJob, err := snap.JobByID(ws, args.RequestNamespace(), args.Job.ID)
  1056  	if err != nil {
  1057  		return err
  1058  	}
  1059  
  1060  	var index uint64
  1061  	var updatedIndex uint64
  1062  
  1063  	if oldJob != nil {
  1064  		index = oldJob.JobModifyIndex
  1065  
  1066  		// We want to reuse deployments where possible, so only insert the job if
  1067  		// it has changed or the job didn't exist
  1068  		if oldJob.SpecChanged(args.Job) {
  1069  			// Insert the updated Job into the snapshot
  1070  			updatedIndex = oldJob.JobModifyIndex + 1
  1071  			snap.UpsertJob(updatedIndex, args.Job)
  1072  		}
  1073  	} else if oldJob == nil {
  1074  		// Insert the updated Job into the snapshot
  1075  		snap.UpsertJob(100, args.Job)
  1076  	}
  1077  
  1078  	// Create an eval and mark it as requiring annotations and insert that as well
  1079  	eval := &structs.Evaluation{
  1080  		ID:             uuid.Generate(),
  1081  		Namespace:      args.RequestNamespace(),
  1082  		Priority:       args.Job.Priority,
  1083  		Type:           args.Job.Type,
  1084  		TriggeredBy:    structs.EvalTriggerJobRegister,
  1085  		JobID:          args.Job.ID,
  1086  		JobModifyIndex: updatedIndex,
  1087  		Status:         structs.EvalStatusPending,
  1088  		AnnotatePlan:   true,
  1089  	}
  1090  
  1091  	snap.UpsertEvals(100, []*structs.Evaluation{eval})
  1092  
  1093  	// Create an in-memory Planner that returns no errors and stores the
  1094  	// submitted plan and created evals.
  1095  	planner := &scheduler.Harness{
  1096  		State: &snap.StateStore,
  1097  	}
  1098  
  1099  	// Create the scheduler and run it
  1100  	sched, err := scheduler.NewScheduler(eval.Type, j.srv.logger, snap, planner)
  1101  	if err != nil {
  1102  		return err
  1103  	}
  1104  
  1105  	if err := sched.Process(eval); err != nil {
  1106  		return err
  1107  	}
  1108  
  1109  	// Annotate and store the diff
  1110  	if plans := len(planner.Plans); plans != 1 {
  1111  		return fmt.Errorf("scheduler resulted in an unexpected number of plans: %v", plans)
  1112  	}
  1113  	annotations := planner.Plans[0].Annotations
  1114  	if args.Diff {
  1115  		jobDiff, err := oldJob.Diff(args.Job, true)
  1116  		if err != nil {
  1117  			return fmt.Errorf("failed to create job diff: %v", err)
  1118  		}
  1119  
  1120  		if err := scheduler.Annotate(jobDiff, annotations); err != nil {
  1121  			return fmt.Errorf("failed to annotate job diff: %v", err)
  1122  		}
  1123  		reply.Diff = jobDiff
  1124  	}
  1125  
  1126  	// Grab the failures
  1127  	if len(planner.Evals) != 1 {
  1128  		return fmt.Errorf("scheduler resulted in an unexpected number of eval updates: %v", planner.Evals)
  1129  	}
  1130  	updatedEval := planner.Evals[0]
  1131  
  1132  	// If it is a periodic job calculate the next launch
  1133  	if args.Job.IsPeriodic() && args.Job.Periodic.Enabled {
  1134  		reply.NextPeriodicLaunch = args.Job.Periodic.Next(time.Now().In(args.Job.Periodic.GetLocation()))
  1135  	}
  1136  
  1137  	reply.FailedTGAllocs = updatedEval.FailedTGAllocs
  1138  	reply.JobModifyIndex = index
  1139  	reply.Annotations = annotations
  1140  	reply.CreatedEvals = planner.CreateEvals
  1141  	reply.Index = index
  1142  	return nil
  1143  }
  1144  
  1145  // validateJob validates a Job and task drivers and returns an error if there is
  1146  // a validation problem or if the Job is of a type a user is not allowed to
  1147  // submit.
  1148  func validateJob(job *structs.Job) (invalid, warnings error) {
  1149  	validationErrors := new(multierror.Error)
  1150  	if err := job.Validate(); err != nil {
  1151  		multierror.Append(validationErrors, err)
  1152  	}
  1153  
  1154  	// Get any warnings
  1155  	warnings = job.Warnings()
  1156  
  1157  	// Get the signals required
  1158  	signals := job.RequiredSignals()
  1159  
  1160  	// Validate the driver configurations.
  1161  	for _, tg := range job.TaskGroups {
  1162  		// Get the signals for the task group
  1163  		tgSignals, tgOk := signals[tg.Name]
  1164  
  1165  		for _, task := range tg.Tasks {
  1166  			d, err := driver.NewDriver(
  1167  				task.Driver,
  1168  				driver.NewEmptyDriverContext(),
  1169  			)
  1170  			if err != nil {
  1171  				msg := "failed to create driver for task %q in group %q for validation: %v"
  1172  				multierror.Append(validationErrors, fmt.Errorf(msg, tg.Name, task.Name, err))
  1173  				continue
  1174  			}
  1175  
  1176  			if err := d.Validate(task.Config); err != nil {
  1177  				formatted := fmt.Errorf("group %q -> task %q -> config: %v", tg.Name, task.Name, err)
  1178  				multierror.Append(validationErrors, formatted)
  1179  			}
  1180  
  1181  			// The task group didn't have any task that required signals
  1182  			if !tgOk {
  1183  				continue
  1184  			}
  1185  
  1186  			// This task requires signals. Ensure the driver is capable
  1187  			if required, ok := tgSignals[task.Name]; ok {
  1188  				abilities := d.Abilities()
  1189  				if !abilities.SendSignals {
  1190  					formatted := fmt.Errorf("group %q -> task %q: driver %q doesn't support sending signals. Requested signals are %v",
  1191  						tg.Name, task.Name, task.Driver, strings.Join(required, ", "))
  1192  					multierror.Append(validationErrors, formatted)
  1193  				}
  1194  			}
  1195  		}
  1196  	}
  1197  
  1198  	if job.Type == structs.JobTypeCore {
  1199  		multierror.Append(validationErrors, fmt.Errorf("job type cannot be core"))
  1200  	}
  1201  
  1202  	if len(job.Payload) != 0 {
  1203  		multierror.Append(validationErrors, fmt.Errorf("job can't be submitted with a payload, only dispatched"))
  1204  	}
  1205  
  1206  	return validationErrors.ErrorOrNil(), warnings
  1207  }
  1208  
  1209  // validateJobUpdate ensures updates to a job are valid.
  1210  func validateJobUpdate(old, new *structs.Job) error {
  1211  	// Type transitions are disallowed
  1212  	if old.Type != new.Type {
  1213  		return fmt.Errorf("cannot update job from type %q to %q", old.Type, new.Type)
  1214  	}
  1215  
  1216  	// Transitioning to/from periodic is disallowed
  1217  	if old.IsPeriodic() && !new.IsPeriodic() {
  1218  		return fmt.Errorf("cannot update periodic job to being non-periodic")
  1219  	}
  1220  	if new.IsPeriodic() && !old.IsPeriodic() {
  1221  		return fmt.Errorf("cannot update non-periodic job to being periodic")
  1222  	}
  1223  
  1224  	// Transitioning to/from parameterized is disallowed
  1225  	if old.IsParameterized() && !new.IsParameterized() {
  1226  		return fmt.Errorf("cannot update non-parameterized job to being parameterized")
  1227  	}
  1228  	if new.IsParameterized() && !old.IsParameterized() {
  1229  		return fmt.Errorf("cannot update parameterized job to being non-parameterized")
  1230  	}
  1231  
  1232  	return nil
  1233  }
  1234  
  1235  // Dispatch a parameterized job.
  1236  func (j *Job) Dispatch(args *structs.JobDispatchRequest, reply *structs.JobDispatchResponse) error {
  1237  	if done, err := j.srv.forward("Job.Dispatch", args, args, reply); done {
  1238  		return err
  1239  	}
  1240  	defer metrics.MeasureSince([]string{"nomad", "job", "dispatch"}, time.Now())
  1241  
  1242  	// Check for submit-job permissions
  1243  	if aclObj, err := j.srv.ResolveToken(args.AuthToken); err != nil {
  1244  		return err
  1245  	} else if aclObj != nil && !aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilityDispatchJob) {
  1246  		return structs.ErrPermissionDenied
  1247  	}
  1248  
  1249  	// Lookup the parameterized job
  1250  	if args.JobID == "" {
  1251  		return fmt.Errorf("missing parameterized job ID")
  1252  	}
  1253  
  1254  	snap, err := j.srv.fsm.State().Snapshot()
  1255  	if err != nil {
  1256  		return err
  1257  	}
  1258  	ws := memdb.NewWatchSet()
  1259  	parameterizedJob, err := snap.JobByID(ws, args.RequestNamespace(), args.JobID)
  1260  	if err != nil {
  1261  		return err
  1262  	}
  1263  	if parameterizedJob == nil {
  1264  		return fmt.Errorf("parameterized job not found")
  1265  	}
  1266  
  1267  	if !parameterizedJob.IsParameterized() {
  1268  		return fmt.Errorf("Specified job %q is not a parameterized job", args.JobID)
  1269  	}
  1270  
  1271  	if parameterizedJob.Stop {
  1272  		return fmt.Errorf("Specified job %q is stopped", args.JobID)
  1273  	}
  1274  
  1275  	// Validate the arguments
  1276  	if err := validateDispatchRequest(args, parameterizedJob); err != nil {
  1277  		return err
  1278  	}
  1279  
  1280  	// Derive the child job and commit it via Raft
  1281  	dispatchJob := parameterizedJob.Copy()
  1282  	dispatchJob.ParameterizedJob = nil
  1283  	dispatchJob.ID = structs.DispatchedID(parameterizedJob.ID, time.Now())
  1284  	dispatchJob.ParentID = parameterizedJob.ID
  1285  	dispatchJob.Name = dispatchJob.ID
  1286  	dispatchJob.SetSubmitTime()
  1287  
  1288  	// Merge in the meta data
  1289  	for k, v := range args.Meta {
  1290  		if dispatchJob.Meta == nil {
  1291  			dispatchJob.Meta = make(map[string]string, len(args.Meta))
  1292  		}
  1293  		dispatchJob.Meta[k] = v
  1294  	}
  1295  
  1296  	// Compress the payload
  1297  	dispatchJob.Payload = snappy.Encode(nil, args.Payload)
  1298  
  1299  	regReq := &structs.JobRegisterRequest{
  1300  		Job:          dispatchJob,
  1301  		WriteRequest: args.WriteRequest,
  1302  	}
  1303  
  1304  	// Commit this update via Raft
  1305  	fsmErr, jobCreateIndex, err := j.srv.raftApply(structs.JobRegisterRequestType, regReq)
  1306  	if err, ok := fsmErr.(error); ok && err != nil {
  1307  		j.srv.logger.Printf("[ERR] nomad.job: Dispatched job register failed: %v", err)
  1308  		return err
  1309  	}
  1310  	if err != nil {
  1311  		j.srv.logger.Printf("[ERR] nomad.job: Dispatched job register failed: %v", err)
  1312  		return err
  1313  	}
  1314  
  1315  	reply.JobCreateIndex = jobCreateIndex
  1316  	reply.DispatchedJobID = dispatchJob.ID
  1317  	reply.Index = jobCreateIndex
  1318  
  1319  	// If the job is periodic, we don't create an eval.
  1320  	if !dispatchJob.IsPeriodic() {
  1321  		// Create a new evaluation
  1322  		eval := &structs.Evaluation{
  1323  			ID:             uuid.Generate(),
  1324  			Namespace:      args.RequestNamespace(),
  1325  			Priority:       dispatchJob.Priority,
  1326  			Type:           dispatchJob.Type,
  1327  			TriggeredBy:    structs.EvalTriggerJobRegister,
  1328  			JobID:          dispatchJob.ID,
  1329  			JobModifyIndex: jobCreateIndex,
  1330  			Status:         structs.EvalStatusPending,
  1331  		}
  1332  		update := &structs.EvalUpdateRequest{
  1333  			Evals:        []*structs.Evaluation{eval},
  1334  			WriteRequest: structs.WriteRequest{Region: args.Region},
  1335  		}
  1336  
  1337  		// Commit this evaluation via Raft
  1338  		_, evalIndex, err := j.srv.raftApply(structs.EvalUpdateRequestType, update)
  1339  		if err != nil {
  1340  			j.srv.logger.Printf("[ERR] nomad.job: Eval create failed: %v", err)
  1341  			return err
  1342  		}
  1343  
  1344  		// Setup the reply
  1345  		reply.EvalID = eval.ID
  1346  		reply.EvalCreateIndex = evalIndex
  1347  		reply.Index = evalIndex
  1348  	}
  1349  
  1350  	return nil
  1351  }
  1352  
  1353  // validateDispatchRequest returns whether the request is valid given the
  1354  // parameterized job.
  1355  func validateDispatchRequest(req *structs.JobDispatchRequest, job *structs.Job) error {
  1356  	// Check the payload constraint is met
  1357  	hasInputData := len(req.Payload) != 0
  1358  	if job.ParameterizedJob.Payload == structs.DispatchPayloadRequired && !hasInputData {
  1359  		return fmt.Errorf("Payload is not provided but required by parameterized job")
  1360  	} else if job.ParameterizedJob.Payload == structs.DispatchPayloadForbidden && hasInputData {
  1361  		return fmt.Errorf("Payload provided but forbidden by parameterized job")
  1362  	}
  1363  
  1364  	// Check the payload doesn't exceed the size limit
  1365  	if l := len(req.Payload); l > DispatchPayloadSizeLimit {
  1366  		return fmt.Errorf("Payload exceeds maximum size; %d > %d", l, DispatchPayloadSizeLimit)
  1367  	}
  1368  
  1369  	// Check if the metadata is a set
  1370  	keys := make(map[string]struct{}, len(req.Meta))
  1371  	for k := range keys {
  1372  		if _, ok := keys[k]; ok {
  1373  			return fmt.Errorf("Duplicate key %q in passed metadata", k)
  1374  		}
  1375  		keys[k] = struct{}{}
  1376  	}
  1377  
  1378  	required := helper.SliceStringToSet(job.ParameterizedJob.MetaRequired)
  1379  	optional := helper.SliceStringToSet(job.ParameterizedJob.MetaOptional)
  1380  
  1381  	// Check the metadata key constraints are met
  1382  	unpermitted := make(map[string]struct{})
  1383  	for k := range req.Meta {
  1384  		_, req := required[k]
  1385  		_, opt := optional[k]
  1386  		if !req && !opt {
  1387  			unpermitted[k] = struct{}{}
  1388  		}
  1389  	}
  1390  
  1391  	if len(unpermitted) != 0 {
  1392  		flat := make([]string, 0, len(unpermitted))
  1393  		for k := range unpermitted {
  1394  			flat = append(flat, k)
  1395  		}
  1396  
  1397  		return fmt.Errorf("Dispatch request included unpermitted metadata keys: %v", flat)
  1398  	}
  1399  
  1400  	missing := make(map[string]struct{})
  1401  	for _, k := range job.ParameterizedJob.MetaRequired {
  1402  		if _, ok := req.Meta[k]; !ok {
  1403  			missing[k] = struct{}{}
  1404  		}
  1405  	}
  1406  
  1407  	if len(missing) != 0 {
  1408  		flat := make([]string, 0, len(missing))
  1409  		for k := range missing {
  1410  			flat = append(flat, k)
  1411  		}
  1412  
  1413  		return fmt.Errorf("Dispatch did not provide required meta keys: %v", flat)
  1414  	}
  1415  
  1416  	return nil
  1417  }