github.com/dkerwin/nomad@v0.3.3-0.20160525181927-74554135514b/nomad/job_endpoint_test.go (about)

     1  package nomad
     2  
     3  import (
     4  	"reflect"
     5  	"strings"
     6  	"testing"
     7  	"time"
     8  
     9  	"github.com/hashicorp/net-rpc-msgpackrpc"
    10  	"github.com/hashicorp/nomad/nomad/mock"
    11  	"github.com/hashicorp/nomad/nomad/structs"
    12  	"github.com/hashicorp/nomad/testutil"
    13  )
    14  
    15  func TestJobEndpoint_Register(t *testing.T) {
    16  	s1 := testServer(t, func(c *Config) {
    17  		c.NumSchedulers = 0 // Prevent automatic dequeue
    18  	})
    19  	defer s1.Shutdown()
    20  	codec := rpcClient(t, s1)
    21  	testutil.WaitForLeader(t, s1.RPC)
    22  
    23  	// Create the register request
    24  	job := mock.Job()
    25  	req := &structs.JobRegisterRequest{
    26  		Job:          job,
    27  		WriteRequest: structs.WriteRequest{Region: "global"},
    28  	}
    29  
    30  	// Fetch the response
    31  	var resp structs.JobRegisterResponse
    32  	if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil {
    33  		t.Fatalf("err: %v", err)
    34  	}
    35  	if resp.Index == 0 {
    36  		t.Fatalf("bad index: %d", resp.Index)
    37  	}
    38  
    39  	// Check for the node in the FSM
    40  	state := s1.fsm.State()
    41  	out, err := state.JobByID(job.ID)
    42  	if err != nil {
    43  		t.Fatalf("err: %v", err)
    44  	}
    45  	if out == nil {
    46  		t.Fatalf("expected job")
    47  	}
    48  	if out.CreateIndex != resp.JobModifyIndex {
    49  		t.Fatalf("index mis-match")
    50  	}
    51  	serviceName := out.TaskGroups[0].Tasks[0].Services[0].Name
    52  	expectedServiceName := "web-frontend"
    53  	if serviceName != expectedServiceName {
    54  		t.Fatalf("Expected Service Name: %s, Actual: %s", expectedServiceName, serviceName)
    55  	}
    56  
    57  	// Lookup the evaluation
    58  	eval, err := state.EvalByID(resp.EvalID)
    59  	if err != nil {
    60  		t.Fatalf("err: %v", err)
    61  	}
    62  	if eval == nil {
    63  		t.Fatalf("expected eval")
    64  	}
    65  	if eval.CreateIndex != resp.EvalCreateIndex {
    66  		t.Fatalf("index mis-match")
    67  	}
    68  
    69  	if eval.Priority != job.Priority {
    70  		t.Fatalf("bad: %#v", eval)
    71  	}
    72  	if eval.Type != job.Type {
    73  		t.Fatalf("bad: %#v", eval)
    74  	}
    75  	if eval.TriggeredBy != structs.EvalTriggerJobRegister {
    76  		t.Fatalf("bad: %#v", eval)
    77  	}
    78  	if eval.JobID != job.ID {
    79  		t.Fatalf("bad: %#v", eval)
    80  	}
    81  	if eval.JobModifyIndex != resp.JobModifyIndex {
    82  		t.Fatalf("bad: %#v", eval)
    83  	}
    84  	if eval.Status != structs.EvalStatusPending {
    85  		t.Fatalf("bad: %#v", eval)
    86  	}
    87  }
    88  
    89  func TestJobEndpoint_Register_InvalidDriverConfig(t *testing.T) {
    90  	s1 := testServer(t, func(c *Config) {
    91  		c.NumSchedulers = 0 // Prevent automatic dequeue
    92  	})
    93  	defer s1.Shutdown()
    94  	codec := rpcClient(t, s1)
    95  	testutil.WaitForLeader(t, s1.RPC)
    96  
    97  	// Create the register request with a job containing an invalid driver
    98  	// config
    99  	job := mock.Job()
   100  	job.TaskGroups[0].Tasks[0].Config["foo"] = 1
   101  	req := &structs.JobRegisterRequest{
   102  		Job:          job,
   103  		WriteRequest: structs.WriteRequest{Region: "global"},
   104  	}
   105  
   106  	// Fetch the response
   107  	var resp structs.JobRegisterResponse
   108  	err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp)
   109  	if err == nil {
   110  		t.Fatalf("expected a validation error")
   111  	}
   112  
   113  	if !strings.Contains(err.Error(), "-> config:") {
   114  		t.Fatalf("expected a driver config validation error but got: %v", err)
   115  	}
   116  }
   117  
   118  func TestJobEndpoint_Register_Existing(t *testing.T) {
   119  	s1 := testServer(t, func(c *Config) {
   120  		c.NumSchedulers = 0 // Prevent automatic dequeue
   121  	})
   122  	defer s1.Shutdown()
   123  	codec := rpcClient(t, s1)
   124  	testutil.WaitForLeader(t, s1.RPC)
   125  
   126  	// Create the register request
   127  	job := mock.Job()
   128  	req := &structs.JobRegisterRequest{
   129  		Job:          job,
   130  		WriteRequest: structs.WriteRequest{Region: "global"},
   131  	}
   132  
   133  	// Fetch the response
   134  	var resp structs.JobRegisterResponse
   135  	if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil {
   136  		t.Fatalf("err: %v", err)
   137  	}
   138  	if resp.Index == 0 {
   139  		t.Fatalf("bad index: %d", resp.Index)
   140  	}
   141  
   142  	// Update the job definition
   143  	job2 := mock.Job()
   144  	job2.Priority = 100
   145  	job2.ID = job.ID
   146  	req.Job = job2
   147  
   148  	// Attempt update
   149  	if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil {
   150  		t.Fatalf("err: %v", err)
   151  	}
   152  	if resp.Index == 0 {
   153  		t.Fatalf("bad index: %d", resp.Index)
   154  	}
   155  
   156  	// Check for the node in the FSM
   157  	state := s1.fsm.State()
   158  	out, err := state.JobByID(job.ID)
   159  	if err != nil {
   160  		t.Fatalf("err: %v", err)
   161  	}
   162  	if out == nil {
   163  		t.Fatalf("expected job")
   164  	}
   165  	if out.ModifyIndex != resp.JobModifyIndex {
   166  		t.Fatalf("index mis-match")
   167  	}
   168  	if out.Priority != 100 {
   169  		t.Fatalf("expected update")
   170  	}
   171  
   172  	// Lookup the evaluation
   173  	eval, err := state.EvalByID(resp.EvalID)
   174  	if err != nil {
   175  		t.Fatalf("err: %v", err)
   176  	}
   177  	if eval == nil {
   178  		t.Fatalf("expected eval")
   179  	}
   180  	if eval.CreateIndex != resp.EvalCreateIndex {
   181  		t.Fatalf("index mis-match")
   182  	}
   183  
   184  	if eval.Priority != job2.Priority {
   185  		t.Fatalf("bad: %#v", eval)
   186  	}
   187  	if eval.Type != job2.Type {
   188  		t.Fatalf("bad: %#v", eval)
   189  	}
   190  	if eval.TriggeredBy != structs.EvalTriggerJobRegister {
   191  		t.Fatalf("bad: %#v", eval)
   192  	}
   193  	if eval.JobID != job2.ID {
   194  		t.Fatalf("bad: %#v", eval)
   195  	}
   196  	if eval.JobModifyIndex != resp.JobModifyIndex {
   197  		t.Fatalf("bad: %#v", eval)
   198  	}
   199  	if eval.Status != structs.EvalStatusPending {
   200  		t.Fatalf("bad: %#v", eval)
   201  	}
   202  }
   203  
   204  func TestJobEndpoint_Register_Periodic(t *testing.T) {
   205  	s1 := testServer(t, func(c *Config) {
   206  		c.NumSchedulers = 0 // Prevent automatic dequeue
   207  	})
   208  	defer s1.Shutdown()
   209  	codec := rpcClient(t, s1)
   210  	testutil.WaitForLeader(t, s1.RPC)
   211  
   212  	// Create the register request for a periodic job.
   213  	job := mock.PeriodicJob()
   214  	req := &structs.JobRegisterRequest{
   215  		Job:          job,
   216  		WriteRequest: structs.WriteRequest{Region: "global"},
   217  	}
   218  
   219  	// Fetch the response
   220  	var resp structs.JobRegisterResponse
   221  	if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil {
   222  		t.Fatalf("err: %v", err)
   223  	}
   224  	if resp.JobModifyIndex == 0 {
   225  		t.Fatalf("bad index: %d", resp.Index)
   226  	}
   227  
   228  	// Check for the node in the FSM
   229  	state := s1.fsm.State()
   230  	out, err := state.JobByID(job.ID)
   231  	if err != nil {
   232  		t.Fatalf("err: %v", err)
   233  	}
   234  	if out == nil {
   235  		t.Fatalf("expected job")
   236  	}
   237  	if out.CreateIndex != resp.JobModifyIndex {
   238  		t.Fatalf("index mis-match")
   239  	}
   240  	serviceName := out.TaskGroups[0].Tasks[0].Services[0].Name
   241  	expectedServiceName := "web-frontend"
   242  	if serviceName != expectedServiceName {
   243  		t.Fatalf("Expected Service Name: %s, Actual: %s", expectedServiceName, serviceName)
   244  	}
   245  
   246  	if resp.EvalID != "" {
   247  		t.Fatalf("Register created an eval for a periodic job")
   248  	}
   249  }
   250  
   251  func TestJobEndpoint_Evaluate(t *testing.T) {
   252  	s1 := testServer(t, func(c *Config) {
   253  		c.NumSchedulers = 0 // Prevent automatic dequeue
   254  	})
   255  	defer s1.Shutdown()
   256  	codec := rpcClient(t, s1)
   257  	testutil.WaitForLeader(t, s1.RPC)
   258  
   259  	// Create the register request
   260  	job := mock.Job()
   261  	req := &structs.JobRegisterRequest{
   262  		Job:          job,
   263  		WriteRequest: structs.WriteRequest{Region: "global"},
   264  	}
   265  
   266  	// Fetch the response
   267  	var resp structs.JobRegisterResponse
   268  	if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil {
   269  		t.Fatalf("err: %v", err)
   270  	}
   271  	if resp.Index == 0 {
   272  		t.Fatalf("bad index: %d", resp.Index)
   273  	}
   274  
   275  	// Force a re-evaluation
   276  	reEval := &structs.JobEvaluateRequest{
   277  		JobID:        job.ID,
   278  		WriteRequest: structs.WriteRequest{Region: "global"},
   279  	}
   280  
   281  	// Fetch the response
   282  	if err := msgpackrpc.CallWithCodec(codec, "Job.Evaluate", reEval, &resp); err != nil {
   283  		t.Fatalf("err: %v", err)
   284  	}
   285  	if resp.Index == 0 {
   286  		t.Fatalf("bad index: %d", resp.Index)
   287  	}
   288  
   289  	// Lookup the evaluation
   290  	state := s1.fsm.State()
   291  	eval, err := state.EvalByID(resp.EvalID)
   292  	if err != nil {
   293  		t.Fatalf("err: %v", err)
   294  	}
   295  	if eval == nil {
   296  		t.Fatalf("expected eval")
   297  	}
   298  	if eval.CreateIndex != resp.EvalCreateIndex {
   299  		t.Fatalf("index mis-match")
   300  	}
   301  
   302  	if eval.Priority != job.Priority {
   303  		t.Fatalf("bad: %#v", eval)
   304  	}
   305  	if eval.Type != job.Type {
   306  		t.Fatalf("bad: %#v", eval)
   307  	}
   308  	if eval.TriggeredBy != structs.EvalTriggerJobRegister {
   309  		t.Fatalf("bad: %#v", eval)
   310  	}
   311  	if eval.JobID != job.ID {
   312  		t.Fatalf("bad: %#v", eval)
   313  	}
   314  	if eval.JobModifyIndex != resp.JobModifyIndex {
   315  		t.Fatalf("bad: %#v", eval)
   316  	}
   317  	if eval.Status != structs.EvalStatusPending {
   318  		t.Fatalf("bad: %#v", eval)
   319  	}
   320  }
   321  
   322  func TestJobEndpoint_Evaluate_Periodic(t *testing.T) {
   323  	s1 := testServer(t, func(c *Config) {
   324  		c.NumSchedulers = 0 // Prevent automatic dequeue
   325  	})
   326  	defer s1.Shutdown()
   327  	codec := rpcClient(t, s1)
   328  	testutil.WaitForLeader(t, s1.RPC)
   329  
   330  	// Create the register request
   331  	job := mock.PeriodicJob()
   332  	req := &structs.JobRegisterRequest{
   333  		Job:          job,
   334  		WriteRequest: structs.WriteRequest{Region: "global"},
   335  	}
   336  
   337  	// Fetch the response
   338  	var resp structs.JobRegisterResponse
   339  	if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil {
   340  		t.Fatalf("err: %v", err)
   341  	}
   342  	if resp.JobModifyIndex == 0 {
   343  		t.Fatalf("bad index: %d", resp.Index)
   344  	}
   345  
   346  	// Force a re-evaluation
   347  	reEval := &structs.JobEvaluateRequest{
   348  		JobID:        job.ID,
   349  		WriteRequest: structs.WriteRequest{Region: "global"},
   350  	}
   351  
   352  	// Fetch the response
   353  	if err := msgpackrpc.CallWithCodec(codec, "Job.Evaluate", reEval, &resp); err == nil {
   354  		t.Fatal("expect an err")
   355  	}
   356  }
   357  
   358  func TestJobEndpoint_Deregister(t *testing.T) {
   359  	s1 := testServer(t, func(c *Config) {
   360  		c.NumSchedulers = 0 // Prevent automatic dequeue
   361  	})
   362  	defer s1.Shutdown()
   363  	codec := rpcClient(t, s1)
   364  	testutil.WaitForLeader(t, s1.RPC)
   365  
   366  	// Create the register request
   367  	job := mock.Job()
   368  	reg := &structs.JobRegisterRequest{
   369  		Job:          job,
   370  		WriteRequest: structs.WriteRequest{Region: "global"},
   371  	}
   372  
   373  	// Fetch the response
   374  	var resp structs.JobRegisterResponse
   375  	if err := msgpackrpc.CallWithCodec(codec, "Job.Register", reg, &resp); err != nil {
   376  		t.Fatalf("err: %v", err)
   377  	}
   378  
   379  	// Deregister
   380  	dereg := &structs.JobDeregisterRequest{
   381  		JobID:        job.ID,
   382  		WriteRequest: structs.WriteRequest{Region: "global"},
   383  	}
   384  	var resp2 structs.JobDeregisterResponse
   385  	if err := msgpackrpc.CallWithCodec(codec, "Job.Deregister", dereg, &resp2); err != nil {
   386  		t.Fatalf("err: %v", err)
   387  	}
   388  	if resp2.Index == 0 {
   389  		t.Fatalf("bad index: %d", resp2.Index)
   390  	}
   391  
   392  	// Check for the node in the FSM
   393  	state := s1.fsm.State()
   394  	out, err := state.JobByID(job.ID)
   395  	if err != nil {
   396  		t.Fatalf("err: %v", err)
   397  	}
   398  	if out != nil {
   399  		t.Fatalf("unexpected job")
   400  	}
   401  
   402  	// Lookup the evaluation
   403  	eval, err := state.EvalByID(resp2.EvalID)
   404  	if err != nil {
   405  		t.Fatalf("err: %v", err)
   406  	}
   407  	if eval == nil {
   408  		t.Fatalf("expected eval")
   409  	}
   410  	if eval.CreateIndex != resp2.EvalCreateIndex {
   411  		t.Fatalf("index mis-match")
   412  	}
   413  
   414  	if eval.Priority != structs.JobDefaultPriority {
   415  		t.Fatalf("bad: %#v", eval)
   416  	}
   417  	if eval.Type != structs.JobTypeService {
   418  		t.Fatalf("bad: %#v", eval)
   419  	}
   420  	if eval.TriggeredBy != structs.EvalTriggerJobDeregister {
   421  		t.Fatalf("bad: %#v", eval)
   422  	}
   423  	if eval.JobID != job.ID {
   424  		t.Fatalf("bad: %#v", eval)
   425  	}
   426  	if eval.JobModifyIndex != resp2.JobModifyIndex {
   427  		t.Fatalf("bad: %#v", eval)
   428  	}
   429  	if eval.Status != structs.EvalStatusPending {
   430  		t.Fatalf("bad: %#v", eval)
   431  	}
   432  }
   433  
   434  func TestJobEndpoint_Deregister_NonExistent(t *testing.T) {
   435  	s1 := testServer(t, func(c *Config) {
   436  		c.NumSchedulers = 0 // Prevent automatic dequeue
   437  	})
   438  	defer s1.Shutdown()
   439  	codec := rpcClient(t, s1)
   440  	testutil.WaitForLeader(t, s1.RPC)
   441  
   442  	// Deregister
   443  	jobID := "foo"
   444  	dereg := &structs.JobDeregisterRequest{
   445  		JobID:        jobID,
   446  		WriteRequest: structs.WriteRequest{Region: "global"},
   447  	}
   448  	var resp2 structs.JobDeregisterResponse
   449  	if err := msgpackrpc.CallWithCodec(codec, "Job.Deregister", dereg, &resp2); err != nil {
   450  		t.Fatalf("err: %v", err)
   451  	}
   452  	if resp2.JobModifyIndex == 0 {
   453  		t.Fatalf("bad index: %d", resp2.Index)
   454  	}
   455  
   456  	// Lookup the evaluation
   457  	state := s1.fsm.State()
   458  	eval, err := state.EvalByID(resp2.EvalID)
   459  	if err != nil {
   460  		t.Fatalf("err: %v", err)
   461  	}
   462  	if eval == nil {
   463  		t.Fatalf("expected eval")
   464  	}
   465  	if eval.CreateIndex != resp2.EvalCreateIndex {
   466  		t.Fatalf("index mis-match")
   467  	}
   468  
   469  	if eval.Priority != structs.JobDefaultPriority {
   470  		t.Fatalf("bad: %#v", eval)
   471  	}
   472  	if eval.Type != structs.JobTypeService {
   473  		t.Fatalf("bad: %#v", eval)
   474  	}
   475  	if eval.TriggeredBy != structs.EvalTriggerJobDeregister {
   476  		t.Fatalf("bad: %#v", eval)
   477  	}
   478  	if eval.JobID != jobID {
   479  		t.Fatalf("bad: %#v", eval)
   480  	}
   481  	if eval.JobModifyIndex != resp2.JobModifyIndex {
   482  		t.Fatalf("bad: %#v", eval)
   483  	}
   484  	if eval.Status != structs.EvalStatusPending {
   485  		t.Fatalf("bad: %#v", eval)
   486  	}
   487  }
   488  
   489  func TestJobEndpoint_Deregister_Periodic(t *testing.T) {
   490  	s1 := testServer(t, func(c *Config) {
   491  		c.NumSchedulers = 0 // Prevent automatic dequeue
   492  	})
   493  	defer s1.Shutdown()
   494  	codec := rpcClient(t, s1)
   495  	testutil.WaitForLeader(t, s1.RPC)
   496  
   497  	// Create the register request
   498  	job := mock.PeriodicJob()
   499  	reg := &structs.JobRegisterRequest{
   500  		Job:          job,
   501  		WriteRequest: structs.WriteRequest{Region: "global"},
   502  	}
   503  
   504  	// Fetch the response
   505  	var resp structs.JobRegisterResponse
   506  	if err := msgpackrpc.CallWithCodec(codec, "Job.Register", reg, &resp); err != nil {
   507  		t.Fatalf("err: %v", err)
   508  	}
   509  
   510  	// Deregister
   511  	dereg := &structs.JobDeregisterRequest{
   512  		JobID:        job.ID,
   513  		WriteRequest: structs.WriteRequest{Region: "global"},
   514  	}
   515  	var resp2 structs.JobDeregisterResponse
   516  	if err := msgpackrpc.CallWithCodec(codec, "Job.Deregister", dereg, &resp2); err != nil {
   517  		t.Fatalf("err: %v", err)
   518  	}
   519  	if resp2.JobModifyIndex == 0 {
   520  		t.Fatalf("bad index: %d", resp2.Index)
   521  	}
   522  
   523  	// Check for the node in the FSM
   524  	state := s1.fsm.State()
   525  	out, err := state.JobByID(job.ID)
   526  	if err != nil {
   527  		t.Fatalf("err: %v", err)
   528  	}
   529  	if out != nil {
   530  		t.Fatalf("unexpected job")
   531  	}
   532  
   533  	if resp.EvalID != "" {
   534  		t.Fatalf("Deregister created an eval for a periodic job")
   535  	}
   536  }
   537  
   538  func TestJobEndpoint_GetJob(t *testing.T) {
   539  	s1 := testServer(t, nil)
   540  	defer s1.Shutdown()
   541  	codec := rpcClient(t, s1)
   542  	testutil.WaitForLeader(t, s1.RPC)
   543  
   544  	// Create the register request
   545  	job := mock.Job()
   546  	reg := &structs.JobRegisterRequest{
   547  		Job:          job,
   548  		WriteRequest: structs.WriteRequest{Region: "global"},
   549  	}
   550  
   551  	// Fetch the response
   552  	var resp structs.JobRegisterResponse
   553  	if err := msgpackrpc.CallWithCodec(codec, "Job.Register", reg, &resp); err != nil {
   554  		t.Fatalf("err: %v", err)
   555  	}
   556  	job.CreateIndex = resp.JobModifyIndex
   557  	job.ModifyIndex = resp.JobModifyIndex
   558  	job.JobModifyIndex = resp.JobModifyIndex
   559  
   560  	// Lookup the job
   561  	get := &structs.JobSpecificRequest{
   562  		JobID:        job.ID,
   563  		QueryOptions: structs.QueryOptions{Region: "global"},
   564  	}
   565  	var resp2 structs.SingleJobResponse
   566  	if err := msgpackrpc.CallWithCodec(codec, "Job.GetJob", get, &resp2); err != nil {
   567  		t.Fatalf("err: %v", err)
   568  	}
   569  	if resp2.Index != resp.JobModifyIndex {
   570  		t.Fatalf("Bad index: %d %d", resp2.Index, resp.Index)
   571  	}
   572  
   573  	// Make a copy of the origin job and change the service name so that we can
   574  	// do a deep equal with the response from the GET JOB Api
   575  	j := job
   576  	j.TaskGroups[0].Tasks[0].Services[0].Name = "web-frontend"
   577  	for tgix, tg := range j.TaskGroups {
   578  		for tidx, t := range tg.Tasks {
   579  			for sidx, service := range t.Services {
   580  				for cidx, check := range service.Checks {
   581  					check.Name = resp2.Job.TaskGroups[tgix].Tasks[tidx].Services[sidx].Checks[cidx].Name
   582  				}
   583  			}
   584  		}
   585  	}
   586  
   587  	if !reflect.DeepEqual(j, resp2.Job) {
   588  		t.Fatalf("bad: %#v %#v", job, resp2.Job)
   589  	}
   590  
   591  	// Lookup non-existing job
   592  	get.JobID = "foobarbaz"
   593  	if err := msgpackrpc.CallWithCodec(codec, "Job.GetJob", get, &resp2); err != nil {
   594  		t.Fatalf("err: %v", err)
   595  	}
   596  	if resp2.Index != resp.JobModifyIndex {
   597  		t.Fatalf("Bad index: %d %d", resp2.Index, resp.Index)
   598  	}
   599  	if resp2.Job != nil {
   600  		t.Fatalf("unexpected job")
   601  	}
   602  }
   603  
   604  func TestJobEndpoint_GetJob_Blocking(t *testing.T) {
   605  	s1 := testServer(t, nil)
   606  	defer s1.Shutdown()
   607  	state := s1.fsm.State()
   608  	codec := rpcClient(t, s1)
   609  	testutil.WaitForLeader(t, s1.RPC)
   610  
   611  	// Create the jobs
   612  	job1 := mock.Job()
   613  	job2 := mock.Job()
   614  
   615  	// Upsert a job we are not interested in first.
   616  	time.AfterFunc(100*time.Millisecond, func() {
   617  		if err := state.UpsertJob(100, job1); err != nil {
   618  			t.Fatalf("err: %v", err)
   619  		}
   620  	})
   621  
   622  	// Upsert another job later which should trigger the watch.
   623  	time.AfterFunc(200*time.Millisecond, func() {
   624  		if err := state.UpsertJob(200, job2); err != nil {
   625  			t.Fatalf("err: %v", err)
   626  		}
   627  	})
   628  
   629  	req := &structs.JobSpecificRequest{
   630  		JobID: job2.ID,
   631  		QueryOptions: structs.QueryOptions{
   632  			Region:        "global",
   633  			MinQueryIndex: 50,
   634  		},
   635  	}
   636  	start := time.Now()
   637  	var resp structs.SingleJobResponse
   638  	if err := msgpackrpc.CallWithCodec(codec, "Job.GetJob", req, &resp); err != nil {
   639  		t.Fatalf("err: %v", err)
   640  	}
   641  
   642  	if elapsed := time.Since(start); elapsed < 200*time.Millisecond {
   643  		t.Fatalf("should block (returned in %s) %#v", elapsed, resp)
   644  	}
   645  	if resp.Index != 200 {
   646  		t.Fatalf("Bad index: %d %d", resp.Index, 200)
   647  	}
   648  	if resp.Job == nil || resp.Job.ID != job2.ID {
   649  		t.Fatalf("bad: %#v", resp.Job)
   650  	}
   651  
   652  	// Job delete fires watches
   653  	time.AfterFunc(100*time.Millisecond, func() {
   654  		if err := state.DeleteJob(300, job2.ID); err != nil {
   655  			t.Fatalf("err: %v", err)
   656  		}
   657  	})
   658  
   659  	req.QueryOptions.MinQueryIndex = 250
   660  	start = time.Now()
   661  
   662  	var resp2 structs.SingleJobResponse
   663  	if err := msgpackrpc.CallWithCodec(codec, "Job.GetJob", req, &resp2); err != nil {
   664  		t.Fatalf("err: %v", err)
   665  	}
   666  
   667  	if elapsed := time.Since(start); elapsed < 100*time.Millisecond {
   668  		t.Fatalf("should block (returned in %s) %#v", elapsed, resp2)
   669  	}
   670  	if resp2.Index != 300 {
   671  		t.Fatalf("Bad index: %d %d", resp2.Index, 300)
   672  	}
   673  	if resp2.Job != nil {
   674  		t.Fatalf("bad: %#v", resp2.Job)
   675  	}
   676  }
   677  
   678  func TestJobEndpoint_ListJobs(t *testing.T) {
   679  	s1 := testServer(t, nil)
   680  	defer s1.Shutdown()
   681  	codec := rpcClient(t, s1)
   682  	testutil.WaitForLeader(t, s1.RPC)
   683  
   684  	// Create the register request
   685  	job := mock.Job()
   686  	state := s1.fsm.State()
   687  	err := state.UpsertJob(1000, job)
   688  	if err != nil {
   689  		t.Fatalf("err: %v", err)
   690  	}
   691  
   692  	// Lookup the jobs
   693  	get := &structs.JobListRequest{
   694  		QueryOptions: structs.QueryOptions{Region: "global"},
   695  	}
   696  	var resp2 structs.JobListResponse
   697  	if err := msgpackrpc.CallWithCodec(codec, "Job.List", get, &resp2); err != nil {
   698  		t.Fatalf("err: %v", err)
   699  	}
   700  	if resp2.Index != 1000 {
   701  		t.Fatalf("Bad index: %d %d", resp2.Index, 1000)
   702  	}
   703  
   704  	if len(resp2.Jobs) != 1 {
   705  		t.Fatalf("bad: %#v", resp2.Jobs)
   706  	}
   707  	if resp2.Jobs[0].ID != job.ID {
   708  		t.Fatalf("bad: %#v", resp2.Jobs[0])
   709  	}
   710  
   711  	// Lookup the jobs by prefix
   712  	get = &structs.JobListRequest{
   713  		QueryOptions: structs.QueryOptions{Region: "global", Prefix: resp2.Jobs[0].ID[:4]},
   714  	}
   715  	var resp3 structs.JobListResponse
   716  	if err := msgpackrpc.CallWithCodec(codec, "Job.List", get, &resp3); err != nil {
   717  		t.Fatalf("err: %v", err)
   718  	}
   719  	if resp3.Index != 1000 {
   720  		t.Fatalf("Bad index: %d %d", resp3.Index, 1000)
   721  	}
   722  
   723  	if len(resp3.Jobs) != 1 {
   724  		t.Fatalf("bad: %#v", resp3.Jobs)
   725  	}
   726  	if resp3.Jobs[0].ID != job.ID {
   727  		t.Fatalf("bad: %#v", resp3.Jobs[0])
   728  	}
   729  }
   730  
   731  func TestJobEndpoint_ListJobs_Blocking(t *testing.T) {
   732  	s1 := testServer(t, nil)
   733  	defer s1.Shutdown()
   734  	state := s1.fsm.State()
   735  	codec := rpcClient(t, s1)
   736  	testutil.WaitForLeader(t, s1.RPC)
   737  
   738  	// Create the job
   739  	job := mock.Job()
   740  
   741  	// Upsert job triggers watches
   742  	time.AfterFunc(100*time.Millisecond, func() {
   743  		if err := state.UpsertJob(100, job); err != nil {
   744  			t.Fatalf("err: %v", err)
   745  		}
   746  	})
   747  
   748  	req := &structs.JobListRequest{
   749  		QueryOptions: structs.QueryOptions{
   750  			Region:        "global",
   751  			MinQueryIndex: 50,
   752  		},
   753  	}
   754  	start := time.Now()
   755  	var resp structs.JobListResponse
   756  	if err := msgpackrpc.CallWithCodec(codec, "Job.List", req, &resp); err != nil {
   757  		t.Fatalf("err: %v", err)
   758  	}
   759  
   760  	if elapsed := time.Since(start); elapsed < 100*time.Millisecond {
   761  		t.Fatalf("should block (returned in %s) %#v", elapsed, resp)
   762  	}
   763  	if resp.Index != 100 {
   764  		t.Fatalf("Bad index: %d %d", resp.Index, 100)
   765  	}
   766  	if len(resp.Jobs) != 1 || resp.Jobs[0].ID != job.ID {
   767  		t.Fatalf("bad: %#v", resp.Jobs)
   768  	}
   769  
   770  	// Job deletion triggers watches
   771  	time.AfterFunc(100*time.Millisecond, func() {
   772  		if err := state.DeleteJob(200, job.ID); err != nil {
   773  			t.Fatalf("err: %v", err)
   774  		}
   775  	})
   776  
   777  	req.MinQueryIndex = 150
   778  	start = time.Now()
   779  	var resp2 structs.JobListResponse
   780  	if err := msgpackrpc.CallWithCodec(codec, "Job.List", req, &resp2); err != nil {
   781  		t.Fatalf("err: %v", err)
   782  	}
   783  
   784  	if elapsed := time.Since(start); elapsed < 100*time.Millisecond {
   785  		t.Fatalf("should block (returned in %s) %#v", elapsed, resp2)
   786  	}
   787  	if resp2.Index != 200 {
   788  		t.Fatalf("Bad index: %d %d", resp2.Index, 200)
   789  	}
   790  	if len(resp2.Jobs) != 0 {
   791  		t.Fatalf("bad: %#v", resp2.Jobs)
   792  	}
   793  }
   794  
   795  func TestJobEndpoint_Allocations(t *testing.T) {
   796  	s1 := testServer(t, nil)
   797  	defer s1.Shutdown()
   798  	codec := rpcClient(t, s1)
   799  	testutil.WaitForLeader(t, s1.RPC)
   800  
   801  	// Create the register request
   802  	alloc1 := mock.Alloc()
   803  	alloc2 := mock.Alloc()
   804  	alloc2.JobID = alloc1.JobID
   805  	state := s1.fsm.State()
   806  	err := state.UpsertAllocs(1000,
   807  		[]*structs.Allocation{alloc1, alloc2})
   808  	if err != nil {
   809  		t.Fatalf("err: %v", err)
   810  	}
   811  
   812  	// Lookup the jobs
   813  	get := &structs.JobSpecificRequest{
   814  		JobID:        alloc1.JobID,
   815  		QueryOptions: structs.QueryOptions{Region: "global"},
   816  	}
   817  	var resp2 structs.JobAllocationsResponse
   818  	if err := msgpackrpc.CallWithCodec(codec, "Job.Allocations", get, &resp2); err != nil {
   819  		t.Fatalf("err: %v", err)
   820  	}
   821  	if resp2.Index != 1000 {
   822  		t.Fatalf("Bad index: %d %d", resp2.Index, 1000)
   823  	}
   824  
   825  	if len(resp2.Allocations) != 2 {
   826  		t.Fatalf("bad: %#v", resp2.Allocations)
   827  	}
   828  }
   829  
   830  func TestJobEndpoint_Allocations_Blocking(t *testing.T) {
   831  	s1 := testServer(t, nil)
   832  	defer s1.Shutdown()
   833  	codec := rpcClient(t, s1)
   834  	testutil.WaitForLeader(t, s1.RPC)
   835  
   836  	// Create the register request
   837  	alloc1 := mock.Alloc()
   838  	alloc2 := mock.Alloc()
   839  	alloc2.JobID = "job1"
   840  	state := s1.fsm.State()
   841  
   842  	// First upsert an unrelated alloc
   843  	time.AfterFunc(100*time.Millisecond, func() {
   844  		err := state.UpsertAllocs(100, []*structs.Allocation{alloc1})
   845  		if err != nil {
   846  			t.Fatalf("err: %v", err)
   847  		}
   848  	})
   849  
   850  	// Upsert an alloc for the job we are interested in later
   851  	time.AfterFunc(200*time.Millisecond, func() {
   852  		err := state.UpsertAllocs(200, []*structs.Allocation{alloc2})
   853  		if err != nil {
   854  			t.Fatalf("err: %v", err)
   855  		}
   856  	})
   857  
   858  	// Lookup the jobs
   859  	get := &structs.JobSpecificRequest{
   860  		JobID: "job1",
   861  		QueryOptions: structs.QueryOptions{
   862  			Region:        "global",
   863  			MinQueryIndex: 50,
   864  		},
   865  	}
   866  	var resp structs.JobAllocationsResponse
   867  	start := time.Now()
   868  	if err := msgpackrpc.CallWithCodec(codec, "Job.Allocations", get, &resp); err != nil {
   869  		t.Fatalf("err: %v", err)
   870  	}
   871  
   872  	if elapsed := time.Since(start); elapsed < 200*time.Millisecond {
   873  		t.Fatalf("should block (returned in %s) %#v", elapsed, resp)
   874  	}
   875  	if resp.Index != 200 {
   876  		t.Fatalf("Bad index: %d %d", resp.Index, 200)
   877  	}
   878  	if len(resp.Allocations) != 1 || resp.Allocations[0].JobID != "job1" {
   879  		t.Fatalf("bad: %#v", resp.Allocations)
   880  	}
   881  }
   882  
   883  func TestJobEndpoint_Evaluations(t *testing.T) {
   884  	s1 := testServer(t, nil)
   885  	defer s1.Shutdown()
   886  	codec := rpcClient(t, s1)
   887  	testutil.WaitForLeader(t, s1.RPC)
   888  
   889  	// Create the register request
   890  	eval1 := mock.Eval()
   891  	eval2 := mock.Eval()
   892  	eval2.JobID = eval1.JobID
   893  	state := s1.fsm.State()
   894  	err := state.UpsertEvals(1000,
   895  		[]*structs.Evaluation{eval1, eval2})
   896  	if err != nil {
   897  		t.Fatalf("err: %v", err)
   898  	}
   899  
   900  	// Lookup the jobs
   901  	get := &structs.JobSpecificRequest{
   902  		JobID:        eval1.JobID,
   903  		QueryOptions: structs.QueryOptions{Region: "global"},
   904  	}
   905  	var resp2 structs.JobEvaluationsResponse
   906  	if err := msgpackrpc.CallWithCodec(codec, "Job.Evaluations", get, &resp2); err != nil {
   907  		t.Fatalf("err: %v", err)
   908  	}
   909  	if resp2.Index != 1000 {
   910  		t.Fatalf("Bad index: %d %d", resp2.Index, 1000)
   911  	}
   912  
   913  	if len(resp2.Evaluations) != 2 {
   914  		t.Fatalf("bad: %#v", resp2.Evaluations)
   915  	}
   916  }
   917  
   918  func TestJobEndpoint_Plan_WithDiff(t *testing.T) {
   919  	s1 := testServer(t, func(c *Config) {
   920  		c.NumSchedulers = 0 // Prevent automatic dequeue
   921  	})
   922  	defer s1.Shutdown()
   923  	codec := rpcClient(t, s1)
   924  	testutil.WaitForLeader(t, s1.RPC)
   925  
   926  	// Create the register request
   927  	job := mock.Job()
   928  	req := &structs.JobRegisterRequest{
   929  		Job:          job,
   930  		WriteRequest: structs.WriteRequest{Region: "global"},
   931  	}
   932  
   933  	// Fetch the response
   934  	var resp structs.JobRegisterResponse
   935  	if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil {
   936  		t.Fatalf("err: %v", err)
   937  	}
   938  	if resp.Index == 0 {
   939  		t.Fatalf("bad index: %d", resp.Index)
   940  	}
   941  
   942  	// Create a plan request
   943  	planReq := &structs.JobPlanRequest{
   944  		Job:          job,
   945  		Diff:         true,
   946  		WriteRequest: structs.WriteRequest{Region: "global"},
   947  	}
   948  
   949  	// Fetch the response
   950  	var planResp structs.JobPlanResponse
   951  	if err := msgpackrpc.CallWithCodec(codec, "Job.Plan", planReq, &planResp); err != nil {
   952  		t.Fatalf("err: %v", err)
   953  	}
   954  
   955  	// Check the response
   956  	if planResp.JobModifyIndex == 0 {
   957  		t.Fatalf("bad cas: %d", planResp.JobModifyIndex)
   958  	}
   959  	if planResp.Annotations == nil {
   960  		t.Fatalf("no annotations")
   961  	}
   962  	if planResp.Diff == nil {
   963  		t.Fatalf("no diff")
   964  	}
   965  }
   966  
   967  func TestJobEndpoint_Plan_NoDiff(t *testing.T) {
   968  	s1 := testServer(t, func(c *Config) {
   969  		c.NumSchedulers = 0 // Prevent automatic dequeue
   970  	})
   971  	defer s1.Shutdown()
   972  	codec := rpcClient(t, s1)
   973  	testutil.WaitForLeader(t, s1.RPC)
   974  
   975  	// Create the register request
   976  	job := mock.Job()
   977  	req := &structs.JobRegisterRequest{
   978  		Job:          job,
   979  		WriteRequest: structs.WriteRequest{Region: "global"},
   980  	}
   981  
   982  	// Fetch the response
   983  	var resp structs.JobRegisterResponse
   984  	if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil {
   985  		t.Fatalf("err: %v", err)
   986  	}
   987  	if resp.Index == 0 {
   988  		t.Fatalf("bad index: %d", resp.Index)
   989  	}
   990  
   991  	// Create a plan request
   992  	planReq := &structs.JobPlanRequest{
   993  		Job:          job,
   994  		Diff:         false,
   995  		WriteRequest: structs.WriteRequest{Region: "global"},
   996  	}
   997  
   998  	// Fetch the response
   999  	var planResp structs.JobPlanResponse
  1000  	if err := msgpackrpc.CallWithCodec(codec, "Job.Plan", planReq, &planResp); err != nil {
  1001  		t.Fatalf("err: %v", err)
  1002  	}
  1003  
  1004  	// Check the response
  1005  	if planResp.JobModifyIndex == 0 {
  1006  		t.Fatalf("bad cas: %d", planResp.JobModifyIndex)
  1007  	}
  1008  	if planResp.Annotations == nil {
  1009  		t.Fatalf("no annotations")
  1010  	}
  1011  	if planResp.Diff != nil {
  1012  		t.Fatalf("got diff")
  1013  	}
  1014  }