github.com/maier/nomad@v0.4.1-0.20161110003312-a9e3d0b8549d/nomad/eval_endpoint_test.go (about)

     1  package nomad
     2  
     3  import (
     4  	"reflect"
     5  	"strings"
     6  	"testing"
     7  	"time"
     8  
     9  	"github.com/hashicorp/net-rpc-msgpackrpc"
    10  	"github.com/hashicorp/nomad/nomad/mock"
    11  	"github.com/hashicorp/nomad/nomad/structs"
    12  	"github.com/hashicorp/nomad/scheduler"
    13  	"github.com/hashicorp/nomad/testutil"
    14  )
    15  
    16  func TestEvalEndpoint_GetEval(t *testing.T) {
    17  	s1 := testServer(t, nil)
    18  	defer s1.Shutdown()
    19  	codec := rpcClient(t, s1)
    20  	testutil.WaitForLeader(t, s1.RPC)
    21  
    22  	// Create the register request
    23  	eval1 := mock.Eval()
    24  	s1.fsm.State().UpsertEvals(1000, []*structs.Evaluation{eval1})
    25  
    26  	// Lookup the eval
    27  	get := &structs.EvalSpecificRequest{
    28  		EvalID:       eval1.ID,
    29  		QueryOptions: structs.QueryOptions{Region: "global"},
    30  	}
    31  	var resp structs.SingleEvalResponse
    32  	if err := msgpackrpc.CallWithCodec(codec, "Eval.GetEval", get, &resp); err != nil {
    33  		t.Fatalf("err: %v", err)
    34  	}
    35  	if resp.Index != 1000 {
    36  		t.Fatalf("Bad index: %d %d", resp.Index, 1000)
    37  	}
    38  
    39  	if !reflect.DeepEqual(eval1, resp.Eval) {
    40  		t.Fatalf("bad: %#v %#v", eval1, resp.Eval)
    41  	}
    42  
    43  	// Lookup non-existing node
    44  	get.EvalID = structs.GenerateUUID()
    45  	if err := msgpackrpc.CallWithCodec(codec, "Eval.GetEval", get, &resp); err != nil {
    46  		t.Fatalf("err: %v", err)
    47  	}
    48  	if resp.Index != 1000 {
    49  		t.Fatalf("Bad index: %d %d", resp.Index, 1000)
    50  	}
    51  	if resp.Eval != nil {
    52  		t.Fatalf("unexpected eval")
    53  	}
    54  }
    55  
    56  func TestEvalEndpoint_GetEval_Blocking(t *testing.T) {
    57  	s1 := testServer(t, nil)
    58  	defer s1.Shutdown()
    59  	state := s1.fsm.State()
    60  	codec := rpcClient(t, s1)
    61  	testutil.WaitForLeader(t, s1.RPC)
    62  
    63  	// Create the evals
    64  	eval1 := mock.Eval()
    65  	eval2 := mock.Eval()
    66  
    67  	// First create an unrelated eval
    68  	time.AfterFunc(100*time.Millisecond, func() {
    69  		err := state.UpsertEvals(100, []*structs.Evaluation{eval1})
    70  		if err != nil {
    71  			t.Fatalf("err: %v", err)
    72  		}
    73  	})
    74  
    75  	// Upsert the eval we are watching later
    76  	time.AfterFunc(200*time.Millisecond, func() {
    77  		err := state.UpsertEvals(200, []*structs.Evaluation{eval2})
    78  		if err != nil {
    79  			t.Fatalf("err: %v", err)
    80  		}
    81  	})
    82  
    83  	// Lookup the eval
    84  	req := &structs.EvalSpecificRequest{
    85  		EvalID: eval2.ID,
    86  		QueryOptions: structs.QueryOptions{
    87  			Region:        "global",
    88  			MinQueryIndex: 50,
    89  		},
    90  	}
    91  	var resp structs.SingleEvalResponse
    92  	start := time.Now()
    93  	if err := msgpackrpc.CallWithCodec(codec, "Eval.GetEval", req, &resp); err != nil {
    94  		t.Fatalf("err: %v", err)
    95  	}
    96  
    97  	if elapsed := time.Since(start); elapsed < 200*time.Millisecond {
    98  		t.Fatalf("should block (returned in %s) %#v", elapsed, resp)
    99  	}
   100  	if resp.Index != 200 {
   101  		t.Fatalf("Bad index: %d %d", resp.Index, 200)
   102  	}
   103  	if resp.Eval == nil || resp.Eval.ID != eval2.ID {
   104  		t.Fatalf("bad: %#v", resp.Eval)
   105  	}
   106  
   107  	// Eval delete triggers watches
   108  	time.AfterFunc(100*time.Millisecond, func() {
   109  		err := state.DeleteEval(300, []string{eval2.ID}, []string{})
   110  		if err != nil {
   111  			t.Fatalf("err: %v", err)
   112  		}
   113  	})
   114  
   115  	req.QueryOptions.MinQueryIndex = 250
   116  	var resp2 structs.SingleEvalResponse
   117  	start = time.Now()
   118  	if err := msgpackrpc.CallWithCodec(codec, "Eval.GetEval", req, &resp2); err != nil {
   119  		t.Fatalf("err: %v", err)
   120  	}
   121  
   122  	if elapsed := time.Since(start); elapsed < 100*time.Millisecond {
   123  		t.Fatalf("should block (returned in %s) %#v", elapsed, resp2)
   124  	}
   125  	if resp2.Index != 300 {
   126  		t.Fatalf("Bad index: %d %d", resp2.Index, 300)
   127  	}
   128  	if resp2.Eval != nil {
   129  		t.Fatalf("bad: %#v", resp2.Eval)
   130  	}
   131  }
   132  
   133  func TestEvalEndpoint_Dequeue(t *testing.T) {
   134  	s1 := testServer(t, func(c *Config) {
   135  		c.NumSchedulers = 0 // Prevent automatic dequeue
   136  	})
   137  	defer s1.Shutdown()
   138  	codec := rpcClient(t, s1)
   139  	testutil.WaitForLeader(t, s1.RPC)
   140  
   141  	// Create the register request
   142  	eval1 := mock.Eval()
   143  	s1.evalBroker.Enqueue(eval1)
   144  
   145  	// Dequeue the eval
   146  	get := &structs.EvalDequeueRequest{
   147  		Schedulers:       defaultSched,
   148  		SchedulerVersion: scheduler.SchedulerVersion,
   149  		WriteRequest:     structs.WriteRequest{Region: "global"},
   150  	}
   151  	var resp structs.EvalDequeueResponse
   152  	if err := msgpackrpc.CallWithCodec(codec, "Eval.Dequeue", get, &resp); err != nil {
   153  		t.Fatalf("err: %v", err)
   154  	}
   155  
   156  	if !reflect.DeepEqual(eval1, resp.Eval) {
   157  		t.Fatalf("bad: %v %v", eval1, resp.Eval)
   158  	}
   159  
   160  	// Ensure outstanding
   161  	token, ok := s1.evalBroker.Outstanding(eval1.ID)
   162  	if !ok {
   163  		t.Fatalf("should be outstanding")
   164  	}
   165  	if token != resp.Token {
   166  		t.Fatalf("bad token: %#v %#v", token, resp.Token)
   167  	}
   168  }
   169  
   170  func TestEvalEndpoint_Dequeue_Version_Mismatch(t *testing.T) {
   171  	s1 := testServer(t, func(c *Config) {
   172  		c.NumSchedulers = 0 // Prevent automatic dequeue
   173  	})
   174  	defer s1.Shutdown()
   175  	codec := rpcClient(t, s1)
   176  	testutil.WaitForLeader(t, s1.RPC)
   177  
   178  	// Create the register request
   179  	eval1 := mock.Eval()
   180  	s1.evalBroker.Enqueue(eval1)
   181  
   182  	// Dequeue the eval
   183  	get := &structs.EvalDequeueRequest{
   184  		Schedulers:       defaultSched,
   185  		SchedulerVersion: 0,
   186  		WriteRequest:     structs.WriteRequest{Region: "global"},
   187  	}
   188  	var resp structs.EvalDequeueResponse
   189  	err := msgpackrpc.CallWithCodec(codec, "Eval.Dequeue", get, &resp)
   190  	if err == nil || !strings.Contains(err.Error(), "scheduler version is 0") {
   191  		t.Fatalf("err: %v", err)
   192  	}
   193  }
   194  
   195  func TestEvalEndpoint_Ack(t *testing.T) {
   196  	s1 := testServer(t, nil)
   197  	defer s1.Shutdown()
   198  	codec := rpcClient(t, s1)
   199  
   200  	testutil.WaitForResult(func() (bool, error) {
   201  		return s1.evalBroker.Enabled(), nil
   202  	}, func(err error) {
   203  		t.Fatalf("should enable eval broker")
   204  	})
   205  
   206  	// Create the register request
   207  	eval1 := mock.Eval()
   208  	s1.evalBroker.Enqueue(eval1)
   209  	out, token, err := s1.evalBroker.Dequeue(defaultSched, time.Second)
   210  	if err != nil {
   211  		t.Fatalf("err: %v", err)
   212  	}
   213  	if out == nil {
   214  		t.Fatalf("missing eval")
   215  	}
   216  
   217  	// Ack the eval
   218  	get := &structs.EvalAckRequest{
   219  		EvalID:       out.ID,
   220  		Token:        token,
   221  		WriteRequest: structs.WriteRequest{Region: "global"},
   222  	}
   223  	var resp structs.GenericResponse
   224  	if err := msgpackrpc.CallWithCodec(codec, "Eval.Ack", get, &resp); err != nil {
   225  		t.Fatalf("err: %v", err)
   226  	}
   227  
   228  	// Ensure outstanding
   229  	if _, ok := s1.evalBroker.Outstanding(eval1.ID); ok {
   230  		t.Fatalf("should not be outstanding")
   231  	}
   232  }
   233  
   234  func TestEvalEndpoint_Nack(t *testing.T) {
   235  	s1 := testServer(t, func(c *Config) {
   236  		// Disable all of the schedulers so we can manually dequeue
   237  		// evals and check the queue status
   238  		c.NumSchedulers = 0
   239  	})
   240  	defer s1.Shutdown()
   241  	codec := rpcClient(t, s1)
   242  
   243  	testutil.WaitForResult(func() (bool, error) {
   244  		return s1.evalBroker.Enabled(), nil
   245  	}, func(err error) {
   246  		t.Fatalf("should enable eval broker")
   247  	})
   248  
   249  	// Create the register request
   250  	eval1 := mock.Eval()
   251  	s1.evalBroker.Enqueue(eval1)
   252  	out, token, _ := s1.evalBroker.Dequeue(defaultSched, time.Second)
   253  	if out == nil {
   254  		t.Fatalf("missing eval")
   255  	}
   256  
   257  	// Nack the eval
   258  	get := &structs.EvalAckRequest{
   259  		EvalID:       out.ID,
   260  		Token:        token,
   261  		WriteRequest: structs.WriteRequest{Region: "global"},
   262  	}
   263  	var resp structs.GenericResponse
   264  	if err := msgpackrpc.CallWithCodec(codec, "Eval.Nack", get, &resp); err != nil {
   265  		t.Fatalf("err: %v", err)
   266  	}
   267  
   268  	// Ensure outstanding
   269  	if _, ok := s1.evalBroker.Outstanding(eval1.ID); ok {
   270  		t.Fatalf("should not be outstanding")
   271  	}
   272  
   273  	// Should get it back
   274  	out2, _, _ := s1.evalBroker.Dequeue(defaultSched, time.Second)
   275  	if out2 != out {
   276  		t.Fatalf("nack failed")
   277  	}
   278  }
   279  
   280  func TestEvalEndpoint_Update(t *testing.T) {
   281  	s1 := testServer(t, nil)
   282  	defer s1.Shutdown()
   283  	codec := rpcClient(t, s1)
   284  
   285  	testutil.WaitForResult(func() (bool, error) {
   286  		return s1.evalBroker.Enabled(), nil
   287  	}, func(err error) {
   288  		t.Fatalf("should enable eval broker")
   289  	})
   290  
   291  	// Create the register request
   292  	eval1 := mock.Eval()
   293  	s1.evalBroker.Enqueue(eval1)
   294  	out, token, err := s1.evalBroker.Dequeue(defaultSched, time.Second)
   295  	if err != nil {
   296  		t.Fatalf("err: %v", err)
   297  	}
   298  	if out == nil {
   299  		t.Fatalf("missing eval")
   300  	}
   301  
   302  	// Update the eval
   303  	eval2 := eval1.Copy()
   304  	eval2.Status = structs.EvalStatusComplete
   305  
   306  	get := &structs.EvalUpdateRequest{
   307  		Evals:        []*structs.Evaluation{eval2},
   308  		EvalToken:    token,
   309  		WriteRequest: structs.WriteRequest{Region: "global"},
   310  	}
   311  	var resp structs.GenericResponse
   312  	if err := msgpackrpc.CallWithCodec(codec, "Eval.Update", get, &resp); err != nil {
   313  		t.Fatalf("err: %v", err)
   314  	}
   315  
   316  	// Ensure updated
   317  	outE, err := s1.fsm.State().EvalByID(eval2.ID)
   318  	if err != nil {
   319  		t.Fatalf("err: %v", err)
   320  	}
   321  	if outE.Status != structs.EvalStatusComplete {
   322  		t.Fatalf("Bad: %#v", out)
   323  	}
   324  }
   325  
   326  func TestEvalEndpoint_Create(t *testing.T) {
   327  	s1 := testServer(t, func(c *Config) {
   328  		c.NumSchedulers = 0 // Prevent automatic dequeue
   329  	})
   330  	defer s1.Shutdown()
   331  	codec := rpcClient(t, s1)
   332  
   333  	testutil.WaitForResult(func() (bool, error) {
   334  		return s1.evalBroker.Enabled(), nil
   335  	}, func(err error) {
   336  		t.Fatalf("should enable eval broker")
   337  	})
   338  
   339  	// Create the register request
   340  	prev := mock.Eval()
   341  	s1.evalBroker.Enqueue(prev)
   342  	out, token, err := s1.evalBroker.Dequeue(defaultSched, time.Second)
   343  	if err != nil {
   344  		t.Fatalf("err: %v", err)
   345  	}
   346  	if out == nil {
   347  		t.Fatalf("missing eval")
   348  	}
   349  
   350  	// Create the register request
   351  	eval1 := mock.Eval()
   352  	eval1.PreviousEval = prev.ID
   353  	get := &structs.EvalUpdateRequest{
   354  		Evals:        []*structs.Evaluation{eval1},
   355  		EvalToken:    token,
   356  		WriteRequest: structs.WriteRequest{Region: "global"},
   357  	}
   358  	var resp structs.GenericResponse
   359  	if err := msgpackrpc.CallWithCodec(codec, "Eval.Create", get, &resp); err != nil {
   360  		t.Fatalf("err: %v", err)
   361  	}
   362  
   363  	// Ensure created
   364  	outE, err := s1.fsm.State().EvalByID(eval1.ID)
   365  	if err != nil {
   366  		t.Fatalf("err: %v", err)
   367  	}
   368  
   369  	eval1.CreateIndex = resp.Index
   370  	eval1.ModifyIndex = resp.Index
   371  	if !reflect.DeepEqual(eval1, outE) {
   372  		t.Fatalf("Bad: %#v %#v", outE, eval1)
   373  	}
   374  }
   375  
   376  func TestEvalEndpoint_Reap(t *testing.T) {
   377  	s1 := testServer(t, nil)
   378  	defer s1.Shutdown()
   379  	codec := rpcClient(t, s1)
   380  	testutil.WaitForLeader(t, s1.RPC)
   381  
   382  	// Create the register request
   383  	eval1 := mock.Eval()
   384  	s1.fsm.State().UpsertEvals(1000, []*structs.Evaluation{eval1})
   385  
   386  	// Reap the eval
   387  	get := &structs.EvalDeleteRequest{
   388  		Evals:        []string{eval1.ID},
   389  		WriteRequest: structs.WriteRequest{Region: "global"},
   390  	}
   391  	var resp structs.GenericResponse
   392  	if err := msgpackrpc.CallWithCodec(codec, "Eval.Reap", get, &resp); err != nil {
   393  		t.Fatalf("err: %v", err)
   394  	}
   395  	if resp.Index == 0 {
   396  		t.Fatalf("Bad index: %d", resp.Index)
   397  	}
   398  
   399  	// Ensure deleted
   400  	outE, err := s1.fsm.State().EvalByID(eval1.ID)
   401  	if err != nil {
   402  		t.Fatalf("err: %v", err)
   403  	}
   404  	if outE != nil {
   405  		t.Fatalf("Bad: %#v", outE)
   406  	}
   407  }
   408  
   409  func TestEvalEndpoint_List(t *testing.T) {
   410  	s1 := testServer(t, nil)
   411  	defer s1.Shutdown()
   412  	codec := rpcClient(t, s1)
   413  	testutil.WaitForLeader(t, s1.RPC)
   414  
   415  	// Create the register request
   416  	eval1 := mock.Eval()
   417  	eval1.ID = "aaaaaaaa-3350-4b4b-d185-0e1992ed43e9"
   418  	eval2 := mock.Eval()
   419  	eval2.ID = "aaaabbbb-3350-4b4b-d185-0e1992ed43e9"
   420  	s1.fsm.State().UpsertEvals(1000, []*structs.Evaluation{eval1, eval2})
   421  
   422  	// Lookup the eval
   423  	get := &structs.EvalListRequest{
   424  		QueryOptions: structs.QueryOptions{Region: "global"},
   425  	}
   426  	var resp structs.EvalListResponse
   427  	if err := msgpackrpc.CallWithCodec(codec, "Eval.List", get, &resp); err != nil {
   428  		t.Fatalf("err: %v", err)
   429  	}
   430  	if resp.Index != 1000 {
   431  		t.Fatalf("Bad index: %d %d", resp.Index, 1000)
   432  	}
   433  
   434  	if len(resp.Evaluations) != 2 {
   435  		t.Fatalf("bad: %#v", resp.Evaluations)
   436  	}
   437  
   438  	// Lookup the eval by prefix
   439  	get = &structs.EvalListRequest{
   440  		QueryOptions: structs.QueryOptions{Region: "global", Prefix: "aaaabb"},
   441  	}
   442  	var resp2 structs.EvalListResponse
   443  	if err := msgpackrpc.CallWithCodec(codec, "Eval.List", get, &resp2); err != nil {
   444  		t.Fatalf("err: %v", err)
   445  	}
   446  	if resp2.Index != 1000 {
   447  		t.Fatalf("Bad index: %d %d", resp2.Index, 1000)
   448  	}
   449  
   450  	if len(resp2.Evaluations) != 1 {
   451  		t.Fatalf("bad: %#v", resp2.Evaluations)
   452  	}
   453  
   454  }
   455  
   456  func TestEvalEndpoint_List_Blocking(t *testing.T) {
   457  	s1 := testServer(t, nil)
   458  	defer s1.Shutdown()
   459  	state := s1.fsm.State()
   460  	codec := rpcClient(t, s1)
   461  	testutil.WaitForLeader(t, s1.RPC)
   462  
   463  	// Create the ieval
   464  	eval := mock.Eval()
   465  
   466  	// Upsert eval triggers watches
   467  	time.AfterFunc(100*time.Millisecond, func() {
   468  		if err := state.UpsertEvals(2, []*structs.Evaluation{eval}); err != nil {
   469  			t.Fatalf("err: %v", err)
   470  		}
   471  	})
   472  
   473  	req := &structs.EvalListRequest{
   474  		QueryOptions: structs.QueryOptions{
   475  			Region:        "global",
   476  			MinQueryIndex: 1,
   477  		},
   478  	}
   479  	start := time.Now()
   480  	var resp structs.EvalListResponse
   481  	if err := msgpackrpc.CallWithCodec(codec, "Eval.List", req, &resp); err != nil {
   482  		t.Fatalf("err: %v", err)
   483  	}
   484  
   485  	if elapsed := time.Since(start); elapsed < 100*time.Millisecond {
   486  		t.Fatalf("should block (returned in %s) %#v", elapsed, resp)
   487  	}
   488  	if resp.Index != 2 {
   489  		t.Fatalf("Bad index: %d %d", resp.Index, 2)
   490  	}
   491  	if len(resp.Evaluations) != 1 || resp.Evaluations[0].ID != eval.ID {
   492  		t.Fatalf("bad: %#v", resp.Evaluations)
   493  	}
   494  
   495  	// Eval deletion triggers watches
   496  	time.AfterFunc(100*time.Millisecond, func() {
   497  		if err := state.DeleteEval(3, []string{eval.ID}, nil); err != nil {
   498  			t.Fatalf("err: %v", err)
   499  		}
   500  	})
   501  
   502  	req.MinQueryIndex = 2
   503  	start = time.Now()
   504  	var resp2 structs.EvalListResponse
   505  	if err := msgpackrpc.CallWithCodec(codec, "Eval.List", req, &resp2); err != nil {
   506  		t.Fatalf("err: %v", err)
   507  	}
   508  
   509  	if elapsed := time.Since(start); elapsed < 100*time.Millisecond {
   510  		t.Fatalf("should block (returned in %s) %#v", elapsed, resp2)
   511  	}
   512  	if resp2.Index != 3 {
   513  		t.Fatalf("Bad index: %d %d", resp2.Index, 3)
   514  	}
   515  	if len(resp2.Evaluations) != 0 {
   516  		t.Fatalf("bad: %#v", resp2.Evaluations)
   517  	}
   518  }
   519  
   520  func TestEvalEndpoint_Allocations(t *testing.T) {
   521  	s1 := testServer(t, nil)
   522  	defer s1.Shutdown()
   523  	codec := rpcClient(t, s1)
   524  	testutil.WaitForLeader(t, s1.RPC)
   525  
   526  	// Create the register request
   527  	alloc1 := mock.Alloc()
   528  	alloc2 := mock.Alloc()
   529  	alloc2.EvalID = alloc1.EvalID
   530  	state := s1.fsm.State()
   531  	state.UpsertJobSummary(998, mock.JobSummary(alloc1.JobID))
   532  	state.UpsertJobSummary(999, mock.JobSummary(alloc2.JobID))
   533  	err := state.UpsertAllocs(1000,
   534  		[]*structs.Allocation{alloc1, alloc2})
   535  	if err != nil {
   536  		t.Fatalf("err: %v", err)
   537  	}
   538  
   539  	// Lookup the eval
   540  	get := &structs.EvalSpecificRequest{
   541  		EvalID:       alloc1.EvalID,
   542  		QueryOptions: structs.QueryOptions{Region: "global"},
   543  	}
   544  	var resp structs.EvalAllocationsResponse
   545  	if err := msgpackrpc.CallWithCodec(codec, "Eval.Allocations", get, &resp); err != nil {
   546  		t.Fatalf("err: %v", err)
   547  	}
   548  	if resp.Index != 1000 {
   549  		t.Fatalf("Bad index: %d %d", resp.Index, 1000)
   550  	}
   551  
   552  	if len(resp.Allocations) != 2 {
   553  		t.Fatalf("bad: %#v", resp.Allocations)
   554  	}
   555  }
   556  
   557  func TestEvalEndpoint_Allocations_Blocking(t *testing.T) {
   558  	s1 := testServer(t, nil)
   559  	defer s1.Shutdown()
   560  	state := s1.fsm.State()
   561  	codec := rpcClient(t, s1)
   562  	testutil.WaitForLeader(t, s1.RPC)
   563  
   564  	// Create the allocs
   565  	alloc1 := mock.Alloc()
   566  	alloc2 := mock.Alloc()
   567  
   568  	// Upsert an unrelated alloc first
   569  	time.AfterFunc(100*time.Millisecond, func() {
   570  		state.UpsertJobSummary(99, mock.JobSummary(alloc1.JobID))
   571  		err := state.UpsertAllocs(100, []*structs.Allocation{alloc1})
   572  		if err != nil {
   573  			t.Fatalf("err: %v", err)
   574  		}
   575  	})
   576  
   577  	// Upsert an alloc which will trigger the watch later
   578  	time.AfterFunc(200*time.Millisecond, func() {
   579  		state.UpsertJobSummary(199, mock.JobSummary(alloc2.JobID))
   580  		err := state.UpsertAllocs(200, []*structs.Allocation{alloc2})
   581  		if err != nil {
   582  			t.Fatalf("err: %v", err)
   583  		}
   584  	})
   585  
   586  	// Lookup the eval
   587  	get := &structs.EvalSpecificRequest{
   588  		EvalID: alloc2.EvalID,
   589  		QueryOptions: structs.QueryOptions{
   590  			Region:        "global",
   591  			MinQueryIndex: 50,
   592  		},
   593  	}
   594  	var resp structs.EvalAllocationsResponse
   595  	start := time.Now()
   596  	if err := msgpackrpc.CallWithCodec(codec, "Eval.Allocations", get, &resp); err != nil {
   597  		t.Fatalf("err: %v", err)
   598  	}
   599  
   600  	if elapsed := time.Since(start); elapsed < 200*time.Millisecond {
   601  		t.Fatalf("should block (returned in %s) %#v", elapsed, resp)
   602  	}
   603  	if resp.Index != 200 {
   604  		t.Fatalf("Bad index: %d %d", resp.Index, 200)
   605  	}
   606  	if len(resp.Allocations) != 1 || resp.Allocations[0].ID != alloc2.ID {
   607  		t.Fatalf("bad: %#v", resp.Allocations)
   608  	}
   609  }
   610  
   611  func TestEvalEndpoint_Reblock_NonExistent(t *testing.T) {
   612  	s1 := testServer(t, func(c *Config) {
   613  		c.NumSchedulers = 0 // Prevent automatic dequeue
   614  	})
   615  	defer s1.Shutdown()
   616  	codec := rpcClient(t, s1)
   617  
   618  	testutil.WaitForResult(func() (bool, error) {
   619  		return s1.evalBroker.Enabled(), nil
   620  	}, func(err error) {
   621  		t.Fatalf("should enable eval broker")
   622  	})
   623  
   624  	// Create the register request
   625  	eval1 := mock.Eval()
   626  	s1.evalBroker.Enqueue(eval1)
   627  	out, token, err := s1.evalBroker.Dequeue(defaultSched, time.Second)
   628  	if err != nil {
   629  		t.Fatalf("err: %v", err)
   630  	}
   631  	if out == nil {
   632  		t.Fatalf("missing eval")
   633  	}
   634  
   635  	get := &structs.EvalUpdateRequest{
   636  		Evals:        []*structs.Evaluation{eval1},
   637  		EvalToken:    token,
   638  		WriteRequest: structs.WriteRequest{Region: "global"},
   639  	}
   640  	var resp structs.GenericResponse
   641  	if err := msgpackrpc.CallWithCodec(codec, "Eval.Reblock", get, &resp); err == nil {
   642  		t.Fatalf("expect error since eval does not exist")
   643  	}
   644  }
   645  
   646  func TestEvalEndpoint_Reblock_NonBlocked(t *testing.T) {
   647  	s1 := testServer(t, func(c *Config) {
   648  		c.NumSchedulers = 0 // Prevent automatic dequeue
   649  	})
   650  	defer s1.Shutdown()
   651  	codec := rpcClient(t, s1)
   652  
   653  	testutil.WaitForResult(func() (bool, error) {
   654  		return s1.evalBroker.Enabled(), nil
   655  	}, func(err error) {
   656  		t.Fatalf("should enable eval broker")
   657  	})
   658  
   659  	// Create the eval
   660  	eval1 := mock.Eval()
   661  	s1.evalBroker.Enqueue(eval1)
   662  
   663  	// Insert it into the state store
   664  	if err := s1.fsm.State().UpsertEvals(1000, []*structs.Evaluation{eval1}); err != nil {
   665  		t.Fatal(err)
   666  	}
   667  
   668  	out, token, err := s1.evalBroker.Dequeue(defaultSched, 2*time.Second)
   669  	if err != nil {
   670  		t.Fatalf("err: %v", err)
   671  	}
   672  	if out == nil {
   673  		t.Fatalf("missing eval")
   674  	}
   675  
   676  	get := &structs.EvalUpdateRequest{
   677  		Evals:        []*structs.Evaluation{eval1},
   678  		EvalToken:    token,
   679  		WriteRequest: structs.WriteRequest{Region: "global"},
   680  	}
   681  	var resp structs.GenericResponse
   682  	if err := msgpackrpc.CallWithCodec(codec, "Eval.Reblock", get, &resp); err == nil {
   683  		t.Fatalf("should error since eval was not in blocked state", err)
   684  	}
   685  }
   686  
   687  func TestEvalEndpoint_Reblock(t *testing.T) {
   688  	s1 := testServer(t, func(c *Config) {
   689  		c.NumSchedulers = 0 // Prevent automatic dequeue
   690  	})
   691  	defer s1.Shutdown()
   692  	codec := rpcClient(t, s1)
   693  
   694  	testutil.WaitForResult(func() (bool, error) {
   695  		return s1.evalBroker.Enabled(), nil
   696  	}, func(err error) {
   697  		t.Fatalf("should enable eval broker")
   698  	})
   699  
   700  	// Create the eval
   701  	eval1 := mock.Eval()
   702  	eval1.Status = structs.EvalStatusBlocked
   703  	s1.evalBroker.Enqueue(eval1)
   704  
   705  	// Insert it into the state store
   706  	if err := s1.fsm.State().UpsertEvals(1000, []*structs.Evaluation{eval1}); err != nil {
   707  		t.Fatal(err)
   708  	}
   709  
   710  	out, token, err := s1.evalBroker.Dequeue(defaultSched, 7*time.Second)
   711  	if err != nil {
   712  		t.Fatalf("err: %v", err)
   713  	}
   714  	if out == nil {
   715  		t.Fatalf("bad: %v", out)
   716  	}
   717  
   718  	get := &structs.EvalUpdateRequest{
   719  		Evals:        []*structs.Evaluation{eval1},
   720  		EvalToken:    token,
   721  		WriteRequest: structs.WriteRequest{Region: "global"},
   722  	}
   723  	var resp structs.GenericResponse
   724  	if err := msgpackrpc.CallWithCodec(codec, "Eval.Reblock", get, &resp); err != nil {
   725  		t.Fatalf("err: %v", err)
   726  	}
   727  
   728  	// Check that it is blocked
   729  	bStats := s1.blockedEvals.Stats()
   730  	if bStats.TotalBlocked+bStats.TotalEscaped == 0 {
   731  		t.Fatalf("ReblockEval didn't insert eval into the blocked eval tracker")
   732  	}
   733  }