github.com/dkerwin/nomad@v0.3.3-0.20160525181927-74554135514b/nomad/eval_endpoint_test.go (about)

     1  package nomad
     2  
     3  import (
     4  	"reflect"
     5  	"testing"
     6  	"time"
     7  
     8  	"github.com/hashicorp/net-rpc-msgpackrpc"
     9  	"github.com/hashicorp/nomad/nomad/mock"
    10  	"github.com/hashicorp/nomad/nomad/structs"
    11  	"github.com/hashicorp/nomad/testutil"
    12  )
    13  
    14  func TestEvalEndpoint_GetEval(t *testing.T) {
    15  	s1 := testServer(t, nil)
    16  	defer s1.Shutdown()
    17  	codec := rpcClient(t, s1)
    18  	testutil.WaitForLeader(t, s1.RPC)
    19  
    20  	// Create the register request
    21  	eval1 := mock.Eval()
    22  	s1.fsm.State().UpsertEvals(1000, []*structs.Evaluation{eval1})
    23  
    24  	// Lookup the eval
    25  	get := &structs.EvalSpecificRequest{
    26  		EvalID:       eval1.ID,
    27  		QueryOptions: structs.QueryOptions{Region: "global"},
    28  	}
    29  	var resp structs.SingleEvalResponse
    30  	if err := msgpackrpc.CallWithCodec(codec, "Eval.GetEval", get, &resp); err != nil {
    31  		t.Fatalf("err: %v", err)
    32  	}
    33  	if resp.Index != 1000 {
    34  		t.Fatalf("Bad index: %d %d", resp.Index, 1000)
    35  	}
    36  
    37  	if !reflect.DeepEqual(eval1, resp.Eval) {
    38  		t.Fatalf("bad: %#v %#v", eval1, resp.Eval)
    39  	}
    40  
    41  	// Lookup non-existing node
    42  	get.EvalID = structs.GenerateUUID()
    43  	if err := msgpackrpc.CallWithCodec(codec, "Eval.GetEval", get, &resp); err != nil {
    44  		t.Fatalf("err: %v", err)
    45  	}
    46  	if resp.Index != 1000 {
    47  		t.Fatalf("Bad index: %d %d", resp.Index, 1000)
    48  	}
    49  	if resp.Eval != nil {
    50  		t.Fatalf("unexpected eval")
    51  	}
    52  }
    53  
    54  func TestEvalEndpoint_GetEval_Blocking(t *testing.T) {
    55  	s1 := testServer(t, nil)
    56  	defer s1.Shutdown()
    57  	state := s1.fsm.State()
    58  	codec := rpcClient(t, s1)
    59  	testutil.WaitForLeader(t, s1.RPC)
    60  
    61  	// Create the evals
    62  	eval1 := mock.Eval()
    63  	eval2 := mock.Eval()
    64  
    65  	// First create an unrelated eval
    66  	time.AfterFunc(100*time.Millisecond, func() {
    67  		err := state.UpsertEvals(100, []*structs.Evaluation{eval1})
    68  		if err != nil {
    69  			t.Fatalf("err: %v", err)
    70  		}
    71  	})
    72  
    73  	// Upsert the eval we are watching later
    74  	time.AfterFunc(200*time.Millisecond, func() {
    75  		err := state.UpsertEvals(200, []*structs.Evaluation{eval2})
    76  		if err != nil {
    77  			t.Fatalf("err: %v", err)
    78  		}
    79  	})
    80  
    81  	// Lookup the eval
    82  	req := &structs.EvalSpecificRequest{
    83  		EvalID: eval2.ID,
    84  		QueryOptions: structs.QueryOptions{
    85  			Region:        "global",
    86  			MinQueryIndex: 50,
    87  		},
    88  	}
    89  	var resp structs.SingleEvalResponse
    90  	start := time.Now()
    91  	if err := msgpackrpc.CallWithCodec(codec, "Eval.GetEval", req, &resp); err != nil {
    92  		t.Fatalf("err: %v", err)
    93  	}
    94  
    95  	if elapsed := time.Since(start); elapsed < 200*time.Millisecond {
    96  		t.Fatalf("should block (returned in %s) %#v", elapsed, resp)
    97  	}
    98  	if resp.Index != 200 {
    99  		t.Fatalf("Bad index: %d %d", resp.Index, 200)
   100  	}
   101  	if resp.Eval == nil || resp.Eval.ID != eval2.ID {
   102  		t.Fatalf("bad: %#v", resp.Eval)
   103  	}
   104  
   105  	// Eval delete triggers watches
   106  	time.AfterFunc(100*time.Millisecond, func() {
   107  		err := state.DeleteEval(300, []string{eval2.ID}, []string{})
   108  		if err != nil {
   109  			t.Fatalf("err: %v", err)
   110  		}
   111  	})
   112  
   113  	req.QueryOptions.MinQueryIndex = 250
   114  	var resp2 structs.SingleEvalResponse
   115  	start = time.Now()
   116  	if err := msgpackrpc.CallWithCodec(codec, "Eval.GetEval", req, &resp2); err != nil {
   117  		t.Fatalf("err: %v", err)
   118  	}
   119  
   120  	if elapsed := time.Since(start); elapsed < 100*time.Millisecond {
   121  		t.Fatalf("should block (returned in %s) %#v", elapsed, resp2)
   122  	}
   123  	if resp2.Index != 300 {
   124  		t.Fatalf("Bad index: %d %d", resp2.Index, 300)
   125  	}
   126  	if resp2.Eval != nil {
   127  		t.Fatalf("bad: %#v", resp2.Eval)
   128  	}
   129  }
   130  
   131  func TestEvalEndpoint_Dequeue(t *testing.T) {
   132  	s1 := testServer(t, func(c *Config) {
   133  		c.NumSchedulers = 0 // Prevent automatic dequeue
   134  	})
   135  	defer s1.Shutdown()
   136  	codec := rpcClient(t, s1)
   137  	testutil.WaitForLeader(t, s1.RPC)
   138  
   139  	// Create the register request
   140  	eval1 := mock.Eval()
   141  	s1.evalBroker.Enqueue(eval1)
   142  
   143  	// Dequeue the eval
   144  	get := &structs.EvalDequeueRequest{
   145  		Schedulers:   defaultSched,
   146  		WriteRequest: structs.WriteRequest{Region: "global"},
   147  	}
   148  	var resp structs.EvalDequeueResponse
   149  	if err := msgpackrpc.CallWithCodec(codec, "Eval.Dequeue", get, &resp); err != nil {
   150  		t.Fatalf("err: %v", err)
   151  	}
   152  
   153  	if !reflect.DeepEqual(eval1, resp.Eval) {
   154  		t.Fatalf("bad: %v %v", eval1, resp.Eval)
   155  	}
   156  
   157  	// Ensure outstanding
   158  	token, ok := s1.evalBroker.Outstanding(eval1.ID)
   159  	if !ok {
   160  		t.Fatalf("should be outstanding")
   161  	}
   162  	if token != resp.Token {
   163  		t.Fatalf("bad token: %#v %#v", token, resp.Token)
   164  	}
   165  }
   166  
   167  func TestEvalEndpoint_Ack(t *testing.T) {
   168  	s1 := testServer(t, nil)
   169  	defer s1.Shutdown()
   170  	codec := rpcClient(t, s1)
   171  
   172  	testutil.WaitForResult(func() (bool, error) {
   173  		return s1.evalBroker.Enabled(), nil
   174  	}, func(err error) {
   175  		t.Fatalf("should enable eval broker")
   176  	})
   177  
   178  	// Create the register request
   179  	eval1 := mock.Eval()
   180  	s1.evalBroker.Enqueue(eval1)
   181  	out, token, err := s1.evalBroker.Dequeue(defaultSched, time.Second)
   182  	if err != nil {
   183  		t.Fatalf("err: %v", err)
   184  	}
   185  	if out == nil {
   186  		t.Fatalf("missing eval")
   187  	}
   188  
   189  	// Ack the eval
   190  	get := &structs.EvalAckRequest{
   191  		EvalID:       out.ID,
   192  		Token:        token,
   193  		WriteRequest: structs.WriteRequest{Region: "global"},
   194  	}
   195  	var resp structs.GenericResponse
   196  	if err := msgpackrpc.CallWithCodec(codec, "Eval.Ack", get, &resp); err != nil {
   197  		t.Fatalf("err: %v", err)
   198  	}
   199  
   200  	// Ensure outstanding
   201  	if _, ok := s1.evalBroker.Outstanding(eval1.ID); ok {
   202  		t.Fatalf("should not be outstanding")
   203  	}
   204  }
   205  
   206  func TestEvalEndpoint_Nack(t *testing.T) {
   207  	s1 := testServer(t, func(c *Config) {
   208  		// Disable all of the schedulers so we can manually dequeue
   209  		// evals and check the queue status
   210  		c.NumSchedulers = 0
   211  	})
   212  	defer s1.Shutdown()
   213  	codec := rpcClient(t, s1)
   214  
   215  	testutil.WaitForResult(func() (bool, error) {
   216  		return s1.evalBroker.Enabled(), nil
   217  	}, func(err error) {
   218  		t.Fatalf("should enable eval broker")
   219  	})
   220  
   221  	// Create the register request
   222  	eval1 := mock.Eval()
   223  	s1.evalBroker.Enqueue(eval1)
   224  	out, token, _ := s1.evalBroker.Dequeue(defaultSched, time.Second)
   225  	if out == nil {
   226  		t.Fatalf("missing eval")
   227  	}
   228  
   229  	// Nack the eval
   230  	get := &structs.EvalAckRequest{
   231  		EvalID:       out.ID,
   232  		Token:        token,
   233  		WriteRequest: structs.WriteRequest{Region: "global"},
   234  	}
   235  	var resp structs.GenericResponse
   236  	if err := msgpackrpc.CallWithCodec(codec, "Eval.Nack", get, &resp); err != nil {
   237  		t.Fatalf("err: %v", err)
   238  	}
   239  
   240  	// Ensure outstanding
   241  	if _, ok := s1.evalBroker.Outstanding(eval1.ID); ok {
   242  		t.Fatalf("should not be outstanding")
   243  	}
   244  
   245  	// Should get it back
   246  	out2, _, _ := s1.evalBroker.Dequeue(defaultSched, time.Second)
   247  	if out2 != out {
   248  		t.Fatalf("nack failed")
   249  	}
   250  }
   251  
   252  func TestEvalEndpoint_Update(t *testing.T) {
   253  	s1 := testServer(t, nil)
   254  	defer s1.Shutdown()
   255  	codec := rpcClient(t, s1)
   256  
   257  	testutil.WaitForResult(func() (bool, error) {
   258  		return s1.evalBroker.Enabled(), nil
   259  	}, func(err error) {
   260  		t.Fatalf("should enable eval broker")
   261  	})
   262  
   263  	// Create the register request
   264  	eval1 := mock.Eval()
   265  	s1.evalBroker.Enqueue(eval1)
   266  	out, token, err := s1.evalBroker.Dequeue(defaultSched, time.Second)
   267  	if err != nil {
   268  		t.Fatalf("err: %v", err)
   269  	}
   270  	if out == nil {
   271  		t.Fatalf("missing eval")
   272  	}
   273  
   274  	// Update the eval
   275  	eval2 := eval1.Copy()
   276  	eval2.Status = structs.EvalStatusComplete
   277  
   278  	get := &structs.EvalUpdateRequest{
   279  		Evals:        []*structs.Evaluation{eval2},
   280  		EvalToken:    token,
   281  		WriteRequest: structs.WriteRequest{Region: "global"},
   282  	}
   283  	var resp structs.GenericResponse
   284  	if err := msgpackrpc.CallWithCodec(codec, "Eval.Update", get, &resp); err != nil {
   285  		t.Fatalf("err: %v", err)
   286  	}
   287  
   288  	// Ensure updated
   289  	outE, err := s1.fsm.State().EvalByID(eval2.ID)
   290  	if err != nil {
   291  		t.Fatalf("err: %v", err)
   292  	}
   293  	if outE.Status != structs.EvalStatusComplete {
   294  		t.Fatalf("Bad: %#v", out)
   295  	}
   296  }
   297  
   298  func TestEvalEndpoint_Create(t *testing.T) {
   299  	s1 := testServer(t, func(c *Config) {
   300  		c.NumSchedulers = 0 // Prevent automatic dequeue
   301  	})
   302  	defer s1.Shutdown()
   303  	codec := rpcClient(t, s1)
   304  
   305  	testutil.WaitForResult(func() (bool, error) {
   306  		return s1.evalBroker.Enabled(), nil
   307  	}, func(err error) {
   308  		t.Fatalf("should enable eval broker")
   309  	})
   310  
   311  	// Create the register request
   312  	prev := mock.Eval()
   313  	s1.evalBroker.Enqueue(prev)
   314  	out, token, err := s1.evalBroker.Dequeue(defaultSched, time.Second)
   315  	if err != nil {
   316  		t.Fatalf("err: %v", err)
   317  	}
   318  	if out == nil {
   319  		t.Fatalf("missing eval")
   320  	}
   321  
   322  	// Create the register request
   323  	eval1 := mock.Eval()
   324  	eval1.PreviousEval = prev.ID
   325  	get := &structs.EvalUpdateRequest{
   326  		Evals:        []*structs.Evaluation{eval1},
   327  		EvalToken:    token,
   328  		WriteRequest: structs.WriteRequest{Region: "global"},
   329  	}
   330  	var resp structs.GenericResponse
   331  	if err := msgpackrpc.CallWithCodec(codec, "Eval.Create", get, &resp); err != nil {
   332  		t.Fatalf("err: %v", err)
   333  	}
   334  
   335  	// Ensure created
   336  	outE, err := s1.fsm.State().EvalByID(eval1.ID)
   337  	if err != nil {
   338  		t.Fatalf("err: %v", err)
   339  	}
   340  
   341  	eval1.CreateIndex = resp.Index
   342  	eval1.ModifyIndex = resp.Index
   343  	if !reflect.DeepEqual(eval1, outE) {
   344  		t.Fatalf("Bad: %#v %#v", outE, eval1)
   345  	}
   346  }
   347  
   348  func TestEvalEndpoint_Reap(t *testing.T) {
   349  	s1 := testServer(t, nil)
   350  	defer s1.Shutdown()
   351  	codec := rpcClient(t, s1)
   352  	testutil.WaitForLeader(t, s1.RPC)
   353  
   354  	// Create the register request
   355  	eval1 := mock.Eval()
   356  	s1.fsm.State().UpsertEvals(1000, []*structs.Evaluation{eval1})
   357  
   358  	// Reap the eval
   359  	get := &structs.EvalDeleteRequest{
   360  		Evals:        []string{eval1.ID},
   361  		WriteRequest: structs.WriteRequest{Region: "global"},
   362  	}
   363  	var resp structs.GenericResponse
   364  	if err := msgpackrpc.CallWithCodec(codec, "Eval.Reap", get, &resp); err != nil {
   365  		t.Fatalf("err: %v", err)
   366  	}
   367  	if resp.Index == 0 {
   368  		t.Fatalf("Bad index: %d", resp.Index)
   369  	}
   370  
   371  	// Ensure deleted
   372  	outE, err := s1.fsm.State().EvalByID(eval1.ID)
   373  	if err != nil {
   374  		t.Fatalf("err: %v", err)
   375  	}
   376  	if outE != nil {
   377  		t.Fatalf("Bad: %#v", outE)
   378  	}
   379  }
   380  
   381  func TestEvalEndpoint_List(t *testing.T) {
   382  	s1 := testServer(t, nil)
   383  	defer s1.Shutdown()
   384  	codec := rpcClient(t, s1)
   385  	testutil.WaitForLeader(t, s1.RPC)
   386  
   387  	// Create the register request
   388  	eval1 := mock.Eval()
   389  	eval1.ID = "aaaaaaaa-3350-4b4b-d185-0e1992ed43e9"
   390  	eval2 := mock.Eval()
   391  	eval2.ID = "aaaabbbb-3350-4b4b-d185-0e1992ed43e9"
   392  	s1.fsm.State().UpsertEvals(1000, []*structs.Evaluation{eval1, eval2})
   393  
   394  	// Lookup the eval
   395  	get := &structs.EvalListRequest{
   396  		QueryOptions: structs.QueryOptions{Region: "global"},
   397  	}
   398  	var resp structs.EvalListResponse
   399  	if err := msgpackrpc.CallWithCodec(codec, "Eval.List", get, &resp); err != nil {
   400  		t.Fatalf("err: %v", err)
   401  	}
   402  	if resp.Index != 1000 {
   403  		t.Fatalf("Bad index: %d %d", resp.Index, 1000)
   404  	}
   405  
   406  	if len(resp.Evaluations) != 2 {
   407  		t.Fatalf("bad: %#v", resp.Evaluations)
   408  	}
   409  
   410  	// Lookup the eval by prefix
   411  	get = &structs.EvalListRequest{
   412  		QueryOptions: structs.QueryOptions{Region: "global", Prefix: "aaaabb"},
   413  	}
   414  	var resp2 structs.EvalListResponse
   415  	if err := msgpackrpc.CallWithCodec(codec, "Eval.List", get, &resp2); err != nil {
   416  		t.Fatalf("err: %v", err)
   417  	}
   418  	if resp2.Index != 1000 {
   419  		t.Fatalf("Bad index: %d %d", resp2.Index, 1000)
   420  	}
   421  
   422  	if len(resp2.Evaluations) != 1 {
   423  		t.Fatalf("bad: %#v", resp2.Evaluations)
   424  	}
   425  
   426  }
   427  
   428  func TestEvalEndpoint_List_Blocking(t *testing.T) {
   429  	s1 := testServer(t, nil)
   430  	defer s1.Shutdown()
   431  	state := s1.fsm.State()
   432  	codec := rpcClient(t, s1)
   433  	testutil.WaitForLeader(t, s1.RPC)
   434  
   435  	// Create the ieval
   436  	eval := mock.Eval()
   437  
   438  	// Upsert eval triggers watches
   439  	time.AfterFunc(100*time.Millisecond, func() {
   440  		if err := state.UpsertEvals(2, []*structs.Evaluation{eval}); err != nil {
   441  			t.Fatalf("err: %v", err)
   442  		}
   443  	})
   444  
   445  	req := &structs.EvalListRequest{
   446  		QueryOptions: structs.QueryOptions{
   447  			Region:        "global",
   448  			MinQueryIndex: 1,
   449  		},
   450  	}
   451  	start := time.Now()
   452  	var resp structs.EvalListResponse
   453  	if err := msgpackrpc.CallWithCodec(codec, "Eval.List", req, &resp); err != nil {
   454  		t.Fatalf("err: %v", err)
   455  	}
   456  
   457  	if elapsed := time.Since(start); elapsed < 100*time.Millisecond {
   458  		t.Fatalf("should block (returned in %s) %#v", elapsed, resp)
   459  	}
   460  	if resp.Index != 2 {
   461  		t.Fatalf("Bad index: %d %d", resp.Index, 2)
   462  	}
   463  	if len(resp.Evaluations) != 1 || resp.Evaluations[0].ID != eval.ID {
   464  		t.Fatalf("bad: %#v", resp.Evaluations)
   465  	}
   466  
   467  	// Eval deletion triggers watches
   468  	time.AfterFunc(100*time.Millisecond, func() {
   469  		if err := state.DeleteEval(3, []string{eval.ID}, nil); err != nil {
   470  			t.Fatalf("err: %v", err)
   471  		}
   472  	})
   473  
   474  	req.MinQueryIndex = 2
   475  	start = time.Now()
   476  	var resp2 structs.EvalListResponse
   477  	if err := msgpackrpc.CallWithCodec(codec, "Eval.List", req, &resp2); err != nil {
   478  		t.Fatalf("err: %v", err)
   479  	}
   480  
   481  	if elapsed := time.Since(start); elapsed < 100*time.Millisecond {
   482  		t.Fatalf("should block (returned in %s) %#v", elapsed, resp2)
   483  	}
   484  	if resp2.Index != 3 {
   485  		t.Fatalf("Bad index: %d %d", resp2.Index, 3)
   486  	}
   487  	if len(resp2.Evaluations) != 0 {
   488  		t.Fatalf("bad: %#v", resp2.Evaluations)
   489  	}
   490  }
   491  
   492  func TestEvalEndpoint_Allocations(t *testing.T) {
   493  	s1 := testServer(t, nil)
   494  	defer s1.Shutdown()
   495  	codec := rpcClient(t, s1)
   496  	testutil.WaitForLeader(t, s1.RPC)
   497  
   498  	// Create the register request
   499  	alloc1 := mock.Alloc()
   500  	alloc2 := mock.Alloc()
   501  	alloc2.EvalID = alloc1.EvalID
   502  	state := s1.fsm.State()
   503  	err := state.UpsertAllocs(1000,
   504  		[]*structs.Allocation{alloc1, alloc2})
   505  	if err != nil {
   506  		t.Fatalf("err: %v", err)
   507  	}
   508  
   509  	// Lookup the eval
   510  	get := &structs.EvalSpecificRequest{
   511  		EvalID:       alloc1.EvalID,
   512  		QueryOptions: structs.QueryOptions{Region: "global"},
   513  	}
   514  	var resp structs.EvalAllocationsResponse
   515  	if err := msgpackrpc.CallWithCodec(codec, "Eval.Allocations", get, &resp); err != nil {
   516  		t.Fatalf("err: %v", err)
   517  	}
   518  	if resp.Index != 1000 {
   519  		t.Fatalf("Bad index: %d %d", resp.Index, 1000)
   520  	}
   521  
   522  	if len(resp.Allocations) != 2 {
   523  		t.Fatalf("bad: %#v", resp.Allocations)
   524  	}
   525  }
   526  
   527  func TestEvalEndpoint_Allocations_Blocking(t *testing.T) {
   528  	s1 := testServer(t, nil)
   529  	defer s1.Shutdown()
   530  	state := s1.fsm.State()
   531  	codec := rpcClient(t, s1)
   532  	testutil.WaitForLeader(t, s1.RPC)
   533  
   534  	// Create the allocs
   535  	alloc1 := mock.Alloc()
   536  	alloc2 := mock.Alloc()
   537  
   538  	// Upsert an unrelated alloc first
   539  	time.AfterFunc(100*time.Millisecond, func() {
   540  		err := state.UpsertAllocs(100, []*structs.Allocation{alloc1})
   541  		if err != nil {
   542  			t.Fatalf("err: %v", err)
   543  		}
   544  	})
   545  
   546  	// Upsert an alloc which will trigger the watch later
   547  	time.AfterFunc(200*time.Millisecond, func() {
   548  		err := state.UpsertAllocs(200, []*structs.Allocation{alloc2})
   549  		if err != nil {
   550  			t.Fatalf("err: %v", err)
   551  		}
   552  	})
   553  
   554  	// Lookup the eval
   555  	get := &structs.EvalSpecificRequest{
   556  		EvalID: alloc2.EvalID,
   557  		QueryOptions: structs.QueryOptions{
   558  			Region:        "global",
   559  			MinQueryIndex: 50,
   560  		},
   561  	}
   562  	var resp structs.EvalAllocationsResponse
   563  	start := time.Now()
   564  	if err := msgpackrpc.CallWithCodec(codec, "Eval.Allocations", get, &resp); err != nil {
   565  		t.Fatalf("err: %v", err)
   566  	}
   567  
   568  	if elapsed := time.Since(start); elapsed < 200*time.Millisecond {
   569  		t.Fatalf("should block (returned in %s) %#v", elapsed, resp)
   570  	}
   571  	if resp.Index != 200 {
   572  		t.Fatalf("Bad index: %d %d", resp.Index, 200)
   573  	}
   574  	if len(resp.Allocations) != 1 || resp.Allocations[0].ID != alloc2.ID {
   575  		t.Fatalf("bad: %#v", resp.Allocations)
   576  	}
   577  }
   578  
   579  func TestEvalEndpoint_Reblock_NonExistent(t *testing.T) {
   580  	s1 := testServer(t, nil)
   581  	defer s1.Shutdown()
   582  	codec := rpcClient(t, s1)
   583  
   584  	testutil.WaitForResult(func() (bool, error) {
   585  		return s1.evalBroker.Enabled(), nil
   586  	}, func(err error) {
   587  		t.Fatalf("should enable eval broker")
   588  	})
   589  
   590  	// Create the register request
   591  	eval1 := mock.Eval()
   592  	s1.evalBroker.Enqueue(eval1)
   593  	out, token, err := s1.evalBroker.Dequeue(defaultSched, time.Second)
   594  	if err != nil {
   595  		t.Fatalf("err: %v", err)
   596  	}
   597  	if out == nil {
   598  		t.Fatalf("missing eval")
   599  	}
   600  
   601  	get := &structs.EvalUpdateRequest{
   602  		Evals:        []*structs.Evaluation{eval1},
   603  		EvalToken:    token,
   604  		WriteRequest: structs.WriteRequest{Region: "global"},
   605  	}
   606  	var resp structs.GenericResponse
   607  	if err := msgpackrpc.CallWithCodec(codec, "Eval.Reblock", get, &resp); err == nil {
   608  		t.Fatalf("expect error since eval does not exist")
   609  	}
   610  }
   611  
   612  func TestEvalEndpoint_Reblock_NonBlocked(t *testing.T) {
   613  	s1 := testServer(t, nil)
   614  	defer s1.Shutdown()
   615  	codec := rpcClient(t, s1)
   616  
   617  	testutil.WaitForResult(func() (bool, error) {
   618  		return s1.evalBroker.Enabled(), nil
   619  	}, func(err error) {
   620  		t.Fatalf("should enable eval broker")
   621  	})
   622  
   623  	// Create the eval
   624  	eval1 := mock.Eval()
   625  	s1.evalBroker.Enqueue(eval1)
   626  
   627  	// Insert it into the state store
   628  	if err := s1.fsm.State().UpsertEvals(1000, []*structs.Evaluation{eval1}); err != nil {
   629  		t.Fatal(err)
   630  	}
   631  
   632  	out, token, err := s1.evalBroker.Dequeue(defaultSched, time.Second)
   633  	if err != nil {
   634  		t.Fatalf("err: %v", err)
   635  	}
   636  	if out == nil {
   637  		t.Fatalf("missing eval")
   638  	}
   639  
   640  	get := &structs.EvalUpdateRequest{
   641  		Evals:        []*structs.Evaluation{eval1},
   642  		EvalToken:    token,
   643  		WriteRequest: structs.WriteRequest{Region: "global"},
   644  	}
   645  	var resp structs.GenericResponse
   646  	if err := msgpackrpc.CallWithCodec(codec, "Eval.Reblock", get, &resp); err == nil {
   647  		t.Fatalf("should error since eval was not in blocked state", err)
   648  	}
   649  }
   650  
   651  func TestEvalEndpoint_Reblock(t *testing.T) {
   652  	s1 := testServer(t, nil)
   653  	defer s1.Shutdown()
   654  	codec := rpcClient(t, s1)
   655  
   656  	testutil.WaitForResult(func() (bool, error) {
   657  		return s1.evalBroker.Enabled(), nil
   658  	}, func(err error) {
   659  		t.Fatalf("should enable eval broker")
   660  	})
   661  
   662  	// Create the eval
   663  	eval1 := mock.Eval()
   664  	eval1.Status = structs.EvalStatusBlocked
   665  	s1.evalBroker.Enqueue(eval1)
   666  
   667  	// Insert it into the state store
   668  	if err := s1.fsm.State().UpsertEvals(1000, []*structs.Evaluation{eval1}); err != nil {
   669  		t.Fatal(err)
   670  	}
   671  
   672  	out, token, err := s1.evalBroker.Dequeue(defaultSched, time.Second)
   673  	if err != nil {
   674  		t.Fatalf("err: %v", err)
   675  	}
   676  	if out == nil {
   677  		t.Fatalf("missing eval")
   678  	}
   679  
   680  	get := &structs.EvalUpdateRequest{
   681  		Evals:        []*structs.Evaluation{eval1},
   682  		EvalToken:    token,
   683  		WriteRequest: structs.WriteRequest{Region: "global"},
   684  	}
   685  	var resp structs.GenericResponse
   686  	if err := msgpackrpc.CallWithCodec(codec, "Eval.Reblock", get, &resp); err != nil {
   687  		t.Fatalf("err: %v", err)
   688  	}
   689  
   690  	// Check that it is blocked
   691  	bStats := s1.blockedEvals.Stats()
   692  	if bStats.TotalBlocked+bStats.TotalEscaped == 0 {
   693  		t.Fatalf("ReblockEval didn't insert eval into the blocked eval tracker")
   694  	}
   695  }