github.com/dkerwin/nomad@v0.3.3-0.20160525181927-74554135514b/scheduler/system_sched_test.go (about)

     1  package scheduler
     2  
     3  import (
     4  	"reflect"
     5  	"testing"
     6  	"time"
     7  
     8  	"github.com/hashicorp/nomad/nomad/mock"
     9  	"github.com/hashicorp/nomad/nomad/structs"
    10  )
    11  
    12  func TestSystemSched_JobRegister(t *testing.T) {
    13  	h := NewHarness(t)
    14  
    15  	// Create some nodes
    16  	for i := 0; i < 10; i++ {
    17  		node := mock.Node()
    18  		noErr(t, h.State.UpsertNode(h.NextIndex(), node))
    19  	}
    20  
    21  	// Create a job
    22  	job := mock.SystemJob()
    23  	noErr(t, h.State.UpsertJob(h.NextIndex(), job))
    24  
    25  	// Create a mock evaluation to deregister the job
    26  	eval := &structs.Evaluation{
    27  		ID:          structs.GenerateUUID(),
    28  		Priority:    job.Priority,
    29  		TriggeredBy: structs.EvalTriggerJobRegister,
    30  		JobID:       job.ID,
    31  	}
    32  
    33  	// Process the evaluation
    34  	err := h.Process(NewSystemScheduler, eval)
    35  	if err != nil {
    36  		t.Fatalf("err: %v", err)
    37  	}
    38  
    39  	// Ensure a single plan
    40  	if len(h.Plans) != 1 {
    41  		t.Fatalf("bad: %#v", h.Plans)
    42  	}
    43  	plan := h.Plans[0]
    44  
    45  	// Ensure the plan doesn't have annotations.
    46  	if plan.Annotations != nil {
    47  		t.Fatalf("expected no annotations")
    48  	}
    49  
    50  	// Ensure the plan allocated
    51  	var planned []*structs.Allocation
    52  	for _, allocList := range plan.NodeAllocation {
    53  		planned = append(planned, allocList...)
    54  	}
    55  	if len(planned) != 10 {
    56  		t.Fatalf("bad: %#v", plan)
    57  	}
    58  
    59  	// Lookup the allocations by JobID
    60  	out, err := h.State.AllocsByJob(job.ID)
    61  	noErr(t, err)
    62  
    63  	// Ensure all allocations placed
    64  	if len(out) != 10 {
    65  		t.Fatalf("bad: %#v", out)
    66  	}
    67  
    68  	// Check the available nodes
    69  	if count, ok := out[0].Metrics.NodesAvailable["dc1"]; !ok || count != 10 {
    70  		t.Fatalf("bad: %#v", out[0].Metrics)
    71  	}
    72  
    73  	h.AssertEvalStatus(t, structs.EvalStatusComplete)
    74  }
    75  
    76  func TestSystemSched_JobRegister_Annotate(t *testing.T) {
    77  	h := NewHarness(t)
    78  
    79  	// Create some nodes
    80  	for i := 0; i < 10; i++ {
    81  		node := mock.Node()
    82  		noErr(t, h.State.UpsertNode(h.NextIndex(), node))
    83  	}
    84  
    85  	// Create a job
    86  	job := mock.SystemJob()
    87  	noErr(t, h.State.UpsertJob(h.NextIndex(), job))
    88  
    89  	// Create a mock evaluation to deregister the job
    90  	eval := &structs.Evaluation{
    91  		ID:           structs.GenerateUUID(),
    92  		Priority:     job.Priority,
    93  		TriggeredBy:  structs.EvalTriggerJobRegister,
    94  		JobID:        job.ID,
    95  		AnnotatePlan: true,
    96  	}
    97  
    98  	// Process the evaluation
    99  	err := h.Process(NewSystemScheduler, eval)
   100  	if err != nil {
   101  		t.Fatalf("err: %v", err)
   102  	}
   103  
   104  	// Ensure a single plan
   105  	if len(h.Plans) != 1 {
   106  		t.Fatalf("bad: %#v", h.Plans)
   107  	}
   108  	plan := h.Plans[0]
   109  
   110  	// Ensure the plan allocated
   111  	var planned []*structs.Allocation
   112  	for _, allocList := range plan.NodeAllocation {
   113  		planned = append(planned, allocList...)
   114  	}
   115  	if len(planned) != 10 {
   116  		t.Fatalf("bad: %#v", plan)
   117  	}
   118  
   119  	// Lookup the allocations by JobID
   120  	out, err := h.State.AllocsByJob(job.ID)
   121  	noErr(t, err)
   122  
   123  	// Ensure all allocations placed
   124  	if len(out) != 10 {
   125  		t.Fatalf("bad: %#v", out)
   126  	}
   127  
   128  	// Check the available nodes
   129  	if count, ok := out[0].Metrics.NodesAvailable["dc1"]; !ok || count != 10 {
   130  		t.Fatalf("bad: %#v", out[0].Metrics)
   131  	}
   132  
   133  	h.AssertEvalStatus(t, structs.EvalStatusComplete)
   134  
   135  	// Ensure the plan had annotations.
   136  	if plan.Annotations == nil {
   137  		t.Fatalf("expected annotations")
   138  	}
   139  
   140  	desiredTGs := plan.Annotations.DesiredTGUpdates
   141  	if l := len(desiredTGs); l != 1 {
   142  		t.Fatalf("incorrect number of task groups; got %v; want %v", l, 1)
   143  	}
   144  
   145  	desiredChanges, ok := desiredTGs["web"]
   146  	if !ok {
   147  		t.Fatalf("expected task group web to have desired changes")
   148  	}
   149  
   150  	expected := &structs.DesiredUpdates{Place: 10}
   151  	if !reflect.DeepEqual(desiredChanges, expected) {
   152  		t.Fatalf("Unexpected desired updates; got %#v; want %#v", desiredChanges, expected)
   153  	}
   154  }
   155  
   156  func TestSystemSched_JobRegister_AddNode(t *testing.T) {
   157  	h := NewHarness(t)
   158  
   159  	// Create some nodes
   160  	var nodes []*structs.Node
   161  	for i := 0; i < 10; i++ {
   162  		node := mock.Node()
   163  		nodes = append(nodes, node)
   164  		noErr(t, h.State.UpsertNode(h.NextIndex(), node))
   165  	}
   166  
   167  	// Generate a fake job with allocations
   168  	job := mock.SystemJob()
   169  	noErr(t, h.State.UpsertJob(h.NextIndex(), job))
   170  
   171  	var allocs []*structs.Allocation
   172  	for _, node := range nodes {
   173  		alloc := mock.Alloc()
   174  		alloc.Job = job
   175  		alloc.JobID = job.ID
   176  		alloc.NodeID = node.ID
   177  		alloc.Name = "my-job.web[0]"
   178  		allocs = append(allocs, alloc)
   179  	}
   180  	noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs))
   181  
   182  	// Add a new node.
   183  	node := mock.Node()
   184  	noErr(t, h.State.UpsertNode(h.NextIndex(), node))
   185  
   186  	// Create a mock evaluation to deal with the node update
   187  	eval := &structs.Evaluation{
   188  		ID:          structs.GenerateUUID(),
   189  		Priority:    50,
   190  		TriggeredBy: structs.EvalTriggerNodeUpdate,
   191  		JobID:       job.ID,
   192  	}
   193  
   194  	// Process the evaluation
   195  	err := h.Process(NewSystemScheduler, eval)
   196  	if err != nil {
   197  		t.Fatalf("err: %v", err)
   198  	}
   199  
   200  	// Ensure a single plan
   201  	if len(h.Plans) != 1 {
   202  		t.Fatalf("bad: %#v", h.Plans)
   203  	}
   204  	plan := h.Plans[0]
   205  
   206  	// Ensure the plan had no node updates
   207  	var update []*structs.Allocation
   208  	for _, updateList := range plan.NodeUpdate {
   209  		update = append(update, updateList...)
   210  	}
   211  	if len(update) != 0 {
   212  		t.Log(len(update))
   213  		t.Fatalf("bad: %#v", plan)
   214  	}
   215  
   216  	// Ensure the plan allocated on the new node
   217  	var planned []*structs.Allocation
   218  	for _, allocList := range plan.NodeAllocation {
   219  		planned = append(planned, allocList...)
   220  	}
   221  	if len(planned) != 1 {
   222  		t.Fatalf("bad: %#v", plan)
   223  	}
   224  
   225  	// Ensure it allocated on the right node
   226  	if _, ok := plan.NodeAllocation[node.ID]; !ok {
   227  		t.Fatalf("allocated on wrong node: %#v", plan)
   228  	}
   229  
   230  	// Lookup the allocations by JobID
   231  	out, err := h.State.AllocsByJob(job.ID)
   232  	noErr(t, err)
   233  
   234  	// Ensure all allocations placed
   235  	out = structs.FilterTerminalAllocs(out)
   236  	if len(out) != 11 {
   237  		t.Fatalf("bad: %#v", out)
   238  	}
   239  
   240  	h.AssertEvalStatus(t, structs.EvalStatusComplete)
   241  }
   242  
   243  func TestSystemSched_JobRegister_AllocFail(t *testing.T) {
   244  	h := NewHarness(t)
   245  
   246  	// Create NO nodes
   247  	// Create a job
   248  	job := mock.SystemJob()
   249  	noErr(t, h.State.UpsertJob(h.NextIndex(), job))
   250  
   251  	// Create a mock evaluation to register the job
   252  	eval := &structs.Evaluation{
   253  		ID:          structs.GenerateUUID(),
   254  		Priority:    job.Priority,
   255  		TriggeredBy: structs.EvalTriggerJobRegister,
   256  		JobID:       job.ID,
   257  	}
   258  
   259  	// Process the evaluation
   260  	err := h.Process(NewSystemScheduler, eval)
   261  	if err != nil {
   262  		t.Fatalf("err: %v", err)
   263  	}
   264  
   265  	// Ensure no plan as this should be a no-op.
   266  	if len(h.Plans) != 0 {
   267  		t.Fatalf("bad: %#v", h.Plans)
   268  	}
   269  
   270  	h.AssertEvalStatus(t, structs.EvalStatusComplete)
   271  }
   272  
   273  func TestSystemSched_JobModify(t *testing.T) {
   274  	h := NewHarness(t)
   275  
   276  	// Create some nodes
   277  	var nodes []*structs.Node
   278  	for i := 0; i < 10; i++ {
   279  		node := mock.Node()
   280  		nodes = append(nodes, node)
   281  		noErr(t, h.State.UpsertNode(h.NextIndex(), node))
   282  	}
   283  
   284  	// Generate a fake job with allocations
   285  	job := mock.SystemJob()
   286  	noErr(t, h.State.UpsertJob(h.NextIndex(), job))
   287  
   288  	var allocs []*structs.Allocation
   289  	for _, node := range nodes {
   290  		alloc := mock.Alloc()
   291  		alloc.Job = job
   292  		alloc.JobID = job.ID
   293  		alloc.NodeID = node.ID
   294  		alloc.Name = "my-job.web[0]"
   295  		allocs = append(allocs, alloc)
   296  	}
   297  	noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs))
   298  
   299  	// Add a few terminal status allocations, these should be ignored
   300  	var terminal []*structs.Allocation
   301  	for i := 0; i < 5; i++ {
   302  		alloc := mock.Alloc()
   303  		alloc.Job = job
   304  		alloc.JobID = job.ID
   305  		alloc.NodeID = nodes[i].ID
   306  		alloc.Name = "my-job.web[0]"
   307  		alloc.DesiredStatus = structs.AllocDesiredStatusFailed
   308  		terminal = append(terminal, alloc)
   309  	}
   310  	noErr(t, h.State.UpsertAllocs(h.NextIndex(), terminal))
   311  
   312  	// Update the job
   313  	job2 := mock.SystemJob()
   314  	job2.ID = job.ID
   315  
   316  	// Update the task, such that it cannot be done in-place
   317  	job2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other"
   318  	noErr(t, h.State.UpsertJob(h.NextIndex(), job2))
   319  
   320  	// Create a mock evaluation to deal with drain
   321  	eval := &structs.Evaluation{
   322  		ID:          structs.GenerateUUID(),
   323  		Priority:    50,
   324  		TriggeredBy: structs.EvalTriggerJobRegister,
   325  		JobID:       job.ID,
   326  	}
   327  
   328  	// Process the evaluation
   329  	err := h.Process(NewSystemScheduler, eval)
   330  	if err != nil {
   331  		t.Fatalf("err: %v", err)
   332  	}
   333  
   334  	// Ensure a single plan
   335  	if len(h.Plans) != 1 {
   336  		t.Fatalf("bad: %#v", h.Plans)
   337  	}
   338  	plan := h.Plans[0]
   339  
   340  	// Ensure the plan evicted all allocs
   341  	var update []*structs.Allocation
   342  	for _, updateList := range plan.NodeUpdate {
   343  		update = append(update, updateList...)
   344  	}
   345  	if len(update) != len(allocs) {
   346  		t.Fatalf("bad: %#v", plan)
   347  	}
   348  
   349  	// Ensure the plan allocated
   350  	var planned []*structs.Allocation
   351  	for _, allocList := range plan.NodeAllocation {
   352  		planned = append(planned, allocList...)
   353  	}
   354  	if len(planned) != 10 {
   355  		t.Fatalf("bad: %#v", plan)
   356  	}
   357  
   358  	// Lookup the allocations by JobID
   359  	out, err := h.State.AllocsByJob(job.ID)
   360  	noErr(t, err)
   361  
   362  	// Ensure all allocations placed
   363  	out = structs.FilterTerminalAllocs(out)
   364  	if len(out) != 10 {
   365  		t.Fatalf("bad: %#v", out)
   366  	}
   367  
   368  	h.AssertEvalStatus(t, structs.EvalStatusComplete)
   369  }
   370  
   371  func TestSystemSched_JobModify_Rolling(t *testing.T) {
   372  	h := NewHarness(t)
   373  
   374  	// Create some nodes
   375  	var nodes []*structs.Node
   376  	for i := 0; i < 10; i++ {
   377  		node := mock.Node()
   378  		nodes = append(nodes, node)
   379  		noErr(t, h.State.UpsertNode(h.NextIndex(), node))
   380  	}
   381  
   382  	// Generate a fake job with allocations
   383  	job := mock.SystemJob()
   384  	noErr(t, h.State.UpsertJob(h.NextIndex(), job))
   385  
   386  	var allocs []*structs.Allocation
   387  	for _, node := range nodes {
   388  		alloc := mock.Alloc()
   389  		alloc.Job = job
   390  		alloc.JobID = job.ID
   391  		alloc.NodeID = node.ID
   392  		alloc.Name = "my-job.web[0]"
   393  		allocs = append(allocs, alloc)
   394  	}
   395  	noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs))
   396  
   397  	// Update the job
   398  	job2 := mock.SystemJob()
   399  	job2.ID = job.ID
   400  	job2.Update = structs.UpdateStrategy{
   401  		Stagger:     30 * time.Second,
   402  		MaxParallel: 5,
   403  	}
   404  
   405  	// Update the task, such that it cannot be done in-place
   406  	job2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other"
   407  	noErr(t, h.State.UpsertJob(h.NextIndex(), job2))
   408  
   409  	// Create a mock evaluation to deal with drain
   410  	eval := &structs.Evaluation{
   411  		ID:          structs.GenerateUUID(),
   412  		Priority:    50,
   413  		TriggeredBy: structs.EvalTriggerJobRegister,
   414  		JobID:       job.ID,
   415  	}
   416  
   417  	// Process the evaluation
   418  	err := h.Process(NewSystemScheduler, eval)
   419  	if err != nil {
   420  		t.Fatalf("err: %v", err)
   421  	}
   422  
   423  	// Ensure a single plan
   424  	if len(h.Plans) != 1 {
   425  		t.Fatalf("bad: %#v", h.Plans)
   426  	}
   427  	plan := h.Plans[0]
   428  
   429  	// Ensure the plan evicted only MaxParallel
   430  	var update []*structs.Allocation
   431  	for _, updateList := range plan.NodeUpdate {
   432  		update = append(update, updateList...)
   433  	}
   434  	if len(update) != job2.Update.MaxParallel {
   435  		t.Fatalf("bad: %#v", plan)
   436  	}
   437  
   438  	// Ensure the plan allocated
   439  	var planned []*structs.Allocation
   440  	for _, allocList := range plan.NodeAllocation {
   441  		planned = append(planned, allocList...)
   442  	}
   443  	if len(planned) != job2.Update.MaxParallel {
   444  		t.Fatalf("bad: %#v", plan)
   445  	}
   446  
   447  	h.AssertEvalStatus(t, structs.EvalStatusComplete)
   448  
   449  	// Ensure a follow up eval was created
   450  	eval = h.Evals[0]
   451  	if eval.NextEval == "" {
   452  		t.Fatalf("missing next eval")
   453  	}
   454  
   455  	// Check for create
   456  	if len(h.CreateEvals) == 0 {
   457  		t.Fatalf("missing created eval")
   458  	}
   459  	create := h.CreateEvals[0]
   460  	if eval.NextEval != create.ID {
   461  		t.Fatalf("ID mismatch")
   462  	}
   463  	if create.PreviousEval != eval.ID {
   464  		t.Fatalf("missing previous eval")
   465  	}
   466  
   467  	if create.TriggeredBy != structs.EvalTriggerRollingUpdate {
   468  		t.Fatalf("bad: %#v", create)
   469  	}
   470  }
   471  
   472  func TestSystemSched_JobModify_InPlace(t *testing.T) {
   473  	h := NewHarness(t)
   474  
   475  	// Create some nodes
   476  	var nodes []*structs.Node
   477  	for i := 0; i < 10; i++ {
   478  		node := mock.Node()
   479  		nodes = append(nodes, node)
   480  		noErr(t, h.State.UpsertNode(h.NextIndex(), node))
   481  	}
   482  
   483  	// Generate a fake job with allocations
   484  	job := mock.SystemJob()
   485  	noErr(t, h.State.UpsertJob(h.NextIndex(), job))
   486  
   487  	var allocs []*structs.Allocation
   488  	for _, node := range nodes {
   489  		alloc := mock.Alloc()
   490  		alloc.Job = job
   491  		alloc.JobID = job.ID
   492  		alloc.NodeID = node.ID
   493  		alloc.Name = "my-job.web[0]"
   494  		allocs = append(allocs, alloc)
   495  	}
   496  	noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs))
   497  
   498  	// Update the job
   499  	job2 := mock.SystemJob()
   500  	job2.ID = job.ID
   501  	noErr(t, h.State.UpsertJob(h.NextIndex(), job2))
   502  
   503  	// Create a mock evaluation to deal with drain
   504  	eval := &structs.Evaluation{
   505  		ID:          structs.GenerateUUID(),
   506  		Priority:    50,
   507  		TriggeredBy: structs.EvalTriggerJobRegister,
   508  		JobID:       job.ID,
   509  	}
   510  
   511  	// Process the evaluation
   512  	err := h.Process(NewSystemScheduler, eval)
   513  	if err != nil {
   514  		t.Fatalf("err: %v", err)
   515  	}
   516  
   517  	// Ensure a single plan
   518  	if len(h.Plans) != 1 {
   519  		t.Fatalf("bad: %#v", h.Plans)
   520  	}
   521  	plan := h.Plans[0]
   522  
   523  	// Ensure the plan did not evict any allocs
   524  	var update []*structs.Allocation
   525  	for _, updateList := range plan.NodeUpdate {
   526  		update = append(update, updateList...)
   527  	}
   528  	if len(update) != 0 {
   529  		t.Fatalf("bad: %#v", plan)
   530  	}
   531  
   532  	// Ensure the plan updated the existing allocs
   533  	var planned []*structs.Allocation
   534  	for _, allocList := range plan.NodeAllocation {
   535  		planned = append(planned, allocList...)
   536  	}
   537  	if len(planned) != 10 {
   538  		t.Fatalf("bad: %#v", plan)
   539  	}
   540  	for _, p := range planned {
   541  		if p.Job != job2 {
   542  			t.Fatalf("should update job")
   543  		}
   544  	}
   545  
   546  	// Lookup the allocations by JobID
   547  	out, err := h.State.AllocsByJob(job.ID)
   548  	noErr(t, err)
   549  
   550  	// Ensure all allocations placed
   551  	if len(out) != 10 {
   552  		t.Fatalf("bad: %#v", out)
   553  	}
   554  	h.AssertEvalStatus(t, structs.EvalStatusComplete)
   555  
   556  	// Verify the network did not change
   557  	rp := structs.Port{Label: "main", Value: 5000}
   558  	for _, alloc := range out {
   559  		for _, resources := range alloc.TaskResources {
   560  			if resources.Networks[0].ReservedPorts[0] != rp {
   561  				t.Fatalf("bad: %#v", alloc)
   562  			}
   563  		}
   564  	}
   565  }
   566  
   567  func TestSystemSched_JobDeregister(t *testing.T) {
   568  	h := NewHarness(t)
   569  
   570  	// Create some nodes
   571  	var nodes []*structs.Node
   572  	for i := 0; i < 10; i++ {
   573  		node := mock.Node()
   574  		nodes = append(nodes, node)
   575  		noErr(t, h.State.UpsertNode(h.NextIndex(), node))
   576  	}
   577  
   578  	// Generate a fake job with allocations
   579  	job := mock.SystemJob()
   580  
   581  	var allocs []*structs.Allocation
   582  	for _, node := range nodes {
   583  		alloc := mock.Alloc()
   584  		alloc.Job = job
   585  		alloc.JobID = job.ID
   586  		alloc.NodeID = node.ID
   587  		alloc.Name = "my-job.web[0]"
   588  		allocs = append(allocs, alloc)
   589  	}
   590  	noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs))
   591  
   592  	// Create a mock evaluation to deregister the job
   593  	eval := &structs.Evaluation{
   594  		ID:          structs.GenerateUUID(),
   595  		Priority:    50,
   596  		TriggeredBy: structs.EvalTriggerJobDeregister,
   597  		JobID:       job.ID,
   598  	}
   599  
   600  	// Process the evaluation
   601  	err := h.Process(NewSystemScheduler, eval)
   602  	if err != nil {
   603  		t.Fatalf("err: %v", err)
   604  	}
   605  
   606  	// Ensure a single plan
   607  	if len(h.Plans) != 1 {
   608  		t.Fatalf("bad: %#v", h.Plans)
   609  	}
   610  	plan := h.Plans[0]
   611  
   612  	// Ensure the plan evicted the job from all nodes.
   613  	for _, node := range nodes {
   614  		if len(plan.NodeUpdate[node.ID]) != 1 {
   615  			t.Fatalf("bad: %#v", plan)
   616  		}
   617  	}
   618  
   619  	// Lookup the allocations by JobID
   620  	out, err := h.State.AllocsByJob(job.ID)
   621  	noErr(t, err)
   622  
   623  	// Ensure no remaining allocations
   624  	out = structs.FilterTerminalAllocs(out)
   625  	if len(out) != 0 {
   626  		t.Fatalf("bad: %#v", out)
   627  	}
   628  
   629  	h.AssertEvalStatus(t, structs.EvalStatusComplete)
   630  }
   631  
   632  func TestSystemSched_NodeDrain(t *testing.T) {
   633  	h := NewHarness(t)
   634  
   635  	// Register a draining node
   636  	node := mock.Node()
   637  	node.Drain = true
   638  	noErr(t, h.State.UpsertNode(h.NextIndex(), node))
   639  
   640  	// Generate a fake job allocated on that node.
   641  	job := mock.SystemJob()
   642  	noErr(t, h.State.UpsertJob(h.NextIndex(), job))
   643  
   644  	alloc := mock.Alloc()
   645  	alloc.Job = job
   646  	alloc.JobID = job.ID
   647  	alloc.NodeID = node.ID
   648  	alloc.Name = "my-job.web[0]"
   649  	noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc}))
   650  
   651  	// Create a mock evaluation to deal with drain
   652  	eval := &structs.Evaluation{
   653  		ID:          structs.GenerateUUID(),
   654  		Priority:    50,
   655  		TriggeredBy: structs.EvalTriggerNodeUpdate,
   656  		JobID:       job.ID,
   657  		NodeID:      node.ID,
   658  	}
   659  
   660  	// Process the evaluation
   661  	err := h.Process(NewSystemScheduler, eval)
   662  	if err != nil {
   663  		t.Fatalf("err: %v", err)
   664  	}
   665  
   666  	// Ensure a single plan
   667  	if len(h.Plans) != 1 {
   668  		t.Fatalf("bad: %#v", h.Plans)
   669  	}
   670  	plan := h.Plans[0]
   671  
   672  	// Ensure the plan evicted all allocs
   673  	if len(plan.NodeUpdate[node.ID]) != 1 {
   674  		t.Fatalf("bad: %#v", plan)
   675  	}
   676  
   677  	// Ensure the plan updated the allocation.
   678  	var planned []*structs.Allocation
   679  	for _, allocList := range plan.NodeUpdate {
   680  		planned = append(planned, allocList...)
   681  	}
   682  	if len(planned) != 1 {
   683  		t.Log(len(planned))
   684  		t.Fatalf("bad: %#v", plan)
   685  	}
   686  
   687  	// Lookup the allocations by JobID
   688  	out, err := h.State.AllocsByJob(job.ID)
   689  	noErr(t, err)
   690  
   691  	// Ensure the allocations is stopped
   692  	if planned[0].DesiredStatus != structs.AllocDesiredStatusStop {
   693  		t.Fatalf("bad: %#v", out)
   694  	}
   695  
   696  	h.AssertEvalStatus(t, structs.EvalStatusComplete)
   697  }
   698  
   699  func TestSystemSched_RetryLimit(t *testing.T) {
   700  	h := NewHarness(t)
   701  	h.Planner = &RejectPlan{h}
   702  
   703  	// Create some nodes
   704  	for i := 0; i < 10; i++ {
   705  		node := mock.Node()
   706  		noErr(t, h.State.UpsertNode(h.NextIndex(), node))
   707  	}
   708  
   709  	// Create a job
   710  	job := mock.SystemJob()
   711  	noErr(t, h.State.UpsertJob(h.NextIndex(), job))
   712  
   713  	// Create a mock evaluation to deregister the job
   714  	eval := &structs.Evaluation{
   715  		ID:          structs.GenerateUUID(),
   716  		Priority:    job.Priority,
   717  		TriggeredBy: structs.EvalTriggerJobRegister,
   718  		JobID:       job.ID,
   719  	}
   720  
   721  	// Process the evaluation
   722  	err := h.Process(NewSystemScheduler, eval)
   723  	if err != nil {
   724  		t.Fatalf("err: %v", err)
   725  	}
   726  
   727  	// Ensure multiple plans
   728  	if len(h.Plans) == 0 {
   729  		t.Fatalf("bad: %#v", h.Plans)
   730  	}
   731  
   732  	// Lookup the allocations by JobID
   733  	out, err := h.State.AllocsByJob(job.ID)
   734  	noErr(t, err)
   735  
   736  	// Ensure no allocations placed
   737  	if len(out) != 0 {
   738  		t.Fatalf("bad: %#v", out)
   739  	}
   740  
   741  	// Should hit the retry limit
   742  	h.AssertEvalStatus(t, structs.EvalStatusFailed)
   743  }