github.com/ranjib/nomad@v0.1.1-0.20160225204057-97751b02f70b/scheduler/generic_sched_test.go (about)

     1  package scheduler
     2  
     3  import (
     4  	"fmt"
     5  	"testing"
     6  	"time"
     7  
     8  	"github.com/hashicorp/nomad/nomad/mock"
     9  	"github.com/hashicorp/nomad/nomad/structs"
    10  )
    11  
    12  func TestServiceSched_JobRegister(t *testing.T) {
    13  	h := NewHarness(t)
    14  
    15  	// Create some nodes
    16  	for i := 0; i < 10; i++ {
    17  		node := mock.Node()
    18  		noErr(t, h.State.UpsertNode(h.NextIndex(), node))
    19  	}
    20  
    21  	// Create a job
    22  	job := mock.Job()
    23  	noErr(t, h.State.UpsertJob(h.NextIndex(), job))
    24  
    25  	// Create a mock evaluation to register the job
    26  	eval := &structs.Evaluation{
    27  		ID:          structs.GenerateUUID(),
    28  		Priority:    job.Priority,
    29  		TriggeredBy: structs.EvalTriggerJobRegister,
    30  		JobID:       job.ID,
    31  	}
    32  
    33  	// Process the evaluation
    34  	err := h.Process(NewServiceScheduler, eval)
    35  	if err != nil {
    36  		t.Fatalf("err: %v", err)
    37  	}
    38  
    39  	// Ensure a single plan
    40  	if len(h.Plans) != 1 {
    41  		t.Fatalf("bad: %#v", h.Plans)
    42  	}
    43  	plan := h.Plans[0]
    44  
    45  	// Ensure the plan allocated
    46  	var planned []*structs.Allocation
    47  	for _, allocList := range plan.NodeAllocation {
    48  		planned = append(planned, allocList...)
    49  	}
    50  	if len(planned) != 10 {
    51  		t.Fatalf("bad: %#v", plan)
    52  	}
    53  
    54  	// Lookup the allocations by JobID
    55  	out, err := h.State.AllocsByJob(job.ID)
    56  	noErr(t, err)
    57  
    58  	// Ensure all allocations placed
    59  	if len(out) != 10 {
    60  		t.Fatalf("bad: %#v", out)
    61  	}
    62  
    63  	// Ensure different ports were used.
    64  	used := make(map[int]struct{})
    65  	for _, alloc := range out {
    66  		for _, resource := range alloc.TaskResources {
    67  			for _, port := range resource.Networks[0].DynamicPorts {
    68  				if _, ok := used[port.Value]; ok {
    69  					t.Fatalf("Port collision %v", port.Value)
    70  				}
    71  				used[port.Value] = struct{}{}
    72  			}
    73  		}
    74  	}
    75  
    76  	h.AssertEvalStatus(t, structs.EvalStatusComplete)
    77  }
    78  
    79  func TestServiceSched_JobRegister_AllocFail(t *testing.T) {
    80  	h := NewHarness(t)
    81  
    82  	// Create NO nodes
    83  	// Create a job
    84  	job := mock.Job()
    85  	noErr(t, h.State.UpsertJob(h.NextIndex(), job))
    86  
    87  	// Create a mock evaluation to register the job
    88  	eval := &structs.Evaluation{
    89  		ID:          structs.GenerateUUID(),
    90  		Priority:    job.Priority,
    91  		TriggeredBy: structs.EvalTriggerJobRegister,
    92  		JobID:       job.ID,
    93  	}
    94  
    95  	// Process the evaluation
    96  	err := h.Process(NewServiceScheduler, eval)
    97  	if err != nil {
    98  		t.Fatalf("err: %v", err)
    99  	}
   100  
   101  	// Ensure a single plan
   102  	if len(h.Plans) != 1 {
   103  		t.Fatalf("bad: %#v", h.Plans)
   104  	}
   105  	plan := h.Plans[0]
   106  
   107  	// Ensure the plan has created a follow up eval.
   108  	if len(h.CreateEvals) != 1 || h.CreateEvals[0].Status != structs.EvalStatusBlocked {
   109  		t.Fatalf("bad: %#v", h.CreateEvals)
   110  	}
   111  
   112  	// Ensure the plan failed to alloc
   113  	if len(plan.FailedAllocs) != 1 {
   114  		t.Fatalf("bad: %#v", plan)
   115  	}
   116  
   117  	// Lookup the allocations by JobID
   118  	out, err := h.State.AllocsByJob(job.ID)
   119  	noErr(t, err)
   120  
   121  	// Ensure all allocations placed
   122  	if len(out) != 1 {
   123  		t.Fatalf("bad: %#v", out)
   124  	}
   125  
   126  	// Check the coalesced failures
   127  	if out[0].Metrics.CoalescedFailures != 9 {
   128  		t.Fatalf("bad: %#v", out[0].Metrics)
   129  	}
   130  
   131  	// Check the available nodes
   132  	if count, ok := out[0].Metrics.NodesAvailable["dc1"]; !ok || count != 0 {
   133  		t.Fatalf("bad: %#v", out[0].Metrics)
   134  	}
   135  
   136  	h.AssertEvalStatus(t, structs.EvalStatusComplete)
   137  }
   138  
   139  func TestServiceSched_JobRegister_BlockedEval(t *testing.T) {
   140  	h := NewHarness(t)
   141  
   142  	// Create a full node
   143  	node := mock.Node()
   144  	node.Reserved = node.Resources
   145  	node.ComputeClass()
   146  	noErr(t, h.State.UpsertNode(h.NextIndex(), node))
   147  
   148  	// Create an ineligible node
   149  	node2 := mock.Node()
   150  	node2.Attributes["kernel.name"] = "windows"
   151  	node2.ComputeClass()
   152  	noErr(t, h.State.UpsertNode(h.NextIndex(), node2))
   153  
   154  	// Create a jobs
   155  	job := mock.Job()
   156  	noErr(t, h.State.UpsertJob(h.NextIndex(), job))
   157  
   158  	// Create a mock evaluation to register the job
   159  	eval := &structs.Evaluation{
   160  		ID:          structs.GenerateUUID(),
   161  		Priority:    job.Priority,
   162  		TriggeredBy: structs.EvalTriggerJobRegister,
   163  		JobID:       job.ID,
   164  	}
   165  
   166  	// Process the evaluation
   167  	err := h.Process(NewServiceScheduler, eval)
   168  	if err != nil {
   169  		t.Fatalf("err: %v", err)
   170  	}
   171  
   172  	// Ensure a single plan
   173  	if len(h.Plans) != 1 {
   174  		t.Fatalf("bad: %#v", h.Plans)
   175  	}
   176  	plan := h.Plans[0]
   177  
   178  	// Ensure the plan has created a follow up eval.
   179  	if len(h.CreateEvals) != 1 {
   180  		t.Fatalf("bad: %#v", h.CreateEvals)
   181  	}
   182  
   183  	created := h.CreateEvals[0]
   184  	if created.Status != structs.EvalStatusBlocked {
   185  		t.Fatalf("bad: %#v", created)
   186  	}
   187  
   188  	classes := created.ClassEligibility
   189  	if len(classes) != 2 || !classes[node.ComputedClass] || classes[node2.ComputedClass] {
   190  		t.Fatalf("bad: %#v", classes)
   191  	}
   192  
   193  	if created.EscapedComputedClass {
   194  		t.Fatalf("bad: %#v", created)
   195  	}
   196  
   197  	// Ensure the plan failed to alloc
   198  	if len(plan.FailedAllocs) != 1 {
   199  		t.Fatalf("bad: %#v", plan)
   200  	}
   201  
   202  	// Lookup the allocations by JobID
   203  	out, err := h.State.AllocsByJob(job.ID)
   204  	noErr(t, err)
   205  
   206  	// Ensure all allocations placed
   207  	if len(out) != 1 {
   208  		for _, a := range out {
   209  			t.Logf("%#v", a)
   210  		}
   211  		t.Fatalf("bad: %#v", out)
   212  	}
   213  
   214  	// Check the coalesced failures
   215  	if out[0].Metrics.CoalescedFailures != 9 {
   216  		t.Fatalf("bad: %#v", out[0].Metrics)
   217  	}
   218  
   219  	// Check the available nodes
   220  	if count, ok := out[0].Metrics.NodesAvailable["dc1"]; !ok || count != 2 {
   221  		t.Fatalf("bad: %#v", out[0].Metrics)
   222  	}
   223  
   224  	h.AssertEvalStatus(t, structs.EvalStatusComplete)
   225  }
   226  
   227  func TestServiceSched_JobRegister_FeasibleAndInfeasibleTG(t *testing.T) {
   228  	h := NewHarness(t)
   229  
   230  	// Create one node
   231  	node := mock.Node()
   232  	node.NodeClass = "class_0"
   233  	noErr(t, node.ComputeClass())
   234  	noErr(t, h.State.UpsertNode(h.NextIndex(), node))
   235  
   236  	// Create a job that constrains on a node class
   237  	job := mock.Job()
   238  	job.TaskGroups[0].Count = 2
   239  	job.TaskGroups[0].Constraints = append(job.Constraints,
   240  		&structs.Constraint{
   241  			LTarget: "${node.class}",
   242  			RTarget: "class_0",
   243  			Operand: "=",
   244  		},
   245  	)
   246  	tg2 := job.TaskGroups[0].Copy()
   247  	tg2.Name = "web2"
   248  	tg2.Constraints[1].RTarget = "class_1"
   249  	job.TaskGroups = append(job.TaskGroups, tg2)
   250  	noErr(t, h.State.UpsertJob(h.NextIndex(), job))
   251  
   252  	// Create a mock evaluation to register the job
   253  	eval := &structs.Evaluation{
   254  		ID:          structs.GenerateUUID(),
   255  		Priority:    job.Priority,
   256  		TriggeredBy: structs.EvalTriggerJobRegister,
   257  		JobID:       job.ID,
   258  	}
   259  
   260  	// Process the evaluation
   261  	err := h.Process(NewServiceScheduler, eval)
   262  	if err != nil {
   263  		t.Fatalf("err: %v", err)
   264  	}
   265  
   266  	// Ensure a single plan
   267  	if len(h.Plans) != 1 {
   268  		t.Fatalf("bad: %#v", h.Plans)
   269  	}
   270  	plan := h.Plans[0]
   271  
   272  	// Ensure the plan allocated
   273  	var planned []*structs.Allocation
   274  	for _, allocList := range plan.NodeAllocation {
   275  		planned = append(planned, allocList...)
   276  	}
   277  	if len(planned) != 2 {
   278  		t.Fatalf("bad: %#v", plan)
   279  	}
   280  	if len(plan.FailedAllocs) != 1 {
   281  		t.Fatalf("bad: %#v", plan)
   282  	}
   283  
   284  	// Lookup the allocations by JobID
   285  	out, err := h.State.AllocsByJob(job.ID)
   286  	noErr(t, err)
   287  
   288  	// Ensure all allocations placed
   289  	if len(out) != 3 {
   290  		t.Fatalf("bad: %#v", out)
   291  	}
   292  
   293  	h.AssertEvalStatus(t, structs.EvalStatusComplete)
   294  }
   295  
   296  func TestServiceSched_JobModify(t *testing.T) {
   297  	h := NewHarness(t)
   298  
   299  	// Create some nodes
   300  	var nodes []*structs.Node
   301  	for i := 0; i < 10; i++ {
   302  		node := mock.Node()
   303  		nodes = append(nodes, node)
   304  		noErr(t, h.State.UpsertNode(h.NextIndex(), node))
   305  	}
   306  
   307  	// Generate a fake job with allocations
   308  	job := mock.Job()
   309  	noErr(t, h.State.UpsertJob(h.NextIndex(), job))
   310  
   311  	var allocs []*structs.Allocation
   312  	for i := 0; i < 10; i++ {
   313  		alloc := mock.Alloc()
   314  		alloc.Job = job
   315  		alloc.JobID = job.ID
   316  		alloc.NodeID = nodes[i].ID
   317  		alloc.Name = fmt.Sprintf("my-job.web[%d]", i)
   318  		allocs = append(allocs, alloc)
   319  	}
   320  	noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs))
   321  
   322  	// Add a few terminal status allocations, these should be ignored
   323  	var terminal []*structs.Allocation
   324  	for i := 0; i < 5; i++ {
   325  		alloc := mock.Alloc()
   326  		alloc.Job = job
   327  		alloc.JobID = job.ID
   328  		alloc.NodeID = nodes[i].ID
   329  		alloc.Name = fmt.Sprintf("my-job.web[%d]", i)
   330  		alloc.DesiredStatus = structs.AllocDesiredStatusFailed
   331  		terminal = append(terminal, alloc)
   332  	}
   333  	noErr(t, h.State.UpsertAllocs(h.NextIndex(), terminal))
   334  
   335  	// Update the job
   336  	job2 := mock.Job()
   337  	job2.ID = job.ID
   338  
   339  	// Update the task, such that it cannot be done in-place
   340  	job2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other"
   341  	noErr(t, h.State.UpsertJob(h.NextIndex(), job2))
   342  
   343  	// Create a mock evaluation to deal with drain
   344  	eval := &structs.Evaluation{
   345  		ID:          structs.GenerateUUID(),
   346  		Priority:    50,
   347  		TriggeredBy: structs.EvalTriggerJobRegister,
   348  		JobID:       job.ID,
   349  	}
   350  
   351  	// Process the evaluation
   352  	err := h.Process(NewServiceScheduler, eval)
   353  	if err != nil {
   354  		t.Fatalf("err: %v", err)
   355  	}
   356  
   357  	// Ensure a single plan
   358  	if len(h.Plans) != 1 {
   359  		t.Fatalf("bad: %#v", h.Plans)
   360  	}
   361  	plan := h.Plans[0]
   362  
   363  	// Ensure the plan evicted all allocs
   364  	var update []*structs.Allocation
   365  	for _, updateList := range plan.NodeUpdate {
   366  		update = append(update, updateList...)
   367  	}
   368  	if len(update) != len(allocs) {
   369  		t.Fatalf("bad: %#v", plan)
   370  	}
   371  
   372  	// Ensure the plan allocated
   373  	var planned []*structs.Allocation
   374  	for _, allocList := range plan.NodeAllocation {
   375  		planned = append(planned, allocList...)
   376  	}
   377  	if len(planned) != 10 {
   378  		t.Fatalf("bad: %#v", plan)
   379  	}
   380  
   381  	// Lookup the allocations by JobID
   382  	out, err := h.State.AllocsByJob(job.ID)
   383  	noErr(t, err)
   384  
   385  	// Ensure all allocations placed
   386  	out = structs.FilterTerminalAllocs(out)
   387  	if len(out) != 10 {
   388  		t.Fatalf("bad: %#v", out)
   389  	}
   390  
   391  	h.AssertEvalStatus(t, structs.EvalStatusComplete)
   392  }
   393  
   394  func TestServiceSched_JobModify_Rolling(t *testing.T) {
   395  	h := NewHarness(t)
   396  
   397  	// Create some nodes
   398  	var nodes []*structs.Node
   399  	for i := 0; i < 10; i++ {
   400  		node := mock.Node()
   401  		nodes = append(nodes, node)
   402  		noErr(t, h.State.UpsertNode(h.NextIndex(), node))
   403  	}
   404  
   405  	// Generate a fake job with allocations
   406  	job := mock.Job()
   407  	noErr(t, h.State.UpsertJob(h.NextIndex(), job))
   408  
   409  	var allocs []*structs.Allocation
   410  	for i := 0; i < 10; i++ {
   411  		alloc := mock.Alloc()
   412  		alloc.Job = job
   413  		alloc.JobID = job.ID
   414  		alloc.NodeID = nodes[i].ID
   415  		alloc.Name = fmt.Sprintf("my-job.web[%d]", i)
   416  		allocs = append(allocs, alloc)
   417  	}
   418  	noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs))
   419  
   420  	// Update the job
   421  	job2 := mock.Job()
   422  	job2.ID = job.ID
   423  	job2.Update = structs.UpdateStrategy{
   424  		Stagger:     30 * time.Second,
   425  		MaxParallel: 5,
   426  	}
   427  
   428  	// Update the task, such that it cannot be done in-place
   429  	job2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other"
   430  	noErr(t, h.State.UpsertJob(h.NextIndex(), job2))
   431  
   432  	// Create a mock evaluation to deal with drain
   433  	eval := &structs.Evaluation{
   434  		ID:          structs.GenerateUUID(),
   435  		Priority:    50,
   436  		TriggeredBy: structs.EvalTriggerJobRegister,
   437  		JobID:       job.ID,
   438  	}
   439  
   440  	// Process the evaluation
   441  	err := h.Process(NewServiceScheduler, eval)
   442  	if err != nil {
   443  		t.Fatalf("err: %v", err)
   444  	}
   445  
   446  	// Ensure a single plan
   447  	if len(h.Plans) != 1 {
   448  		t.Fatalf("bad: %#v", h.Plans)
   449  	}
   450  	plan := h.Plans[0]
   451  
   452  	// Ensure the plan evicted only MaxParallel
   453  	var update []*structs.Allocation
   454  	for _, updateList := range plan.NodeUpdate {
   455  		update = append(update, updateList...)
   456  	}
   457  	if len(update) != job2.Update.MaxParallel {
   458  		t.Fatalf("bad: %#v", plan)
   459  	}
   460  
   461  	// Ensure the plan allocated
   462  	var planned []*structs.Allocation
   463  	for _, allocList := range plan.NodeAllocation {
   464  		planned = append(planned, allocList...)
   465  	}
   466  	if len(planned) != job2.Update.MaxParallel {
   467  		t.Fatalf("bad: %#v", plan)
   468  	}
   469  
   470  	h.AssertEvalStatus(t, structs.EvalStatusComplete)
   471  
   472  	// Ensure a follow up eval was created
   473  	eval = h.Evals[0]
   474  	if eval.NextEval == "" {
   475  		t.Fatalf("missing next eval")
   476  	}
   477  
   478  	// Check for create
   479  	if len(h.CreateEvals) == 0 {
   480  		t.Fatalf("missing created eval")
   481  	}
   482  	create := h.CreateEvals[0]
   483  	if eval.NextEval != create.ID {
   484  		t.Fatalf("ID mismatch")
   485  	}
   486  	if create.PreviousEval != eval.ID {
   487  		t.Fatalf("missing previous eval")
   488  	}
   489  
   490  	if create.TriggeredBy != structs.EvalTriggerRollingUpdate {
   491  		t.Fatalf("bad: %#v", create)
   492  	}
   493  }
   494  
   495  func TestServiceSched_JobModify_InPlace(t *testing.T) {
   496  	h := NewHarness(t)
   497  
   498  	// Create some nodes
   499  	var nodes []*structs.Node
   500  	for i := 0; i < 10; i++ {
   501  		node := mock.Node()
   502  		nodes = append(nodes, node)
   503  		noErr(t, h.State.UpsertNode(h.NextIndex(), node))
   504  	}
   505  
   506  	// Generate a fake job with allocations
   507  	job := mock.Job()
   508  	noErr(t, h.State.UpsertJob(h.NextIndex(), job))
   509  
   510  	var allocs []*structs.Allocation
   511  	for i := 0; i < 10; i++ {
   512  		alloc := mock.Alloc()
   513  		alloc.Job = job
   514  		alloc.JobID = job.ID
   515  		alloc.NodeID = nodes[i].ID
   516  		alloc.Name = fmt.Sprintf("my-job.web[%d]", i)
   517  		allocs = append(allocs, alloc)
   518  	}
   519  	noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs))
   520  
   521  	// Update the job
   522  	job2 := mock.Job()
   523  	job2.ID = job.ID
   524  	noErr(t, h.State.UpsertJob(h.NextIndex(), job2))
   525  
   526  	// Create a mock evaluation to deal with drain
   527  	eval := &structs.Evaluation{
   528  		ID:          structs.GenerateUUID(),
   529  		Priority:    50,
   530  		TriggeredBy: structs.EvalTriggerJobRegister,
   531  		JobID:       job.ID,
   532  	}
   533  
   534  	// Process the evaluation
   535  	err := h.Process(NewServiceScheduler, eval)
   536  	if err != nil {
   537  		t.Fatalf("err: %v", err)
   538  	}
   539  
   540  	// Ensure a single plan
   541  	if len(h.Plans) != 1 {
   542  		t.Fatalf("bad: %#v", h.Plans)
   543  	}
   544  	plan := h.Plans[0]
   545  
   546  	// Ensure the plan did not evict any allocs
   547  	var update []*structs.Allocation
   548  	for _, updateList := range plan.NodeUpdate {
   549  		update = append(update, updateList...)
   550  	}
   551  	if len(update) != 0 {
   552  		t.Fatalf("bad: %#v", plan)
   553  	}
   554  
   555  	// Ensure the plan updated the existing allocs
   556  	var planned []*structs.Allocation
   557  	for _, allocList := range plan.NodeAllocation {
   558  		planned = append(planned, allocList...)
   559  	}
   560  	if len(planned) != 10 {
   561  		t.Fatalf("bad: %#v", plan)
   562  	}
   563  	for _, p := range planned {
   564  		if p.Job != job2 {
   565  			t.Fatalf("should update job")
   566  		}
   567  	}
   568  
   569  	// Lookup the allocations by JobID
   570  	out, err := h.State.AllocsByJob(job.ID)
   571  	noErr(t, err)
   572  
   573  	// Ensure all allocations placed
   574  	if len(out) != 10 {
   575  		for _, alloc := range out {
   576  			t.Logf("%#v", alloc)
   577  		}
   578  		t.Fatalf("bad: %#v", out)
   579  	}
   580  	h.AssertEvalStatus(t, structs.EvalStatusComplete)
   581  
   582  	// Verify the network did not change
   583  	rp := structs.Port{Label: "main", Value: 5000}
   584  	for _, alloc := range out {
   585  		for _, resources := range alloc.TaskResources {
   586  			if resources.Networks[0].ReservedPorts[0] != rp {
   587  				t.Fatalf("bad: %#v", alloc)
   588  			}
   589  		}
   590  	}
   591  }
   592  
   593  func TestServiceSched_JobDeregister(t *testing.T) {
   594  	h := NewHarness(t)
   595  
   596  	// Generate a fake job with allocations
   597  	job := mock.Job()
   598  
   599  	var allocs []*structs.Allocation
   600  	for i := 0; i < 10; i++ {
   601  		alloc := mock.Alloc()
   602  		alloc.Job = job
   603  		alloc.JobID = job.ID
   604  		allocs = append(allocs, alloc)
   605  	}
   606  	noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs))
   607  
   608  	// Create a mock evaluation to deregister the job
   609  	eval := &structs.Evaluation{
   610  		ID:          structs.GenerateUUID(),
   611  		Priority:    50,
   612  		TriggeredBy: structs.EvalTriggerJobDeregister,
   613  		JobID:       job.ID,
   614  	}
   615  
   616  	// Process the evaluation
   617  	err := h.Process(NewServiceScheduler, eval)
   618  	if err != nil {
   619  		t.Fatalf("err: %v", err)
   620  	}
   621  
   622  	// Ensure a single plan
   623  	if len(h.Plans) != 1 {
   624  		t.Fatalf("bad: %#v", h.Plans)
   625  	}
   626  	plan := h.Plans[0]
   627  
   628  	// Ensure the plan evicted all nodes
   629  	if len(plan.NodeUpdate["12345678-abcd-efab-cdef-123456789abc"]) != len(allocs) {
   630  		t.Fatalf("bad: %#v", plan)
   631  	}
   632  
   633  	// Lookup the allocations by JobID
   634  	out, err := h.State.AllocsByJob(job.ID)
   635  	noErr(t, err)
   636  
   637  	// Ensure that the job field on the allocation is still populated
   638  	for _, alloc := range out {
   639  		if alloc.Job == nil {
   640  			t.Fatalf("bad: %#v", alloc)
   641  		}
   642  	}
   643  
   644  	// Ensure no remaining allocations
   645  	out = structs.FilterTerminalAllocs(out)
   646  	if len(out) != 0 {
   647  		t.Fatalf("bad: %#v", out)
   648  	}
   649  
   650  	h.AssertEvalStatus(t, structs.EvalStatusComplete)
   651  }
   652  
   653  func TestServiceSched_NodeDrain(t *testing.T) {
   654  	h := NewHarness(t)
   655  
   656  	// Register a draining node
   657  	node := mock.Node()
   658  	node.Drain = true
   659  	noErr(t, h.State.UpsertNode(h.NextIndex(), node))
   660  
   661  	// Create some nodes
   662  	for i := 0; i < 10; i++ {
   663  		node := mock.Node()
   664  		noErr(t, h.State.UpsertNode(h.NextIndex(), node))
   665  	}
   666  
   667  	// Generate a fake job with allocations and an update policy.
   668  	job := mock.Job()
   669  	noErr(t, h.State.UpsertJob(h.NextIndex(), job))
   670  
   671  	var allocs []*structs.Allocation
   672  	for i := 0; i < 10; i++ {
   673  		alloc := mock.Alloc()
   674  		alloc.Job = job
   675  		alloc.JobID = job.ID
   676  		alloc.NodeID = node.ID
   677  		alloc.Name = fmt.Sprintf("my-job.web[%d]", i)
   678  		allocs = append(allocs, alloc)
   679  	}
   680  	noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs))
   681  
   682  	// Create a mock evaluation to deal with drain
   683  	eval := &structs.Evaluation{
   684  		ID:          structs.GenerateUUID(),
   685  		Priority:    50,
   686  		TriggeredBy: structs.EvalTriggerNodeUpdate,
   687  		JobID:       job.ID,
   688  		NodeID:      node.ID,
   689  	}
   690  
   691  	// Process the evaluation
   692  	err := h.Process(NewServiceScheduler, eval)
   693  	if err != nil {
   694  		t.Fatalf("err: %v", err)
   695  	}
   696  
   697  	// Ensure a single plan
   698  	if len(h.Plans) != 1 {
   699  		t.Fatalf("bad: %#v", h.Plans)
   700  	}
   701  	plan := h.Plans[0]
   702  
   703  	// Ensure the plan evicted all allocs
   704  	if len(plan.NodeUpdate[node.ID]) != len(allocs) {
   705  		t.Fatalf("bad: %#v", plan)
   706  	}
   707  
   708  	// Ensure the plan allocated
   709  	var planned []*structs.Allocation
   710  	for _, allocList := range plan.NodeAllocation {
   711  		planned = append(planned, allocList...)
   712  	}
   713  	if len(planned) != 10 {
   714  		t.Fatalf("bad: %#v", plan)
   715  	}
   716  
   717  	// Lookup the allocations by JobID
   718  	out, err := h.State.AllocsByJob(job.ID)
   719  	noErr(t, err)
   720  
   721  	// Ensure all allocations placed
   722  	out = structs.FilterTerminalAllocs(out)
   723  	if len(out) != 10 {
   724  		t.Fatalf("bad: %#v", out)
   725  	}
   726  
   727  	h.AssertEvalStatus(t, structs.EvalStatusComplete)
   728  }
   729  
   730  func TestServiceSched_NodeDrain_UpdateStrategy(t *testing.T) {
   731  	h := NewHarness(t)
   732  
   733  	// Register a draining node
   734  	node := mock.Node()
   735  	node.Drain = true
   736  	noErr(t, h.State.UpsertNode(h.NextIndex(), node))
   737  
   738  	// Create some nodes
   739  	for i := 0; i < 10; i++ {
   740  		node := mock.Node()
   741  		noErr(t, h.State.UpsertNode(h.NextIndex(), node))
   742  	}
   743  
   744  	// Generate a fake job with allocations and an update policy.
   745  	job := mock.Job()
   746  	mp := 5
   747  	job.Update = structs.UpdateStrategy{
   748  		Stagger:     time.Second,
   749  		MaxParallel: mp,
   750  	}
   751  	noErr(t, h.State.UpsertJob(h.NextIndex(), job))
   752  
   753  	var allocs []*structs.Allocation
   754  	for i := 0; i < 10; i++ {
   755  		alloc := mock.Alloc()
   756  		alloc.Job = job
   757  		alloc.JobID = job.ID
   758  		alloc.NodeID = node.ID
   759  		alloc.Name = fmt.Sprintf("my-job.web[%d]", i)
   760  		allocs = append(allocs, alloc)
   761  	}
   762  	noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs))
   763  
   764  	// Create a mock evaluation to deal with drain
   765  	eval := &structs.Evaluation{
   766  		ID:          structs.GenerateUUID(),
   767  		Priority:    50,
   768  		TriggeredBy: structs.EvalTriggerNodeUpdate,
   769  		JobID:       job.ID,
   770  		NodeID:      node.ID,
   771  	}
   772  
   773  	// Process the evaluation
   774  	err := h.Process(NewServiceScheduler, eval)
   775  	if err != nil {
   776  		t.Fatalf("err: %v", err)
   777  	}
   778  
   779  	// Ensure a single plan
   780  	if len(h.Plans) != 1 {
   781  		t.Fatalf("bad: %#v", h.Plans)
   782  	}
   783  	plan := h.Plans[0]
   784  
   785  	// Ensure the plan evicted all allocs
   786  	if len(plan.NodeUpdate[node.ID]) != mp {
   787  		t.Fatalf("bad: %#v", plan)
   788  	}
   789  
   790  	// Ensure the plan allocated
   791  	var planned []*structs.Allocation
   792  	for _, allocList := range plan.NodeAllocation {
   793  		planned = append(planned, allocList...)
   794  	}
   795  	if len(planned) != mp {
   796  		t.Fatalf("bad: %#v", plan)
   797  	}
   798  
   799  	// Ensure there is a followup eval.
   800  	if len(h.CreateEvals) != 1 ||
   801  		h.CreateEvals[0].TriggeredBy != structs.EvalTriggerRollingUpdate {
   802  		t.Fatalf("bad: %#v", h.CreateEvals)
   803  	}
   804  
   805  	h.AssertEvalStatus(t, structs.EvalStatusComplete)
   806  }
   807  
   808  func TestServiceSched_RetryLimit(t *testing.T) {
   809  	h := NewHarness(t)
   810  	h.Planner = &RejectPlan{h}
   811  
   812  	// Create some nodes
   813  	for i := 0; i < 10; i++ {
   814  		node := mock.Node()
   815  		noErr(t, h.State.UpsertNode(h.NextIndex(), node))
   816  	}
   817  
   818  	// Create a job
   819  	job := mock.Job()
   820  	noErr(t, h.State.UpsertJob(h.NextIndex(), job))
   821  
   822  	// Create a mock evaluation to register the job
   823  	eval := &structs.Evaluation{
   824  		ID:          structs.GenerateUUID(),
   825  		Priority:    job.Priority,
   826  		TriggeredBy: structs.EvalTriggerJobRegister,
   827  		JobID:       job.ID,
   828  	}
   829  
   830  	// Process the evaluation
   831  	err := h.Process(NewServiceScheduler, eval)
   832  	if err != nil {
   833  		t.Fatalf("err: %v", err)
   834  	}
   835  
   836  	// Ensure multiple plans
   837  	if len(h.Plans) == 0 {
   838  		t.Fatalf("bad: %#v", h.Plans)
   839  	}
   840  
   841  	// Lookup the allocations by JobID
   842  	out, err := h.State.AllocsByJob(job.ID)
   843  	noErr(t, err)
   844  
   845  	// Ensure no allocations placed
   846  	if len(out) != 0 {
   847  		t.Fatalf("bad: %#v", out)
   848  	}
   849  
   850  	// Should hit the retry limit
   851  	h.AssertEvalStatus(t, structs.EvalStatusFailed)
   852  }
   853  
   854  func TestBatchSched_Run_DeadAlloc(t *testing.T) {
   855  	h := NewHarness(t)
   856  
   857  	// Create a node
   858  	node := mock.Node()
   859  	noErr(t, h.State.UpsertNode(h.NextIndex(), node))
   860  
   861  	// Create a job
   862  	job := mock.Job()
   863  	job.TaskGroups[0].Count = 1
   864  	noErr(t, h.State.UpsertJob(h.NextIndex(), job))
   865  
   866  	// Create a failed alloc
   867  	alloc := mock.Alloc()
   868  	alloc.Job = job
   869  	alloc.JobID = job.ID
   870  	alloc.NodeID = node.ID
   871  	alloc.Name = "my-job.web[0]"
   872  	alloc.ClientStatus = structs.AllocClientStatusDead
   873  	noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc}))
   874  
   875  	// Create a mock evaluation to register the job
   876  	eval := &structs.Evaluation{
   877  		ID:          structs.GenerateUUID(),
   878  		Priority:    job.Priority,
   879  		TriggeredBy: structs.EvalTriggerJobRegister,
   880  		JobID:       job.ID,
   881  	}
   882  
   883  	// Process the evaluation
   884  	err := h.Process(NewBatchScheduler, eval)
   885  	if err != nil {
   886  		t.Fatalf("err: %v", err)
   887  	}
   888  
   889  	// Ensure no plan as it should be a no-op
   890  	if len(h.Plans) != 0 {
   891  		t.Fatalf("bad: %#v", h.Plans)
   892  	}
   893  
   894  	// Lookup the allocations by JobID
   895  	out, err := h.State.AllocsByJob(job.ID)
   896  	noErr(t, err)
   897  
   898  	// Ensure no allocations placed
   899  	if len(out) != 1 {
   900  		t.Fatalf("bad: %#v", out)
   901  	}
   902  
   903  	h.AssertEvalStatus(t, structs.EvalStatusComplete)
   904  }
   905  
   906  func TestBatchSched_Run_FailedAlloc(t *testing.T) {
   907  	h := NewHarness(t)
   908  
   909  	// Create a node
   910  	node := mock.Node()
   911  	noErr(t, h.State.UpsertNode(h.NextIndex(), node))
   912  
   913  	// Create a job
   914  	job := mock.Job()
   915  	job.TaskGroups[0].Count = 1
   916  	noErr(t, h.State.UpsertJob(h.NextIndex(), job))
   917  
   918  	// Create a failed alloc
   919  	alloc := mock.Alloc()
   920  	alloc.Job = job
   921  	alloc.JobID = job.ID
   922  	alloc.NodeID = node.ID
   923  	alloc.Name = "my-job.web[0]"
   924  	alloc.ClientStatus = structs.AllocClientStatusFailed
   925  	noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc}))
   926  
   927  	// Create a mock evaluation to register the job
   928  	eval := &structs.Evaluation{
   929  		ID:          structs.GenerateUUID(),
   930  		Priority:    job.Priority,
   931  		TriggeredBy: structs.EvalTriggerJobRegister,
   932  		JobID:       job.ID,
   933  	}
   934  
   935  	// Process the evaluation
   936  	err := h.Process(NewBatchScheduler, eval)
   937  	if err != nil {
   938  		t.Fatalf("err: %v", err)
   939  	}
   940  
   941  	// Ensure no plan as it should be a no-op
   942  	if len(h.Plans) != 1 {
   943  		t.Fatalf("bad: %#v", h.Plans)
   944  	}
   945  
   946  	// Lookup the allocations by JobID
   947  	out, err := h.State.AllocsByJob(job.ID)
   948  	noErr(t, err)
   949  
   950  	// Ensure a replacement alloc was placed.
   951  	if len(out) != 2 {
   952  		t.Fatalf("bad: %#v", out)
   953  	}
   954  
   955  	h.AssertEvalStatus(t, structs.EvalStatusComplete)
   956  }