github.com/dkerwin/nomad@v0.3.3-0.20160525181927-74554135514b/nomad/core_sched_test.go (about)

     1  package nomad
     2  
     3  import (
     4  	"testing"
     5  	"time"
     6  
     7  	"github.com/hashicorp/nomad/nomad/mock"
     8  	"github.com/hashicorp/nomad/nomad/structs"
     9  	"github.com/hashicorp/nomad/testutil"
    10  )
    11  
    12  func TestCoreScheduler_EvalGC(t *testing.T) {
    13  	s1 := testServer(t, nil)
    14  	defer s1.Shutdown()
    15  	testutil.WaitForLeader(t, s1.RPC)
    16  
    17  	// Insert "dead" eval
    18  	state := s1.fsm.State()
    19  	eval := mock.Eval()
    20  	eval.Status = structs.EvalStatusFailed
    21  	err := state.UpsertEvals(1000, []*structs.Evaluation{eval})
    22  	if err != nil {
    23  		t.Fatalf("err: %v", err)
    24  	}
    25  
    26  	// Insert "dead" alloc
    27  	alloc := mock.Alloc()
    28  	alloc.EvalID = eval.ID
    29  	alloc.DesiredStatus = structs.AllocDesiredStatusFailed
    30  	err = state.UpsertAllocs(1001, []*structs.Allocation{alloc})
    31  	if err != nil {
    32  		t.Fatalf("err: %v", err)
    33  	}
    34  
    35  	// Update the time tables to make this work
    36  	tt := s1.fsm.TimeTable()
    37  	tt.Witness(2000, time.Now().UTC().Add(-1*s1.config.EvalGCThreshold))
    38  
    39  	// Create a core scheduler
    40  	snap, err := state.Snapshot()
    41  	if err != nil {
    42  		t.Fatalf("err: %v", err)
    43  	}
    44  	core := NewCoreScheduler(s1, snap)
    45  
    46  	// Attempt the GC
    47  	gc := s1.coreJobEval(structs.CoreJobEvalGC)
    48  	gc.ModifyIndex = 2000
    49  	err = core.Process(gc)
    50  	if err != nil {
    51  		t.Fatalf("err: %v", err)
    52  	}
    53  
    54  	// Should be gone
    55  	out, err := state.EvalByID(eval.ID)
    56  	if err != nil {
    57  		t.Fatalf("err: %v", err)
    58  	}
    59  	if out != nil {
    60  		t.Fatalf("bad: %v", out)
    61  	}
    62  
    63  	outA, err := state.AllocByID(alloc.ID)
    64  	if err != nil {
    65  		t.Fatalf("err: %v", err)
    66  	}
    67  	if outA != nil {
    68  		t.Fatalf("bad: %v", outA)
    69  	}
    70  }
    71  
    72  func TestCoreScheduler_EvalGC_Batch_NoAllocs(t *testing.T) {
    73  	s1 := testServer(t, nil)
    74  	defer s1.Shutdown()
    75  	testutil.WaitForLeader(t, s1.RPC)
    76  
    77  	// Insert "dead" eval
    78  	state := s1.fsm.State()
    79  	eval := mock.Eval()
    80  	eval.Type = structs.JobTypeBatch
    81  	eval.Status = structs.EvalStatusFailed
    82  	err := state.UpsertEvals(1000, []*structs.Evaluation{eval})
    83  	if err != nil {
    84  		t.Fatalf("err: %v", err)
    85  	}
    86  
    87  	// Update the time tables to make this work
    88  	tt := s1.fsm.TimeTable()
    89  	tt.Witness(2000, time.Now().UTC().Add(-1*s1.config.EvalGCThreshold))
    90  
    91  	// Create a core scheduler
    92  	snap, err := state.Snapshot()
    93  	if err != nil {
    94  		t.Fatalf("err: %v", err)
    95  	}
    96  	core := NewCoreScheduler(s1, snap)
    97  
    98  	// Attempt the GC
    99  	gc := s1.coreJobEval(structs.CoreJobEvalGC)
   100  	gc.ModifyIndex = 2000
   101  	err = core.Process(gc)
   102  	if err != nil {
   103  		t.Fatalf("err: %v", err)
   104  	}
   105  
   106  	// Should be gone because there is no alloc associated
   107  	out, err := state.EvalByID(eval.ID)
   108  	if err != nil {
   109  		t.Fatalf("err: %v", err)
   110  	}
   111  	if out != nil {
   112  		t.Fatalf("bad: %v", out)
   113  	}
   114  }
   115  
   116  func TestCoreScheduler_EvalGC_Batch_Allocs_WithJob(t *testing.T) {
   117  	s1 := testServer(t, nil)
   118  	defer s1.Shutdown()
   119  	testutil.WaitForLeader(t, s1.RPC)
   120  
   121  	// Insert job.
   122  	state := s1.fsm.State()
   123  	job := mock.Job()
   124  	job.Type = structs.JobTypeBatch
   125  	err := state.UpsertJob(1000, job)
   126  	if err != nil {
   127  		t.Fatalf("err: %v", err)
   128  	}
   129  
   130  	// Insert "dead" eval
   131  	eval := mock.Eval()
   132  	eval.Type = structs.JobTypeBatch
   133  	eval.Status = structs.EvalStatusFailed
   134  	eval.JobID = job.ID
   135  	if err := state.UpsertEvals(1001, []*structs.Evaluation{eval}); err != nil {
   136  		t.Fatalf("err: %v", err)
   137  	}
   138  
   139  	// Insert "dead" alloc
   140  	alloc := mock.Alloc()
   141  	alloc.EvalID = eval.ID
   142  	alloc.JobID = job.ID
   143  	alloc.DesiredStatus = structs.AllocDesiredStatusFailed
   144  	err = state.UpsertAllocs(1002, []*structs.Allocation{alloc})
   145  	if err != nil {
   146  		t.Fatalf("err: %v", err)
   147  	}
   148  
   149  	// Update the time tables to make this work
   150  	tt := s1.fsm.TimeTable()
   151  	tt.Witness(2000, time.Now().UTC().Add(-1*s1.config.EvalGCThreshold))
   152  
   153  	// Create a core scheduler
   154  	snap, err := state.Snapshot()
   155  	if err != nil {
   156  		t.Fatalf("err: %v", err)
   157  	}
   158  	core := NewCoreScheduler(s1, snap)
   159  
   160  	// Attempt the GC
   161  	gc := s1.coreJobEval(structs.CoreJobEvalGC)
   162  	gc.ModifyIndex = 2000
   163  	err = core.Process(gc)
   164  	if err != nil {
   165  		t.Fatalf("err: %v", err)
   166  	}
   167  
   168  	// Shouldn't be gone because there are associated allocs.
   169  	out, err := state.EvalByID(eval.ID)
   170  	if err != nil {
   171  		t.Fatalf("err: %v", err)
   172  	}
   173  	if out == nil {
   174  		t.Fatalf("bad: %v", out)
   175  	}
   176  
   177  	outA, err := state.AllocByID(alloc.ID)
   178  	if err != nil {
   179  		t.Fatalf("err: %v", err)
   180  	}
   181  	if outA == nil {
   182  		t.Fatalf("bad: %v", outA)
   183  	}
   184  }
   185  
   186  func TestCoreScheduler_EvalGC_Batch_Allocs_NoJob(t *testing.T) {
   187  	s1 := testServer(t, nil)
   188  	defer s1.Shutdown()
   189  	testutil.WaitForLeader(t, s1.RPC)
   190  
   191  	// Insert "dead" eval
   192  	state := s1.fsm.State()
   193  	eval := mock.Eval()
   194  	eval.Type = structs.JobTypeBatch
   195  	eval.Status = structs.EvalStatusFailed
   196  	err := state.UpsertEvals(1000, []*structs.Evaluation{eval})
   197  	if err != nil {
   198  		t.Fatalf("err: %v", err)
   199  	}
   200  
   201  	// Insert "dead" alloc
   202  	alloc := mock.Alloc()
   203  	alloc.EvalID = eval.ID
   204  	alloc.DesiredStatus = structs.AllocDesiredStatusFailed
   205  	err = state.UpsertAllocs(1001, []*structs.Allocation{alloc})
   206  	if err != nil {
   207  		t.Fatalf("err: %v", err)
   208  	}
   209  
   210  	// Update the time tables to make this work
   211  	tt := s1.fsm.TimeTable()
   212  	tt.Witness(2000, time.Now().UTC().Add(-1*s1.config.EvalGCThreshold))
   213  
   214  	// Create a core scheduler
   215  	snap, err := state.Snapshot()
   216  	if err != nil {
   217  		t.Fatalf("err: %v", err)
   218  	}
   219  	core := NewCoreScheduler(s1, snap)
   220  
   221  	// Attempt the GC
   222  	gc := s1.coreJobEval(structs.CoreJobEvalGC)
   223  	gc.ModifyIndex = 2000
   224  	err = core.Process(gc)
   225  	if err != nil {
   226  		t.Fatalf("err: %v", err)
   227  	}
   228  
   229  	// Should be gone because the job is deregistered.
   230  	out, err := state.EvalByID(eval.ID)
   231  	if err != nil {
   232  		t.Fatalf("err: %v", err)
   233  	}
   234  	if out != nil {
   235  		t.Fatalf("bad: %v", out)
   236  	}
   237  }
   238  
   239  func TestCoreScheduler_EvalGC_Force(t *testing.T) {
   240  	s1 := testServer(t, nil)
   241  	defer s1.Shutdown()
   242  	testutil.WaitForLeader(t, s1.RPC)
   243  
   244  	// Insert "dead" eval
   245  	state := s1.fsm.State()
   246  	eval := mock.Eval()
   247  	eval.Status = structs.EvalStatusFailed
   248  	err := state.UpsertEvals(1000, []*structs.Evaluation{eval})
   249  	if err != nil {
   250  		t.Fatalf("err: %v", err)
   251  	}
   252  
   253  	// Insert "dead" alloc
   254  	alloc := mock.Alloc()
   255  	alloc.EvalID = eval.ID
   256  	alloc.DesiredStatus = structs.AllocDesiredStatusFailed
   257  	err = state.UpsertAllocs(1001, []*structs.Allocation{alloc})
   258  	if err != nil {
   259  		t.Fatalf("err: %v", err)
   260  	}
   261  
   262  	// Create a core scheduler
   263  	snap, err := state.Snapshot()
   264  	if err != nil {
   265  		t.Fatalf("err: %v", err)
   266  	}
   267  	core := NewCoreScheduler(s1, snap)
   268  
   269  	// Attempt the GC
   270  	gc := s1.coreJobEval(structs.CoreJobForceGC)
   271  	err = core.Process(gc)
   272  	if err != nil {
   273  		t.Fatalf("err: %v", err)
   274  	}
   275  
   276  	// Should be gone
   277  	out, err := state.EvalByID(eval.ID)
   278  	if err != nil {
   279  		t.Fatalf("err: %v", err)
   280  	}
   281  	if out != nil {
   282  		t.Fatalf("bad: %v", out)
   283  	}
   284  
   285  	outA, err := state.AllocByID(alloc.ID)
   286  	if err != nil {
   287  		t.Fatalf("err: %v", err)
   288  	}
   289  	if outA != nil {
   290  		t.Fatalf("bad: %v", outA)
   291  	}
   292  }
   293  
   294  func TestCoreScheduler_NodeGC(t *testing.T) {
   295  	s1 := testServer(t, nil)
   296  	defer s1.Shutdown()
   297  	testutil.WaitForLeader(t, s1.RPC)
   298  
   299  	// Insert "dead" node
   300  	state := s1.fsm.State()
   301  	node := mock.Node()
   302  	node.Status = structs.NodeStatusDown
   303  	err := state.UpsertNode(1000, node)
   304  	if err != nil {
   305  		t.Fatalf("err: %v", err)
   306  	}
   307  
   308  	// Update the time tables to make this work
   309  	tt := s1.fsm.TimeTable()
   310  	tt.Witness(2000, time.Now().UTC().Add(-1*s1.config.NodeGCThreshold))
   311  
   312  	// Create a core scheduler
   313  	snap, err := state.Snapshot()
   314  	if err != nil {
   315  		t.Fatalf("err: %v", err)
   316  	}
   317  	core := NewCoreScheduler(s1, snap)
   318  
   319  	// Attempt the GC
   320  	gc := s1.coreJobEval(structs.CoreJobNodeGC)
   321  	gc.ModifyIndex = 2000
   322  	err = core.Process(gc)
   323  	if err != nil {
   324  		t.Fatalf("err: %v", err)
   325  	}
   326  
   327  	// Should be gone
   328  	out, err := state.NodeByID(node.ID)
   329  	if err != nil {
   330  		t.Fatalf("err: %v", err)
   331  	}
   332  	if out != nil {
   333  		t.Fatalf("bad: %v", out)
   334  	}
   335  }
   336  
   337  func TestCoreScheduler_NodeGC_Force(t *testing.T) {
   338  	s1 := testServer(t, nil)
   339  	defer s1.Shutdown()
   340  	testutil.WaitForLeader(t, s1.RPC)
   341  
   342  	// Insert "dead" node
   343  	state := s1.fsm.State()
   344  	node := mock.Node()
   345  	node.Status = structs.NodeStatusDown
   346  	err := state.UpsertNode(1000, node)
   347  	if err != nil {
   348  		t.Fatalf("err: %v", err)
   349  	}
   350  
   351  	// Create a core scheduler
   352  	snap, err := state.Snapshot()
   353  	if err != nil {
   354  		t.Fatalf("err: %v", err)
   355  	}
   356  	core := NewCoreScheduler(s1, snap)
   357  
   358  	// Attempt the GC
   359  	gc := s1.coreJobEval(structs.CoreJobForceGC)
   360  	err = core.Process(gc)
   361  	if err != nil {
   362  		t.Fatalf("err: %v", err)
   363  	}
   364  
   365  	// Should be gone
   366  	out, err := state.NodeByID(node.ID)
   367  	if err != nil {
   368  		t.Fatalf("err: %v", err)
   369  	}
   370  	if out != nil {
   371  		t.Fatalf("bad: %v", out)
   372  	}
   373  }
   374  
   375  func TestCoreScheduler_JobGC(t *testing.T) {
   376  	tests := []struct {
   377  		test, evalStatus, allocStatus string
   378  		shouldExist                   bool
   379  	}{
   380  		{
   381  			test:        "Terminal",
   382  			evalStatus:  structs.EvalStatusFailed,
   383  			allocStatus: structs.AllocDesiredStatusFailed,
   384  			shouldExist: false,
   385  		},
   386  		{
   387  			test:        "Has Alloc",
   388  			evalStatus:  structs.EvalStatusFailed,
   389  			allocStatus: structs.AllocDesiredStatusRun,
   390  			shouldExist: true,
   391  		},
   392  		{
   393  			test:        "Has Eval",
   394  			evalStatus:  structs.EvalStatusPending,
   395  			allocStatus: structs.AllocDesiredStatusFailed,
   396  			shouldExist: true,
   397  		},
   398  	}
   399  
   400  	for _, test := range tests {
   401  		s1 := testServer(t, nil)
   402  		defer s1.Shutdown()
   403  		testutil.WaitForLeader(t, s1.RPC)
   404  
   405  		// Insert job.
   406  		state := s1.fsm.State()
   407  		job := mock.Job()
   408  		job.Type = structs.JobTypeBatch
   409  		err := state.UpsertJob(1000, job)
   410  		if err != nil {
   411  			t.Fatalf("test(%s) err: %v", test.test, err)
   412  		}
   413  
   414  		// Insert eval
   415  		eval := mock.Eval()
   416  		eval.JobID = job.ID
   417  		eval.Status = test.evalStatus
   418  		err = state.UpsertEvals(1001, []*structs.Evaluation{eval})
   419  		if err != nil {
   420  			t.Fatalf("test(%s) err: %v", test.test, err)
   421  		}
   422  
   423  		// Insert alloc
   424  		alloc := mock.Alloc()
   425  		alloc.JobID = job.ID
   426  		alloc.EvalID = eval.ID
   427  		alloc.DesiredStatus = test.allocStatus
   428  		err = state.UpsertAllocs(1002, []*structs.Allocation{alloc})
   429  		if err != nil {
   430  			t.Fatalf("test(%s) err: %v", test.test, err)
   431  		}
   432  
   433  		// Update the time tables to make this work
   434  		tt := s1.fsm.TimeTable()
   435  		tt.Witness(2000, time.Now().UTC().Add(-1*s1.config.JobGCThreshold))
   436  
   437  		// Create a core scheduler
   438  		snap, err := state.Snapshot()
   439  		if err != nil {
   440  			t.Fatalf("test(%s) err: %v", test.test, err)
   441  		}
   442  		core := NewCoreScheduler(s1, snap)
   443  
   444  		// Attempt the GC
   445  		gc := s1.coreJobEval(structs.CoreJobJobGC)
   446  		gc.ModifyIndex = 2000
   447  		err = core.Process(gc)
   448  		if err != nil {
   449  			t.Fatalf("test(%s) err: %v", test.test, err)
   450  		}
   451  
   452  		// Should still exist
   453  		out, err := state.JobByID(job.ID)
   454  		if err != nil {
   455  			t.Fatalf("test(%s) err: %v", test.test, err)
   456  		}
   457  		if (test.shouldExist && out == nil) || (!test.shouldExist && out != nil) {
   458  			t.Fatalf("test(%s) bad: %v", test.test, out)
   459  		}
   460  
   461  		outE, err := state.EvalByID(eval.ID)
   462  		if err != nil {
   463  			t.Fatalf("test(%s) err: %v", test.test, err)
   464  		}
   465  		if (test.shouldExist && outE == nil) || (!test.shouldExist && outE != nil) {
   466  			t.Fatalf("test(%s) bad: %v", test.test, out)
   467  		}
   468  
   469  		outA, err := state.AllocByID(alloc.ID)
   470  		if err != nil {
   471  			t.Fatalf("test(%s) err: %v", test.test, err)
   472  		}
   473  		if (test.shouldExist && outA == nil) || (!test.shouldExist && outA != nil) {
   474  			t.Fatalf("test(%s) bad: %v", test.test, outA)
   475  		}
   476  	}
   477  }
   478  
   479  func TestCoreScheduler_JobGC_Force(t *testing.T) {
   480  	tests := []struct {
   481  		test, evalStatus, allocStatus string
   482  		shouldExist                   bool
   483  	}{
   484  		{
   485  			test:        "Terminal",
   486  			evalStatus:  structs.EvalStatusFailed,
   487  			allocStatus: structs.AllocDesiredStatusFailed,
   488  			shouldExist: false,
   489  		},
   490  		{
   491  			test:        "Has Alloc",
   492  			evalStatus:  structs.EvalStatusFailed,
   493  			allocStatus: structs.AllocDesiredStatusRun,
   494  			shouldExist: true,
   495  		},
   496  		{
   497  			test:        "Has Eval",
   498  			evalStatus:  structs.EvalStatusPending,
   499  			allocStatus: structs.AllocDesiredStatusFailed,
   500  			shouldExist: true,
   501  		},
   502  	}
   503  
   504  	for _, test := range tests {
   505  		s1 := testServer(t, nil)
   506  		defer s1.Shutdown()
   507  		testutil.WaitForLeader(t, s1.RPC)
   508  
   509  		// Insert job.
   510  		state := s1.fsm.State()
   511  		job := mock.Job()
   512  		job.Type = structs.JobTypeBatch
   513  		err := state.UpsertJob(1000, job)
   514  		if err != nil {
   515  			t.Fatalf("test(%s) err: %v", test.test, err)
   516  		}
   517  
   518  		// Insert eval
   519  		eval := mock.Eval()
   520  		eval.JobID = job.ID
   521  		eval.Status = test.evalStatus
   522  		err = state.UpsertEvals(1001, []*structs.Evaluation{eval})
   523  		if err != nil {
   524  			t.Fatalf("test(%s) err: %v", test.test, err)
   525  		}
   526  
   527  		// Insert alloc
   528  		alloc := mock.Alloc()
   529  		alloc.JobID = job.ID
   530  		alloc.EvalID = eval.ID
   531  		alloc.DesiredStatus = test.allocStatus
   532  		err = state.UpsertAllocs(1002, []*structs.Allocation{alloc})
   533  		if err != nil {
   534  			t.Fatalf("test(%s) err: %v", test.test, err)
   535  		}
   536  
   537  		// Create a core scheduler
   538  		snap, err := state.Snapshot()
   539  		if err != nil {
   540  			t.Fatalf("test(%s) err: %v", test.test, err)
   541  		}
   542  		core := NewCoreScheduler(s1, snap)
   543  
   544  		// Attempt the GC
   545  		gc := s1.coreJobEval(structs.CoreJobForceGC)
   546  		err = core.Process(gc)
   547  		if err != nil {
   548  			t.Fatalf("test(%s) err: %v", test.test, err)
   549  		}
   550  
   551  		// Should still exist
   552  		out, err := state.JobByID(job.ID)
   553  		if err != nil {
   554  			t.Fatalf("test(%s) err: %v", test.test, err)
   555  		}
   556  		if (test.shouldExist && out == nil) || (!test.shouldExist && out != nil) {
   557  			t.Fatalf("test(%s) bad: %v", test.test, out)
   558  		}
   559  
   560  		outE, err := state.EvalByID(eval.ID)
   561  		if err != nil {
   562  			t.Fatalf("test(%s) err: %v", test.test, err)
   563  		}
   564  		if (test.shouldExist && outE == nil) || (!test.shouldExist && outE != nil) {
   565  			t.Fatalf("test(%s) bad: %v", test.test, out)
   566  		}
   567  
   568  		outA, err := state.AllocByID(alloc.ID)
   569  		if err != nil {
   570  			t.Fatalf("test(%s) err: %v", test.test, err)
   571  		}
   572  		if (test.shouldExist && outA == nil) || (!test.shouldExist && outA != nil) {
   573  			t.Fatalf("test(%s) bad: %v", test.test, outA)
   574  		}
   575  	}
   576  }
   577  
   578  func TestCoreScheduler_PartitionReap(t *testing.T) {
   579  	s1 := testServer(t, nil)
   580  	defer s1.Shutdown()
   581  	testutil.WaitForLeader(t, s1.RPC)
   582  
   583  	// Create a core scheduler
   584  	snap, err := s1.fsm.State().Snapshot()
   585  	if err != nil {
   586  		t.Fatalf("err: %v", err)
   587  	}
   588  	core := NewCoreScheduler(s1, snap)
   589  
   590  	// Set the max ids per reap to something lower.
   591  	maxIdsPerReap = 2
   592  
   593  	evals := []string{"a", "b", "c"}
   594  	allocs := []string{"1", "2", "3"}
   595  	requests := core.(*CoreScheduler).partitionReap(evals, allocs)
   596  	if len(requests) != 3 {
   597  		t.Fatalf("Expected 3 requests got: %v", requests)
   598  	}
   599  
   600  	first := requests[0]
   601  	if len(first.Allocs) != 2 && len(first.Evals) != 0 {
   602  		t.Fatalf("Unexpected first request: %v", first)
   603  	}
   604  
   605  	second := requests[1]
   606  	if len(second.Allocs) != 1 && len(second.Evals) != 1 {
   607  		t.Fatalf("Unexpected second request: %v", second)
   608  	}
   609  
   610  	third := requests[2]
   611  	if len(third.Allocs) != 0 && len(third.Evals) != 2 {
   612  		t.Fatalf("Unexpected third request: %v", third)
   613  	}
   614  }