github.com/jrxfive/nomad@v0.6.1-0.20170802162750-1fef470e89bf/nomad/core_sched_test.go (about)

     1  package nomad
     2  
     3  import (
     4  	"testing"
     5  	"time"
     6  
     7  	memdb "github.com/hashicorp/go-memdb"
     8  	"github.com/hashicorp/nomad/nomad/mock"
     9  	"github.com/hashicorp/nomad/nomad/structs"
    10  	"github.com/hashicorp/nomad/testutil"
    11  	"github.com/stretchr/testify/assert"
    12  )
    13  
    14  func TestCoreScheduler_EvalGC(t *testing.T) {
    15  	t.Parallel()
    16  	s1 := testServer(t, nil)
    17  	defer s1.Shutdown()
    18  	testutil.WaitForLeader(t, s1.RPC)
    19  
    20  	// COMPAT Remove in 0.6: Reset the FSM time table since we reconcile which sets index 0
    21  	s1.fsm.timetable.table = make([]TimeTableEntry, 1, 10)
    22  
    23  	// Insert "dead" eval
    24  	state := s1.fsm.State()
    25  	eval := mock.Eval()
    26  	eval.Status = structs.EvalStatusFailed
    27  	state.UpsertJobSummary(999, mock.JobSummary(eval.JobID))
    28  	err := state.UpsertEvals(1000, []*structs.Evaluation{eval})
    29  	if err != nil {
    30  		t.Fatalf("err: %v", err)
    31  	}
    32  
    33  	// Insert "dead" alloc
    34  	alloc := mock.Alloc()
    35  	alloc.EvalID = eval.ID
    36  	alloc.DesiredStatus = structs.AllocDesiredStatusStop
    37  	alloc.JobID = eval.JobID
    38  
    39  	// Insert "lost" alloc
    40  	alloc2 := mock.Alloc()
    41  	alloc2.EvalID = eval.ID
    42  	alloc2.DesiredStatus = structs.AllocDesiredStatusRun
    43  	alloc2.ClientStatus = structs.AllocClientStatusLost
    44  	alloc2.JobID = eval.JobID
    45  	err = state.UpsertAllocs(1001, []*structs.Allocation{alloc, alloc2})
    46  	if err != nil {
    47  		t.Fatalf("err: %v", err)
    48  	}
    49  
    50  	// Update the time tables to make this work
    51  	tt := s1.fsm.TimeTable()
    52  	tt.Witness(2000, time.Now().UTC().Add(-1*s1.config.EvalGCThreshold))
    53  
    54  	// Create a core scheduler
    55  	snap, err := state.Snapshot()
    56  	if err != nil {
    57  		t.Fatalf("err: %v", err)
    58  	}
    59  	core := NewCoreScheduler(s1, snap)
    60  
    61  	// Attempt the GC
    62  	gc := s1.coreJobEval(structs.CoreJobEvalGC, 2000)
    63  	err = core.Process(gc)
    64  	if err != nil {
    65  		t.Fatalf("err: %v", err)
    66  	}
    67  
    68  	// Should be gone
    69  	ws := memdb.NewWatchSet()
    70  	out, err := state.EvalByID(ws, eval.ID)
    71  	if err != nil {
    72  		t.Fatalf("err: %v", err)
    73  	}
    74  	if out != nil {
    75  		t.Fatalf("bad: %v", out)
    76  	}
    77  
    78  	outA, err := state.AllocByID(ws, alloc.ID)
    79  	if err != nil {
    80  		t.Fatalf("err: %v", err)
    81  	}
    82  	if outA != nil {
    83  		t.Fatalf("bad: %v", outA)
    84  	}
    85  
    86  	outA2, err := state.AllocByID(ws, alloc2.ID)
    87  	if err != nil {
    88  		t.Fatalf("err: %v", err)
    89  	}
    90  	if outA2 != nil {
    91  		t.Fatalf("bad: %v", outA2)
    92  	}
    93  }
    94  
    95  // An EvalGC should never reap a batch job that has not been stopped
    96  func TestCoreScheduler_EvalGC_Batch(t *testing.T) {
    97  	t.Parallel()
    98  	s1 := testServer(t, nil)
    99  	defer s1.Shutdown()
   100  	testutil.WaitForLeader(t, s1.RPC)
   101  
   102  	// COMPAT Remove in 0.6: Reset the FSM time table since we reconcile which sets index 0
   103  	s1.fsm.timetable.table = make([]TimeTableEntry, 1, 10)
   104  
   105  	// Insert a "dead" job
   106  	state := s1.fsm.State()
   107  	job := mock.Job()
   108  	job.Type = structs.JobTypeBatch
   109  	job.Status = structs.JobStatusDead
   110  	err := state.UpsertJob(1000, job)
   111  	if err != nil {
   112  		t.Fatalf("err: %v", err)
   113  	}
   114  
   115  	// Insert "complete" eval
   116  	eval := mock.Eval()
   117  	eval.Status = structs.EvalStatusComplete
   118  	eval.Type = structs.JobTypeBatch
   119  	eval.JobID = job.ID
   120  	err = state.UpsertEvals(1001, []*structs.Evaluation{eval})
   121  	if err != nil {
   122  		t.Fatalf("err: %v", err)
   123  	}
   124  
   125  	// Insert "failed" alloc
   126  	alloc := mock.Alloc()
   127  	alloc.JobID = job.ID
   128  	alloc.EvalID = eval.ID
   129  	alloc.DesiredStatus = structs.AllocDesiredStatusStop
   130  
   131  	// Insert "lost" alloc
   132  	alloc2 := mock.Alloc()
   133  	alloc2.JobID = job.ID
   134  	alloc2.EvalID = eval.ID
   135  	alloc2.DesiredStatus = structs.AllocDesiredStatusRun
   136  	alloc2.ClientStatus = structs.AllocClientStatusLost
   137  
   138  	err = state.UpsertAllocs(1002, []*structs.Allocation{alloc, alloc2})
   139  	if err != nil {
   140  		t.Fatalf("err: %v", err)
   141  	}
   142  
   143  	// Update the time tables to make this work
   144  	tt := s1.fsm.TimeTable()
   145  	tt.Witness(2000, time.Now().UTC().Add(-1*s1.config.EvalGCThreshold))
   146  
   147  	// Create a core scheduler
   148  	snap, err := state.Snapshot()
   149  	if err != nil {
   150  		t.Fatalf("err: %v", err)
   151  	}
   152  	core := NewCoreScheduler(s1, snap)
   153  
   154  	// Attempt the GC
   155  	gc := s1.coreJobEval(structs.CoreJobEvalGC, 2000)
   156  	err = core.Process(gc)
   157  	if err != nil {
   158  		t.Fatalf("err: %v", err)
   159  	}
   160  
   161  	// Nothing should be gone
   162  	ws := memdb.NewWatchSet()
   163  	out, err := state.EvalByID(ws, eval.ID)
   164  	if err != nil {
   165  		t.Fatalf("err: %v", err)
   166  	}
   167  	if out == nil {
   168  		t.Fatalf("bad: %v", out)
   169  	}
   170  
   171  	outA, err := state.AllocByID(ws, alloc.ID)
   172  	if err != nil {
   173  		t.Fatalf("err: %v", err)
   174  	}
   175  	if outA == nil {
   176  		t.Fatalf("bad: %v", outA)
   177  	}
   178  
   179  	outA2, err := state.AllocByID(ws, alloc2.ID)
   180  	if err != nil {
   181  		t.Fatalf("err: %v", err)
   182  	}
   183  	if outA2 == nil {
   184  		t.Fatalf("bad: %v", outA2)
   185  	}
   186  
   187  	outB, err := state.JobByID(ws, job.ID)
   188  	if err != nil {
   189  		t.Fatalf("err: %v", err)
   190  	}
   191  	if outB == nil {
   192  		t.Fatalf("bad: %v", outB)
   193  	}
   194  }
   195  
   196  // An EvalGC should  reap a batch job that has been stopped
   197  func TestCoreScheduler_EvalGC_BatchStopped(t *testing.T) {
   198  	t.Parallel()
   199  	s1 := testServer(t, nil)
   200  	defer s1.Shutdown()
   201  	testutil.WaitForLeader(t, s1.RPC)
   202  
   203  	// COMPAT Remove in 0.6: Reset the FSM time table since we reconcile which sets index 0
   204  	s1.fsm.timetable.table = make([]TimeTableEntry, 1, 10)
   205  
   206  	// Create a "dead" job
   207  	state := s1.fsm.State()
   208  	job := mock.Job()
   209  	job.Type = structs.JobTypeBatch
   210  	job.Status = structs.JobStatusDead
   211  
   212  	// Insert "complete" eval
   213  	eval := mock.Eval()
   214  	eval.Status = structs.EvalStatusComplete
   215  	eval.Type = structs.JobTypeBatch
   216  	eval.JobID = job.ID
   217  	err := state.UpsertEvals(1001, []*structs.Evaluation{eval})
   218  	if err != nil {
   219  		t.Fatalf("err: %v", err)
   220  	}
   221  
   222  	// Insert "failed" alloc
   223  	alloc := mock.Alloc()
   224  	alloc.JobID = job.ID
   225  	alloc.EvalID = eval.ID
   226  	alloc.DesiredStatus = structs.AllocDesiredStatusStop
   227  
   228  	// Insert "lost" alloc
   229  	alloc2 := mock.Alloc()
   230  	alloc2.JobID = job.ID
   231  	alloc2.EvalID = eval.ID
   232  	alloc2.DesiredStatus = structs.AllocDesiredStatusRun
   233  	alloc2.ClientStatus = structs.AllocClientStatusLost
   234  
   235  	err = state.UpsertAllocs(1002, []*structs.Allocation{alloc, alloc2})
   236  	if err != nil {
   237  		t.Fatalf("err: %v", err)
   238  	}
   239  
   240  	// Update the time tables to make this work
   241  	tt := s1.fsm.TimeTable()
   242  	tt.Witness(2000, time.Now().UTC().Add(-1*s1.config.EvalGCThreshold))
   243  
   244  	// Create a core scheduler
   245  	snap, err := state.Snapshot()
   246  	if err != nil {
   247  		t.Fatalf("err: %v", err)
   248  	}
   249  	core := NewCoreScheduler(s1, snap)
   250  
   251  	// Attempt the GC
   252  	gc := s1.coreJobEval(structs.CoreJobEvalGC, 2000)
   253  	err = core.Process(gc)
   254  	if err != nil {
   255  		t.Fatalf("err: %v", err)
   256  	}
   257  
   258  	// Everything should be gone
   259  	ws := memdb.NewWatchSet()
   260  	out, err := state.EvalByID(ws, eval.ID)
   261  	if err != nil {
   262  		t.Fatalf("err: %v", err)
   263  	}
   264  	if out != nil {
   265  		t.Fatalf("bad: %v", out)
   266  	}
   267  
   268  	outA, err := state.AllocByID(ws, alloc.ID)
   269  	if err != nil {
   270  		t.Fatalf("err: %v", err)
   271  	}
   272  	if outA != nil {
   273  		t.Fatalf("bad: %v", outA)
   274  	}
   275  
   276  	outA2, err := state.AllocByID(ws, alloc2.ID)
   277  	if err != nil {
   278  		t.Fatalf("err: %v", err)
   279  	}
   280  	if outA2 != nil {
   281  		t.Fatalf("bad: %v", outA2)
   282  	}
   283  }
   284  
   285  func TestCoreScheduler_EvalGC_Partial(t *testing.T) {
   286  	t.Parallel()
   287  	s1 := testServer(t, nil)
   288  	defer s1.Shutdown()
   289  	testutil.WaitForLeader(t, s1.RPC)
   290  
   291  	// COMPAT Remove in 0.6: Reset the FSM time table since we reconcile which sets index 0
   292  	s1.fsm.timetable.table = make([]TimeTableEntry, 1, 10)
   293  
   294  	// Insert "dead" eval
   295  	state := s1.fsm.State()
   296  	eval := mock.Eval()
   297  	eval.Status = structs.EvalStatusComplete
   298  	state.UpsertJobSummary(999, mock.JobSummary(eval.JobID))
   299  	err := state.UpsertEvals(1000, []*structs.Evaluation{eval})
   300  	if err != nil {
   301  		t.Fatalf("err: %v", err)
   302  	}
   303  
   304  	// Insert "dead" alloc
   305  	alloc := mock.Alloc()
   306  	alloc.EvalID = eval.ID
   307  	alloc.DesiredStatus = structs.AllocDesiredStatusStop
   308  	state.UpsertJobSummary(1001, mock.JobSummary(alloc.JobID))
   309  
   310  	// Insert "lost" alloc
   311  	alloc2 := mock.Alloc()
   312  	alloc2.JobID = alloc.JobID
   313  	alloc2.EvalID = eval.ID
   314  	alloc2.DesiredStatus = structs.AllocDesiredStatusRun
   315  	alloc2.ClientStatus = structs.AllocClientStatusLost
   316  
   317  	err = state.UpsertAllocs(1002, []*structs.Allocation{alloc, alloc2})
   318  	if err != nil {
   319  		t.Fatalf("err: %v", err)
   320  	}
   321  
   322  	// Insert "running" alloc
   323  	alloc3 := mock.Alloc()
   324  	alloc3.EvalID = eval.ID
   325  	state.UpsertJobSummary(1003, mock.JobSummary(alloc3.JobID))
   326  	err = state.UpsertAllocs(1004, []*structs.Allocation{alloc3})
   327  	if err != nil {
   328  		t.Fatalf("err: %v", err)
   329  	}
   330  
   331  	// Update the time tables to make this work
   332  	tt := s1.fsm.TimeTable()
   333  	tt.Witness(2000, time.Now().UTC().Add(-1*s1.config.EvalGCThreshold))
   334  
   335  	// Create a core scheduler
   336  	snap, err := state.Snapshot()
   337  	if err != nil {
   338  		t.Fatalf("err: %v", err)
   339  	}
   340  	core := NewCoreScheduler(s1, snap)
   341  
   342  	// Attempt the GC
   343  	gc := s1.coreJobEval(structs.CoreJobEvalGC, 2000)
   344  	err = core.Process(gc)
   345  	if err != nil {
   346  		t.Fatalf("err: %v", err)
   347  	}
   348  
   349  	// Should not be gone
   350  	ws := memdb.NewWatchSet()
   351  	out, err := state.EvalByID(ws, eval.ID)
   352  	if err != nil {
   353  		t.Fatalf("err: %v", err)
   354  	}
   355  	if out == nil {
   356  		t.Fatalf("bad: %v", out)
   357  	}
   358  
   359  	outA, err := state.AllocByID(ws, alloc3.ID)
   360  	if err != nil {
   361  		t.Fatalf("err: %v", err)
   362  	}
   363  	if outA == nil {
   364  		t.Fatalf("bad: %v", outA)
   365  	}
   366  
   367  	// Should be gone
   368  	outB, err := state.AllocByID(ws, alloc.ID)
   369  	if err != nil {
   370  		t.Fatalf("err: %v", err)
   371  	}
   372  	if outB != nil {
   373  		t.Fatalf("bad: %v", outB)
   374  	}
   375  
   376  	outC, err := state.AllocByID(ws, alloc2.ID)
   377  	if err != nil {
   378  		t.Fatalf("err: %v", err)
   379  	}
   380  	if outC != nil {
   381  		t.Fatalf("bad: %v", outC)
   382  	}
   383  }
   384  
   385  func TestCoreScheduler_EvalGC_Force(t *testing.T) {
   386  	t.Parallel()
   387  	s1 := testServer(t, nil)
   388  	defer s1.Shutdown()
   389  	testutil.WaitForLeader(t, s1.RPC)
   390  
   391  	// COMPAT Remove in 0.6: Reset the FSM time table since we reconcile which sets index 0
   392  	s1.fsm.timetable.table = make([]TimeTableEntry, 1, 10)
   393  
   394  	// Insert "dead" eval
   395  	state := s1.fsm.State()
   396  	eval := mock.Eval()
   397  	eval.Status = structs.EvalStatusFailed
   398  	state.UpsertJobSummary(999, mock.JobSummary(eval.JobID))
   399  	err := state.UpsertEvals(1000, []*structs.Evaluation{eval})
   400  	if err != nil {
   401  		t.Fatalf("err: %v", err)
   402  	}
   403  
   404  	// Insert "dead" alloc
   405  	alloc := mock.Alloc()
   406  	alloc.EvalID = eval.ID
   407  	alloc.DesiredStatus = structs.AllocDesiredStatusStop
   408  	state.UpsertJobSummary(1001, mock.JobSummary(alloc.JobID))
   409  	err = state.UpsertAllocs(1002, []*structs.Allocation{alloc})
   410  	if err != nil {
   411  		t.Fatalf("err: %v", err)
   412  	}
   413  
   414  	// Create a core scheduler
   415  	snap, err := state.Snapshot()
   416  	if err != nil {
   417  		t.Fatalf("err: %v", err)
   418  	}
   419  	core := NewCoreScheduler(s1, snap)
   420  
   421  	// Attempt the GC
   422  	gc := s1.coreJobEval(structs.CoreJobForceGC, 1002)
   423  	err = core.Process(gc)
   424  	if err != nil {
   425  		t.Fatalf("err: %v", err)
   426  	}
   427  
   428  	// Should be gone
   429  	ws := memdb.NewWatchSet()
   430  	out, err := state.EvalByID(ws, eval.ID)
   431  	if err != nil {
   432  		t.Fatalf("err: %v", err)
   433  	}
   434  	if out != nil {
   435  		t.Fatalf("bad: %v", out)
   436  	}
   437  
   438  	outA, err := state.AllocByID(ws, alloc.ID)
   439  	if err != nil {
   440  		t.Fatalf("err: %v", err)
   441  	}
   442  	if outA != nil {
   443  		t.Fatalf("bad: %v", outA)
   444  	}
   445  }
   446  
   447  func TestCoreScheduler_NodeGC(t *testing.T) {
   448  	t.Parallel()
   449  	s1 := testServer(t, nil)
   450  	defer s1.Shutdown()
   451  	testutil.WaitForLeader(t, s1.RPC)
   452  
   453  	// COMPAT Remove in 0.6: Reset the FSM time table since we reconcile which sets index 0
   454  	s1.fsm.timetable.table = make([]TimeTableEntry, 1, 10)
   455  
   456  	// Insert "dead" node
   457  	state := s1.fsm.State()
   458  	node := mock.Node()
   459  	node.Status = structs.NodeStatusDown
   460  	err := state.UpsertNode(1000, node)
   461  	if err != nil {
   462  		t.Fatalf("err: %v", err)
   463  	}
   464  
   465  	// Update the time tables to make this work
   466  	tt := s1.fsm.TimeTable()
   467  	tt.Witness(2000, time.Now().UTC().Add(-1*s1.config.NodeGCThreshold))
   468  
   469  	// Create a core scheduler
   470  	snap, err := state.Snapshot()
   471  	if err != nil {
   472  		t.Fatalf("err: %v", err)
   473  	}
   474  	core := NewCoreScheduler(s1, snap)
   475  
   476  	// Attempt the GC
   477  	gc := s1.coreJobEval(structs.CoreJobNodeGC, 2000)
   478  	err = core.Process(gc)
   479  	if err != nil {
   480  		t.Fatalf("err: %v", err)
   481  	}
   482  
   483  	// Should be gone
   484  	ws := memdb.NewWatchSet()
   485  	out, err := state.NodeByID(ws, node.ID)
   486  	if err != nil {
   487  		t.Fatalf("err: %v", err)
   488  	}
   489  	if out != nil {
   490  		t.Fatalf("bad: %v", out)
   491  	}
   492  }
   493  
   494  func TestCoreScheduler_NodeGC_TerminalAllocs(t *testing.T) {
   495  	t.Parallel()
   496  	s1 := testServer(t, nil)
   497  	defer s1.Shutdown()
   498  	testutil.WaitForLeader(t, s1.RPC)
   499  
   500  	// COMPAT Remove in 0.6: Reset the FSM time table since we reconcile which sets index 0
   501  	s1.fsm.timetable.table = make([]TimeTableEntry, 1, 10)
   502  
   503  	// Insert "dead" node
   504  	state := s1.fsm.State()
   505  	node := mock.Node()
   506  	node.Status = structs.NodeStatusDown
   507  	err := state.UpsertNode(1000, node)
   508  	if err != nil {
   509  		t.Fatalf("err: %v", err)
   510  	}
   511  
   512  	// Insert a terminal alloc on that node
   513  	alloc := mock.Alloc()
   514  	alloc.DesiredStatus = structs.AllocDesiredStatusStop
   515  	state.UpsertJobSummary(1001, mock.JobSummary(alloc.JobID))
   516  	if err := state.UpsertAllocs(1002, []*structs.Allocation{alloc}); err != nil {
   517  		t.Fatalf("err: %v", err)
   518  	}
   519  
   520  	// Update the time tables to make this work
   521  	tt := s1.fsm.TimeTable()
   522  	tt.Witness(2000, time.Now().UTC().Add(-1*s1.config.NodeGCThreshold))
   523  
   524  	// Create a core scheduler
   525  	snap, err := state.Snapshot()
   526  	if err != nil {
   527  		t.Fatalf("err: %v", err)
   528  	}
   529  	core := NewCoreScheduler(s1, snap)
   530  
   531  	// Attempt the GC
   532  	gc := s1.coreJobEval(structs.CoreJobNodeGC, 2000)
   533  	err = core.Process(gc)
   534  	if err != nil {
   535  		t.Fatalf("err: %v", err)
   536  	}
   537  
   538  	// Should be gone
   539  	ws := memdb.NewWatchSet()
   540  	out, err := state.NodeByID(ws, node.ID)
   541  	if err != nil {
   542  		t.Fatalf("err: %v", err)
   543  	}
   544  	if out != nil {
   545  		t.Fatalf("bad: %v", out)
   546  	}
   547  }
   548  
   549  func TestCoreScheduler_NodeGC_RunningAllocs(t *testing.T) {
   550  	t.Parallel()
   551  	s1 := testServer(t, nil)
   552  	defer s1.Shutdown()
   553  	testutil.WaitForLeader(t, s1.RPC)
   554  
   555  	// COMPAT Remove in 0.6: Reset the FSM time table since we reconcile which sets index 0
   556  	s1.fsm.timetable.table = make([]TimeTableEntry, 1, 10)
   557  
   558  	// Insert "dead" node
   559  	state := s1.fsm.State()
   560  	node := mock.Node()
   561  	node.Status = structs.NodeStatusDown
   562  	err := state.UpsertNode(1000, node)
   563  	if err != nil {
   564  		t.Fatalf("err: %v", err)
   565  	}
   566  
   567  	// Insert a running alloc on that node
   568  	alloc := mock.Alloc()
   569  	alloc.NodeID = node.ID
   570  	alloc.DesiredStatus = structs.AllocDesiredStatusRun
   571  	alloc.ClientStatus = structs.AllocClientStatusRunning
   572  	state.UpsertJobSummary(1001, mock.JobSummary(alloc.JobID))
   573  	if err := state.UpsertAllocs(1002, []*structs.Allocation{alloc}); err != nil {
   574  		t.Fatalf("err: %v", err)
   575  	}
   576  
   577  	// Update the time tables to make this work
   578  	tt := s1.fsm.TimeTable()
   579  	tt.Witness(2000, time.Now().UTC().Add(-1*s1.config.NodeGCThreshold))
   580  
   581  	// Create a core scheduler
   582  	snap, err := state.Snapshot()
   583  	if err != nil {
   584  		t.Fatalf("err: %v", err)
   585  	}
   586  	core := NewCoreScheduler(s1, snap)
   587  
   588  	// Attempt the GC
   589  	gc := s1.coreJobEval(structs.CoreJobNodeGC, 2000)
   590  	err = core.Process(gc)
   591  	if err != nil {
   592  		t.Fatalf("err: %v", err)
   593  	}
   594  
   595  	// Should still be here
   596  	ws := memdb.NewWatchSet()
   597  	out, err := state.NodeByID(ws, node.ID)
   598  	if err != nil {
   599  		t.Fatalf("err: %v", err)
   600  	}
   601  	if out == nil {
   602  		t.Fatalf("bad: %v", out)
   603  	}
   604  }
   605  
   606  func TestCoreScheduler_NodeGC_Force(t *testing.T) {
   607  	t.Parallel()
   608  	s1 := testServer(t, nil)
   609  	defer s1.Shutdown()
   610  	testutil.WaitForLeader(t, s1.RPC)
   611  
   612  	// COMPAT Remove in 0.6: Reset the FSM time table since we reconcile which sets index 0
   613  	s1.fsm.timetable.table = make([]TimeTableEntry, 1, 10)
   614  
   615  	// Insert "dead" node
   616  	state := s1.fsm.State()
   617  	node := mock.Node()
   618  	node.Status = structs.NodeStatusDown
   619  	err := state.UpsertNode(1000, node)
   620  	if err != nil {
   621  		t.Fatalf("err: %v", err)
   622  	}
   623  
   624  	// Create a core scheduler
   625  	snap, err := state.Snapshot()
   626  	if err != nil {
   627  		t.Fatalf("err: %v", err)
   628  	}
   629  	core := NewCoreScheduler(s1, snap)
   630  
   631  	// Attempt the GC
   632  	gc := s1.coreJobEval(structs.CoreJobForceGC, 1000)
   633  	err = core.Process(gc)
   634  	if err != nil {
   635  		t.Fatalf("err: %v", err)
   636  	}
   637  
   638  	// Should be gone
   639  	ws := memdb.NewWatchSet()
   640  	out, err := state.NodeByID(ws, node.ID)
   641  	if err != nil {
   642  		t.Fatalf("err: %v", err)
   643  	}
   644  	if out != nil {
   645  		t.Fatalf("bad: %v", out)
   646  	}
   647  }
   648  
   649  func TestCoreScheduler_JobGC_OutstandingEvals(t *testing.T) {
   650  	t.Parallel()
   651  	s1 := testServer(t, nil)
   652  	defer s1.Shutdown()
   653  	testutil.WaitForLeader(t, s1.RPC)
   654  
   655  	// COMPAT Remove in 0.6: Reset the FSM time table since we reconcile which sets index 0
   656  	s1.fsm.timetable.table = make([]TimeTableEntry, 1, 10)
   657  
   658  	// Insert job.
   659  	state := s1.fsm.State()
   660  	job := mock.Job()
   661  	job.Type = structs.JobTypeBatch
   662  	job.Status = structs.JobStatusDead
   663  	err := state.UpsertJob(1000, job)
   664  	if err != nil {
   665  		t.Fatalf("err: %v", err)
   666  	}
   667  
   668  	// Insert two evals, one terminal and one not
   669  	eval := mock.Eval()
   670  	eval.JobID = job.ID
   671  	eval.Status = structs.EvalStatusComplete
   672  
   673  	eval2 := mock.Eval()
   674  	eval2.JobID = job.ID
   675  	eval2.Status = structs.EvalStatusPending
   676  	err = state.UpsertEvals(1001, []*structs.Evaluation{eval, eval2})
   677  	if err != nil {
   678  		t.Fatalf("err: %v", err)
   679  	}
   680  
   681  	// Update the time tables to make this work
   682  	tt := s1.fsm.TimeTable()
   683  	tt.Witness(2000, time.Now().UTC().Add(-1*s1.config.JobGCThreshold))
   684  
   685  	// Create a core scheduler
   686  	snap, err := state.Snapshot()
   687  	if err != nil {
   688  		t.Fatalf("err: %v", err)
   689  	}
   690  	core := NewCoreScheduler(s1, snap)
   691  
   692  	// Attempt the GC
   693  	gc := s1.coreJobEval(structs.CoreJobJobGC, 2000)
   694  	err = core.Process(gc)
   695  	if err != nil {
   696  		t.Fatalf("err: %v", err)
   697  	}
   698  
   699  	// Should still exist
   700  	ws := memdb.NewWatchSet()
   701  	out, err := state.JobByID(ws, job.ID)
   702  	if err != nil {
   703  		t.Fatalf("err: %v", err)
   704  	}
   705  	if out == nil {
   706  		t.Fatalf("bad: %v", out)
   707  	}
   708  
   709  	outE, err := state.EvalByID(ws, eval.ID)
   710  	if err != nil {
   711  		t.Fatalf("err: %v", err)
   712  	}
   713  	if outE == nil {
   714  		t.Fatalf("bad: %v", outE)
   715  	}
   716  
   717  	outE2, err := state.EvalByID(ws, eval2.ID)
   718  	if err != nil {
   719  		t.Fatalf("err: %v", err)
   720  	}
   721  	if outE2 == nil {
   722  		t.Fatalf("bad: %v", outE2)
   723  	}
   724  
   725  	// Update the second eval to be terminal
   726  	eval2.Status = structs.EvalStatusComplete
   727  	err = state.UpsertEvals(1003, []*structs.Evaluation{eval2})
   728  	if err != nil {
   729  		t.Fatalf("err: %v", err)
   730  	}
   731  
   732  	// Create a core scheduler
   733  	snap, err = state.Snapshot()
   734  	if err != nil {
   735  		t.Fatalf("err: %v", err)
   736  	}
   737  	core = NewCoreScheduler(s1, snap)
   738  
   739  	// Attempt the GC
   740  	gc = s1.coreJobEval(structs.CoreJobJobGC, 2000)
   741  	err = core.Process(gc)
   742  	if err != nil {
   743  		t.Fatalf("err: %v", err)
   744  	}
   745  
   746  	// Should not still exist
   747  	out, err = state.JobByID(ws, job.ID)
   748  	if err != nil {
   749  		t.Fatalf("err: %v", err)
   750  	}
   751  	if out != nil {
   752  		t.Fatalf("bad: %v", out)
   753  	}
   754  
   755  	outE, err = state.EvalByID(ws, eval.ID)
   756  	if err != nil {
   757  		t.Fatalf("err: %v", err)
   758  	}
   759  	if outE != nil {
   760  		t.Fatalf("bad: %v", outE)
   761  	}
   762  
   763  	outE2, err = state.EvalByID(ws, eval2.ID)
   764  	if err != nil {
   765  		t.Fatalf("err: %v", err)
   766  	}
   767  	if outE2 != nil {
   768  		t.Fatalf("bad: %v", outE2)
   769  	}
   770  }
   771  
   772  func TestCoreScheduler_JobGC_OutstandingAllocs(t *testing.T) {
   773  	t.Parallel()
   774  	s1 := testServer(t, nil)
   775  	defer s1.Shutdown()
   776  	testutil.WaitForLeader(t, s1.RPC)
   777  
   778  	// COMPAT Remove in 0.6: Reset the FSM time table since we reconcile which sets index 0
   779  	s1.fsm.timetable.table = make([]TimeTableEntry, 1, 10)
   780  
   781  	// Insert job.
   782  	state := s1.fsm.State()
   783  	job := mock.Job()
   784  	job.Type = structs.JobTypeBatch
   785  	job.Status = structs.JobStatusDead
   786  	err := state.UpsertJob(1000, job)
   787  	if err != nil {
   788  		t.Fatalf("err: %v", err)
   789  	}
   790  
   791  	// Insert an eval
   792  	eval := mock.Eval()
   793  	eval.JobID = job.ID
   794  	eval.Status = structs.EvalStatusComplete
   795  	err = state.UpsertEvals(1001, []*structs.Evaluation{eval})
   796  	if err != nil {
   797  		t.Fatalf("err: %v", err)
   798  	}
   799  
   800  	// Insert two allocs, one terminal and one not
   801  	alloc := mock.Alloc()
   802  	alloc.JobID = job.ID
   803  	alloc.EvalID = eval.ID
   804  	alloc.DesiredStatus = structs.AllocDesiredStatusRun
   805  	alloc.ClientStatus = structs.AllocClientStatusComplete
   806  
   807  	alloc2 := mock.Alloc()
   808  	alloc2.JobID = job.ID
   809  	alloc2.EvalID = eval.ID
   810  	alloc2.DesiredStatus = structs.AllocDesiredStatusRun
   811  	alloc2.ClientStatus = structs.AllocClientStatusRunning
   812  
   813  	err = state.UpsertAllocs(1002, []*structs.Allocation{alloc, alloc2})
   814  	if err != nil {
   815  		t.Fatalf("err: %v", err)
   816  	}
   817  
   818  	// Update the time tables to make this work
   819  	tt := s1.fsm.TimeTable()
   820  	tt.Witness(2000, time.Now().UTC().Add(-1*s1.config.JobGCThreshold))
   821  
   822  	// Create a core scheduler
   823  	snap, err := state.Snapshot()
   824  	if err != nil {
   825  		t.Fatalf("err: %v", err)
   826  	}
   827  	core := NewCoreScheduler(s1, snap)
   828  
   829  	// Attempt the GC
   830  	gc := s1.coreJobEval(structs.CoreJobJobGC, 2000)
   831  	err = core.Process(gc)
   832  	if err != nil {
   833  		t.Fatalf("err: %v", err)
   834  	}
   835  
   836  	// Should still exist
   837  	ws := memdb.NewWatchSet()
   838  	out, err := state.JobByID(ws, job.ID)
   839  	if err != nil {
   840  		t.Fatalf("err: %v", err)
   841  	}
   842  	if out == nil {
   843  		t.Fatalf("bad: %v", out)
   844  	}
   845  
   846  	outA, err := state.AllocByID(ws, alloc.ID)
   847  	if err != nil {
   848  		t.Fatalf("err: %v", err)
   849  	}
   850  	if outA == nil {
   851  		t.Fatalf("bad: %v", outA)
   852  	}
   853  
   854  	outA2, err := state.AllocByID(ws, alloc2.ID)
   855  	if err != nil {
   856  		t.Fatalf("err: %v", err)
   857  	}
   858  	if outA2 == nil {
   859  		t.Fatalf("bad: %v", outA2)
   860  	}
   861  
   862  	// Update the second alloc to be terminal
   863  	alloc2.ClientStatus = structs.AllocClientStatusComplete
   864  	err = state.UpsertAllocs(1003, []*structs.Allocation{alloc2})
   865  	if err != nil {
   866  		t.Fatalf("err: %v", err)
   867  	}
   868  
   869  	// Create a core scheduler
   870  	snap, err = state.Snapshot()
   871  	if err != nil {
   872  		t.Fatalf("err: %v", err)
   873  	}
   874  	core = NewCoreScheduler(s1, snap)
   875  
   876  	// Attempt the GC
   877  	gc = s1.coreJobEval(structs.CoreJobJobGC, 2000)
   878  	err = core.Process(gc)
   879  	if err != nil {
   880  		t.Fatalf("err: %v", err)
   881  	}
   882  
   883  	// Should not still exist
   884  	out, err = state.JobByID(ws, job.ID)
   885  	if err != nil {
   886  		t.Fatalf("err: %v", err)
   887  	}
   888  	if out != nil {
   889  		t.Fatalf("bad: %v", out)
   890  	}
   891  
   892  	outA, err = state.AllocByID(ws, alloc.ID)
   893  	if err != nil {
   894  		t.Fatalf("err: %v", err)
   895  	}
   896  	if outA != nil {
   897  		t.Fatalf("bad: %v", outA)
   898  	}
   899  
   900  	outA2, err = state.AllocByID(ws, alloc2.ID)
   901  	if err != nil {
   902  		t.Fatalf("err: %v", err)
   903  	}
   904  	if outA2 != nil {
   905  		t.Fatalf("bad: %v", outA2)
   906  	}
   907  }
   908  
   909  // This test ensures that batch jobs are GC'd in one shot, meaning it all
   910  // allocs/evals and job or nothing
   911  func TestCoreScheduler_JobGC_OneShot(t *testing.T) {
   912  	t.Parallel()
   913  	s1 := testServer(t, nil)
   914  	defer s1.Shutdown()
   915  	testutil.WaitForLeader(t, s1.RPC)
   916  
   917  	// COMPAT Remove in 0.6: Reset the FSM time table since we reconcile which sets index 0
   918  	s1.fsm.timetable.table = make([]TimeTableEntry, 1, 10)
   919  
   920  	// Insert job.
   921  	state := s1.fsm.State()
   922  	job := mock.Job()
   923  	job.Type = structs.JobTypeBatch
   924  	err := state.UpsertJob(1000, job)
   925  	if err != nil {
   926  		t.Fatalf("err: %v", err)
   927  	}
   928  
   929  	// Insert two complete evals
   930  	eval := mock.Eval()
   931  	eval.JobID = job.ID
   932  	eval.Status = structs.EvalStatusComplete
   933  
   934  	eval2 := mock.Eval()
   935  	eval2.JobID = job.ID
   936  	eval2.Status = structs.EvalStatusComplete
   937  
   938  	err = state.UpsertEvals(1001, []*structs.Evaluation{eval, eval2})
   939  	if err != nil {
   940  		t.Fatalf("err: %v", err)
   941  	}
   942  
   943  	// Insert one complete alloc and one running on distinct evals
   944  	alloc := mock.Alloc()
   945  	alloc.JobID = job.ID
   946  	alloc.EvalID = eval.ID
   947  	alloc.DesiredStatus = structs.AllocDesiredStatusStop
   948  
   949  	alloc2 := mock.Alloc()
   950  	alloc2.JobID = job.ID
   951  	alloc2.EvalID = eval2.ID
   952  	alloc2.DesiredStatus = structs.AllocDesiredStatusRun
   953  
   954  	err = state.UpsertAllocs(1002, []*structs.Allocation{alloc, alloc2})
   955  	if err != nil {
   956  		t.Fatalf("err: %v", err)
   957  	}
   958  
   959  	// Force the jobs state to dead
   960  	job.Status = structs.JobStatusDead
   961  
   962  	// Update the time tables to make this work
   963  	tt := s1.fsm.TimeTable()
   964  	tt.Witness(2000, time.Now().UTC().Add(-1*s1.config.JobGCThreshold))
   965  
   966  	// Create a core scheduler
   967  	snap, err := state.Snapshot()
   968  	if err != nil {
   969  		t.Fatalf("err: %v", err)
   970  	}
   971  	core := NewCoreScheduler(s1, snap)
   972  
   973  	// Attempt the GC
   974  	gc := s1.coreJobEval(structs.CoreJobJobGC, 2000)
   975  	err = core.Process(gc)
   976  	if err != nil {
   977  		t.Fatalf("err: %v", err)
   978  	}
   979  
   980  	// Should still exist
   981  	ws := memdb.NewWatchSet()
   982  	out, err := state.JobByID(ws, job.ID)
   983  	if err != nil {
   984  		t.Fatalf("err: %v", err)
   985  	}
   986  	if out == nil {
   987  		t.Fatalf("bad: %v", out)
   988  	}
   989  
   990  	outE, err := state.EvalByID(ws, eval.ID)
   991  	if err != nil {
   992  		t.Fatalf("err: %v", err)
   993  	}
   994  	if outE == nil {
   995  		t.Fatalf("bad: %v", outE)
   996  	}
   997  
   998  	outE2, err := state.EvalByID(ws, eval2.ID)
   999  	if err != nil {
  1000  		t.Fatalf("err: %v", err)
  1001  	}
  1002  	if outE2 == nil {
  1003  		t.Fatalf("bad: %v", outE2)
  1004  	}
  1005  
  1006  	outA, err := state.AllocByID(ws, alloc.ID)
  1007  	if err != nil {
  1008  		t.Fatalf("err: %v", err)
  1009  	}
  1010  	if outA == nil {
  1011  		t.Fatalf("bad: %v", outA)
  1012  	}
  1013  	outA2, err := state.AllocByID(ws, alloc2.ID)
  1014  	if err != nil {
  1015  		t.Fatalf("err: %v", err)
  1016  	}
  1017  	if outA2 == nil {
  1018  		t.Fatalf("bad: %v", outA2)
  1019  	}
  1020  }
  1021  
  1022  // This test ensures that stopped jobs are GCd
  1023  func TestCoreScheduler_JobGC_Stopped(t *testing.T) {
  1024  	t.Parallel()
  1025  	s1 := testServer(t, nil)
  1026  	defer s1.Shutdown()
  1027  	testutil.WaitForLeader(t, s1.RPC)
  1028  
  1029  	// COMPAT Remove in 0.6: Reset the FSM time table since we reconcile which sets index 0
  1030  	s1.fsm.timetable.table = make([]TimeTableEntry, 1, 10)
  1031  
  1032  	// Insert job.
  1033  	state := s1.fsm.State()
  1034  	job := mock.Job()
  1035  	//job.Status = structs.JobStatusDead
  1036  	job.Stop = true
  1037  	err := state.UpsertJob(1000, job)
  1038  	if err != nil {
  1039  		t.Fatalf("err: %v", err)
  1040  	}
  1041  
  1042  	// Insert two complete evals
  1043  	eval := mock.Eval()
  1044  	eval.JobID = job.ID
  1045  	eval.Status = structs.EvalStatusComplete
  1046  
  1047  	eval2 := mock.Eval()
  1048  	eval2.JobID = job.ID
  1049  	eval2.Status = structs.EvalStatusComplete
  1050  
  1051  	err = state.UpsertEvals(1001, []*structs.Evaluation{eval, eval2})
  1052  	if err != nil {
  1053  		t.Fatalf("err: %v", err)
  1054  	}
  1055  
  1056  	// Insert one complete alloc
  1057  	alloc := mock.Alloc()
  1058  	alloc.JobID = job.ID
  1059  	alloc.EvalID = eval.ID
  1060  	alloc.DesiredStatus = structs.AllocDesiredStatusStop
  1061  
  1062  	err = state.UpsertAllocs(1002, []*structs.Allocation{alloc})
  1063  	if err != nil {
  1064  		t.Fatalf("err: %v", err)
  1065  	}
  1066  
  1067  	// Update the time tables to make this work
  1068  	tt := s1.fsm.TimeTable()
  1069  	tt.Witness(2000, time.Now().UTC().Add(-1*s1.config.JobGCThreshold))
  1070  
  1071  	// Create a core scheduler
  1072  	snap, err := state.Snapshot()
  1073  	if err != nil {
  1074  		t.Fatalf("err: %v", err)
  1075  	}
  1076  	core := NewCoreScheduler(s1, snap)
  1077  
  1078  	// Attempt the GC
  1079  	gc := s1.coreJobEval(structs.CoreJobJobGC, 2000)
  1080  	err = core.Process(gc)
  1081  	if err != nil {
  1082  		t.Fatalf("err: %v", err)
  1083  	}
  1084  
  1085  	// Shouldn't still exist
  1086  	ws := memdb.NewWatchSet()
  1087  	out, err := state.JobByID(ws, job.ID)
  1088  	if err != nil {
  1089  		t.Fatalf("err: %v", err)
  1090  	}
  1091  	if out != nil {
  1092  		t.Fatalf("bad: %v", out)
  1093  	}
  1094  
  1095  	outE, err := state.EvalByID(ws, eval.ID)
  1096  	if err != nil {
  1097  		t.Fatalf("err: %v", err)
  1098  	}
  1099  	if outE != nil {
  1100  		t.Fatalf("bad: %v", outE)
  1101  	}
  1102  
  1103  	outE2, err := state.EvalByID(ws, eval2.ID)
  1104  	if err != nil {
  1105  		t.Fatalf("err: %v", err)
  1106  	}
  1107  	if outE2 != nil {
  1108  		t.Fatalf("bad: %v", outE2)
  1109  	}
  1110  
  1111  	outA, err := state.AllocByID(ws, alloc.ID)
  1112  	if err != nil {
  1113  		t.Fatalf("err: %v", err)
  1114  	}
  1115  	if outA != nil {
  1116  		t.Fatalf("bad: %v", outA)
  1117  	}
  1118  }
  1119  
  1120  func TestCoreScheduler_JobGC_Force(t *testing.T) {
  1121  	t.Parallel()
  1122  	s1 := testServer(t, nil)
  1123  	defer s1.Shutdown()
  1124  	testutil.WaitForLeader(t, s1.RPC)
  1125  
  1126  	// COMPAT Remove in 0.6: Reset the FSM time table since we reconcile which sets index 0
  1127  	s1.fsm.timetable.table = make([]TimeTableEntry, 1, 10)
  1128  
  1129  	// Insert job.
  1130  	state := s1.fsm.State()
  1131  	job := mock.Job()
  1132  	job.Type = structs.JobTypeBatch
  1133  	job.Status = structs.JobStatusDead
  1134  	err := state.UpsertJob(1000, job)
  1135  	if err != nil {
  1136  		t.Fatalf("err: %v", err)
  1137  	}
  1138  
  1139  	// Insert a terminal eval
  1140  	eval := mock.Eval()
  1141  	eval.JobID = job.ID
  1142  	eval.Status = structs.EvalStatusComplete
  1143  	err = state.UpsertEvals(1001, []*structs.Evaluation{eval})
  1144  	if err != nil {
  1145  		t.Fatalf("err: %v", err)
  1146  	}
  1147  
  1148  	// Create a core scheduler
  1149  	snap, err := state.Snapshot()
  1150  	if err != nil {
  1151  		t.Fatalf("err: %v", err)
  1152  	}
  1153  	core := NewCoreScheduler(s1, snap)
  1154  
  1155  	// Attempt the GC
  1156  	gc := s1.coreJobEval(structs.CoreJobForceGC, 1002)
  1157  	err = core.Process(gc)
  1158  	if err != nil {
  1159  		t.Fatalf("err: %v", err)
  1160  	}
  1161  
  1162  	// Shouldn't still exist
  1163  	ws := memdb.NewWatchSet()
  1164  	out, err := state.JobByID(ws, job.ID)
  1165  	if err != nil {
  1166  		t.Fatalf("err: %v", err)
  1167  	}
  1168  	if out != nil {
  1169  		t.Fatalf("bad: %v", out)
  1170  	}
  1171  
  1172  	outE, err := state.EvalByID(ws, eval.ID)
  1173  	if err != nil {
  1174  		t.Fatalf("err: %v", err)
  1175  	}
  1176  	if outE != nil {
  1177  		t.Fatalf("bad: %v", outE)
  1178  	}
  1179  }
  1180  
  1181  // This test ensures parameterized jobs only get gc'd when stopped
  1182  func TestCoreScheduler_JobGC_Parameterized(t *testing.T) {
  1183  	t.Parallel()
  1184  	s1 := testServer(t, nil)
  1185  	defer s1.Shutdown()
  1186  	testutil.WaitForLeader(t, s1.RPC)
  1187  
  1188  	// COMPAT Remove in 0.6: Reset the FSM time table since we reconcile which sets index 0
  1189  	s1.fsm.timetable.table = make([]TimeTableEntry, 1, 10)
  1190  
  1191  	// Insert a parameterized job.
  1192  	state := s1.fsm.State()
  1193  	job := mock.Job()
  1194  	job.Type = structs.JobTypeBatch
  1195  	job.Status = structs.JobStatusRunning
  1196  	job.ParameterizedJob = &structs.ParameterizedJobConfig{
  1197  		Payload: structs.DispatchPayloadRequired,
  1198  	}
  1199  	err := state.UpsertJob(1000, job)
  1200  	if err != nil {
  1201  		t.Fatalf("err: %v", err)
  1202  	}
  1203  
  1204  	// Create a core scheduler
  1205  	snap, err := state.Snapshot()
  1206  	if err != nil {
  1207  		t.Fatalf("err: %v", err)
  1208  	}
  1209  	core := NewCoreScheduler(s1, snap)
  1210  
  1211  	// Attempt the GC
  1212  	gc := s1.coreJobEval(structs.CoreJobForceGC, 1002)
  1213  	err = core.Process(gc)
  1214  	if err != nil {
  1215  		t.Fatalf("err: %v", err)
  1216  	}
  1217  
  1218  	// Should still exist
  1219  	ws := memdb.NewWatchSet()
  1220  	out, err := state.JobByID(ws, job.ID)
  1221  	if err != nil {
  1222  		t.Fatalf("err: %v", err)
  1223  	}
  1224  	if out == nil {
  1225  		t.Fatalf("bad: %v", out)
  1226  	}
  1227  
  1228  	// Mark the job as stopped and try again
  1229  	job2 := job.Copy()
  1230  	job2.Stop = true
  1231  	err = state.UpsertJob(2000, job2)
  1232  	if err != nil {
  1233  		t.Fatalf("err: %v", err)
  1234  	}
  1235  
  1236  	// Create a core scheduler
  1237  	snap, err = state.Snapshot()
  1238  	if err != nil {
  1239  		t.Fatalf("err: %v", err)
  1240  	}
  1241  	core = NewCoreScheduler(s1, snap)
  1242  
  1243  	// Attempt the GC
  1244  	gc = s1.coreJobEval(structs.CoreJobForceGC, 2002)
  1245  	err = core.Process(gc)
  1246  	if err != nil {
  1247  		t.Fatalf("err: %v", err)
  1248  	}
  1249  
  1250  	// Should not exist
  1251  	out, err = state.JobByID(ws, job.ID)
  1252  	if err != nil {
  1253  		t.Fatalf("err: %v", err)
  1254  	}
  1255  	if out != nil {
  1256  		t.Fatalf("bad: %+v", out)
  1257  	}
  1258  }
  1259  
  1260  // This test ensures periodic jobs don't get GCd til they are stopped
  1261  func TestCoreScheduler_JobGC_Periodic(t *testing.T) {
  1262  	t.Parallel()
  1263  
  1264  	s1 := testServer(t, nil)
  1265  	defer s1.Shutdown()
  1266  	testutil.WaitForLeader(t, s1.RPC)
  1267  
  1268  	// COMPAT Remove in 0.6: Reset the FSM time table since we reconcile which sets index 0
  1269  	s1.fsm.timetable.table = make([]TimeTableEntry, 1, 10)
  1270  
  1271  	// Insert a parameterized job.
  1272  	state := s1.fsm.State()
  1273  	job := mock.PeriodicJob()
  1274  	err := state.UpsertJob(1000, job)
  1275  	if err != nil {
  1276  		t.Fatalf("err: %v", err)
  1277  	}
  1278  
  1279  	// Create a core scheduler
  1280  	snap, err := state.Snapshot()
  1281  	if err != nil {
  1282  		t.Fatalf("err: %v", err)
  1283  	}
  1284  	core := NewCoreScheduler(s1, snap)
  1285  
  1286  	// Attempt the GC
  1287  	gc := s1.coreJobEval(structs.CoreJobForceGC, 1002)
  1288  	err = core.Process(gc)
  1289  	if err != nil {
  1290  		t.Fatalf("err: %v", err)
  1291  	}
  1292  
  1293  	// Should still exist
  1294  	ws := memdb.NewWatchSet()
  1295  	out, err := state.JobByID(ws, job.ID)
  1296  	if err != nil {
  1297  		t.Fatalf("err: %v", err)
  1298  	}
  1299  	if out == nil {
  1300  		t.Fatalf("bad: %v", out)
  1301  	}
  1302  
  1303  	// Mark the job as stopped and try again
  1304  	job2 := job.Copy()
  1305  	job2.Stop = true
  1306  	err = state.UpsertJob(2000, job2)
  1307  	if err != nil {
  1308  		t.Fatalf("err: %v", err)
  1309  	}
  1310  
  1311  	// Create a core scheduler
  1312  	snap, err = state.Snapshot()
  1313  	if err != nil {
  1314  		t.Fatalf("err: %v", err)
  1315  	}
  1316  	core = NewCoreScheduler(s1, snap)
  1317  
  1318  	// Attempt the GC
  1319  	gc = s1.coreJobEval(structs.CoreJobForceGC, 2002)
  1320  	err = core.Process(gc)
  1321  	if err != nil {
  1322  		t.Fatalf("err: %v", err)
  1323  	}
  1324  
  1325  	// Should not exist
  1326  	out, err = state.JobByID(ws, job.ID)
  1327  	if err != nil {
  1328  		t.Fatalf("err: %v", err)
  1329  	}
  1330  	if out != nil {
  1331  		t.Fatalf("bad: %+v", out)
  1332  	}
  1333  }
  1334  
  1335  func TestCoreScheduler_DeploymentGC(t *testing.T) {
  1336  	t.Parallel()
  1337  	s1 := testServer(t, nil)
  1338  	defer s1.Shutdown()
  1339  	testutil.WaitForLeader(t, s1.RPC)
  1340  	assert := assert.New(t)
  1341  
  1342  	// COMPAT Remove in 0.6: Reset the FSM time table since we reconcile which sets index 0
  1343  	s1.fsm.timetable.table = make([]TimeTableEntry, 1, 10)
  1344  
  1345  	// Insert an active, terminal, and terminal with allocations edeployment
  1346  	state := s1.fsm.State()
  1347  	d1, d2, d3 := mock.Deployment(), mock.Deployment(), mock.Deployment()
  1348  	d1.Status = structs.DeploymentStatusFailed
  1349  	d3.Status = structs.DeploymentStatusSuccessful
  1350  	assert.Nil(state.UpsertDeployment(1000, d1), "UpsertDeployment")
  1351  	assert.Nil(state.UpsertDeployment(1001, d2), "UpsertDeployment")
  1352  	assert.Nil(state.UpsertDeployment(1002, d3), "UpsertDeployment")
  1353  
  1354  	a := mock.Alloc()
  1355  	a.JobID = d3.JobID
  1356  	a.DeploymentID = d3.ID
  1357  	assert.Nil(state.UpsertAllocs(1003, []*structs.Allocation{a}), "UpsertAllocs")
  1358  
  1359  	// Update the time tables to make this work
  1360  	tt := s1.fsm.TimeTable()
  1361  	tt.Witness(2000, time.Now().UTC().Add(-1*s1.config.DeploymentGCThreshold))
  1362  
  1363  	// Create a core scheduler
  1364  	snap, err := state.Snapshot()
  1365  	assert.Nil(err, "Snapshot")
  1366  	core := NewCoreScheduler(s1, snap)
  1367  
  1368  	// Attempt the GC
  1369  	gc := s1.coreJobEval(structs.CoreJobDeploymentGC, 2000)
  1370  	assert.Nil(core.Process(gc), "Process GC")
  1371  
  1372  	// Should be gone
  1373  	ws := memdb.NewWatchSet()
  1374  	out, err := state.DeploymentByID(ws, d1.ID)
  1375  	assert.Nil(err, "DeploymentByID")
  1376  	assert.Nil(out, "Terminal Deployment")
  1377  	out2, err := state.DeploymentByID(ws, d2.ID)
  1378  	assert.Nil(err, "DeploymentByID")
  1379  	assert.NotNil(out2, "Active Deployment")
  1380  	out3, err := state.DeploymentByID(ws, d3.ID)
  1381  	assert.Nil(err, "DeploymentByID")
  1382  	assert.NotNil(out3, "Terminal Deployment With Allocs")
  1383  }
  1384  
  1385  func TestCoreScheduler_DeploymentGC_Force(t *testing.T) {
  1386  	t.Parallel()
  1387  	s1 := testServer(t, nil)
  1388  	defer s1.Shutdown()
  1389  	testutil.WaitForLeader(t, s1.RPC)
  1390  	assert := assert.New(t)
  1391  
  1392  	// COMPAT Remove in 0.6: Reset the FSM time table since we reconcile which sets index 0
  1393  	s1.fsm.timetable.table = make([]TimeTableEntry, 1, 10)
  1394  
  1395  	// Insert terminal and active deployment
  1396  	state := s1.fsm.State()
  1397  	d1, d2 := mock.Deployment(), mock.Deployment()
  1398  	d1.Status = structs.DeploymentStatusFailed
  1399  	assert.Nil(state.UpsertDeployment(1000, d1), "UpsertDeployment")
  1400  	assert.Nil(state.UpsertDeployment(1001, d2), "UpsertDeployment")
  1401  
  1402  	// Create a core scheduler
  1403  	snap, err := state.Snapshot()
  1404  	assert.Nil(err, "Snapshot")
  1405  	core := NewCoreScheduler(s1, snap)
  1406  
  1407  	// Attempt the GC
  1408  	gc := s1.coreJobEval(structs.CoreJobForceGC, 1000)
  1409  	assert.Nil(core.Process(gc), "Process Force GC")
  1410  
  1411  	// Should be gone
  1412  	ws := memdb.NewWatchSet()
  1413  	out, err := state.DeploymentByID(ws, d1.ID)
  1414  	assert.Nil(err, "DeploymentByID")
  1415  	assert.Nil(out, "Terminal Deployment")
  1416  	out2, err := state.DeploymentByID(ws, d2.ID)
  1417  	assert.Nil(err, "DeploymentByID")
  1418  	assert.NotNil(out2, "Active Deployment")
  1419  }
  1420  
  1421  func TestCoreScheduler_PartitionEvalReap(t *testing.T) {
  1422  	t.Parallel()
  1423  	s1 := testServer(t, nil)
  1424  	defer s1.Shutdown()
  1425  	testutil.WaitForLeader(t, s1.RPC)
  1426  
  1427  	// COMPAT Remove in 0.6: Reset the FSM time table since we reconcile which sets index 0
  1428  	s1.fsm.timetable.table = make([]TimeTableEntry, 1, 10)
  1429  
  1430  	// Create a core scheduler
  1431  	snap, err := s1.fsm.State().Snapshot()
  1432  	if err != nil {
  1433  		t.Fatalf("err: %v", err)
  1434  	}
  1435  	core := NewCoreScheduler(s1, snap)
  1436  
  1437  	// Set the max ids per reap to something lower.
  1438  	maxIdsPerReap = 2
  1439  
  1440  	evals := []string{"a", "b", "c"}
  1441  	allocs := []string{"1", "2", "3"}
  1442  	requests := core.(*CoreScheduler).partitionEvalReap(evals, allocs)
  1443  	if len(requests) != 3 {
  1444  		t.Fatalf("Expected 3 requests got: %v", requests)
  1445  	}
  1446  
  1447  	first := requests[0]
  1448  	if len(first.Allocs) != 2 && len(first.Evals) != 0 {
  1449  		t.Fatalf("Unexpected first request: %v", first)
  1450  	}
  1451  
  1452  	second := requests[1]
  1453  	if len(second.Allocs) != 1 && len(second.Evals) != 1 {
  1454  		t.Fatalf("Unexpected second request: %v", second)
  1455  	}
  1456  
  1457  	third := requests[2]
  1458  	if len(third.Allocs) != 0 && len(third.Evals) != 2 {
  1459  		t.Fatalf("Unexpected third request: %v", third)
  1460  	}
  1461  }
  1462  
  1463  func TestCoreScheduler_PartitionDeploymentReap(t *testing.T) {
  1464  	t.Parallel()
  1465  	s1 := testServer(t, nil)
  1466  	defer s1.Shutdown()
  1467  	testutil.WaitForLeader(t, s1.RPC)
  1468  
  1469  	// COMPAT Remove in 0.6: Reset the FSM time table since we reconcile which sets index 0
  1470  	s1.fsm.timetable.table = make([]TimeTableEntry, 1, 10)
  1471  
  1472  	// Create a core scheduler
  1473  	snap, err := s1.fsm.State().Snapshot()
  1474  	if err != nil {
  1475  		t.Fatalf("err: %v", err)
  1476  	}
  1477  	core := NewCoreScheduler(s1, snap)
  1478  
  1479  	// Set the max ids per reap to something lower.
  1480  	maxIdsPerReap = 2
  1481  
  1482  	deployments := []string{"a", "b", "c"}
  1483  	requests := core.(*CoreScheduler).partitionDeploymentReap(deployments)
  1484  	if len(requests) != 2 {
  1485  		t.Fatalf("Expected 2 requests got: %v", requests)
  1486  	}
  1487  
  1488  	first := requests[0]
  1489  	if len(first.Deployments) != 2 {
  1490  		t.Fatalf("Unexpected first request: %v", first)
  1491  	}
  1492  
  1493  	second := requests[1]
  1494  	if len(second.Deployments) != 1 {
  1495  		t.Fatalf("Unexpected second request: %v", second)
  1496  	}
  1497  }