github.com/hhrutter/nomad@v0.6.0-rc2.0.20170723054333-80c4b03f0705/nomad/deploymentwatcher/deployments_watcher_test.go (about)

     1  package deploymentwatcher
     2  
     3  import (
     4  	"fmt"
     5  	"testing"
     6  	"time"
     7  
     8  	memdb "github.com/hashicorp/go-memdb"
     9  	"github.com/hashicorp/nomad/helper"
    10  	"github.com/hashicorp/nomad/nomad/mock"
    11  	"github.com/hashicorp/nomad/nomad/structs"
    12  	"github.com/hashicorp/nomad/testutil"
    13  	"github.com/stretchr/testify/assert"
    14  	mocker "github.com/stretchr/testify/mock"
    15  )
    16  
    17  func testDeploymentWatcher(t *testing.T, qps float64, batchDur time.Duration) (*Watcher, *mockBackend) {
    18  	m := newMockBackend(t)
    19  	w := NewDeploymentsWatcher(testLogger(), m, m, qps, batchDur)
    20  	return w, m
    21  }
    22  
    23  func defaultTestDeploymentWatcher(t *testing.T) (*Watcher, *mockBackend) {
    24  	return testDeploymentWatcher(t, LimitStateQueriesPerSecond, CrossDeploymentEvalBatchDuration)
    25  }
    26  
    27  // Tests that the watcher properly watches for deployments and reconciles them
    28  func TestWatcher_WatchDeployments(t *testing.T) {
    29  	t.Parallel()
    30  	assert := assert.New(t)
    31  	w, m := defaultTestDeploymentWatcher(t)
    32  
    33  	// Return no allocations or evals
    34  	m.On("Allocations", mocker.Anything, mocker.Anything).Return(nil).Run(func(args mocker.Arguments) {
    35  		reply := args.Get(1).(*structs.AllocListResponse)
    36  		reply.Index = m.nextIndex()
    37  	})
    38  	m.On("Evaluations", mocker.Anything, mocker.Anything).Return(nil).Run(func(args mocker.Arguments) {
    39  		reply := args.Get(1).(*structs.JobEvaluationsResponse)
    40  		reply.Index = m.nextIndex()
    41  	})
    42  
    43  	// Create three jobs
    44  	j1, j2, j3 := mock.Job(), mock.Job(), mock.Job()
    45  	jobs := map[string]*structs.Job{
    46  		j1.ID: j1,
    47  		j2.ID: j2,
    48  		j3.ID: j3,
    49  	}
    50  
    51  	// Create three deployments all running
    52  	d1, d2, d3 := mock.Deployment(), mock.Deployment(), mock.Deployment()
    53  	d1.JobID = j1.ID
    54  	d2.JobID = j2.ID
    55  	d3.JobID = j3.ID
    56  
    57  	m.On("GetJob", mocker.Anything, mocker.Anything).
    58  		Return(nil).Run(func(args mocker.Arguments) {
    59  		in := args.Get(0).(*structs.JobSpecificRequest)
    60  		reply := args.Get(1).(*structs.SingleJobResponse)
    61  		reply.Job = jobs[in.JobID]
    62  		reply.Index = reply.Job.ModifyIndex
    63  	})
    64  
    65  	// Set up the calls for retrieving deployments
    66  	m.On("List", mocker.Anything, mocker.Anything).Return(nil).Run(func(args mocker.Arguments) {
    67  		reply := args.Get(1).(*structs.DeploymentListResponse)
    68  		reply.Deployments = []*structs.Deployment{d1}
    69  		reply.Index = m.nextIndex()
    70  	}).Once()
    71  
    72  	// Next list 3
    73  	block1 := make(chan time.Time)
    74  	m.On("List", mocker.Anything, mocker.Anything).Return(nil).Run(func(args mocker.Arguments) {
    75  		reply := args.Get(1).(*structs.DeploymentListResponse)
    76  		reply.Deployments = []*structs.Deployment{d1, d2, d3}
    77  		reply.Index = m.nextIndex()
    78  	}).Once().WaitUntil(block1)
    79  
    80  	//// Next list 3 but have one be terminal
    81  	block2 := make(chan time.Time)
    82  	d3terminal := d3.Copy()
    83  	d3terminal.Status = structs.DeploymentStatusFailed
    84  	m.On("List", mocker.Anything, mocker.Anything).Return(nil).Run(func(args mocker.Arguments) {
    85  		reply := args.Get(1).(*structs.DeploymentListResponse)
    86  		reply.Deployments = []*structs.Deployment{d1, d2, d3terminal}
    87  		reply.Index = m.nextIndex()
    88  	}).WaitUntil(block2)
    89  
    90  	m.On("List", mocker.Anything, mocker.Anything).Return(nil).Run(func(args mocker.Arguments) {
    91  		reply := args.Get(1).(*structs.DeploymentListResponse)
    92  		reply.Deployments = []*structs.Deployment{d1, d2, d3terminal}
    93  		reply.Index = m.nextIndex()
    94  	})
    95  
    96  	w.SetEnabled(true)
    97  	testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil },
    98  		func(err error) { assert.Equal(1, len(w.watchers), "1 deployment returned") })
    99  
   100  	close(block1)
   101  	testutil.WaitForResult(func() (bool, error) { return 3 == len(w.watchers), nil },
   102  		func(err error) { assert.Equal(3, len(w.watchers), "3 deployment returned") })
   103  
   104  	close(block2)
   105  	testutil.WaitForResult(func() (bool, error) { return 2 == len(w.watchers), nil },
   106  		func(err error) { assert.Equal(3, len(w.watchers), "3 deployment returned - 1 terminal") })
   107  }
   108  
   109  // Tests that calls against an unknown deployment fail
   110  func TestWatcher_UnknownDeployment(t *testing.T) {
   111  	t.Parallel()
   112  	assert := assert.New(t)
   113  	w, m := defaultTestDeploymentWatcher(t)
   114  	w.SetEnabled(true)
   115  
   116  	// Set up the calls for retrieving deployments
   117  	m.On("List", mocker.Anything, mocker.Anything).Return(nil).Run(func(args mocker.Arguments) {
   118  		reply := args.Get(1).(*structs.DeploymentListResponse)
   119  		reply.Index = m.nextIndex()
   120  	})
   121  	m.On("GetDeployment", mocker.Anything, mocker.Anything).Return(nil).Run(func(args mocker.Arguments) {
   122  		reply := args.Get(1).(*structs.SingleDeploymentResponse)
   123  		reply.Index = m.nextIndex()
   124  	})
   125  
   126  	// The expected error is that it should be an unknown deployment
   127  	dID := structs.GenerateUUID()
   128  	expected := fmt.Sprintf("unknown deployment %q", dID)
   129  
   130  	// Request setting the health against an unknown deployment
   131  	req := &structs.DeploymentAllocHealthRequest{
   132  		DeploymentID:         dID,
   133  		HealthyAllocationIDs: []string{structs.GenerateUUID()},
   134  	}
   135  	var resp structs.DeploymentUpdateResponse
   136  	err := w.SetAllocHealth(req, &resp)
   137  	if assert.NotNil(err, "should have error for unknown deployment") {
   138  		assert.Contains(err.Error(), expected)
   139  	}
   140  
   141  	// Request promoting against an unknown deployment
   142  	req2 := &structs.DeploymentPromoteRequest{
   143  		DeploymentID: dID,
   144  		All:          true,
   145  	}
   146  	err = w.PromoteDeployment(req2, &resp)
   147  	if assert.NotNil(err, "should have error for unknown deployment") {
   148  		assert.Contains(err.Error(), expected)
   149  	}
   150  
   151  	// Request pausing against an unknown deployment
   152  	req3 := &structs.DeploymentPauseRequest{
   153  		DeploymentID: dID,
   154  		Pause:        true,
   155  	}
   156  	err = w.PauseDeployment(req3, &resp)
   157  	if assert.NotNil(err, "should have error for unknown deployment") {
   158  		assert.Contains(err.Error(), expected)
   159  	}
   160  
   161  	// Request failing against an unknown deployment
   162  	req4 := &structs.DeploymentFailRequest{
   163  		DeploymentID: dID,
   164  	}
   165  	err = w.FailDeployment(req4, &resp)
   166  	if assert.NotNil(err, "should have error for unknown deployment") {
   167  		assert.Contains(err.Error(), expected)
   168  	}
   169  }
   170  
   171  // Test setting an unknown allocation's health
   172  func TestWatcher_SetAllocHealth_Unknown(t *testing.T) {
   173  	t.Parallel()
   174  	assert := assert.New(t)
   175  	w, m := defaultTestDeploymentWatcher(t)
   176  
   177  	// Create a job, and a deployment
   178  	j := mock.Job()
   179  	d := mock.Deployment()
   180  	d.JobID = j.ID
   181  	assert.Nil(m.state.UpsertJob(m.nextIndex(), j), "UpsertJob")
   182  	assert.Nil(m.state.UpsertDeployment(m.nextIndex(), d), "UpsertDeployment")
   183  
   184  	// Assert the following methods will be called
   185  	m.On("List", mocker.Anything, mocker.Anything).Return(nil).Run(m.listFromState)
   186  	m.On("Allocations", mocker.MatchedBy(matchDeploymentSpecificRequest(d.ID)),
   187  		mocker.Anything).Return(nil).Run(m.allocationsFromState)
   188  	m.On("Evaluations", mocker.MatchedBy(matchJobSpecificRequest(j.ID)),
   189  		mocker.Anything).Return(nil).Run(m.evaluationsFromState)
   190  	m.On("GetJob", mocker.MatchedBy(matchJobSpecificRequest(j.ID)),
   191  		mocker.Anything).Return(nil).Run(m.getJobFromState)
   192  
   193  	w.SetEnabled(true)
   194  	testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil },
   195  		func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") })
   196  
   197  	// Assert that we get a call to UpsertDeploymentAllocHealth
   198  	a := mock.Alloc()
   199  	matchConfig := &matchDeploymentAllocHealthRequestConfig{
   200  		DeploymentID: d.ID,
   201  		Healthy:      []string{a.ID},
   202  		Eval:         true,
   203  	}
   204  	matcher := matchDeploymentAllocHealthRequest(matchConfig)
   205  	m.On("UpdateDeploymentAllocHealth", mocker.MatchedBy(matcher)).Return(nil)
   206  
   207  	// Call SetAllocHealth
   208  	req := &structs.DeploymentAllocHealthRequest{
   209  		DeploymentID:         d.ID,
   210  		HealthyAllocationIDs: []string{a.ID},
   211  	}
   212  	var resp structs.DeploymentUpdateResponse
   213  	err := w.SetAllocHealth(req, &resp)
   214  	if assert.NotNil(err, "Set health of unknown allocation") {
   215  		assert.Contains(err.Error(), "unknown")
   216  	}
   217  	assert.Equal(1, len(w.watchers), "Deployment should still be active")
   218  }
   219  
   220  // Test setting allocation health
   221  func TestWatcher_SetAllocHealth_Healthy(t *testing.T) {
   222  	t.Parallel()
   223  	assert := assert.New(t)
   224  	w, m := defaultTestDeploymentWatcher(t)
   225  
   226  	// Create a job, alloc, and a deployment
   227  	j := mock.Job()
   228  	d := mock.Deployment()
   229  	d.JobID = j.ID
   230  	a := mock.Alloc()
   231  	a.DeploymentID = d.ID
   232  	assert.Nil(m.state.UpsertJob(m.nextIndex(), j), "UpsertJob")
   233  	assert.Nil(m.state.UpsertDeployment(m.nextIndex(), d), "UpsertDeployment")
   234  	assert.Nil(m.state.UpsertAllocs(m.nextIndex(), []*structs.Allocation{a}), "UpsertAllocs")
   235  
   236  	// Assert the following methods will be called
   237  	m.On("List", mocker.Anything, mocker.Anything).Return(nil).Run(m.listFromState)
   238  	m.On("Allocations", mocker.MatchedBy(matchDeploymentSpecificRequest(d.ID)),
   239  		mocker.Anything).Return(nil).Run(m.allocationsFromState)
   240  	m.On("Evaluations", mocker.MatchedBy(matchJobSpecificRequest(j.ID)),
   241  		mocker.Anything).Return(nil).Run(m.evaluationsFromState)
   242  	m.On("GetJob", mocker.MatchedBy(matchJobSpecificRequest(j.ID)),
   243  		mocker.Anything).Return(nil).Run(m.getJobFromState)
   244  
   245  	w.SetEnabled(true)
   246  	testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil },
   247  		func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") })
   248  
   249  	// Assert that we get a call to UpsertDeploymentAllocHealth
   250  	matchConfig := &matchDeploymentAllocHealthRequestConfig{
   251  		DeploymentID: d.ID,
   252  		Healthy:      []string{a.ID},
   253  		Eval:         true,
   254  	}
   255  	matcher := matchDeploymentAllocHealthRequest(matchConfig)
   256  	m.On("UpdateDeploymentAllocHealth", mocker.MatchedBy(matcher)).Return(nil)
   257  
   258  	// Call SetAllocHealth
   259  	req := &structs.DeploymentAllocHealthRequest{
   260  		DeploymentID:         d.ID,
   261  		HealthyAllocationIDs: []string{a.ID},
   262  	}
   263  	var resp structs.DeploymentUpdateResponse
   264  	err := w.SetAllocHealth(req, &resp)
   265  	assert.Nil(err, "SetAllocHealth")
   266  	assert.Equal(1, len(w.watchers), "Deployment should still be active")
   267  	m.AssertCalled(t, "UpdateDeploymentAllocHealth", mocker.MatchedBy(matcher))
   268  }
   269  
   270  // Test setting allocation unhealthy
   271  func TestWatcher_SetAllocHealth_Unhealthy(t *testing.T) {
   272  	t.Parallel()
   273  	assert := assert.New(t)
   274  	w, m := defaultTestDeploymentWatcher(t)
   275  
   276  	// Create a job, alloc, and a deployment
   277  	j := mock.Job()
   278  	d := mock.Deployment()
   279  	d.JobID = j.ID
   280  	a := mock.Alloc()
   281  	a.DeploymentID = d.ID
   282  	assert.Nil(m.state.UpsertJob(m.nextIndex(), j), "UpsertJob")
   283  	assert.Nil(m.state.UpsertDeployment(m.nextIndex(), d), "UpsertDeployment")
   284  	assert.Nil(m.state.UpsertAllocs(m.nextIndex(), []*structs.Allocation{a}), "UpsertAllocs")
   285  
   286  	// Assert the following methods will be called
   287  	m.On("List", mocker.Anything, mocker.Anything).Return(nil).Run(m.listFromState)
   288  	m.On("Allocations", mocker.MatchedBy(matchDeploymentSpecificRequest(d.ID)),
   289  		mocker.Anything).Return(nil).Run(m.allocationsFromState)
   290  	m.On("Evaluations", mocker.MatchedBy(matchJobSpecificRequest(j.ID)),
   291  		mocker.Anything).Return(nil).Run(m.evaluationsFromState)
   292  	m.On("GetJob", mocker.MatchedBy(matchJobSpecificRequest(j.ID)),
   293  		mocker.Anything).Return(nil).Run(m.getJobFromState)
   294  
   295  	w.SetEnabled(true)
   296  	testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil },
   297  		func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") })
   298  
   299  	// Assert that we get a call to UpsertDeploymentAllocHealth
   300  	matchConfig := &matchDeploymentAllocHealthRequestConfig{
   301  		DeploymentID: d.ID,
   302  		Unhealthy:    []string{a.ID},
   303  		Eval:         true,
   304  		DeploymentUpdate: &structs.DeploymentStatusUpdate{
   305  			DeploymentID:      d.ID,
   306  			Status:            structs.DeploymentStatusFailed,
   307  			StatusDescription: structs.DeploymentStatusDescriptionFailedAllocations,
   308  		},
   309  	}
   310  	matcher := matchDeploymentAllocHealthRequest(matchConfig)
   311  	m.On("UpdateDeploymentAllocHealth", mocker.MatchedBy(matcher)).Return(nil)
   312  
   313  	// Call SetAllocHealth
   314  	req := &structs.DeploymentAllocHealthRequest{
   315  		DeploymentID:           d.ID,
   316  		UnhealthyAllocationIDs: []string{a.ID},
   317  	}
   318  	var resp structs.DeploymentUpdateResponse
   319  	err := w.SetAllocHealth(req, &resp)
   320  	assert.Nil(err, "SetAllocHealth")
   321  
   322  	testutil.WaitForResult(func() (bool, error) { return 0 == len(w.watchers), nil },
   323  		func(err error) { assert.Equal(0, len(w.watchers), "Should have no deployment") })
   324  	m.AssertNumberOfCalls(t, "UpdateDeploymentAllocHealth", 1)
   325  }
   326  
   327  // Test setting allocation unhealthy and that there should be a rollback
   328  func TestWatcher_SetAllocHealth_Unhealthy_Rollback(t *testing.T) {
   329  	t.Parallel()
   330  	assert := assert.New(t)
   331  	w, m := defaultTestDeploymentWatcher(t)
   332  
   333  	// Create a job, alloc, and a deployment
   334  	j := mock.Job()
   335  	j.TaskGroups[0].Update = structs.DefaultUpdateStrategy.Copy()
   336  	j.TaskGroups[0].Update.MaxParallel = 2
   337  	j.TaskGroups[0].Update.AutoRevert = true
   338  	j.Stable = true
   339  	d := mock.Deployment()
   340  	d.JobID = j.ID
   341  	d.TaskGroups["web"].AutoRevert = true
   342  	a := mock.Alloc()
   343  	a.DeploymentID = d.ID
   344  	assert.Nil(m.state.UpsertJob(m.nextIndex(), j), "UpsertJob")
   345  	assert.Nil(m.state.UpsertDeployment(m.nextIndex(), d), "UpsertDeployment")
   346  	assert.Nil(m.state.UpsertAllocs(m.nextIndex(), []*structs.Allocation{a}), "UpsertAllocs")
   347  
   348  	// Upsert the job again to get a new version
   349  	j2 := j.Copy()
   350  	j2.Stable = false
   351  	assert.Nil(m.state.UpsertJob(m.nextIndex(), j2), "UpsertJob2")
   352  
   353  	// Assert the following methods will be called
   354  	m.On("List", mocker.Anything, mocker.Anything).Return(nil).Run(m.listFromState)
   355  	m.On("Allocations", mocker.MatchedBy(matchDeploymentSpecificRequest(d.ID)),
   356  		mocker.Anything).Return(nil).Run(m.allocationsFromState)
   357  	m.On("Evaluations", mocker.MatchedBy(matchJobSpecificRequest(j.ID)),
   358  		mocker.Anything).Return(nil).Run(m.evaluationsFromState)
   359  	m.On("GetJob", mocker.MatchedBy(matchJobSpecificRequest(j.ID)),
   360  		mocker.Anything).Return(nil).Run(m.getJobFromState)
   361  	m.On("GetJobVersions", mocker.MatchedBy(matchJobVersionsRequest(j.ID)),
   362  		mocker.Anything).Return(nil).Run(m.getJobVersionsFromState)
   363  
   364  	w.SetEnabled(true)
   365  	testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil },
   366  		func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") })
   367  
   368  	// Assert that we get a call to UpsertDeploymentAllocHealth
   369  	matchConfig := &matchDeploymentAllocHealthRequestConfig{
   370  		DeploymentID: d.ID,
   371  		Unhealthy:    []string{a.ID},
   372  		Eval:         true,
   373  		DeploymentUpdate: &structs.DeploymentStatusUpdate{
   374  			DeploymentID:      d.ID,
   375  			Status:            structs.DeploymentStatusFailed,
   376  			StatusDescription: structs.DeploymentStatusDescriptionFailedAllocations,
   377  		},
   378  		JobVersion: helper.Uint64ToPtr(0),
   379  	}
   380  	matcher := matchDeploymentAllocHealthRequest(matchConfig)
   381  	m.On("UpdateDeploymentAllocHealth", mocker.MatchedBy(matcher)).Return(nil)
   382  
   383  	// Call SetAllocHealth
   384  	req := &structs.DeploymentAllocHealthRequest{
   385  		DeploymentID:           d.ID,
   386  		UnhealthyAllocationIDs: []string{a.ID},
   387  	}
   388  	var resp structs.DeploymentUpdateResponse
   389  	err := w.SetAllocHealth(req, &resp)
   390  	assert.Nil(err, "SetAllocHealth")
   391  
   392  	testutil.WaitForResult(func() (bool, error) { return 0 == len(w.watchers), nil },
   393  		func(err error) { assert.Equal(0, len(w.watchers), "Should have no deployment") })
   394  	m.AssertNumberOfCalls(t, "UpdateDeploymentAllocHealth", 1)
   395  }
   396  
   397  // Test promoting a deployment
   398  func TestWatcher_PromoteDeployment_HealthyCanaries(t *testing.T) {
   399  	t.Parallel()
   400  	assert := assert.New(t)
   401  	w, m := defaultTestDeploymentWatcher(t)
   402  
   403  	// Create a job, canary alloc, and a deployment
   404  	j := mock.Job()
   405  	j.TaskGroups[0].Update = structs.DefaultUpdateStrategy.Copy()
   406  	j.TaskGroups[0].Update.MaxParallel = 2
   407  	j.TaskGroups[0].Update.Canary = 2
   408  	d := mock.Deployment()
   409  	d.JobID = j.ID
   410  	a := mock.Alloc()
   411  	d.TaskGroups[a.TaskGroup].PlacedCanaries = []string{a.ID}
   412  	a.DeploymentStatus = &structs.AllocDeploymentStatus{
   413  		Healthy: helper.BoolToPtr(true),
   414  	}
   415  	a.DeploymentID = d.ID
   416  	assert.Nil(m.state.UpsertJob(m.nextIndex(), j), "UpsertJob")
   417  	assert.Nil(m.state.UpsertDeployment(m.nextIndex(), d), "UpsertDeployment")
   418  	assert.Nil(m.state.UpsertAllocs(m.nextIndex(), []*structs.Allocation{a}), "UpsertAllocs")
   419  
   420  	// Assert the following methods will be called
   421  	m.On("List", mocker.Anything, mocker.Anything).Return(nil).Run(m.listFromState)
   422  	m.On("Allocations", mocker.MatchedBy(matchDeploymentSpecificRequest(d.ID)),
   423  		mocker.Anything).Return(nil).Run(m.allocationsFromState)
   424  	m.On("Evaluations", mocker.MatchedBy(matchJobSpecificRequest(j.ID)),
   425  		mocker.Anything).Return(nil).Run(m.evaluationsFromState)
   426  	m.On("GetJob", mocker.MatchedBy(matchJobSpecificRequest(j.ID)),
   427  		mocker.Anything).Return(nil).Run(m.getJobFromState)
   428  
   429  	w.SetEnabled(true)
   430  	testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil },
   431  		func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") })
   432  
   433  	// Assert that we get a call to UpsertDeploymentPromotion
   434  	matchConfig := &matchDeploymentPromoteRequestConfig{
   435  		Promotion: &structs.DeploymentPromoteRequest{
   436  			DeploymentID: d.ID,
   437  			All:          true,
   438  		},
   439  		Eval: true,
   440  	}
   441  	matcher := matchDeploymentPromoteRequest(matchConfig)
   442  	m.On("UpdateDeploymentPromotion", mocker.MatchedBy(matcher)).Return(nil)
   443  
   444  	// Call PromoteDeployment
   445  	req := &structs.DeploymentPromoteRequest{
   446  		DeploymentID: d.ID,
   447  		All:          true,
   448  	}
   449  	var resp structs.DeploymentUpdateResponse
   450  	err := w.PromoteDeployment(req, &resp)
   451  	assert.Nil(err, "PromoteDeployment")
   452  	assert.Equal(1, len(w.watchers), "Deployment should still be active")
   453  	m.AssertCalled(t, "UpdateDeploymentPromotion", mocker.MatchedBy(matcher))
   454  }
   455  
   456  // Test promoting a deployment with unhealthy canaries
   457  func TestWatcher_PromoteDeployment_UnhealthyCanaries(t *testing.T) {
   458  	t.Parallel()
   459  	assert := assert.New(t)
   460  	w, m := defaultTestDeploymentWatcher(t)
   461  
   462  	// Create a job, canary alloc, and a deployment
   463  	j := mock.Job()
   464  	j.TaskGroups[0].Update = structs.DefaultUpdateStrategy.Copy()
   465  	j.TaskGroups[0].Update.MaxParallel = 2
   466  	j.TaskGroups[0].Update.Canary = 2
   467  	d := mock.Deployment()
   468  	d.JobID = j.ID
   469  	a := mock.Alloc()
   470  	d.TaskGroups[a.TaskGroup].PlacedCanaries = []string{a.ID}
   471  	a.DeploymentID = d.ID
   472  	assert.Nil(m.state.UpsertJob(m.nextIndex(), j), "UpsertJob")
   473  	assert.Nil(m.state.UpsertDeployment(m.nextIndex(), d), "UpsertDeployment")
   474  	assert.Nil(m.state.UpsertAllocs(m.nextIndex(), []*structs.Allocation{a}), "UpsertAllocs")
   475  
   476  	// Assert the following methods will be called
   477  	m.On("List", mocker.Anything, mocker.Anything).Return(nil).Run(m.listFromState)
   478  	m.On("Allocations", mocker.MatchedBy(matchDeploymentSpecificRequest(d.ID)),
   479  		mocker.Anything).Return(nil).Run(m.allocationsFromState)
   480  	m.On("Evaluations", mocker.MatchedBy(matchJobSpecificRequest(j.ID)),
   481  		mocker.Anything).Return(nil).Run(m.evaluationsFromState)
   482  	m.On("GetJob", mocker.MatchedBy(matchJobSpecificRequest(j.ID)),
   483  		mocker.Anything).Return(nil).Run(m.getJobFromState)
   484  
   485  	w.SetEnabled(true)
   486  	testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil },
   487  		func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") })
   488  
   489  	// Assert that we get a call to UpsertDeploymentPromotion
   490  	matchConfig := &matchDeploymentPromoteRequestConfig{
   491  		Promotion: &structs.DeploymentPromoteRequest{
   492  			DeploymentID: d.ID,
   493  			All:          true,
   494  		},
   495  		Eval: true,
   496  	}
   497  	matcher := matchDeploymentPromoteRequest(matchConfig)
   498  	m.On("UpdateDeploymentPromotion", mocker.MatchedBy(matcher)).Return(nil)
   499  
   500  	// Call SetAllocHealth
   501  	req := &structs.DeploymentPromoteRequest{
   502  		DeploymentID: d.ID,
   503  		All:          true,
   504  	}
   505  	var resp structs.DeploymentUpdateResponse
   506  	err := w.PromoteDeployment(req, &resp)
   507  	if assert.NotNil(err, "PromoteDeployment") {
   508  		assert.Contains(err.Error(), "is not healthy", "Should error because canary isn't marked healthy")
   509  	}
   510  
   511  	assert.Equal(1, len(w.watchers), "Deployment should still be active")
   512  	m.AssertCalled(t, "UpdateDeploymentPromotion", mocker.MatchedBy(matcher))
   513  }
   514  
   515  // Test pausing a deployment that is running
   516  func TestWatcher_PauseDeployment_Pause_Running(t *testing.T) {
   517  	t.Parallel()
   518  	assert := assert.New(t)
   519  	w, m := defaultTestDeploymentWatcher(t)
   520  
   521  	// Create a job and a deployment
   522  	j := mock.Job()
   523  	d := mock.Deployment()
   524  	d.JobID = j.ID
   525  	assert.Nil(m.state.UpsertJob(m.nextIndex(), j), "UpsertJob")
   526  	assert.Nil(m.state.UpsertDeployment(m.nextIndex(), d), "UpsertDeployment")
   527  
   528  	// Assert the following methods will be called
   529  	m.On("List", mocker.Anything, mocker.Anything).Return(nil).Run(m.listFromState)
   530  	m.On("Allocations", mocker.MatchedBy(matchDeploymentSpecificRequest(d.ID)),
   531  		mocker.Anything).Return(nil).Run(m.allocationsFromState)
   532  	m.On("Evaluations", mocker.MatchedBy(matchJobSpecificRequest(j.ID)),
   533  		mocker.Anything).Return(nil).Run(m.evaluationsFromState)
   534  	m.On("GetJob", mocker.MatchedBy(matchJobSpecificRequest(j.ID)),
   535  		mocker.Anything).Return(nil).Run(m.getJobFromState)
   536  
   537  	w.SetEnabled(true)
   538  	testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil },
   539  		func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") })
   540  
   541  	// Assert that we get a call to UpsertDeploymentStatusUpdate
   542  	matchConfig := &matchDeploymentStatusUpdateConfig{
   543  		DeploymentID:      d.ID,
   544  		Status:            structs.DeploymentStatusPaused,
   545  		StatusDescription: structs.DeploymentStatusDescriptionPaused,
   546  	}
   547  	matcher := matchDeploymentStatusUpdateRequest(matchConfig)
   548  	m.On("UpdateDeploymentStatus", mocker.MatchedBy(matcher)).Return(nil)
   549  
   550  	// Call PauseDeployment
   551  	req := &structs.DeploymentPauseRequest{
   552  		DeploymentID: d.ID,
   553  		Pause:        true,
   554  	}
   555  	var resp structs.DeploymentUpdateResponse
   556  	err := w.PauseDeployment(req, &resp)
   557  	assert.Nil(err, "PauseDeployment")
   558  
   559  	assert.Equal(1, len(w.watchers), "Deployment should still be active")
   560  	m.AssertCalled(t, "UpdateDeploymentStatus", mocker.MatchedBy(matcher))
   561  }
   562  
   563  // Test pausing a deployment that is paused
   564  func TestWatcher_PauseDeployment_Pause_Paused(t *testing.T) {
   565  	t.Parallel()
   566  	assert := assert.New(t)
   567  	w, m := defaultTestDeploymentWatcher(t)
   568  
   569  	// Create a job and a deployment
   570  	j := mock.Job()
   571  	d := mock.Deployment()
   572  	d.JobID = j.ID
   573  	d.Status = structs.DeploymentStatusPaused
   574  	assert.Nil(m.state.UpsertJob(m.nextIndex(), j), "UpsertJob")
   575  	assert.Nil(m.state.UpsertDeployment(m.nextIndex(), d), "UpsertDeployment")
   576  
   577  	// Assert the following methods will be called
   578  	m.On("List", mocker.Anything, mocker.Anything).Return(nil).Run(m.listFromState)
   579  	m.On("Allocations", mocker.MatchedBy(matchDeploymentSpecificRequest(d.ID)),
   580  		mocker.Anything).Return(nil).Run(m.allocationsFromState)
   581  	m.On("Evaluations", mocker.MatchedBy(matchJobSpecificRequest(j.ID)),
   582  		mocker.Anything).Return(nil).Run(m.evaluationsFromState)
   583  	m.On("GetJob", mocker.MatchedBy(matchJobSpecificRequest(j.ID)),
   584  		mocker.Anything).Return(nil).Run(m.getJobFromState)
   585  
   586  	w.SetEnabled(true)
   587  	testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil },
   588  		func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") })
   589  
   590  	// Assert that we get a call to UpsertDeploymentStatusUpdate
   591  	matchConfig := &matchDeploymentStatusUpdateConfig{
   592  		DeploymentID:      d.ID,
   593  		Status:            structs.DeploymentStatusPaused,
   594  		StatusDescription: structs.DeploymentStatusDescriptionPaused,
   595  	}
   596  	matcher := matchDeploymentStatusUpdateRequest(matchConfig)
   597  	m.On("UpdateDeploymentStatus", mocker.MatchedBy(matcher)).Return(nil)
   598  
   599  	// Call PauseDeployment
   600  	req := &structs.DeploymentPauseRequest{
   601  		DeploymentID: d.ID,
   602  		Pause:        true,
   603  	}
   604  	var resp structs.DeploymentUpdateResponse
   605  	err := w.PauseDeployment(req, &resp)
   606  	assert.Nil(err, "PauseDeployment")
   607  
   608  	assert.Equal(1, len(w.watchers), "Deployment should still be active")
   609  	m.AssertCalled(t, "UpdateDeploymentStatus", mocker.MatchedBy(matcher))
   610  }
   611  
   612  // Test unpausing a deployment that is paused
   613  func TestWatcher_PauseDeployment_Unpause_Paused(t *testing.T) {
   614  	t.Parallel()
   615  	assert := assert.New(t)
   616  	w, m := defaultTestDeploymentWatcher(t)
   617  
   618  	// Create a job and a deployment
   619  	j := mock.Job()
   620  	d := mock.Deployment()
   621  	d.JobID = j.ID
   622  	d.Status = structs.DeploymentStatusPaused
   623  	assert.Nil(m.state.UpsertJob(m.nextIndex(), j), "UpsertJob")
   624  	assert.Nil(m.state.UpsertDeployment(m.nextIndex(), d), "UpsertDeployment")
   625  
   626  	// Assert the following methods will be called
   627  	m.On("List", mocker.Anything, mocker.Anything).Return(nil).Run(m.listFromState)
   628  	m.On("Allocations", mocker.MatchedBy(matchDeploymentSpecificRequest(d.ID)),
   629  		mocker.Anything).Return(nil).Run(m.allocationsFromState)
   630  	m.On("Evaluations", mocker.MatchedBy(matchJobSpecificRequest(j.ID)),
   631  		mocker.Anything).Return(nil).Run(m.evaluationsFromState)
   632  	m.On("GetJob", mocker.MatchedBy(matchJobSpecificRequest(j.ID)),
   633  		mocker.Anything).Return(nil).Run(m.getJobFromState)
   634  
   635  	w.SetEnabled(true)
   636  	testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil },
   637  		func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") })
   638  
   639  	// Assert that we get a call to UpsertDeploymentStatusUpdate
   640  	matchConfig := &matchDeploymentStatusUpdateConfig{
   641  		DeploymentID:      d.ID,
   642  		Status:            structs.DeploymentStatusRunning,
   643  		StatusDescription: structs.DeploymentStatusDescriptionRunning,
   644  		Eval:              true,
   645  	}
   646  	matcher := matchDeploymentStatusUpdateRequest(matchConfig)
   647  	m.On("UpdateDeploymentStatus", mocker.MatchedBy(matcher)).Return(nil)
   648  
   649  	// Call PauseDeployment
   650  	req := &structs.DeploymentPauseRequest{
   651  		DeploymentID: d.ID,
   652  		Pause:        false,
   653  	}
   654  	var resp structs.DeploymentUpdateResponse
   655  	err := w.PauseDeployment(req, &resp)
   656  	assert.Nil(err, "PauseDeployment")
   657  
   658  	assert.Equal(1, len(w.watchers), "Deployment should still be active")
   659  	m.AssertCalled(t, "UpdateDeploymentStatus", mocker.MatchedBy(matcher))
   660  }
   661  
   662  // Test unpausing a deployment that is running
   663  func TestWatcher_PauseDeployment_Unpause_Running(t *testing.T) {
   664  	t.Parallel()
   665  	assert := assert.New(t)
   666  	w, m := defaultTestDeploymentWatcher(t)
   667  
   668  	// Create a job and a deployment
   669  	j := mock.Job()
   670  	d := mock.Deployment()
   671  	d.JobID = j.ID
   672  	assert.Nil(m.state.UpsertJob(m.nextIndex(), j), "UpsertJob")
   673  	assert.Nil(m.state.UpsertDeployment(m.nextIndex(), d), "UpsertDeployment")
   674  
   675  	// Assert the following methods will be called
   676  	m.On("List", mocker.Anything, mocker.Anything).Return(nil).Run(m.listFromState)
   677  	m.On("Allocations", mocker.MatchedBy(matchDeploymentSpecificRequest(d.ID)),
   678  		mocker.Anything).Return(nil).Run(m.allocationsFromState)
   679  	m.On("Evaluations", mocker.MatchedBy(matchJobSpecificRequest(j.ID)),
   680  		mocker.Anything).Return(nil).Run(m.evaluationsFromState)
   681  	m.On("GetJob", mocker.MatchedBy(matchJobSpecificRequest(j.ID)),
   682  		mocker.Anything).Return(nil).Run(m.getJobFromState)
   683  
   684  	w.SetEnabled(true)
   685  	testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil },
   686  		func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") })
   687  
   688  	// Assert that we get a call to UpsertDeploymentStatusUpdate
   689  	matchConfig := &matchDeploymentStatusUpdateConfig{
   690  		DeploymentID:      d.ID,
   691  		Status:            structs.DeploymentStatusRunning,
   692  		StatusDescription: structs.DeploymentStatusDescriptionRunning,
   693  		Eval:              true,
   694  	}
   695  	matcher := matchDeploymentStatusUpdateRequest(matchConfig)
   696  	m.On("UpdateDeploymentStatus", mocker.MatchedBy(matcher)).Return(nil)
   697  
   698  	// Call PauseDeployment
   699  	req := &structs.DeploymentPauseRequest{
   700  		DeploymentID: d.ID,
   701  		Pause:        false,
   702  	}
   703  	var resp structs.DeploymentUpdateResponse
   704  	err := w.PauseDeployment(req, &resp)
   705  	assert.Nil(err, "PauseDeployment")
   706  
   707  	assert.Equal(1, len(w.watchers), "Deployment should still be active")
   708  	m.AssertCalled(t, "UpdateDeploymentStatus", mocker.MatchedBy(matcher))
   709  }
   710  
   711  // Test failing a deployment that is running
   712  func TestWatcher_FailDeployment_Running(t *testing.T) {
   713  	t.Parallel()
   714  	assert := assert.New(t)
   715  	w, m := defaultTestDeploymentWatcher(t)
   716  
   717  	// Create a job and a deployment
   718  	j := mock.Job()
   719  	d := mock.Deployment()
   720  	d.JobID = j.ID
   721  	assert.Nil(m.state.UpsertJob(m.nextIndex(), j), "UpsertJob")
   722  	assert.Nil(m.state.UpsertDeployment(m.nextIndex(), d), "UpsertDeployment")
   723  
   724  	// Assert the following methods will be called
   725  	m.On("List", mocker.Anything, mocker.Anything).Return(nil).Run(m.listFromState)
   726  	m.On("Allocations", mocker.MatchedBy(matchDeploymentSpecificRequest(d.ID)),
   727  		mocker.Anything).Return(nil).Run(m.allocationsFromState)
   728  	m.On("Evaluations", mocker.MatchedBy(matchJobSpecificRequest(j.ID)),
   729  		mocker.Anything).Return(nil).Run(m.evaluationsFromState)
   730  	m.On("GetJob", mocker.MatchedBy(matchJobSpecificRequest(j.ID)),
   731  		mocker.Anything).Return(nil).Run(m.getJobFromState)
   732  
   733  	w.SetEnabled(true)
   734  	testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil },
   735  		func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") })
   736  
   737  	// Assert that we get a call to UpsertDeploymentStatusUpdate
   738  	matchConfig := &matchDeploymentStatusUpdateConfig{
   739  		DeploymentID:      d.ID,
   740  		Status:            structs.DeploymentStatusFailed,
   741  		StatusDescription: structs.DeploymentStatusDescriptionFailedByUser,
   742  		Eval:              true,
   743  	}
   744  	matcher := matchDeploymentStatusUpdateRequest(matchConfig)
   745  	m.On("UpdateDeploymentStatus", mocker.MatchedBy(matcher)).Return(nil)
   746  
   747  	// Call PauseDeployment
   748  	req := &structs.DeploymentFailRequest{
   749  		DeploymentID: d.ID,
   750  	}
   751  	var resp structs.DeploymentUpdateResponse
   752  	err := w.FailDeployment(req, &resp)
   753  	assert.Nil(err, "FailDeployment")
   754  
   755  	assert.Equal(1, len(w.watchers), "Deployment should still be active")
   756  	m.AssertCalled(t, "UpdateDeploymentStatus", mocker.MatchedBy(matcher))
   757  }
   758  
   759  // Tests that the watcher properly watches for allocation changes and takes the
   760  // proper actions
   761  func TestDeploymentWatcher_Watch(t *testing.T) {
   762  	t.Parallel()
   763  	assert := assert.New(t)
   764  	w, m := testDeploymentWatcher(t, 1000.0, 1*time.Millisecond)
   765  
   766  	// Create a job, alloc, and a deployment
   767  	j := mock.Job()
   768  	j.TaskGroups[0].Update = structs.DefaultUpdateStrategy.Copy()
   769  	j.TaskGroups[0].Update.MaxParallel = 2
   770  	j.TaskGroups[0].Update.AutoRevert = true
   771  	j.Stable = true
   772  	d := mock.Deployment()
   773  	d.JobID = j.ID
   774  	d.TaskGroups["web"].AutoRevert = true
   775  	a := mock.Alloc()
   776  	a.DeploymentID = d.ID
   777  	assert.Nil(m.state.UpsertJob(m.nextIndex(), j), "UpsertJob")
   778  	assert.Nil(m.state.UpsertDeployment(m.nextIndex(), d), "UpsertDeployment")
   779  	assert.Nil(m.state.UpsertAllocs(m.nextIndex(), []*structs.Allocation{a}), "UpsertAllocs")
   780  
   781  	// Upsert the job again to get a new version
   782  	j2 := j.Copy()
   783  	j2.Stable = false
   784  	assert.Nil(m.state.UpsertJob(m.nextIndex(), j2), "UpsertJob2")
   785  
   786  	// Assert the following methods will be called
   787  	m.On("List", mocker.Anything, mocker.Anything).Return(nil).Run(m.listFromState)
   788  	m.On("Allocations", mocker.MatchedBy(matchDeploymentSpecificRequest(d.ID)),
   789  		mocker.Anything).Return(nil).Run(m.allocationsFromState)
   790  	m.On("Evaluations", mocker.MatchedBy(matchJobSpecificRequest(j.ID)),
   791  		mocker.Anything).Return(nil).Run(m.evaluationsFromState)
   792  	m.On("GetJob", mocker.MatchedBy(matchJobSpecificRequest(j.ID)),
   793  		mocker.Anything).Return(nil).Run(m.getJobFromState)
   794  	m.On("GetJobVersions", mocker.MatchedBy(matchJobVersionsRequest(j.ID)),
   795  		mocker.Anything).Return(nil).Run(m.getJobVersionsFromState)
   796  
   797  	w.SetEnabled(true)
   798  	testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil },
   799  		func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") })
   800  
   801  	// Assert that we will get a createEvaluation call only once. This will
   802  	// verify that the watcher is batching allocation changes
   803  	m1 := matchUpsertEvals([]string{d.ID})
   804  	m.On("UpsertEvals", mocker.MatchedBy(m1)).Return(nil).Once()
   805  
   806  	// Update the allocs health to healthy which should create an evaluation
   807  	for i := 0; i < 5; i++ {
   808  		req := &structs.ApplyDeploymentAllocHealthRequest{
   809  			DeploymentAllocHealthRequest: structs.DeploymentAllocHealthRequest{
   810  				DeploymentID:         d.ID,
   811  				HealthyAllocationIDs: []string{a.ID},
   812  			},
   813  		}
   814  		assert.Nil(m.state.UpdateDeploymentAllocHealth(m.nextIndex(), req), "UpsertDeploymentAllocHealth")
   815  	}
   816  
   817  	// Wait for there to be one eval
   818  	testutil.WaitForResult(func() (bool, error) {
   819  		ws := memdb.NewWatchSet()
   820  		evals, err := m.state.EvalsByJob(ws, j.ID)
   821  		if err != nil {
   822  			return false, err
   823  		}
   824  
   825  		if l := len(evals); l != 1 {
   826  			return false, fmt.Errorf("Got %d evals; want 1", l)
   827  		}
   828  
   829  		return true, nil
   830  	}, func(err error) {
   831  		t.Fatal(err)
   832  	})
   833  
   834  	// Assert that we get a call to UpsertDeploymentStatusUpdate
   835  	c := &matchDeploymentStatusUpdateConfig{
   836  		DeploymentID:      d.ID,
   837  		Status:            structs.DeploymentStatusFailed,
   838  		StatusDescription: structs.DeploymentStatusDescriptionRollback(structs.DeploymentStatusDescriptionFailedAllocations, 0),
   839  		JobVersion:        helper.Uint64ToPtr(0),
   840  		Eval:              true,
   841  	}
   842  	m2 := matchDeploymentStatusUpdateRequest(c)
   843  	m.On("UpdateDeploymentStatus", mocker.MatchedBy(m2)).Return(nil)
   844  
   845  	// Update the allocs health to unhealthy which should create a job rollback,
   846  	// status update and eval
   847  	req2 := &structs.ApplyDeploymentAllocHealthRequest{
   848  		DeploymentAllocHealthRequest: structs.DeploymentAllocHealthRequest{
   849  			DeploymentID:           d.ID,
   850  			UnhealthyAllocationIDs: []string{a.ID},
   851  		},
   852  	}
   853  	assert.Nil(m.state.UpdateDeploymentAllocHealth(m.nextIndex(), req2), "UpsertDeploymentAllocHealth")
   854  
   855  	// Wait for there to be one eval
   856  	testutil.WaitForResult(func() (bool, error) {
   857  		ws := memdb.NewWatchSet()
   858  		evals, err := m.state.EvalsByJob(ws, j.ID)
   859  		if err != nil {
   860  			return false, err
   861  		}
   862  
   863  		if l := len(evals); l != 2 {
   864  			return false, fmt.Errorf("Got %d evals; want 1", l)
   865  		}
   866  
   867  		return true, nil
   868  	}, func(err error) {
   869  		t.Fatal(err)
   870  	})
   871  
   872  	m.AssertCalled(t, "UpsertEvals", mocker.MatchedBy(m1))
   873  
   874  	// After we upsert the job version will go to 2. So use this to assert the
   875  	// original call happened.
   876  	c2 := &matchDeploymentStatusUpdateConfig{
   877  		DeploymentID:      d.ID,
   878  		Status:            structs.DeploymentStatusFailed,
   879  		StatusDescription: structs.DeploymentStatusDescriptionRollback(structs.DeploymentStatusDescriptionFailedAllocations, 0),
   880  		JobVersion:        helper.Uint64ToPtr(2),
   881  		Eval:              true,
   882  	}
   883  	m3 := matchDeploymentStatusUpdateRequest(c2)
   884  	m.AssertCalled(t, "UpdateDeploymentStatus", mocker.MatchedBy(m3))
   885  	testutil.WaitForResult(func() (bool, error) { return 0 == len(w.watchers), nil },
   886  		func(err error) { assert.Equal(0, len(w.watchers), "Should have no deployment") })
   887  }
   888  
   889  // Test evaluations are batched between watchers
   890  func TestWatcher_BatchEvals(t *testing.T) {
   891  	t.Parallel()
   892  	assert := assert.New(t)
   893  	w, m := testDeploymentWatcher(t, 1000.0, 1*time.Second)
   894  
   895  	// Create a job, alloc, for two deployments
   896  	j1 := mock.Job()
   897  	d1 := mock.Deployment()
   898  	d1.JobID = j1.ID
   899  	a1 := mock.Alloc()
   900  	a1.DeploymentID = d1.ID
   901  
   902  	j2 := mock.Job()
   903  	d2 := mock.Deployment()
   904  	d2.JobID = j2.ID
   905  	a2 := mock.Alloc()
   906  	a2.DeploymentID = d2.ID
   907  
   908  	assert.Nil(m.state.UpsertJob(m.nextIndex(), j1), "UpsertJob")
   909  	assert.Nil(m.state.UpsertJob(m.nextIndex(), j2), "UpsertJob")
   910  	assert.Nil(m.state.UpsertDeployment(m.nextIndex(), d1), "UpsertDeployment")
   911  	assert.Nil(m.state.UpsertDeployment(m.nextIndex(), d2), "UpsertDeployment")
   912  	assert.Nil(m.state.UpsertAllocs(m.nextIndex(), []*structs.Allocation{a1}), "UpsertAllocs")
   913  	assert.Nil(m.state.UpsertAllocs(m.nextIndex(), []*structs.Allocation{a2}), "UpsertAllocs")
   914  
   915  	// Assert the following methods will be called
   916  	m.On("List", mocker.Anything, mocker.Anything).Return(nil).Run(m.listFromState)
   917  
   918  	m.On("Allocations", mocker.MatchedBy(matchDeploymentSpecificRequest(d1.ID)),
   919  		mocker.Anything).Return(nil).Run(m.allocationsFromState)
   920  	m.On("Allocations", mocker.MatchedBy(matchDeploymentSpecificRequest(d2.ID)),
   921  		mocker.Anything).Return(nil).Run(m.allocationsFromState)
   922  
   923  	m.On("Evaluations", mocker.MatchedBy(matchJobSpecificRequest(j1.ID)),
   924  		mocker.Anything).Return(nil).Run(m.evaluationsFromState)
   925  	m.On("Evaluations", mocker.MatchedBy(matchJobSpecificRequest(j2.ID)),
   926  		mocker.Anything).Return(nil).Run(m.evaluationsFromState)
   927  
   928  	m.On("GetJob", mocker.MatchedBy(matchJobSpecificRequest(j1.ID)),
   929  		mocker.Anything).Return(nil).Run(m.getJobFromState)
   930  	m.On("GetJob", mocker.MatchedBy(matchJobSpecificRequest(j2.ID)),
   931  		mocker.Anything).Return(nil).Run(m.getJobFromState)
   932  
   933  	m.On("GetJobVersions", mocker.MatchedBy(matchJobVersionsRequest(j1.ID)),
   934  		mocker.Anything).Return(nil).Run(m.getJobVersionsFromState)
   935  	m.On("GetJobVersions", mocker.MatchedBy(matchJobVersionsRequest(j2.ID)),
   936  		mocker.Anything).Return(nil).Run(m.getJobVersionsFromState)
   937  
   938  	w.SetEnabled(true)
   939  	testutil.WaitForResult(func() (bool, error) { return 2 == len(w.watchers), nil },
   940  		func(err error) { assert.Equal(2, len(w.watchers), "Should have 2 deployment") })
   941  
   942  	// Assert that we will get a createEvaluation call only once and it contains
   943  	// both deployments. This will verify that the watcher is batching
   944  	// allocation changes
   945  	m1 := matchUpsertEvals([]string{d1.ID, d2.ID})
   946  	m.On("UpsertEvals", mocker.MatchedBy(m1)).Return(nil).Once()
   947  
   948  	// Update the allocs health to healthy which should create an evaluation
   949  	req := &structs.ApplyDeploymentAllocHealthRequest{
   950  		DeploymentAllocHealthRequest: structs.DeploymentAllocHealthRequest{
   951  			DeploymentID:         d1.ID,
   952  			HealthyAllocationIDs: []string{a1.ID},
   953  		},
   954  	}
   955  	assert.Nil(m.state.UpdateDeploymentAllocHealth(m.nextIndex(), req), "UpsertDeploymentAllocHealth")
   956  
   957  	req2 := &structs.ApplyDeploymentAllocHealthRequest{
   958  		DeploymentAllocHealthRequest: structs.DeploymentAllocHealthRequest{
   959  			DeploymentID:         d2.ID,
   960  			HealthyAllocationIDs: []string{a2.ID},
   961  		},
   962  	}
   963  	assert.Nil(m.state.UpdateDeploymentAllocHealth(m.nextIndex(), req2), "UpsertDeploymentAllocHealth")
   964  
   965  	// Wait for there to be one eval for each job
   966  	testutil.WaitForResult(func() (bool, error) {
   967  		ws := memdb.NewWatchSet()
   968  		evals1, err := m.state.EvalsByJob(ws, j1.ID)
   969  		if err != nil {
   970  			return false, err
   971  		}
   972  
   973  		evals2, err := m.state.EvalsByJob(ws, j2.ID)
   974  		if err != nil {
   975  			return false, err
   976  		}
   977  
   978  		if l := len(evals1); l != 1 {
   979  			return false, fmt.Errorf("Got %d evals; want 1", l)
   980  		}
   981  
   982  		if l := len(evals2); l != 1 {
   983  			return false, fmt.Errorf("Got %d evals; want 1", l)
   984  		}
   985  
   986  		return true, nil
   987  	}, func(err error) {
   988  		t.Fatal(err)
   989  	})
   990  
   991  	m.AssertCalled(t, "UpsertEvals", mocker.MatchedBy(m1))
   992  	testutil.WaitForResult(func() (bool, error) { return 2 == len(w.watchers), nil },
   993  		func(err error) { assert.Equal(2, len(w.watchers), "Should have 2 deployment") })
   994  }