github.com/hhrutter/nomad@v0.6.0-rc2.0.20170723054333-80c4b03f0705/nomad/deploymentwatcher/deployments_watcher_test.go (about) 1 package deploymentwatcher 2 3 import ( 4 "fmt" 5 "testing" 6 "time" 7 8 memdb "github.com/hashicorp/go-memdb" 9 "github.com/hashicorp/nomad/helper" 10 "github.com/hashicorp/nomad/nomad/mock" 11 "github.com/hashicorp/nomad/nomad/structs" 12 "github.com/hashicorp/nomad/testutil" 13 "github.com/stretchr/testify/assert" 14 mocker "github.com/stretchr/testify/mock" 15 ) 16 17 func testDeploymentWatcher(t *testing.T, qps float64, batchDur time.Duration) (*Watcher, *mockBackend) { 18 m := newMockBackend(t) 19 w := NewDeploymentsWatcher(testLogger(), m, m, qps, batchDur) 20 return w, m 21 } 22 23 func defaultTestDeploymentWatcher(t *testing.T) (*Watcher, *mockBackend) { 24 return testDeploymentWatcher(t, LimitStateQueriesPerSecond, CrossDeploymentEvalBatchDuration) 25 } 26 27 // Tests that the watcher properly watches for deployments and reconciles them 28 func TestWatcher_WatchDeployments(t *testing.T) { 29 t.Parallel() 30 assert := assert.New(t) 31 w, m := defaultTestDeploymentWatcher(t) 32 33 // Return no allocations or evals 34 m.On("Allocations", mocker.Anything, mocker.Anything).Return(nil).Run(func(args mocker.Arguments) { 35 reply := args.Get(1).(*structs.AllocListResponse) 36 reply.Index = m.nextIndex() 37 }) 38 m.On("Evaluations", mocker.Anything, mocker.Anything).Return(nil).Run(func(args mocker.Arguments) { 39 reply := args.Get(1).(*structs.JobEvaluationsResponse) 40 reply.Index = m.nextIndex() 41 }) 42 43 // Create three jobs 44 j1, j2, j3 := mock.Job(), mock.Job(), mock.Job() 45 jobs := map[string]*structs.Job{ 46 j1.ID: j1, 47 j2.ID: j2, 48 j3.ID: j3, 49 } 50 51 // Create three deployments all running 52 d1, d2, d3 := mock.Deployment(), mock.Deployment(), mock.Deployment() 53 d1.JobID = j1.ID 54 d2.JobID = j2.ID 55 d3.JobID = j3.ID 56 57 m.On("GetJob", mocker.Anything, mocker.Anything). 58 Return(nil).Run(func(args mocker.Arguments) { 59 in := args.Get(0).(*structs.JobSpecificRequest) 60 reply := args.Get(1).(*structs.SingleJobResponse) 61 reply.Job = jobs[in.JobID] 62 reply.Index = reply.Job.ModifyIndex 63 }) 64 65 // Set up the calls for retrieving deployments 66 m.On("List", mocker.Anything, mocker.Anything).Return(nil).Run(func(args mocker.Arguments) { 67 reply := args.Get(1).(*structs.DeploymentListResponse) 68 reply.Deployments = []*structs.Deployment{d1} 69 reply.Index = m.nextIndex() 70 }).Once() 71 72 // Next list 3 73 block1 := make(chan time.Time) 74 m.On("List", mocker.Anything, mocker.Anything).Return(nil).Run(func(args mocker.Arguments) { 75 reply := args.Get(1).(*structs.DeploymentListResponse) 76 reply.Deployments = []*structs.Deployment{d1, d2, d3} 77 reply.Index = m.nextIndex() 78 }).Once().WaitUntil(block1) 79 80 //// Next list 3 but have one be terminal 81 block2 := make(chan time.Time) 82 d3terminal := d3.Copy() 83 d3terminal.Status = structs.DeploymentStatusFailed 84 m.On("List", mocker.Anything, mocker.Anything).Return(nil).Run(func(args mocker.Arguments) { 85 reply := args.Get(1).(*structs.DeploymentListResponse) 86 reply.Deployments = []*structs.Deployment{d1, d2, d3terminal} 87 reply.Index = m.nextIndex() 88 }).WaitUntil(block2) 89 90 m.On("List", mocker.Anything, mocker.Anything).Return(nil).Run(func(args mocker.Arguments) { 91 reply := args.Get(1).(*structs.DeploymentListResponse) 92 reply.Deployments = []*structs.Deployment{d1, d2, d3terminal} 93 reply.Index = m.nextIndex() 94 }) 95 96 w.SetEnabled(true) 97 testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil }, 98 func(err error) { assert.Equal(1, len(w.watchers), "1 deployment returned") }) 99 100 close(block1) 101 testutil.WaitForResult(func() (bool, error) { return 3 == len(w.watchers), nil }, 102 func(err error) { assert.Equal(3, len(w.watchers), "3 deployment returned") }) 103 104 close(block2) 105 testutil.WaitForResult(func() (bool, error) { return 2 == len(w.watchers), nil }, 106 func(err error) { assert.Equal(3, len(w.watchers), "3 deployment returned - 1 terminal") }) 107 } 108 109 // Tests that calls against an unknown deployment fail 110 func TestWatcher_UnknownDeployment(t *testing.T) { 111 t.Parallel() 112 assert := assert.New(t) 113 w, m := defaultTestDeploymentWatcher(t) 114 w.SetEnabled(true) 115 116 // Set up the calls for retrieving deployments 117 m.On("List", mocker.Anything, mocker.Anything).Return(nil).Run(func(args mocker.Arguments) { 118 reply := args.Get(1).(*structs.DeploymentListResponse) 119 reply.Index = m.nextIndex() 120 }) 121 m.On("GetDeployment", mocker.Anything, mocker.Anything).Return(nil).Run(func(args mocker.Arguments) { 122 reply := args.Get(1).(*structs.SingleDeploymentResponse) 123 reply.Index = m.nextIndex() 124 }) 125 126 // The expected error is that it should be an unknown deployment 127 dID := structs.GenerateUUID() 128 expected := fmt.Sprintf("unknown deployment %q", dID) 129 130 // Request setting the health against an unknown deployment 131 req := &structs.DeploymentAllocHealthRequest{ 132 DeploymentID: dID, 133 HealthyAllocationIDs: []string{structs.GenerateUUID()}, 134 } 135 var resp structs.DeploymentUpdateResponse 136 err := w.SetAllocHealth(req, &resp) 137 if assert.NotNil(err, "should have error for unknown deployment") { 138 assert.Contains(err.Error(), expected) 139 } 140 141 // Request promoting against an unknown deployment 142 req2 := &structs.DeploymentPromoteRequest{ 143 DeploymentID: dID, 144 All: true, 145 } 146 err = w.PromoteDeployment(req2, &resp) 147 if assert.NotNil(err, "should have error for unknown deployment") { 148 assert.Contains(err.Error(), expected) 149 } 150 151 // Request pausing against an unknown deployment 152 req3 := &structs.DeploymentPauseRequest{ 153 DeploymentID: dID, 154 Pause: true, 155 } 156 err = w.PauseDeployment(req3, &resp) 157 if assert.NotNil(err, "should have error for unknown deployment") { 158 assert.Contains(err.Error(), expected) 159 } 160 161 // Request failing against an unknown deployment 162 req4 := &structs.DeploymentFailRequest{ 163 DeploymentID: dID, 164 } 165 err = w.FailDeployment(req4, &resp) 166 if assert.NotNil(err, "should have error for unknown deployment") { 167 assert.Contains(err.Error(), expected) 168 } 169 } 170 171 // Test setting an unknown allocation's health 172 func TestWatcher_SetAllocHealth_Unknown(t *testing.T) { 173 t.Parallel() 174 assert := assert.New(t) 175 w, m := defaultTestDeploymentWatcher(t) 176 177 // Create a job, and a deployment 178 j := mock.Job() 179 d := mock.Deployment() 180 d.JobID = j.ID 181 assert.Nil(m.state.UpsertJob(m.nextIndex(), j), "UpsertJob") 182 assert.Nil(m.state.UpsertDeployment(m.nextIndex(), d), "UpsertDeployment") 183 184 // Assert the following methods will be called 185 m.On("List", mocker.Anything, mocker.Anything).Return(nil).Run(m.listFromState) 186 m.On("Allocations", mocker.MatchedBy(matchDeploymentSpecificRequest(d.ID)), 187 mocker.Anything).Return(nil).Run(m.allocationsFromState) 188 m.On("Evaluations", mocker.MatchedBy(matchJobSpecificRequest(j.ID)), 189 mocker.Anything).Return(nil).Run(m.evaluationsFromState) 190 m.On("GetJob", mocker.MatchedBy(matchJobSpecificRequest(j.ID)), 191 mocker.Anything).Return(nil).Run(m.getJobFromState) 192 193 w.SetEnabled(true) 194 testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil }, 195 func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") }) 196 197 // Assert that we get a call to UpsertDeploymentAllocHealth 198 a := mock.Alloc() 199 matchConfig := &matchDeploymentAllocHealthRequestConfig{ 200 DeploymentID: d.ID, 201 Healthy: []string{a.ID}, 202 Eval: true, 203 } 204 matcher := matchDeploymentAllocHealthRequest(matchConfig) 205 m.On("UpdateDeploymentAllocHealth", mocker.MatchedBy(matcher)).Return(nil) 206 207 // Call SetAllocHealth 208 req := &structs.DeploymentAllocHealthRequest{ 209 DeploymentID: d.ID, 210 HealthyAllocationIDs: []string{a.ID}, 211 } 212 var resp structs.DeploymentUpdateResponse 213 err := w.SetAllocHealth(req, &resp) 214 if assert.NotNil(err, "Set health of unknown allocation") { 215 assert.Contains(err.Error(), "unknown") 216 } 217 assert.Equal(1, len(w.watchers), "Deployment should still be active") 218 } 219 220 // Test setting allocation health 221 func TestWatcher_SetAllocHealth_Healthy(t *testing.T) { 222 t.Parallel() 223 assert := assert.New(t) 224 w, m := defaultTestDeploymentWatcher(t) 225 226 // Create a job, alloc, and a deployment 227 j := mock.Job() 228 d := mock.Deployment() 229 d.JobID = j.ID 230 a := mock.Alloc() 231 a.DeploymentID = d.ID 232 assert.Nil(m.state.UpsertJob(m.nextIndex(), j), "UpsertJob") 233 assert.Nil(m.state.UpsertDeployment(m.nextIndex(), d), "UpsertDeployment") 234 assert.Nil(m.state.UpsertAllocs(m.nextIndex(), []*structs.Allocation{a}), "UpsertAllocs") 235 236 // Assert the following methods will be called 237 m.On("List", mocker.Anything, mocker.Anything).Return(nil).Run(m.listFromState) 238 m.On("Allocations", mocker.MatchedBy(matchDeploymentSpecificRequest(d.ID)), 239 mocker.Anything).Return(nil).Run(m.allocationsFromState) 240 m.On("Evaluations", mocker.MatchedBy(matchJobSpecificRequest(j.ID)), 241 mocker.Anything).Return(nil).Run(m.evaluationsFromState) 242 m.On("GetJob", mocker.MatchedBy(matchJobSpecificRequest(j.ID)), 243 mocker.Anything).Return(nil).Run(m.getJobFromState) 244 245 w.SetEnabled(true) 246 testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil }, 247 func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") }) 248 249 // Assert that we get a call to UpsertDeploymentAllocHealth 250 matchConfig := &matchDeploymentAllocHealthRequestConfig{ 251 DeploymentID: d.ID, 252 Healthy: []string{a.ID}, 253 Eval: true, 254 } 255 matcher := matchDeploymentAllocHealthRequest(matchConfig) 256 m.On("UpdateDeploymentAllocHealth", mocker.MatchedBy(matcher)).Return(nil) 257 258 // Call SetAllocHealth 259 req := &structs.DeploymentAllocHealthRequest{ 260 DeploymentID: d.ID, 261 HealthyAllocationIDs: []string{a.ID}, 262 } 263 var resp structs.DeploymentUpdateResponse 264 err := w.SetAllocHealth(req, &resp) 265 assert.Nil(err, "SetAllocHealth") 266 assert.Equal(1, len(w.watchers), "Deployment should still be active") 267 m.AssertCalled(t, "UpdateDeploymentAllocHealth", mocker.MatchedBy(matcher)) 268 } 269 270 // Test setting allocation unhealthy 271 func TestWatcher_SetAllocHealth_Unhealthy(t *testing.T) { 272 t.Parallel() 273 assert := assert.New(t) 274 w, m := defaultTestDeploymentWatcher(t) 275 276 // Create a job, alloc, and a deployment 277 j := mock.Job() 278 d := mock.Deployment() 279 d.JobID = j.ID 280 a := mock.Alloc() 281 a.DeploymentID = d.ID 282 assert.Nil(m.state.UpsertJob(m.nextIndex(), j), "UpsertJob") 283 assert.Nil(m.state.UpsertDeployment(m.nextIndex(), d), "UpsertDeployment") 284 assert.Nil(m.state.UpsertAllocs(m.nextIndex(), []*structs.Allocation{a}), "UpsertAllocs") 285 286 // Assert the following methods will be called 287 m.On("List", mocker.Anything, mocker.Anything).Return(nil).Run(m.listFromState) 288 m.On("Allocations", mocker.MatchedBy(matchDeploymentSpecificRequest(d.ID)), 289 mocker.Anything).Return(nil).Run(m.allocationsFromState) 290 m.On("Evaluations", mocker.MatchedBy(matchJobSpecificRequest(j.ID)), 291 mocker.Anything).Return(nil).Run(m.evaluationsFromState) 292 m.On("GetJob", mocker.MatchedBy(matchJobSpecificRequest(j.ID)), 293 mocker.Anything).Return(nil).Run(m.getJobFromState) 294 295 w.SetEnabled(true) 296 testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil }, 297 func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") }) 298 299 // Assert that we get a call to UpsertDeploymentAllocHealth 300 matchConfig := &matchDeploymentAllocHealthRequestConfig{ 301 DeploymentID: d.ID, 302 Unhealthy: []string{a.ID}, 303 Eval: true, 304 DeploymentUpdate: &structs.DeploymentStatusUpdate{ 305 DeploymentID: d.ID, 306 Status: structs.DeploymentStatusFailed, 307 StatusDescription: structs.DeploymentStatusDescriptionFailedAllocations, 308 }, 309 } 310 matcher := matchDeploymentAllocHealthRequest(matchConfig) 311 m.On("UpdateDeploymentAllocHealth", mocker.MatchedBy(matcher)).Return(nil) 312 313 // Call SetAllocHealth 314 req := &structs.DeploymentAllocHealthRequest{ 315 DeploymentID: d.ID, 316 UnhealthyAllocationIDs: []string{a.ID}, 317 } 318 var resp structs.DeploymentUpdateResponse 319 err := w.SetAllocHealth(req, &resp) 320 assert.Nil(err, "SetAllocHealth") 321 322 testutil.WaitForResult(func() (bool, error) { return 0 == len(w.watchers), nil }, 323 func(err error) { assert.Equal(0, len(w.watchers), "Should have no deployment") }) 324 m.AssertNumberOfCalls(t, "UpdateDeploymentAllocHealth", 1) 325 } 326 327 // Test setting allocation unhealthy and that there should be a rollback 328 func TestWatcher_SetAllocHealth_Unhealthy_Rollback(t *testing.T) { 329 t.Parallel() 330 assert := assert.New(t) 331 w, m := defaultTestDeploymentWatcher(t) 332 333 // Create a job, alloc, and a deployment 334 j := mock.Job() 335 j.TaskGroups[0].Update = structs.DefaultUpdateStrategy.Copy() 336 j.TaskGroups[0].Update.MaxParallel = 2 337 j.TaskGroups[0].Update.AutoRevert = true 338 j.Stable = true 339 d := mock.Deployment() 340 d.JobID = j.ID 341 d.TaskGroups["web"].AutoRevert = true 342 a := mock.Alloc() 343 a.DeploymentID = d.ID 344 assert.Nil(m.state.UpsertJob(m.nextIndex(), j), "UpsertJob") 345 assert.Nil(m.state.UpsertDeployment(m.nextIndex(), d), "UpsertDeployment") 346 assert.Nil(m.state.UpsertAllocs(m.nextIndex(), []*structs.Allocation{a}), "UpsertAllocs") 347 348 // Upsert the job again to get a new version 349 j2 := j.Copy() 350 j2.Stable = false 351 assert.Nil(m.state.UpsertJob(m.nextIndex(), j2), "UpsertJob2") 352 353 // Assert the following methods will be called 354 m.On("List", mocker.Anything, mocker.Anything).Return(nil).Run(m.listFromState) 355 m.On("Allocations", mocker.MatchedBy(matchDeploymentSpecificRequest(d.ID)), 356 mocker.Anything).Return(nil).Run(m.allocationsFromState) 357 m.On("Evaluations", mocker.MatchedBy(matchJobSpecificRequest(j.ID)), 358 mocker.Anything).Return(nil).Run(m.evaluationsFromState) 359 m.On("GetJob", mocker.MatchedBy(matchJobSpecificRequest(j.ID)), 360 mocker.Anything).Return(nil).Run(m.getJobFromState) 361 m.On("GetJobVersions", mocker.MatchedBy(matchJobVersionsRequest(j.ID)), 362 mocker.Anything).Return(nil).Run(m.getJobVersionsFromState) 363 364 w.SetEnabled(true) 365 testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil }, 366 func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") }) 367 368 // Assert that we get a call to UpsertDeploymentAllocHealth 369 matchConfig := &matchDeploymentAllocHealthRequestConfig{ 370 DeploymentID: d.ID, 371 Unhealthy: []string{a.ID}, 372 Eval: true, 373 DeploymentUpdate: &structs.DeploymentStatusUpdate{ 374 DeploymentID: d.ID, 375 Status: structs.DeploymentStatusFailed, 376 StatusDescription: structs.DeploymentStatusDescriptionFailedAllocations, 377 }, 378 JobVersion: helper.Uint64ToPtr(0), 379 } 380 matcher := matchDeploymentAllocHealthRequest(matchConfig) 381 m.On("UpdateDeploymentAllocHealth", mocker.MatchedBy(matcher)).Return(nil) 382 383 // Call SetAllocHealth 384 req := &structs.DeploymentAllocHealthRequest{ 385 DeploymentID: d.ID, 386 UnhealthyAllocationIDs: []string{a.ID}, 387 } 388 var resp structs.DeploymentUpdateResponse 389 err := w.SetAllocHealth(req, &resp) 390 assert.Nil(err, "SetAllocHealth") 391 392 testutil.WaitForResult(func() (bool, error) { return 0 == len(w.watchers), nil }, 393 func(err error) { assert.Equal(0, len(w.watchers), "Should have no deployment") }) 394 m.AssertNumberOfCalls(t, "UpdateDeploymentAllocHealth", 1) 395 } 396 397 // Test promoting a deployment 398 func TestWatcher_PromoteDeployment_HealthyCanaries(t *testing.T) { 399 t.Parallel() 400 assert := assert.New(t) 401 w, m := defaultTestDeploymentWatcher(t) 402 403 // Create a job, canary alloc, and a deployment 404 j := mock.Job() 405 j.TaskGroups[0].Update = structs.DefaultUpdateStrategy.Copy() 406 j.TaskGroups[0].Update.MaxParallel = 2 407 j.TaskGroups[0].Update.Canary = 2 408 d := mock.Deployment() 409 d.JobID = j.ID 410 a := mock.Alloc() 411 d.TaskGroups[a.TaskGroup].PlacedCanaries = []string{a.ID} 412 a.DeploymentStatus = &structs.AllocDeploymentStatus{ 413 Healthy: helper.BoolToPtr(true), 414 } 415 a.DeploymentID = d.ID 416 assert.Nil(m.state.UpsertJob(m.nextIndex(), j), "UpsertJob") 417 assert.Nil(m.state.UpsertDeployment(m.nextIndex(), d), "UpsertDeployment") 418 assert.Nil(m.state.UpsertAllocs(m.nextIndex(), []*structs.Allocation{a}), "UpsertAllocs") 419 420 // Assert the following methods will be called 421 m.On("List", mocker.Anything, mocker.Anything).Return(nil).Run(m.listFromState) 422 m.On("Allocations", mocker.MatchedBy(matchDeploymentSpecificRequest(d.ID)), 423 mocker.Anything).Return(nil).Run(m.allocationsFromState) 424 m.On("Evaluations", mocker.MatchedBy(matchJobSpecificRequest(j.ID)), 425 mocker.Anything).Return(nil).Run(m.evaluationsFromState) 426 m.On("GetJob", mocker.MatchedBy(matchJobSpecificRequest(j.ID)), 427 mocker.Anything).Return(nil).Run(m.getJobFromState) 428 429 w.SetEnabled(true) 430 testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil }, 431 func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") }) 432 433 // Assert that we get a call to UpsertDeploymentPromotion 434 matchConfig := &matchDeploymentPromoteRequestConfig{ 435 Promotion: &structs.DeploymentPromoteRequest{ 436 DeploymentID: d.ID, 437 All: true, 438 }, 439 Eval: true, 440 } 441 matcher := matchDeploymentPromoteRequest(matchConfig) 442 m.On("UpdateDeploymentPromotion", mocker.MatchedBy(matcher)).Return(nil) 443 444 // Call PromoteDeployment 445 req := &structs.DeploymentPromoteRequest{ 446 DeploymentID: d.ID, 447 All: true, 448 } 449 var resp structs.DeploymentUpdateResponse 450 err := w.PromoteDeployment(req, &resp) 451 assert.Nil(err, "PromoteDeployment") 452 assert.Equal(1, len(w.watchers), "Deployment should still be active") 453 m.AssertCalled(t, "UpdateDeploymentPromotion", mocker.MatchedBy(matcher)) 454 } 455 456 // Test promoting a deployment with unhealthy canaries 457 func TestWatcher_PromoteDeployment_UnhealthyCanaries(t *testing.T) { 458 t.Parallel() 459 assert := assert.New(t) 460 w, m := defaultTestDeploymentWatcher(t) 461 462 // Create a job, canary alloc, and a deployment 463 j := mock.Job() 464 j.TaskGroups[0].Update = structs.DefaultUpdateStrategy.Copy() 465 j.TaskGroups[0].Update.MaxParallel = 2 466 j.TaskGroups[0].Update.Canary = 2 467 d := mock.Deployment() 468 d.JobID = j.ID 469 a := mock.Alloc() 470 d.TaskGroups[a.TaskGroup].PlacedCanaries = []string{a.ID} 471 a.DeploymentID = d.ID 472 assert.Nil(m.state.UpsertJob(m.nextIndex(), j), "UpsertJob") 473 assert.Nil(m.state.UpsertDeployment(m.nextIndex(), d), "UpsertDeployment") 474 assert.Nil(m.state.UpsertAllocs(m.nextIndex(), []*structs.Allocation{a}), "UpsertAllocs") 475 476 // Assert the following methods will be called 477 m.On("List", mocker.Anything, mocker.Anything).Return(nil).Run(m.listFromState) 478 m.On("Allocations", mocker.MatchedBy(matchDeploymentSpecificRequest(d.ID)), 479 mocker.Anything).Return(nil).Run(m.allocationsFromState) 480 m.On("Evaluations", mocker.MatchedBy(matchJobSpecificRequest(j.ID)), 481 mocker.Anything).Return(nil).Run(m.evaluationsFromState) 482 m.On("GetJob", mocker.MatchedBy(matchJobSpecificRequest(j.ID)), 483 mocker.Anything).Return(nil).Run(m.getJobFromState) 484 485 w.SetEnabled(true) 486 testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil }, 487 func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") }) 488 489 // Assert that we get a call to UpsertDeploymentPromotion 490 matchConfig := &matchDeploymentPromoteRequestConfig{ 491 Promotion: &structs.DeploymentPromoteRequest{ 492 DeploymentID: d.ID, 493 All: true, 494 }, 495 Eval: true, 496 } 497 matcher := matchDeploymentPromoteRequest(matchConfig) 498 m.On("UpdateDeploymentPromotion", mocker.MatchedBy(matcher)).Return(nil) 499 500 // Call SetAllocHealth 501 req := &structs.DeploymentPromoteRequest{ 502 DeploymentID: d.ID, 503 All: true, 504 } 505 var resp structs.DeploymentUpdateResponse 506 err := w.PromoteDeployment(req, &resp) 507 if assert.NotNil(err, "PromoteDeployment") { 508 assert.Contains(err.Error(), "is not healthy", "Should error because canary isn't marked healthy") 509 } 510 511 assert.Equal(1, len(w.watchers), "Deployment should still be active") 512 m.AssertCalled(t, "UpdateDeploymentPromotion", mocker.MatchedBy(matcher)) 513 } 514 515 // Test pausing a deployment that is running 516 func TestWatcher_PauseDeployment_Pause_Running(t *testing.T) { 517 t.Parallel() 518 assert := assert.New(t) 519 w, m := defaultTestDeploymentWatcher(t) 520 521 // Create a job and a deployment 522 j := mock.Job() 523 d := mock.Deployment() 524 d.JobID = j.ID 525 assert.Nil(m.state.UpsertJob(m.nextIndex(), j), "UpsertJob") 526 assert.Nil(m.state.UpsertDeployment(m.nextIndex(), d), "UpsertDeployment") 527 528 // Assert the following methods will be called 529 m.On("List", mocker.Anything, mocker.Anything).Return(nil).Run(m.listFromState) 530 m.On("Allocations", mocker.MatchedBy(matchDeploymentSpecificRequest(d.ID)), 531 mocker.Anything).Return(nil).Run(m.allocationsFromState) 532 m.On("Evaluations", mocker.MatchedBy(matchJobSpecificRequest(j.ID)), 533 mocker.Anything).Return(nil).Run(m.evaluationsFromState) 534 m.On("GetJob", mocker.MatchedBy(matchJobSpecificRequest(j.ID)), 535 mocker.Anything).Return(nil).Run(m.getJobFromState) 536 537 w.SetEnabled(true) 538 testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil }, 539 func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") }) 540 541 // Assert that we get a call to UpsertDeploymentStatusUpdate 542 matchConfig := &matchDeploymentStatusUpdateConfig{ 543 DeploymentID: d.ID, 544 Status: structs.DeploymentStatusPaused, 545 StatusDescription: structs.DeploymentStatusDescriptionPaused, 546 } 547 matcher := matchDeploymentStatusUpdateRequest(matchConfig) 548 m.On("UpdateDeploymentStatus", mocker.MatchedBy(matcher)).Return(nil) 549 550 // Call PauseDeployment 551 req := &structs.DeploymentPauseRequest{ 552 DeploymentID: d.ID, 553 Pause: true, 554 } 555 var resp structs.DeploymentUpdateResponse 556 err := w.PauseDeployment(req, &resp) 557 assert.Nil(err, "PauseDeployment") 558 559 assert.Equal(1, len(w.watchers), "Deployment should still be active") 560 m.AssertCalled(t, "UpdateDeploymentStatus", mocker.MatchedBy(matcher)) 561 } 562 563 // Test pausing a deployment that is paused 564 func TestWatcher_PauseDeployment_Pause_Paused(t *testing.T) { 565 t.Parallel() 566 assert := assert.New(t) 567 w, m := defaultTestDeploymentWatcher(t) 568 569 // Create a job and a deployment 570 j := mock.Job() 571 d := mock.Deployment() 572 d.JobID = j.ID 573 d.Status = structs.DeploymentStatusPaused 574 assert.Nil(m.state.UpsertJob(m.nextIndex(), j), "UpsertJob") 575 assert.Nil(m.state.UpsertDeployment(m.nextIndex(), d), "UpsertDeployment") 576 577 // Assert the following methods will be called 578 m.On("List", mocker.Anything, mocker.Anything).Return(nil).Run(m.listFromState) 579 m.On("Allocations", mocker.MatchedBy(matchDeploymentSpecificRequest(d.ID)), 580 mocker.Anything).Return(nil).Run(m.allocationsFromState) 581 m.On("Evaluations", mocker.MatchedBy(matchJobSpecificRequest(j.ID)), 582 mocker.Anything).Return(nil).Run(m.evaluationsFromState) 583 m.On("GetJob", mocker.MatchedBy(matchJobSpecificRequest(j.ID)), 584 mocker.Anything).Return(nil).Run(m.getJobFromState) 585 586 w.SetEnabled(true) 587 testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil }, 588 func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") }) 589 590 // Assert that we get a call to UpsertDeploymentStatusUpdate 591 matchConfig := &matchDeploymentStatusUpdateConfig{ 592 DeploymentID: d.ID, 593 Status: structs.DeploymentStatusPaused, 594 StatusDescription: structs.DeploymentStatusDescriptionPaused, 595 } 596 matcher := matchDeploymentStatusUpdateRequest(matchConfig) 597 m.On("UpdateDeploymentStatus", mocker.MatchedBy(matcher)).Return(nil) 598 599 // Call PauseDeployment 600 req := &structs.DeploymentPauseRequest{ 601 DeploymentID: d.ID, 602 Pause: true, 603 } 604 var resp structs.DeploymentUpdateResponse 605 err := w.PauseDeployment(req, &resp) 606 assert.Nil(err, "PauseDeployment") 607 608 assert.Equal(1, len(w.watchers), "Deployment should still be active") 609 m.AssertCalled(t, "UpdateDeploymentStatus", mocker.MatchedBy(matcher)) 610 } 611 612 // Test unpausing a deployment that is paused 613 func TestWatcher_PauseDeployment_Unpause_Paused(t *testing.T) { 614 t.Parallel() 615 assert := assert.New(t) 616 w, m := defaultTestDeploymentWatcher(t) 617 618 // Create a job and a deployment 619 j := mock.Job() 620 d := mock.Deployment() 621 d.JobID = j.ID 622 d.Status = structs.DeploymentStatusPaused 623 assert.Nil(m.state.UpsertJob(m.nextIndex(), j), "UpsertJob") 624 assert.Nil(m.state.UpsertDeployment(m.nextIndex(), d), "UpsertDeployment") 625 626 // Assert the following methods will be called 627 m.On("List", mocker.Anything, mocker.Anything).Return(nil).Run(m.listFromState) 628 m.On("Allocations", mocker.MatchedBy(matchDeploymentSpecificRequest(d.ID)), 629 mocker.Anything).Return(nil).Run(m.allocationsFromState) 630 m.On("Evaluations", mocker.MatchedBy(matchJobSpecificRequest(j.ID)), 631 mocker.Anything).Return(nil).Run(m.evaluationsFromState) 632 m.On("GetJob", mocker.MatchedBy(matchJobSpecificRequest(j.ID)), 633 mocker.Anything).Return(nil).Run(m.getJobFromState) 634 635 w.SetEnabled(true) 636 testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil }, 637 func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") }) 638 639 // Assert that we get a call to UpsertDeploymentStatusUpdate 640 matchConfig := &matchDeploymentStatusUpdateConfig{ 641 DeploymentID: d.ID, 642 Status: structs.DeploymentStatusRunning, 643 StatusDescription: structs.DeploymentStatusDescriptionRunning, 644 Eval: true, 645 } 646 matcher := matchDeploymentStatusUpdateRequest(matchConfig) 647 m.On("UpdateDeploymentStatus", mocker.MatchedBy(matcher)).Return(nil) 648 649 // Call PauseDeployment 650 req := &structs.DeploymentPauseRequest{ 651 DeploymentID: d.ID, 652 Pause: false, 653 } 654 var resp structs.DeploymentUpdateResponse 655 err := w.PauseDeployment(req, &resp) 656 assert.Nil(err, "PauseDeployment") 657 658 assert.Equal(1, len(w.watchers), "Deployment should still be active") 659 m.AssertCalled(t, "UpdateDeploymentStatus", mocker.MatchedBy(matcher)) 660 } 661 662 // Test unpausing a deployment that is running 663 func TestWatcher_PauseDeployment_Unpause_Running(t *testing.T) { 664 t.Parallel() 665 assert := assert.New(t) 666 w, m := defaultTestDeploymentWatcher(t) 667 668 // Create a job and a deployment 669 j := mock.Job() 670 d := mock.Deployment() 671 d.JobID = j.ID 672 assert.Nil(m.state.UpsertJob(m.nextIndex(), j), "UpsertJob") 673 assert.Nil(m.state.UpsertDeployment(m.nextIndex(), d), "UpsertDeployment") 674 675 // Assert the following methods will be called 676 m.On("List", mocker.Anything, mocker.Anything).Return(nil).Run(m.listFromState) 677 m.On("Allocations", mocker.MatchedBy(matchDeploymentSpecificRequest(d.ID)), 678 mocker.Anything).Return(nil).Run(m.allocationsFromState) 679 m.On("Evaluations", mocker.MatchedBy(matchJobSpecificRequest(j.ID)), 680 mocker.Anything).Return(nil).Run(m.evaluationsFromState) 681 m.On("GetJob", mocker.MatchedBy(matchJobSpecificRequest(j.ID)), 682 mocker.Anything).Return(nil).Run(m.getJobFromState) 683 684 w.SetEnabled(true) 685 testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil }, 686 func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") }) 687 688 // Assert that we get a call to UpsertDeploymentStatusUpdate 689 matchConfig := &matchDeploymentStatusUpdateConfig{ 690 DeploymentID: d.ID, 691 Status: structs.DeploymentStatusRunning, 692 StatusDescription: structs.DeploymentStatusDescriptionRunning, 693 Eval: true, 694 } 695 matcher := matchDeploymentStatusUpdateRequest(matchConfig) 696 m.On("UpdateDeploymentStatus", mocker.MatchedBy(matcher)).Return(nil) 697 698 // Call PauseDeployment 699 req := &structs.DeploymentPauseRequest{ 700 DeploymentID: d.ID, 701 Pause: false, 702 } 703 var resp structs.DeploymentUpdateResponse 704 err := w.PauseDeployment(req, &resp) 705 assert.Nil(err, "PauseDeployment") 706 707 assert.Equal(1, len(w.watchers), "Deployment should still be active") 708 m.AssertCalled(t, "UpdateDeploymentStatus", mocker.MatchedBy(matcher)) 709 } 710 711 // Test failing a deployment that is running 712 func TestWatcher_FailDeployment_Running(t *testing.T) { 713 t.Parallel() 714 assert := assert.New(t) 715 w, m := defaultTestDeploymentWatcher(t) 716 717 // Create a job and a deployment 718 j := mock.Job() 719 d := mock.Deployment() 720 d.JobID = j.ID 721 assert.Nil(m.state.UpsertJob(m.nextIndex(), j), "UpsertJob") 722 assert.Nil(m.state.UpsertDeployment(m.nextIndex(), d), "UpsertDeployment") 723 724 // Assert the following methods will be called 725 m.On("List", mocker.Anything, mocker.Anything).Return(nil).Run(m.listFromState) 726 m.On("Allocations", mocker.MatchedBy(matchDeploymentSpecificRequest(d.ID)), 727 mocker.Anything).Return(nil).Run(m.allocationsFromState) 728 m.On("Evaluations", mocker.MatchedBy(matchJobSpecificRequest(j.ID)), 729 mocker.Anything).Return(nil).Run(m.evaluationsFromState) 730 m.On("GetJob", mocker.MatchedBy(matchJobSpecificRequest(j.ID)), 731 mocker.Anything).Return(nil).Run(m.getJobFromState) 732 733 w.SetEnabled(true) 734 testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil }, 735 func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") }) 736 737 // Assert that we get a call to UpsertDeploymentStatusUpdate 738 matchConfig := &matchDeploymentStatusUpdateConfig{ 739 DeploymentID: d.ID, 740 Status: structs.DeploymentStatusFailed, 741 StatusDescription: structs.DeploymentStatusDescriptionFailedByUser, 742 Eval: true, 743 } 744 matcher := matchDeploymentStatusUpdateRequest(matchConfig) 745 m.On("UpdateDeploymentStatus", mocker.MatchedBy(matcher)).Return(nil) 746 747 // Call PauseDeployment 748 req := &structs.DeploymentFailRequest{ 749 DeploymentID: d.ID, 750 } 751 var resp structs.DeploymentUpdateResponse 752 err := w.FailDeployment(req, &resp) 753 assert.Nil(err, "FailDeployment") 754 755 assert.Equal(1, len(w.watchers), "Deployment should still be active") 756 m.AssertCalled(t, "UpdateDeploymentStatus", mocker.MatchedBy(matcher)) 757 } 758 759 // Tests that the watcher properly watches for allocation changes and takes the 760 // proper actions 761 func TestDeploymentWatcher_Watch(t *testing.T) { 762 t.Parallel() 763 assert := assert.New(t) 764 w, m := testDeploymentWatcher(t, 1000.0, 1*time.Millisecond) 765 766 // Create a job, alloc, and a deployment 767 j := mock.Job() 768 j.TaskGroups[0].Update = structs.DefaultUpdateStrategy.Copy() 769 j.TaskGroups[0].Update.MaxParallel = 2 770 j.TaskGroups[0].Update.AutoRevert = true 771 j.Stable = true 772 d := mock.Deployment() 773 d.JobID = j.ID 774 d.TaskGroups["web"].AutoRevert = true 775 a := mock.Alloc() 776 a.DeploymentID = d.ID 777 assert.Nil(m.state.UpsertJob(m.nextIndex(), j), "UpsertJob") 778 assert.Nil(m.state.UpsertDeployment(m.nextIndex(), d), "UpsertDeployment") 779 assert.Nil(m.state.UpsertAllocs(m.nextIndex(), []*structs.Allocation{a}), "UpsertAllocs") 780 781 // Upsert the job again to get a new version 782 j2 := j.Copy() 783 j2.Stable = false 784 assert.Nil(m.state.UpsertJob(m.nextIndex(), j2), "UpsertJob2") 785 786 // Assert the following methods will be called 787 m.On("List", mocker.Anything, mocker.Anything).Return(nil).Run(m.listFromState) 788 m.On("Allocations", mocker.MatchedBy(matchDeploymentSpecificRequest(d.ID)), 789 mocker.Anything).Return(nil).Run(m.allocationsFromState) 790 m.On("Evaluations", mocker.MatchedBy(matchJobSpecificRequest(j.ID)), 791 mocker.Anything).Return(nil).Run(m.evaluationsFromState) 792 m.On("GetJob", mocker.MatchedBy(matchJobSpecificRequest(j.ID)), 793 mocker.Anything).Return(nil).Run(m.getJobFromState) 794 m.On("GetJobVersions", mocker.MatchedBy(matchJobVersionsRequest(j.ID)), 795 mocker.Anything).Return(nil).Run(m.getJobVersionsFromState) 796 797 w.SetEnabled(true) 798 testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil }, 799 func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") }) 800 801 // Assert that we will get a createEvaluation call only once. This will 802 // verify that the watcher is batching allocation changes 803 m1 := matchUpsertEvals([]string{d.ID}) 804 m.On("UpsertEvals", mocker.MatchedBy(m1)).Return(nil).Once() 805 806 // Update the allocs health to healthy which should create an evaluation 807 for i := 0; i < 5; i++ { 808 req := &structs.ApplyDeploymentAllocHealthRequest{ 809 DeploymentAllocHealthRequest: structs.DeploymentAllocHealthRequest{ 810 DeploymentID: d.ID, 811 HealthyAllocationIDs: []string{a.ID}, 812 }, 813 } 814 assert.Nil(m.state.UpdateDeploymentAllocHealth(m.nextIndex(), req), "UpsertDeploymentAllocHealth") 815 } 816 817 // Wait for there to be one eval 818 testutil.WaitForResult(func() (bool, error) { 819 ws := memdb.NewWatchSet() 820 evals, err := m.state.EvalsByJob(ws, j.ID) 821 if err != nil { 822 return false, err 823 } 824 825 if l := len(evals); l != 1 { 826 return false, fmt.Errorf("Got %d evals; want 1", l) 827 } 828 829 return true, nil 830 }, func(err error) { 831 t.Fatal(err) 832 }) 833 834 // Assert that we get a call to UpsertDeploymentStatusUpdate 835 c := &matchDeploymentStatusUpdateConfig{ 836 DeploymentID: d.ID, 837 Status: structs.DeploymentStatusFailed, 838 StatusDescription: structs.DeploymentStatusDescriptionRollback(structs.DeploymentStatusDescriptionFailedAllocations, 0), 839 JobVersion: helper.Uint64ToPtr(0), 840 Eval: true, 841 } 842 m2 := matchDeploymentStatusUpdateRequest(c) 843 m.On("UpdateDeploymentStatus", mocker.MatchedBy(m2)).Return(nil) 844 845 // Update the allocs health to unhealthy which should create a job rollback, 846 // status update and eval 847 req2 := &structs.ApplyDeploymentAllocHealthRequest{ 848 DeploymentAllocHealthRequest: structs.DeploymentAllocHealthRequest{ 849 DeploymentID: d.ID, 850 UnhealthyAllocationIDs: []string{a.ID}, 851 }, 852 } 853 assert.Nil(m.state.UpdateDeploymentAllocHealth(m.nextIndex(), req2), "UpsertDeploymentAllocHealth") 854 855 // Wait for there to be one eval 856 testutil.WaitForResult(func() (bool, error) { 857 ws := memdb.NewWatchSet() 858 evals, err := m.state.EvalsByJob(ws, j.ID) 859 if err != nil { 860 return false, err 861 } 862 863 if l := len(evals); l != 2 { 864 return false, fmt.Errorf("Got %d evals; want 1", l) 865 } 866 867 return true, nil 868 }, func(err error) { 869 t.Fatal(err) 870 }) 871 872 m.AssertCalled(t, "UpsertEvals", mocker.MatchedBy(m1)) 873 874 // After we upsert the job version will go to 2. So use this to assert the 875 // original call happened. 876 c2 := &matchDeploymentStatusUpdateConfig{ 877 DeploymentID: d.ID, 878 Status: structs.DeploymentStatusFailed, 879 StatusDescription: structs.DeploymentStatusDescriptionRollback(structs.DeploymentStatusDescriptionFailedAllocations, 0), 880 JobVersion: helper.Uint64ToPtr(2), 881 Eval: true, 882 } 883 m3 := matchDeploymentStatusUpdateRequest(c2) 884 m.AssertCalled(t, "UpdateDeploymentStatus", mocker.MatchedBy(m3)) 885 testutil.WaitForResult(func() (bool, error) { return 0 == len(w.watchers), nil }, 886 func(err error) { assert.Equal(0, len(w.watchers), "Should have no deployment") }) 887 } 888 889 // Test evaluations are batched between watchers 890 func TestWatcher_BatchEvals(t *testing.T) { 891 t.Parallel() 892 assert := assert.New(t) 893 w, m := testDeploymentWatcher(t, 1000.0, 1*time.Second) 894 895 // Create a job, alloc, for two deployments 896 j1 := mock.Job() 897 d1 := mock.Deployment() 898 d1.JobID = j1.ID 899 a1 := mock.Alloc() 900 a1.DeploymentID = d1.ID 901 902 j2 := mock.Job() 903 d2 := mock.Deployment() 904 d2.JobID = j2.ID 905 a2 := mock.Alloc() 906 a2.DeploymentID = d2.ID 907 908 assert.Nil(m.state.UpsertJob(m.nextIndex(), j1), "UpsertJob") 909 assert.Nil(m.state.UpsertJob(m.nextIndex(), j2), "UpsertJob") 910 assert.Nil(m.state.UpsertDeployment(m.nextIndex(), d1), "UpsertDeployment") 911 assert.Nil(m.state.UpsertDeployment(m.nextIndex(), d2), "UpsertDeployment") 912 assert.Nil(m.state.UpsertAllocs(m.nextIndex(), []*structs.Allocation{a1}), "UpsertAllocs") 913 assert.Nil(m.state.UpsertAllocs(m.nextIndex(), []*structs.Allocation{a2}), "UpsertAllocs") 914 915 // Assert the following methods will be called 916 m.On("List", mocker.Anything, mocker.Anything).Return(nil).Run(m.listFromState) 917 918 m.On("Allocations", mocker.MatchedBy(matchDeploymentSpecificRequest(d1.ID)), 919 mocker.Anything).Return(nil).Run(m.allocationsFromState) 920 m.On("Allocations", mocker.MatchedBy(matchDeploymentSpecificRequest(d2.ID)), 921 mocker.Anything).Return(nil).Run(m.allocationsFromState) 922 923 m.On("Evaluations", mocker.MatchedBy(matchJobSpecificRequest(j1.ID)), 924 mocker.Anything).Return(nil).Run(m.evaluationsFromState) 925 m.On("Evaluations", mocker.MatchedBy(matchJobSpecificRequest(j2.ID)), 926 mocker.Anything).Return(nil).Run(m.evaluationsFromState) 927 928 m.On("GetJob", mocker.MatchedBy(matchJobSpecificRequest(j1.ID)), 929 mocker.Anything).Return(nil).Run(m.getJobFromState) 930 m.On("GetJob", mocker.MatchedBy(matchJobSpecificRequest(j2.ID)), 931 mocker.Anything).Return(nil).Run(m.getJobFromState) 932 933 m.On("GetJobVersions", mocker.MatchedBy(matchJobVersionsRequest(j1.ID)), 934 mocker.Anything).Return(nil).Run(m.getJobVersionsFromState) 935 m.On("GetJobVersions", mocker.MatchedBy(matchJobVersionsRequest(j2.ID)), 936 mocker.Anything).Return(nil).Run(m.getJobVersionsFromState) 937 938 w.SetEnabled(true) 939 testutil.WaitForResult(func() (bool, error) { return 2 == len(w.watchers), nil }, 940 func(err error) { assert.Equal(2, len(w.watchers), "Should have 2 deployment") }) 941 942 // Assert that we will get a createEvaluation call only once and it contains 943 // both deployments. This will verify that the watcher is batching 944 // allocation changes 945 m1 := matchUpsertEvals([]string{d1.ID, d2.ID}) 946 m.On("UpsertEvals", mocker.MatchedBy(m1)).Return(nil).Once() 947 948 // Update the allocs health to healthy which should create an evaluation 949 req := &structs.ApplyDeploymentAllocHealthRequest{ 950 DeploymentAllocHealthRequest: structs.DeploymentAllocHealthRequest{ 951 DeploymentID: d1.ID, 952 HealthyAllocationIDs: []string{a1.ID}, 953 }, 954 } 955 assert.Nil(m.state.UpdateDeploymentAllocHealth(m.nextIndex(), req), "UpsertDeploymentAllocHealth") 956 957 req2 := &structs.ApplyDeploymentAllocHealthRequest{ 958 DeploymentAllocHealthRequest: structs.DeploymentAllocHealthRequest{ 959 DeploymentID: d2.ID, 960 HealthyAllocationIDs: []string{a2.ID}, 961 }, 962 } 963 assert.Nil(m.state.UpdateDeploymentAllocHealth(m.nextIndex(), req2), "UpsertDeploymentAllocHealth") 964 965 // Wait for there to be one eval for each job 966 testutil.WaitForResult(func() (bool, error) { 967 ws := memdb.NewWatchSet() 968 evals1, err := m.state.EvalsByJob(ws, j1.ID) 969 if err != nil { 970 return false, err 971 } 972 973 evals2, err := m.state.EvalsByJob(ws, j2.ID) 974 if err != nil { 975 return false, err 976 } 977 978 if l := len(evals1); l != 1 { 979 return false, fmt.Errorf("Got %d evals; want 1", l) 980 } 981 982 if l := len(evals2); l != 1 { 983 return false, fmt.Errorf("Got %d evals; want 1", l) 984 } 985 986 return true, nil 987 }, func(err error) { 988 t.Fatal(err) 989 }) 990 991 m.AssertCalled(t, "UpsertEvals", mocker.MatchedBy(m1)) 992 testutil.WaitForResult(func() (bool, error) { return 2 == len(w.watchers), nil }, 993 func(err error) { assert.Equal(2, len(w.watchers), "Should have 2 deployment") }) 994 }