launchpad.net/~rogpeppe/juju-core/500-errgo-fix@v0.0.0-20140213181702-000000002356/worker/provisioner/provisioner_test.go (about) 1 // Copyright 2012, 2013 Canonical Ltd. 2 // Licensed under the AGPLv3, see LICENCE file for details. 3 4 package provisioner_test 5 6 import ( 7 "fmt" 8 "strings" 9 "time" 10 11 gc "launchpad.net/gocheck" 12 13 "launchpad.net/juju-core/constraints" 14 "launchpad.net/juju-core/environs" 15 "launchpad.net/juju-core/environs/config" 16 "launchpad.net/juju-core/errors" 17 "launchpad.net/juju-core/instance" 18 "launchpad.net/juju-core/juju/testing" 19 "launchpad.net/juju-core/names" 20 "launchpad.net/juju-core/provider/dummy" 21 "launchpad.net/juju-core/state" 22 "launchpad.net/juju-core/state/api" 23 "launchpad.net/juju-core/state/api/params" 24 apiprovisioner "launchpad.net/juju-core/state/api/provisioner" 25 coretesting "launchpad.net/juju-core/testing" 26 jc "launchpad.net/juju-core/testing/checkers" 27 "launchpad.net/juju-core/utils" 28 "launchpad.net/juju-core/utils/set" 29 "launchpad.net/juju-core/worker/provisioner" 30 ) 31 32 type CommonProvisionerSuite struct { 33 testing.JujuConnSuite 34 op <-chan dummy.Operation 35 cfg *config.Config 36 // // defaultConstraints are used when adding a machine and then later in test assertions. 37 defaultConstraints constraints.Value 38 39 st *api.State 40 provisioner *apiprovisioner.State 41 } 42 43 type ProvisionerSuite struct { 44 CommonProvisionerSuite 45 } 46 47 var _ = gc.Suite(&ProvisionerSuite{}) 48 49 var veryShortAttempt = utils.AttemptStrategy{ 50 Total: 1 * time.Second, 51 Delay: 80 * time.Millisecond, 52 } 53 54 func (s *CommonProvisionerSuite) SetUpSuite(c *gc.C) { 55 s.JujuConnSuite.SetUpSuite(c) 56 s.defaultConstraints = constraints.MustParse("arch=amd64 mem=4G cpu-cores=1 root-disk=8G") 57 } 58 59 func (s *CommonProvisionerSuite) SetUpTest(c *gc.C) { 60 s.JujuConnSuite.SetUpTest(c) 61 // Create the operations channel with more than enough space 62 // for those tests that don't listen on it. 63 op := make(chan dummy.Operation, 500) 64 dummy.Listen(op) 65 s.op = op 66 67 cfg, err := s.State.EnvironConfig() 68 c.Assert(err, gc.IsNil) 69 s.cfg = cfg 70 } 71 72 func (s *CommonProvisionerSuite) APILogin(c *gc.C, machine *state.Machine) { 73 if s.st != nil { 74 c.Assert(s.st.Close(), gc.IsNil) 75 } 76 password, err := utils.RandomPassword() 77 c.Assert(err, gc.IsNil) 78 err = machine.SetPassword(password) 79 c.Assert(err, gc.IsNil) 80 err = machine.SetProvisioned("i-fake", "fake_nonce", nil) 81 c.Assert(err, gc.IsNil) 82 s.st = s.OpenAPIAsMachine(c, machine.Tag(), password, "fake_nonce") 83 c.Assert(s.st, gc.NotNil) 84 c.Logf("API: login as %q successful", machine.Tag()) 85 s.provisioner = s.st.Provisioner() 86 c.Assert(s.provisioner, gc.NotNil) 87 } 88 89 // breakDummyProvider changes the environment config in state in a way 90 // that causes the given environMethod of the dummy provider to return 91 // an error, which is also returned as a message to be checked. 92 func breakDummyProvider(c *gc.C, st *state.State, environMethod string) string { 93 oldCfg, err := st.EnvironConfig() 94 c.Assert(err, gc.IsNil) 95 cfg, err := oldCfg.Apply(map[string]interface{}{"broken": environMethod}) 96 c.Assert(err, gc.IsNil) 97 err = st.SetEnvironConfig(cfg, oldCfg) 98 c.Assert(err, gc.IsNil) 99 return fmt.Sprintf("dummy.%s is broken", environMethod) 100 } 101 102 // setupEnvironment adds an environment manager machine and login to the API. 103 func (s *CommonProvisionerSuite) setupEnvironmentManager(c *gc.C) { 104 machine, err := s.State.AddMachine("quantal", state.JobManageEnviron) 105 c.Assert(err, gc.IsNil) 106 c.Assert(machine.Id(), gc.Equals, "0") 107 err = machine.SetAddresses([]instance.Address{ 108 instance.NewAddress("0.1.2.3"), 109 }) 110 c.Assert(err, gc.IsNil) 111 s.APILogin(c, machine) 112 } 113 114 // invalidateEnvironment alters the environment configuration 115 // so the Settings returned from the watcher will not pass 116 // validation. 117 func (s *CommonProvisionerSuite) invalidateEnvironment(c *gc.C) { 118 attrs := s.cfg.AllAttrs() 119 attrs["type"] = "unknown" 120 invalidCfg, err := config.New(config.NoDefaults, attrs) 121 c.Assert(err, gc.IsNil) 122 err = s.State.SetEnvironConfig(invalidCfg, s.cfg) 123 c.Assert(err, gc.IsNil) 124 } 125 126 // fixEnvironment undoes the work of invalidateEnvironment. 127 func (s *CommonProvisionerSuite) fixEnvironment() error { 128 cfg, err := s.State.EnvironConfig() 129 if err != nil { 130 return err 131 } 132 return s.State.SetEnvironConfig(s.cfg, cfg) 133 } 134 135 // stopper is stoppable. 136 type stopper interface { 137 Stop() error 138 } 139 140 // stop stops a stopper. 141 func stop(c *gc.C, s stopper) { 142 c.Assert(s.Stop(), gc.IsNil) 143 } 144 145 func (s *CommonProvisionerSuite) startUnknownInstance(c *gc.C, id string) instance.Instance { 146 instance, _ := testing.AssertStartInstance(c, s.Conn.Environ, id) 147 select { 148 case o := <-s.op: 149 switch o := o.(type) { 150 case dummy.OpStartInstance: 151 default: 152 c.Fatalf("unexpected operation %#v", o) 153 } 154 case <-time.After(coretesting.LongWait): 155 c.Fatalf("timed out waiting for startinstance operation") 156 } 157 return instance 158 } 159 160 func (s *CommonProvisionerSuite) checkStartInstance(c *gc.C, m *state.Machine) instance.Instance { 161 return s.checkStartInstanceCustom(c, m, "pork", s.defaultConstraints) 162 } 163 164 func (s *CommonProvisionerSuite) checkStartInstanceCustom(c *gc.C, m *state.Machine, secret string, cons constraints.Value) (inst instance.Instance) { 165 s.BackingState.StartSync() 166 for { 167 select { 168 case o := <-s.op: 169 switch o := o.(type) { 170 case dummy.OpStartInstance: 171 inst = o.Instance 172 s.waitInstanceId(c, m, inst.Id()) 173 174 // Check the instance was started with the expected params. 175 c.Assert(o.MachineId, gc.Equals, m.Id()) 176 nonceParts := strings.SplitN(o.MachineNonce, ":", 2) 177 c.Assert(nonceParts, gc.HasLen, 2) 178 c.Assert(nonceParts[0], gc.Equals, names.MachineTag("0")) 179 c.Assert(nonceParts[1], jc.Satisfies, utils.IsValidUUIDString) 180 c.Assert(o.Secret, gc.Equals, secret) 181 c.Assert(o.Constraints, gc.DeepEquals, cons) 182 183 // All provisioned machines in this test suite have their hardware characteristics 184 // attributes set to the same values as the constraints due to the dummy environment being used. 185 hc, err := m.HardwareCharacteristics() 186 c.Assert(err, gc.IsNil) 187 c.Assert(*hc, gc.DeepEquals, instance.HardwareCharacteristics{ 188 Arch: cons.Arch, 189 Mem: cons.Mem, 190 RootDisk: cons.RootDisk, 191 CpuCores: cons.CpuCores, 192 CpuPower: cons.CpuPower, 193 Tags: cons.Tags, 194 }) 195 return 196 default: 197 c.Logf("ignoring unexpected operation %#v", o) 198 } 199 case <-time.After(2 * time.Second): 200 c.Fatalf("provisioner did not start an instance") 201 return 202 } 203 } 204 return 205 } 206 207 // checkNoOperations checks that the environ was not operated upon. 208 func (s *CommonProvisionerSuite) checkNoOperations(c *gc.C) { 209 s.BackingState.StartSync() 210 select { 211 case o := <-s.op: 212 c.Fatalf("unexpected operation %#v", o) 213 case <-time.After(coretesting.ShortWait): 214 return 215 } 216 } 217 218 // checkStopInstances checks that an instance has been stopped. 219 func (s *CommonProvisionerSuite) checkStopInstances(c *gc.C, instances ...instance.Instance) { 220 s.checkStopSomeInstances(c, instances, nil) 221 } 222 223 // checkStopSomeInstances checks that instancesToStop are stopped while instancesToKeep are not. 224 func (s *CommonProvisionerSuite) checkStopSomeInstances(c *gc.C, 225 instancesToStop []instance.Instance, instancesToKeep []instance.Instance) { 226 227 s.BackingState.StartSync() 228 instanceIdsToStop := set.NewStrings() 229 for _, instance := range instancesToStop { 230 instanceIdsToStop.Add(string(instance.Id())) 231 } 232 instanceIdsToKeep := set.NewStrings() 233 for _, instance := range instancesToKeep { 234 instanceIdsToKeep.Add(string(instance.Id())) 235 } 236 // Continue checking for stop instance calls until all the instances we 237 // are waiting on to finish, actually finish, or we time out. 238 for !instanceIdsToStop.IsEmpty() { 239 select { 240 case o := <-s.op: 241 switch o := o.(type) { 242 case dummy.OpStopInstances: 243 for _, stoppedInstance := range o.Instances { 244 instId := string(stoppedInstance.Id()) 245 instanceIdsToStop.Remove(instId) 246 if instanceIdsToKeep.Contains(instId) { 247 c.Errorf("provisioner unexpectedly stopped instance %s", instId) 248 } 249 } 250 default: 251 c.Fatalf("unexpected operation %#v", o) 252 return 253 } 254 case <-time.After(2 * time.Second): 255 c.Fatalf("provisioner did not stop an instance") 256 return 257 } 258 } 259 } 260 261 func (s *CommonProvisionerSuite) waitMachine(c *gc.C, m *state.Machine, check func() bool) { 262 // TODO(jam): We need to grow a new method on NotifyWatcherC 263 // that calls StartSync while waiting for changes, then 264 // waitMachine and waitHardwareCharacteristics can use that 265 // instead 266 w := m.Watch() 267 defer stop(c, w) 268 timeout := time.After(coretesting.LongWait) 269 resync := time.After(0) 270 for { 271 select { 272 case <-w.Changes(): 273 if check() { 274 return 275 } 276 case <-resync: 277 resync = time.After(coretesting.ShortWait) 278 s.BackingState.StartSync() 279 case <-timeout: 280 c.Fatalf("machine %v wait timed out", m) 281 } 282 } 283 } 284 285 func (s *CommonProvisionerSuite) waitHardwareCharacteristics(c *gc.C, m *state.Machine, check func() bool) { 286 w := m.WatchHardwareCharacteristics() 287 defer stop(c, w) 288 timeout := time.After(coretesting.LongWait) 289 resync := time.After(0) 290 for { 291 select { 292 case <-w.Changes(): 293 if check() { 294 return 295 } 296 case <-resync: 297 resync = time.After(coretesting.ShortWait) 298 s.BackingState.StartSync() 299 case <-timeout: 300 c.Fatalf("hardware characteristics for machine %v wait timed out", m) 301 } 302 } 303 } 304 305 // waitRemoved waits for the supplied machine to be removed from state. 306 func (s *CommonProvisionerSuite) waitRemoved(c *gc.C, m *state.Machine) { 307 s.waitMachine(c, m, func() bool { 308 err := m.Refresh() 309 if errors.IsNotFoundError(err) { 310 return true 311 } 312 c.Assert(err, gc.IsNil) 313 c.Logf("machine %v is still %s", m, m.Life()) 314 return false 315 }) 316 } 317 318 // waitInstanceId waits until the supplied machine has an instance id, then 319 // asserts it is as expected. 320 func (s *CommonProvisionerSuite) waitInstanceId(c *gc.C, m *state.Machine, expect instance.Id) { 321 s.waitHardwareCharacteristics(c, m, func() bool { 322 if actual, err := m.InstanceId(); err == nil { 323 c.Assert(actual, gc.Equals, expect) 324 return true 325 } else if !state.IsNotProvisionedError(err) { 326 // We don't expect any errors. 327 panic(err) 328 } 329 c.Logf("machine %v is still unprovisioned", m) 330 return false 331 }) 332 } 333 334 func (s *ProvisionerSuite) SetUpTest(c *gc.C) { 335 s.CommonProvisionerSuite.SetUpTest(c) 336 s.CommonProvisionerSuite.setupEnvironmentManager(c) 337 } 338 339 func (s *ProvisionerSuite) newEnvironProvisioner(c *gc.C) provisioner.Provisioner { 340 machineTag := "machine-0" 341 agentConfig := s.AgentConfigForTag(c, machineTag) 342 return provisioner.NewEnvironProvisioner(s.provisioner, agentConfig) 343 } 344 345 func (s *ProvisionerSuite) TestProvisionerStartStop(c *gc.C) { 346 p := s.newEnvironProvisioner(c) 347 c.Assert(p.Stop(), gc.IsNil) 348 } 349 350 func (s *ProvisionerSuite) addMachine() (*state.Machine, error) { 351 return s.BackingState.AddOneMachine(state.MachineTemplate{ 352 Series: config.DefaultSeries, 353 Jobs: []state.MachineJob{state.JobHostUnits}, 354 Constraints: s.defaultConstraints, 355 }) 356 } 357 358 func (s *ProvisionerSuite) TestSimple(c *gc.C) { 359 p := s.newEnvironProvisioner(c) 360 defer stop(c, p) 361 362 // Check that an instance is provisioned when the machine is created... 363 m, err := s.addMachine() 364 c.Assert(err, gc.IsNil) 365 instance := s.checkStartInstance(c, m) 366 367 // ...and removed, along with the machine, when the machine is Dead. 368 c.Assert(m.EnsureDead(), gc.IsNil) 369 s.checkStopInstances(c, instance) 370 s.waitRemoved(c, m) 371 } 372 373 func (s *ProvisionerSuite) TestConstraints(c *gc.C) { 374 // Create a machine with non-standard constraints. 375 m, err := s.addMachine() 376 c.Assert(err, gc.IsNil) 377 cons := constraints.MustParse("mem=8G arch=amd64 cpu-cores=2 root-disk=10G") 378 err = m.SetConstraints(cons) 379 c.Assert(err, gc.IsNil) 380 381 // Start a provisioner and check those constraints are used. 382 p := s.newEnvironProvisioner(c) 383 defer stop(c, p) 384 s.checkStartInstanceCustom(c, m, "pork", cons) 385 } 386 387 func (s *ProvisionerSuite) TestProvisionerSetsErrorStatusWhenStartInstanceFailed(c *gc.C) { 388 brokenMsg := breakDummyProvider(c, s.State, "StartInstance") 389 p := s.newEnvironProvisioner(c) 390 defer stop(c, p) 391 392 // Check that an instance is not provisioned when the machine is created... 393 m, err := s.addMachine() 394 c.Assert(err, gc.IsNil) 395 s.checkNoOperations(c) 396 397 t0 := time.Now() 398 for time.Since(t0) < coretesting.LongWait { 399 // And check the machine status is set to error. 400 status, info, _, err := m.Status() 401 c.Assert(err, gc.IsNil) 402 if status == params.StatusPending { 403 time.Sleep(coretesting.ShortWait) 404 continue 405 } 406 c.Assert(status, gc.Equals, params.StatusError) 407 c.Assert(info, gc.Equals, brokenMsg) 408 break 409 } 410 411 // Unbreak the environ config. 412 err = s.fixEnvironment() 413 c.Assert(err, gc.IsNil) 414 415 // Restart the PA to make sure the machine is skipped again. 416 stop(c, p) 417 p = s.newEnvironProvisioner(c) 418 defer stop(c, p) 419 s.checkNoOperations(c) 420 } 421 422 func (s *ProvisionerSuite) TestProvisioningDoesNotOccurForContainers(c *gc.C) { 423 p := s.newEnvironProvisioner(c) 424 defer stop(c, p) 425 426 // create a machine to host the container. 427 m, err := s.addMachine() 428 c.Assert(err, gc.IsNil) 429 inst := s.checkStartInstance(c, m) 430 431 // make a container on the machine we just created 432 template := state.MachineTemplate{ 433 Series: config.DefaultSeries, 434 Jobs: []state.MachineJob{state.JobHostUnits}, 435 } 436 container, err := s.State.AddMachineInsideMachine(template, m.Id(), instance.LXC) 437 c.Assert(err, gc.IsNil) 438 439 // the PA should not attempt to create it 440 s.checkNoOperations(c) 441 442 // cleanup 443 c.Assert(container.EnsureDead(), gc.IsNil) 444 c.Assert(container.Remove(), gc.IsNil) 445 c.Assert(m.EnsureDead(), gc.IsNil) 446 s.checkStopInstances(c, inst) 447 s.waitRemoved(c, m) 448 } 449 450 func (s *ProvisionerSuite) TestProvisioningDoesNotOccurWithAnInvalidEnvironment(c *gc.C) { 451 s.invalidateEnvironment(c) 452 453 p := s.newEnvironProvisioner(c) 454 defer stop(c, p) 455 456 // try to create a machine 457 _, err := s.addMachine() 458 c.Assert(err, gc.IsNil) 459 460 // the PA should not create it 461 s.checkNoOperations(c) 462 } 463 464 func (s *ProvisionerSuite) TestProvisioningOccursWithFixedEnvironment(c *gc.C) { 465 s.invalidateEnvironment(c) 466 467 p := s.newEnvironProvisioner(c) 468 defer stop(c, p) 469 470 // try to create a machine 471 m, err := s.addMachine() 472 c.Assert(err, gc.IsNil) 473 474 // the PA should not create it 475 s.checkNoOperations(c) 476 477 err = s.fixEnvironment() 478 c.Assert(err, gc.IsNil) 479 480 s.checkStartInstance(c, m) 481 } 482 483 func (s *ProvisionerSuite) TestProvisioningDoesOccurAfterInvalidEnvironmentPublished(c *gc.C) { 484 p := s.newEnvironProvisioner(c) 485 defer stop(c, p) 486 487 // place a new machine into the state 488 m, err := s.addMachine() 489 c.Assert(err, gc.IsNil) 490 491 s.checkStartInstance(c, m) 492 493 s.invalidateEnvironment(c) 494 495 // create a second machine 496 m, err = s.addMachine() 497 c.Assert(err, gc.IsNil) 498 499 // the PA should create it using the old environment 500 s.checkStartInstance(c, m) 501 } 502 503 func (s *ProvisionerSuite) TestProvisioningDoesNotProvisionTheSameMachineAfterRestart(c *gc.C) { 504 p := s.newEnvironProvisioner(c) 505 defer stop(c, p) 506 507 // create a machine 508 m, err := s.addMachine() 509 c.Assert(err, gc.IsNil) 510 s.checkStartInstance(c, m) 511 512 // restart the PA 513 stop(c, p) 514 p = s.newEnvironProvisioner(c) 515 defer stop(c, p) 516 517 // check that there is only one machine provisioned. 518 machines, err := s.State.AllMachines() 519 c.Assert(err, gc.IsNil) 520 c.Check(len(machines), gc.Equals, 2) 521 c.Check(machines[0].Id(), gc.Equals, "0") 522 c.Check(machines[1].CheckProvisioned("fake_nonce"), jc.IsFalse) 523 524 // the PA should not create it a second time 525 s.checkNoOperations(c) 526 } 527 528 func (s *ProvisionerSuite) TestProvisioningStopsInstances(c *gc.C) { 529 p := s.newEnvironProvisioner(c) 530 defer stop(c, p) 531 532 // create a machine 533 m0, err := s.addMachine() 534 c.Assert(err, gc.IsNil) 535 i0 := s.checkStartInstance(c, m0) 536 537 // create a second machine 538 m1, err := s.addMachine() 539 c.Assert(err, gc.IsNil) 540 i1 := s.checkStartInstance(c, m1) 541 stop(c, p) 542 543 // mark the first machine as dead 544 c.Assert(m0.EnsureDead(), gc.IsNil) 545 546 // remove the second machine entirely 547 c.Assert(m1.EnsureDead(), gc.IsNil) 548 c.Assert(m1.Remove(), gc.IsNil) 549 550 // start a new provisioner to shut them both down 551 p = s.newEnvironProvisioner(c) 552 defer stop(c, p) 553 s.checkStopInstances(c, i0, i1) 554 s.waitRemoved(c, m0) 555 } 556 557 func (s *ProvisionerSuite) TestDyingMachines(c *gc.C) { 558 p := s.newEnvironProvisioner(c) 559 defer stop(c, p) 560 561 // provision a machine 562 m0, err := s.addMachine() 563 c.Assert(err, gc.IsNil) 564 s.checkStartInstance(c, m0) 565 566 // stop the provisioner and make the machine dying 567 stop(c, p) 568 err = m0.Destroy() 569 c.Assert(err, gc.IsNil) 570 571 // add a new, dying, unprovisioned machine 572 m1, err := s.addMachine() 573 c.Assert(err, gc.IsNil) 574 err = m1.Destroy() 575 c.Assert(err, gc.IsNil) 576 577 // start the provisioner and wait for it to reap the useless machine 578 p = s.newEnvironProvisioner(c) 579 defer stop(c, p) 580 s.checkNoOperations(c) 581 s.waitRemoved(c, m1) 582 583 // verify the other one's still fine 584 err = m0.Refresh() 585 c.Assert(err, gc.IsNil) 586 c.Assert(m0.Life(), gc.Equals, state.Dying) 587 } 588 589 func (s *ProvisionerSuite) TestProvisioningRecoversAfterInvalidEnvironmentPublished(c *gc.C) { 590 p := s.newEnvironProvisioner(c) 591 defer stop(c, p) 592 593 // place a new machine into the state 594 m, err := s.addMachine() 595 c.Assert(err, gc.IsNil) 596 s.checkStartInstance(c, m) 597 598 s.invalidateEnvironment(c) 599 s.BackingState.StartSync() 600 601 // create a second machine 602 m, err = s.addMachine() 603 c.Assert(err, gc.IsNil) 604 605 // the PA should create it using the old environment 606 s.checkStartInstance(c, m) 607 608 err = s.fixEnvironment() 609 c.Assert(err, gc.IsNil) 610 611 // insert our observer 612 cfgObserver := make(chan *config.Config, 1) 613 provisioner.SetObserver(p, cfgObserver) 614 615 oldcfg, err := s.State.EnvironConfig() 616 c.Assert(err, gc.IsNil) 617 attrs := oldcfg.AllAttrs() 618 attrs["secret"] = "beef" 619 cfg, err := config.New(config.NoDefaults, attrs) 620 c.Assert(err, gc.IsNil) 621 err = s.State.SetEnvironConfig(cfg, oldcfg) 622 623 s.BackingState.StartSync() 624 625 // wait for the PA to load the new configuration 626 select { 627 case <-cfgObserver: 628 case <-time.After(coretesting.LongWait): 629 c.Fatalf("PA did not action config change") 630 } 631 632 // create a third machine 633 m, err = s.addMachine() 634 c.Assert(err, gc.IsNil) 635 636 // the PA should create it using the new environment 637 s.checkStartInstanceCustom(c, m, "beef", s.defaultConstraints) 638 } 639 640 func (s *ProvisionerSuite) TestProvisioningSafeMode(c *gc.C) { 641 p := s.newEnvironProvisioner(c) 642 defer stop(c, p) 643 644 // create a machine 645 m0, err := s.addMachine() 646 c.Assert(err, gc.IsNil) 647 i0 := s.checkStartInstance(c, m0) 648 649 // create a second machine 650 m1, err := s.addMachine() 651 c.Assert(err, gc.IsNil) 652 i1 := s.checkStartInstance(c, m1) 653 stop(c, p) 654 655 // mark the first machine as dead 656 c.Assert(m0.EnsureDead(), gc.IsNil) 657 658 // remove the second machine entirely from state 659 c.Assert(m1.EnsureDead(), gc.IsNil) 660 c.Assert(m1.Remove(), gc.IsNil) 661 662 // turn on safe mode 663 oldcfg, err := s.State.EnvironConfig() 664 c.Assert(err, gc.IsNil) 665 attrs := oldcfg.AllAttrs() 666 attrs["provisioner-safe-mode"] = true 667 cfg, err := config.New(config.NoDefaults, attrs) 668 c.Assert(err, gc.IsNil) 669 err = s.State.SetEnvironConfig(cfg, oldcfg) 670 671 // start a new provisioner to shut down only the machine still in state. 672 p = s.newEnvironProvisioner(c) 673 defer stop(c, p) 674 s.checkStopSomeInstances(c, []instance.Instance{i0}, []instance.Instance{i1}) 675 s.waitRemoved(c, m0) 676 } 677 678 func (s *ProvisionerSuite) TestProvisioningSafeModeChange(c *gc.C) { 679 p := s.newEnvironProvisioner(c) 680 defer stop(c, p) 681 682 // First check that safe mode is initially off. 683 684 // create a machine 685 m0, err := s.addMachine() 686 c.Assert(err, gc.IsNil) 687 i0 := s.checkStartInstance(c, m0) 688 689 // create a second machine 690 m1, err := s.addMachine() 691 c.Assert(err, gc.IsNil) 692 i1 := s.checkStartInstance(c, m1) 693 694 // mark the first machine as dead 695 c.Assert(m0.EnsureDead(), gc.IsNil) 696 697 // remove the second machine entirely from state 698 c.Assert(m1.EnsureDead(), gc.IsNil) 699 c.Assert(m1.Remove(), gc.IsNil) 700 701 s.checkStopInstances(c, i0, i1) 702 s.waitRemoved(c, m0) 703 704 // insert our observer 705 cfgObserver := make(chan *config.Config, 1) 706 provisioner.SetObserver(p, cfgObserver) 707 708 // turn on safe mode 709 oldcfg, err := s.State.EnvironConfig() 710 c.Assert(err, gc.IsNil) 711 attrs := oldcfg.AllAttrs() 712 attrs["provisioner-safe-mode"] = true 713 cfg, err := config.New(config.NoDefaults, attrs) 714 c.Assert(err, gc.IsNil) 715 err = s.State.SetEnvironConfig(cfg, oldcfg) 716 717 s.BackingState.StartSync() 718 719 // wait for the PA to load the new configuration 720 select { 721 case <-cfgObserver: 722 case <-time.After(coretesting.LongWait): 723 c.Fatalf("PA did not action config change") 724 } 725 726 // Now check that the provisioner has noticed safe mode is on. 727 728 // create a machine 729 m3, err := s.addMachine() 730 c.Assert(err, gc.IsNil) 731 i3 := s.checkStartInstance(c, m3) 732 733 // create an instance out of band 734 i4 := s.startUnknownInstance(c, "999") 735 736 // mark the machine as dead 737 c.Assert(m3.EnsureDead(), gc.IsNil) 738 739 // check the machine's instance is stopped, and the other isn't 740 s.checkStopSomeInstances(c, []instance.Instance{i3}, []instance.Instance{i4}) 741 s.waitRemoved(c, m3) 742 } 743 744 func (s *ProvisionerSuite) newProvisionerTask(c *gc.C, safeMode bool) provisioner.ProvisionerTask { 745 env := s.APIConn.Environ 746 watcher, err := s.provisioner.WatchEnvironMachines() 747 c.Assert(err, gc.IsNil) 748 auth, err := environs.NewAPIAuthenticator(s.provisioner) 749 c.Assert(err, gc.IsNil) 750 return provisioner.NewProvisionerTask("machine-0", safeMode, s.provisioner, watcher, env, auth) 751 } 752 753 func (s *ProvisionerSuite) TestTurningOffSafeModeReapsUnknownInstances(c *gc.C) { 754 task := s.newProvisionerTask(c, true) 755 defer stop(c, task) 756 757 // Initially create a machine, and an unknown instance, with safe mode on. 758 m0, err := s.addMachine() 759 c.Assert(err, gc.IsNil) 760 i0 := s.checkStartInstance(c, m0) 761 i1 := s.startUnknownInstance(c, "999") 762 763 // mark the first machine as dead 764 c.Assert(m0.EnsureDead(), gc.IsNil) 765 766 // with safe mode on, only one of the machines is stopped. 767 s.checkStopSomeInstances(c, []instance.Instance{i0}, []instance.Instance{i1}) 768 s.waitRemoved(c, m0) 769 770 // turn off safe mode and check that the other machine is now stopped also. 771 task.SetSafeMode(false) 772 s.checkStopInstances(c, i1) 773 }