github.com/altoros/juju-vmware@v0.0.0-20150312064031-f19ae857ccca/worker/provisioner/provisioner_test.go (about) 1 // Copyright 2012, 2013 Canonical Ltd. 2 // Licensed under the AGPLv3, see LICENCE file for details. 3 4 package provisioner_test 5 6 import ( 7 "fmt" 8 "strconv" 9 "strings" 10 "time" 11 12 "github.com/juju/errors" 13 "github.com/juju/names" 14 jc "github.com/juju/testing/checkers" 15 "github.com/juju/utils" 16 "github.com/juju/utils/set" 17 gc "gopkg.in/check.v1" 18 19 "github.com/juju/juju/agent" 20 "github.com/juju/juju/api" 21 apiprovisioner "github.com/juju/juju/api/provisioner" 22 "github.com/juju/juju/apiserver/params" 23 apiserverprovisioner "github.com/juju/juju/apiserver/provisioner" 24 "github.com/juju/juju/constraints" 25 "github.com/juju/juju/environmentserver/authentication" 26 "github.com/juju/juju/environs" 27 "github.com/juju/juju/environs/config" 28 "github.com/juju/juju/environs/filestorage" 29 "github.com/juju/juju/environs/imagemetadata" 30 envtesting "github.com/juju/juju/environs/testing" 31 "github.com/juju/juju/environs/tools" 32 "github.com/juju/juju/instance" 33 "github.com/juju/juju/juju/testing" 34 "github.com/juju/juju/mongo" 35 "github.com/juju/juju/network" 36 "github.com/juju/juju/provider/dummy" 37 "github.com/juju/juju/state" 38 "github.com/juju/juju/state/multiwatcher" 39 coretesting "github.com/juju/juju/testing" 40 coretools "github.com/juju/juju/tools" 41 "github.com/juju/juju/version" 42 "github.com/juju/juju/worker/provisioner" 43 ) 44 45 type CommonProvisionerSuite struct { 46 testing.JujuConnSuite 47 op <-chan dummy.Operation 48 cfg *config.Config 49 // defaultConstraints are used when adding a machine and then later in test assertions. 50 defaultConstraints constraints.Value 51 52 st *api.State 53 provisioner *apiprovisioner.State 54 } 55 56 type ProvisionerSuite struct { 57 CommonProvisionerSuite 58 } 59 60 var _ = gc.Suite(&ProvisionerSuite{}) 61 62 var veryShortAttempt = utils.AttemptStrategy{ 63 Total: 1 * time.Second, 64 Delay: 80 * time.Millisecond, 65 } 66 67 func (s *CommonProvisionerSuite) SetUpSuite(c *gc.C) { 68 s.JujuConnSuite.SetUpSuite(c) 69 s.defaultConstraints = constraints.MustParse("arch=amd64 mem=4G cpu-cores=1 root-disk=8G") 70 } 71 72 func (s *CommonProvisionerSuite) SetUpTest(c *gc.C) { 73 // Disable the default state policy, because the 74 // provisioner needs to be able to test pathological 75 // scenarios where a machine exists in state with 76 // invalid environment config. 77 dummy.SetStatePolicy(nil) 78 79 s.JujuConnSuite.SetUpTest(c) 80 81 // Create the operations channel with more than enough space 82 // for those tests that don't listen on it. 83 op := make(chan dummy.Operation, 500) 84 dummy.Listen(op) 85 s.op = op 86 87 cfg, err := s.State.EnvironConfig() 88 c.Assert(err, jc.ErrorIsNil) 89 s.cfg = cfg 90 91 // Create a machine for the dummy bootstrap instance, 92 // so the provisioner doesn't destroy it. 93 insts, err := s.Environ.Instances([]instance.Id{dummy.BootstrapInstanceId}) 94 c.Assert(err, jc.ErrorIsNil) 95 addrs, err := insts[0].Addresses() 96 c.Assert(err, jc.ErrorIsNil) 97 machine, err := s.State.AddOneMachine(state.MachineTemplate{ 98 Addresses: addrs, 99 Series: "quantal", 100 Nonce: agent.BootstrapNonce, 101 InstanceId: dummy.BootstrapInstanceId, 102 Jobs: []state.MachineJob{state.JobManageEnviron}, 103 }) 104 c.Assert(err, jc.ErrorIsNil) 105 c.Assert(machine.Id(), gc.Equals, "0") 106 107 err = machine.SetAgentVersion(version.Current) 108 c.Assert(err, jc.ErrorIsNil) 109 110 password, err := utils.RandomPassword() 111 c.Assert(err, jc.ErrorIsNil) 112 err = machine.SetPassword(password) 113 c.Assert(err, jc.ErrorIsNil) 114 115 s.st = s.OpenAPIAsMachine(c, machine.Tag(), password, agent.BootstrapNonce) 116 c.Assert(s.st, gc.NotNil) 117 c.Logf("API: login as %q successful", machine.Tag()) 118 s.provisioner = s.st.Provisioner() 119 c.Assert(s.provisioner, gc.NotNil) 120 } 121 122 // breakDummyProvider changes the environment config in state in a way 123 // that causes the given environMethod of the dummy provider to return 124 // an error, which is also returned as a message to be checked. 125 func breakDummyProvider(c *gc.C, st *state.State, environMethod string) string { 126 attrs := map[string]interface{}{"broken": environMethod} 127 err := st.UpdateEnvironConfig(attrs, nil, nil) 128 c.Assert(err, jc.ErrorIsNil) 129 return fmt.Sprintf("dummy.%s is broken", environMethod) 130 } 131 132 // invalidateEnvironment alters the environment configuration 133 // so the Settings returned from the watcher will not pass 134 // validation. 135 func (s *CommonProvisionerSuite) invalidateEnvironment(c *gc.C) { 136 st, err := state.Open(s.MongoInfo(c), mongo.DefaultDialOpts(), state.Policy(nil)) 137 c.Assert(err, jc.ErrorIsNil) 138 defer st.Close() 139 attrs := map[string]interface{}{"type": "unknown"} 140 err = st.UpdateEnvironConfig(attrs, nil, nil) 141 c.Assert(err, jc.ErrorIsNil) 142 } 143 144 // fixEnvironment undoes the work of invalidateEnvironment. 145 func (s *CommonProvisionerSuite) fixEnvironment(c *gc.C) error { 146 st, err := state.Open(s.MongoInfo(c), mongo.DefaultDialOpts(), state.Policy(nil)) 147 c.Assert(err, jc.ErrorIsNil) 148 defer st.Close() 149 attrs := map[string]interface{}{"type": s.cfg.AllAttrs()["type"]} 150 return st.UpdateEnvironConfig(attrs, nil, nil) 151 } 152 153 // stopper is stoppable. 154 type stopper interface { 155 Stop() error 156 } 157 158 // stop stops a stopper. 159 func stop(c *gc.C, s stopper) { 160 c.Assert(s.Stop(), gc.IsNil) 161 } 162 163 func (s *CommonProvisionerSuite) startUnknownInstance(c *gc.C, id string) instance.Instance { 164 instance, _ := testing.AssertStartInstance(c, s.Environ, id) 165 select { 166 case o := <-s.op: 167 switch o := o.(type) { 168 case dummy.OpStartInstance: 169 default: 170 c.Fatalf("unexpected operation %#v", o) 171 } 172 case <-time.After(coretesting.LongWait): 173 c.Fatalf("timed out waiting for startinstance operation") 174 } 175 return instance 176 } 177 178 func (s *CommonProvisionerSuite) checkStartInstance(c *gc.C, m *state.Machine) instance.Instance { 179 return s.checkStartInstanceCustom(c, m, "pork", s.defaultConstraints, nil, nil, true, nil, true) 180 } 181 182 func (s *CommonProvisionerSuite) checkStartInstanceNoSecureConnection(c *gc.C, m *state.Machine) instance.Instance { 183 return s.checkStartInstanceCustom(c, m, "pork", s.defaultConstraints, nil, nil, false, nil, true) 184 } 185 186 func (s *CommonProvisionerSuite) checkStartInstanceCustom( 187 c *gc.C, m *state.Machine, 188 secret string, cons constraints.Value, 189 networks []string, networkInfo []network.InterfaceInfo, 190 secureServerConnection bool, 191 checkPossibleTools coretools.List, 192 waitInstanceId bool, 193 ) ( 194 inst instance.Instance, 195 ) { 196 s.BackingState.StartSync() 197 for { 198 select { 199 case o := <-s.op: 200 switch o := o.(type) { 201 case dummy.OpStartInstance: 202 inst = o.Instance 203 if waitInstanceId { 204 s.waitInstanceId(c, m, inst.Id()) 205 } 206 207 // Check the instance was started with the expected params. 208 c.Assert(o.MachineId, gc.Equals, m.Id()) 209 nonceParts := strings.SplitN(o.MachineNonce, ":", 2) 210 c.Assert(nonceParts, gc.HasLen, 2) 211 c.Assert(nonceParts[0], gc.Equals, names.NewMachineTag("0").String()) 212 c.Assert(nonceParts[1], jc.Satisfies, utils.IsValidUUIDString) 213 c.Assert(o.Secret, gc.Equals, secret) 214 c.Assert(o.Networks, jc.DeepEquals, networks) 215 c.Assert(o.NetworkInfo, jc.DeepEquals, networkInfo) 216 c.Assert(o.AgentEnvironment["SECURE_STATESERVER_CONNECTION"], gc.Equals, strconv.FormatBool(secureServerConnection)) 217 218 var jobs []multiwatcher.MachineJob 219 for _, job := range m.Jobs() { 220 jobs = append(jobs, job.ToParams()) 221 } 222 c.Assert(o.Jobs, jc.SameContents, jobs) 223 224 if checkPossibleTools != nil { 225 for _, t := range o.PossibleTools { 226 url := fmt.Sprintf("https://%s/environment/%s/tools/%s", 227 s.st.Addr(), coretesting.EnvironmentTag.Id(), t.Version) 228 c.Check(t.URL, gc.Equals, url) 229 t.URL = "" 230 } 231 for _, t := range checkPossibleTools { 232 t.URL = "" 233 } 234 c.Assert(o.PossibleTools, gc.DeepEquals, checkPossibleTools) 235 } 236 237 // All provisioned machines in this test suite have 238 // their hardware characteristics attributes set to 239 // the same values as the constraints due to the dummy 240 // environment being used. 241 if !constraints.IsEmpty(&cons) { 242 c.Assert(o.Constraints, gc.DeepEquals, cons) 243 hc, err := m.HardwareCharacteristics() 244 c.Assert(err, jc.ErrorIsNil) 245 c.Assert(*hc, gc.DeepEquals, instance.HardwareCharacteristics{ 246 Arch: cons.Arch, 247 Mem: cons.Mem, 248 RootDisk: cons.RootDisk, 249 CpuCores: cons.CpuCores, 250 CpuPower: cons.CpuPower, 251 Tags: cons.Tags, 252 }) 253 } 254 return 255 default: 256 c.Logf("ignoring unexpected operation %#v", o) 257 } 258 case <-time.After(2 * time.Second): 259 c.Fatalf("provisioner did not start an instance") 260 return 261 } 262 } 263 } 264 265 // checkNoOperations checks that the environ was not operated upon. 266 func (s *CommonProvisionerSuite) checkNoOperations(c *gc.C) { 267 s.BackingState.StartSync() 268 select { 269 case o := <-s.op: 270 c.Fatalf("unexpected operation %+v", o) 271 case <-time.After(coretesting.ShortWait): 272 return 273 } 274 } 275 276 // checkStopInstances checks that an instance has been stopped. 277 func (s *CommonProvisionerSuite) checkStopInstances(c *gc.C, instances ...instance.Instance) { 278 s.checkStopSomeInstances(c, instances, nil) 279 } 280 281 // checkStopSomeInstances checks that instancesToStop are stopped while instancesToKeep are not. 282 func (s *CommonProvisionerSuite) checkStopSomeInstances(c *gc.C, 283 instancesToStop []instance.Instance, instancesToKeep []instance.Instance) { 284 285 s.BackingState.StartSync() 286 instanceIdsToStop := set.NewStrings() 287 for _, instance := range instancesToStop { 288 instanceIdsToStop.Add(string(instance.Id())) 289 } 290 instanceIdsToKeep := set.NewStrings() 291 for _, instance := range instancesToKeep { 292 instanceIdsToKeep.Add(string(instance.Id())) 293 } 294 // Continue checking for stop instance calls until all the instances we 295 // are waiting on to finish, actually finish, or we time out. 296 for !instanceIdsToStop.IsEmpty() { 297 select { 298 case o := <-s.op: 299 switch o := o.(type) { 300 case dummy.OpStopInstances: 301 for _, id := range o.Ids { 302 instId := string(id) 303 instanceIdsToStop.Remove(instId) 304 if instanceIdsToKeep.Contains(instId) { 305 c.Errorf("provisioner unexpectedly stopped instance %s", instId) 306 } 307 } 308 default: 309 c.Fatalf("unexpected operation %#v", o) 310 return 311 } 312 case <-time.After(2 * time.Second): 313 c.Fatalf("provisioner did not stop an instance") 314 return 315 } 316 } 317 } 318 319 func (s *CommonProvisionerSuite) waitMachine(c *gc.C, m *state.Machine, check func() bool) { 320 // TODO(jam): We need to grow a new method on NotifyWatcherC 321 // that calls StartSync while waiting for changes, then 322 // waitMachine and waitHardwareCharacteristics can use that 323 // instead 324 w := m.Watch() 325 defer stop(c, w) 326 timeout := time.After(coretesting.LongWait) 327 resync := time.After(0) 328 for { 329 select { 330 case <-w.Changes(): 331 if check() { 332 return 333 } 334 case <-resync: 335 resync = time.After(coretesting.ShortWait) 336 s.BackingState.StartSync() 337 case <-timeout: 338 c.Fatalf("machine %v wait timed out", m) 339 } 340 } 341 } 342 343 func (s *CommonProvisionerSuite) waitHardwareCharacteristics(c *gc.C, m *state.Machine, check func() bool) { 344 w := m.WatchHardwareCharacteristics() 345 defer stop(c, w) 346 timeout := time.After(coretesting.LongWait) 347 resync := time.After(0) 348 for { 349 select { 350 case <-w.Changes(): 351 if check() { 352 return 353 } 354 case <-resync: 355 resync = time.After(coretesting.ShortWait) 356 s.BackingState.StartSync() 357 case <-timeout: 358 c.Fatalf("hardware characteristics for machine %v wait timed out", m) 359 } 360 } 361 } 362 363 // waitRemoved waits for the supplied machine to be removed from state. 364 func (s *CommonProvisionerSuite) waitRemoved(c *gc.C, m *state.Machine) { 365 s.waitMachine(c, m, func() bool { 366 err := m.Refresh() 367 if errors.IsNotFound(err) { 368 return true 369 } 370 c.Assert(err, jc.ErrorIsNil) 371 c.Logf("machine %v is still %s", m, m.Life()) 372 return false 373 }) 374 } 375 376 // waitInstanceId waits until the supplied machine has an instance id, then 377 // asserts it is as expected. 378 func (s *CommonProvisionerSuite) waitInstanceId(c *gc.C, m *state.Machine, expect instance.Id) { 379 s.waitHardwareCharacteristics(c, m, func() bool { 380 if actual, err := m.InstanceId(); err == nil { 381 c.Assert(actual, gc.Equals, expect) 382 return true 383 } else if !errors.IsNotProvisioned(err) { 384 // We don't expect any errors. 385 panic(err) 386 } 387 c.Logf("machine %v is still unprovisioned", m) 388 return false 389 }) 390 } 391 392 func (s *CommonProvisionerSuite) newEnvironProvisioner(c *gc.C) provisioner.Provisioner { 393 machineTag := names.NewMachineTag("0") 394 agentConfig := s.AgentConfigForTag(c, machineTag) 395 return provisioner.NewEnvironProvisioner(s.provisioner, agentConfig) 396 } 397 398 func (s *CommonProvisionerSuite) addMachine() (*state.Machine, error) { 399 return s.addMachineWithRequestedNetworks(nil, s.defaultConstraints) 400 } 401 402 func (s *CommonProvisionerSuite) addMachineWithRequestedNetworks(networks []string, cons constraints.Value) (*state.Machine, error) { 403 return s.BackingState.AddOneMachine(state.MachineTemplate{ 404 Series: coretesting.FakeDefaultSeries, 405 Jobs: []state.MachineJob{state.JobHostUnits}, 406 Constraints: cons, 407 RequestedNetworks: networks, 408 }) 409 } 410 411 func (s *CommonProvisionerSuite) ensureAvailability(c *gc.C, n int) []*state.Machine { 412 changes, err := s.BackingState.EnsureAvailability(n, s.defaultConstraints, coretesting.FakeDefaultSeries, nil) 413 c.Assert(err, jc.ErrorIsNil) 414 added := make([]*state.Machine, len(changes.Added)) 415 for i, mid := range changes.Added { 416 m, err := s.BackingState.Machine(mid) 417 c.Assert(err, jc.ErrorIsNil) 418 added[i] = m 419 } 420 return added 421 } 422 423 func (s *ProvisionerSuite) TestProvisionerStartStop(c *gc.C) { 424 p := s.newEnvironProvisioner(c) 425 c.Assert(p.Stop(), gc.IsNil) 426 } 427 428 func (s *ProvisionerSuite) TestSimple(c *gc.C) { 429 p := s.newEnvironProvisioner(c) 430 defer stop(c, p) 431 432 // Check that an instance is provisioned when the machine is created... 433 m, err := s.addMachine() 434 c.Assert(err, jc.ErrorIsNil) 435 instance := s.checkStartInstanceNoSecureConnection(c, m) 436 437 // ...and removed, along with the machine, when the machine is Dead. 438 c.Assert(m.EnsureDead(), gc.IsNil) 439 s.checkStopInstances(c, instance) 440 s.waitRemoved(c, m) 441 } 442 443 func (s *ProvisionerSuite) TestConstraints(c *gc.C) { 444 // Create a machine with non-standard constraints. 445 m, err := s.addMachine() 446 c.Assert(err, jc.ErrorIsNil) 447 cons := constraints.MustParse("mem=8G arch=amd64 cpu-cores=2 root-disk=10G") 448 err = m.SetConstraints(cons) 449 c.Assert(err, jc.ErrorIsNil) 450 451 // Start a provisioner and check those constraints are used. 452 p := s.newEnvironProvisioner(c) 453 defer stop(c, p) 454 s.checkStartInstanceCustom(c, m, "pork", cons, nil, nil, false, nil, true) 455 } 456 457 func (s *ProvisionerSuite) TestPossibleTools(c *gc.C) { 458 459 storageDir := c.MkDir() 460 s.PatchValue(&tools.DefaultBaseURL, storageDir) 461 stor, err := filestorage.NewFileStorageWriter(storageDir) 462 c.Assert(err, jc.ErrorIsNil) 463 464 // Set a current version that does not match the 465 // agent-version in the environ config. 466 currentVersion := version.MustParseBinary("1.2.3-quantal-arm64") 467 s.PatchValue(&version.Current, currentVersion) 468 469 // Upload some plausible matches, and some that should be filtered out. 470 compatibleVersion := version.MustParseBinary("1.2.3-quantal-amd64") 471 ignoreVersion1 := version.MustParseBinary("1.2.4-quantal-arm64") 472 ignoreVersion2 := version.MustParseBinary("1.2.3-precise-arm64") 473 availableVersions := []version.Binary{ 474 currentVersion, compatibleVersion, ignoreVersion1, ignoreVersion2, 475 } 476 envtesting.AssertUploadFakeToolsVersions(c, stor, s.cfg.AgentStream(), s.cfg.AgentStream(), availableVersions...) 477 478 // Extract the tools that we expect to actually match. 479 expectedList, err := tools.FindTools(s.Environ, -1, -1, coretools.Filter{ 480 Number: currentVersion.Number, 481 Series: currentVersion.Series, 482 }) 483 c.Assert(err, jc.ErrorIsNil) 484 485 // Create the machine and check the tools that get passed into StartInstance. 486 machine, err := s.BackingState.AddOneMachine(state.MachineTemplate{ 487 Series: "quantal", 488 Jobs: []state.MachineJob{state.JobHostUnits}, 489 }) 490 c.Assert(err, jc.ErrorIsNil) 491 492 provisioner := s.newEnvironProvisioner(c) 493 defer stop(c, provisioner) 494 s.checkStartInstanceCustom( 495 c, machine, "pork", constraints.Value{}, 496 nil, nil, false, expectedList, true, 497 ) 498 } 499 500 func (s *ProvisionerSuite) TestProvisionerSetsErrorStatusWhenNoToolsAreAvailable(c *gc.C) { 501 p := s.newEnvironProvisioner(c) 502 defer stop(c, p) 503 504 // Check that an instance is not provisioned when the machine is created... 505 m, err := s.BackingState.AddOneMachine(state.MachineTemplate{ 506 // We need a valid series that has no tools uploaded 507 Series: "raring", 508 Jobs: []state.MachineJob{state.JobHostUnits}, 509 Constraints: s.defaultConstraints, 510 }) 511 c.Assert(err, jc.ErrorIsNil) 512 s.checkNoOperations(c) 513 514 t0 := time.Now() 515 for time.Since(t0) < coretesting.LongWait { 516 // And check the machine status is set to error. 517 status, info, _, err := m.Status() 518 c.Assert(err, jc.ErrorIsNil) 519 if status == state.StatusPending { 520 time.Sleep(coretesting.ShortWait) 521 continue 522 } 523 c.Assert(status, gc.Equals, state.StatusError) 524 c.Assert(info, gc.Equals, "no matching tools available") 525 break 526 } 527 528 // Restart the PA to make sure the machine is skipped again. 529 stop(c, p) 530 p = s.newEnvironProvisioner(c) 531 defer stop(c, p) 532 s.checkNoOperations(c) 533 } 534 535 func (s *ProvisionerSuite) TestProvisionerSetsErrorStatusWhenStartInstanceFailed(c *gc.C) { 536 brokenMsg := breakDummyProvider(c, s.State, "StartInstance") 537 p := s.newEnvironProvisioner(c) 538 defer stop(c, p) 539 540 // Check that an instance is not provisioned when the machine is created... 541 m, err := s.addMachine() 542 c.Assert(err, jc.ErrorIsNil) 543 s.checkNoOperations(c) 544 545 t0 := time.Now() 546 for time.Since(t0) < coretesting.LongWait { 547 // And check the machine status is set to error. 548 status, info, _, err := m.Status() 549 c.Assert(err, jc.ErrorIsNil) 550 if status == state.StatusPending { 551 time.Sleep(coretesting.ShortWait) 552 continue 553 } 554 c.Assert(status, gc.Equals, state.StatusError) 555 c.Assert(info, gc.Equals, brokenMsg) 556 break 557 } 558 559 // Unbreak the environ config. 560 err = s.fixEnvironment(c) 561 c.Assert(err, jc.ErrorIsNil) 562 563 // Restart the PA to make sure the machine is skipped again. 564 stop(c, p) 565 p = s.newEnvironProvisioner(c) 566 defer stop(c, p) 567 s.checkNoOperations(c) 568 } 569 570 func (s *ProvisionerSuite) TestProvisionerFailedStartInstanceWithInjectedCreationError(c *gc.C) { 571 // create the error injection channel 572 errorInjectionChannel := make(chan error, 2) 573 574 p := s.newEnvironProvisioner(c) 575 defer stop(c, p) 576 577 // patch the dummy provider error injection channel 578 cleanup := dummy.PatchTransientErrorInjectionChannel(errorInjectionChannel) 579 defer cleanup() 580 581 retryableError := instance.NewRetryableCreationError("container failed to start and was destroyed") 582 destroyError := errors.New("container failed to start and failed to destroy: manual cleanup of containers needed") 583 // send the error message TWICE, because the provisioner will retry only ONCE 584 errorInjectionChannel <- retryableError 585 errorInjectionChannel <- destroyError 586 587 m, err := s.addMachine() 588 c.Assert(err, jc.ErrorIsNil) 589 s.checkNoOperations(c) 590 591 t0 := time.Now() 592 for time.Since(t0) < coretesting.LongWait { 593 // And check the machine status is set to error. 594 status, info, _, err := m.Status() 595 c.Assert(err, jc.ErrorIsNil) 596 if status == state.StatusPending { 597 time.Sleep(coretesting.ShortWait) 598 continue 599 } 600 c.Assert(status, gc.Equals, state.StatusError) 601 // check that the status matches the error message 602 c.Assert(info, gc.Equals, destroyError.Error()) 603 break 604 } 605 606 } 607 608 func (s *ProvisionerSuite) TestProvisionerSucceedStartInstanceWithInjectedRetryableCreationError(c *gc.C) { 609 // create the error injection channel 610 errorInjectionChannel := make(chan error, 1) 611 c.Assert(errorInjectionChannel, gc.NotNil) 612 613 p := s.newEnvironProvisioner(c) 614 defer stop(c, p) 615 616 // patch the dummy provider error injection channel 617 cleanup := dummy.PatchTransientErrorInjectionChannel(errorInjectionChannel) 618 defer cleanup() 619 620 // send the error message once 621 // - instance creation should succeed 622 retryableError := instance.NewRetryableCreationError("container failed to start and was destroyed") 623 errorInjectionChannel <- retryableError 624 625 m, err := s.addMachine() 626 c.Assert(err, jc.ErrorIsNil) 627 s.checkStartInstanceNoSecureConnection(c, m) 628 } 629 630 func (s *ProvisionerSuite) TestProvisionerSucceedStartInstanceWithInjectedWrappedRetryableCreationError(c *gc.C) { 631 // create the error injection channel 632 errorInjectionChannel := make(chan error, 1) 633 c.Assert(errorInjectionChannel, gc.NotNil) 634 635 p := s.newEnvironProvisioner(c) 636 defer stop(c, p) 637 638 // patch the dummy provider error injection channel 639 cleanup := dummy.PatchTransientErrorInjectionChannel(errorInjectionChannel) 640 defer cleanup() 641 642 // send the error message once 643 // - instance creation should succeed 644 retryableError := errors.Wrap(errors.New(""), instance.NewRetryableCreationError("container failed to start and was destroyed")) 645 errorInjectionChannel <- retryableError 646 647 m, err := s.addMachine() 648 c.Assert(err, jc.ErrorIsNil) 649 s.checkStartInstanceNoSecureConnection(c, m) 650 } 651 652 func (s *ProvisionerSuite) TestProvisionerFailStartInstanceWithInjectedNonRetryableCreationError(c *gc.C) { 653 // create the error injection channel 654 errorInjectionChannel := make(chan error, 1) 655 c.Assert(errorInjectionChannel, gc.NotNil) 656 657 p := s.newEnvironProvisioner(c) 658 defer stop(c, p) 659 660 // patch the dummy provider error injection channel 661 cleanup := dummy.PatchTransientErrorInjectionChannel(errorInjectionChannel) 662 defer cleanup() 663 664 // send the error message once 665 // - instance creation should succeed 666 nonRetryableError := errors.New("some nonretryable error") 667 errorInjectionChannel <- nonRetryableError 668 669 m, err := s.addMachine() 670 c.Assert(err, jc.ErrorIsNil) 671 s.checkNoOperations(c) 672 673 t0 := time.Now() 674 for time.Since(t0) < coretesting.LongWait { 675 // And check the machine status is set to error. 676 status, info, _, err := m.Status() 677 c.Assert(err, jc.ErrorIsNil) 678 if status == state.StatusPending { 679 time.Sleep(coretesting.ShortWait) 680 continue 681 } 682 c.Assert(status, gc.Equals, state.StatusError) 683 // check that the status matches the error message 684 c.Assert(info, gc.Equals, nonRetryableError.Error()) 685 break 686 } 687 } 688 689 func (s *ProvisionerSuite) TestProvisioningDoesNotOccurForContainers(c *gc.C) { 690 p := s.newEnvironProvisioner(c) 691 defer stop(c, p) 692 693 // create a machine to host the container. 694 m, err := s.addMachine() 695 c.Assert(err, jc.ErrorIsNil) 696 inst := s.checkStartInstanceNoSecureConnection(c, m) 697 698 // make a container on the machine we just created 699 template := state.MachineTemplate{ 700 Series: coretesting.FakeDefaultSeries, 701 Jobs: []state.MachineJob{state.JobHostUnits}, 702 } 703 container, err := s.State.AddMachineInsideMachine(template, m.Id(), instance.LXC) 704 c.Assert(err, jc.ErrorIsNil) 705 706 // the PA should not attempt to create it 707 s.checkNoOperations(c) 708 709 // cleanup 710 c.Assert(container.EnsureDead(), gc.IsNil) 711 c.Assert(container.Remove(), gc.IsNil) 712 c.Assert(m.EnsureDead(), gc.IsNil) 713 s.checkStopInstances(c, inst) 714 s.waitRemoved(c, m) 715 } 716 717 func (s *ProvisionerSuite) TestProvisioningMachinesWithRequestedNetworks(c *gc.C) { 718 p := s.newEnvironProvisioner(c) 719 defer stop(c, p) 720 721 // Add and provision a machine with networks specified. 722 requestedNetworks := []string{"net1", "net2"} 723 cons := constraints.MustParse(s.defaultConstraints.String(), "networks=^net3,^net4") 724 expectNetworkInfo := []network.InterfaceInfo{{ 725 MACAddress: "aa:bb:cc:dd:ee:f0", 726 InterfaceName: "eth0", 727 ProviderId: "net1", 728 NetworkName: "net1", 729 VLANTag: 0, 730 CIDR: "0.1.2.0/24", 731 }, { 732 MACAddress: "aa:bb:cc:dd:ee:f1", 733 InterfaceName: "eth1", 734 ProviderId: "net2", 735 NetworkName: "net2", 736 VLANTag: 1, 737 CIDR: "0.2.2.0/24", 738 }} 739 m, err := s.addMachineWithRequestedNetworks(requestedNetworks, cons) 740 c.Assert(err, jc.ErrorIsNil) 741 inst := s.checkStartInstanceCustom( 742 c, m, "pork", cons, 743 requestedNetworks, expectNetworkInfo, false, 744 nil, true, 745 ) 746 747 _, err = s.State.Network("net1") 748 c.Assert(err, jc.ErrorIsNil) 749 _, err = s.State.Network("net2") 750 c.Assert(err, jc.ErrorIsNil) 751 _, err = s.State.Network("net3") 752 c.Assert(err, jc.Satisfies, errors.IsNotFound) 753 _, err = s.State.Network("net4") 754 c.Assert(err, jc.Satisfies, errors.IsNotFound) 755 ifaces, err := m.NetworkInterfaces() 756 c.Assert(err, jc.ErrorIsNil) 757 c.Assert(ifaces, gc.HasLen, 2) 758 759 // Cleanup. 760 c.Assert(m.EnsureDead(), gc.IsNil) 761 s.checkStopInstances(c, inst) 762 s.waitRemoved(c, m) 763 } 764 765 func (s *ProvisionerSuite) TestSetInstanceInfoFailureSetsErrorStatusAndStopsInstanceButKeepsGoing(c *gc.C) { 766 p := s.newEnvironProvisioner(c) 767 defer stop(c, p) 768 769 // Add and provision a machine with networks specified. 770 networks := []string{"bad-net1"} 771 // "bad-" prefix for networks causes dummy provider to report 772 // invalid network.InterfaceInfo. 773 expectNetworkInfo := []network.InterfaceInfo{ 774 {ProviderId: "bad-net1", NetworkName: "bad-net1", CIDR: "invalid"}, 775 } 776 m, err := s.addMachineWithRequestedNetworks(networks, constraints.Value{}) 777 c.Assert(err, jc.ErrorIsNil) 778 inst := s.checkStartInstanceCustom( 779 c, m, "pork", constraints.Value{}, 780 networks, expectNetworkInfo, false, 781 nil, false, 782 ) 783 784 // Ensure machine error status was set. 785 t0 := time.Now() 786 for time.Since(t0) < coretesting.LongWait { 787 // And check the machine status is set to error. 788 status, info, _, err := m.Status() 789 c.Assert(err, jc.ErrorIsNil) 790 if status == state.StatusPending { 791 time.Sleep(coretesting.ShortWait) 792 continue 793 } 794 c.Assert(status, gc.Equals, state.StatusError) 795 c.Assert(info, gc.Matches, `cannot record provisioning info for "dummyenv-0": cannot add network "bad-net1": invalid CIDR address: invalid`) 796 break 797 } 798 s.checkStopInstances(c, inst) 799 800 // Make sure the task didn't stop with an error 801 died := make(chan error) 802 go func() { 803 died <- p.Wait() 804 }() 805 select { 806 case <-time.After(coretesting.LongWait): 807 case err = <-died: 808 c.Fatalf("provisioner task died unexpectedly with err: %v", err) 809 } 810 811 // Restart the PA to make sure the machine is not retried. 812 stop(c, p) 813 p = s.newEnvironProvisioner(c) 814 defer stop(c, p) 815 816 s.checkNoOperations(c) 817 } 818 819 func (s *ProvisionerSuite) TestProvisioningDoesNotOccurWithAnInvalidEnvironment(c *gc.C) { 820 s.invalidateEnvironment(c) 821 822 p := s.newEnvironProvisioner(c) 823 defer stop(c, p) 824 825 // try to create a machine 826 _, err := s.addMachine() 827 c.Assert(err, jc.ErrorIsNil) 828 829 // the PA should not create it 830 s.checkNoOperations(c) 831 } 832 833 func (s *ProvisionerSuite) TestProvisioningOccursWithFixedEnvironment(c *gc.C) { 834 s.invalidateEnvironment(c) 835 836 p := s.newEnvironProvisioner(c) 837 defer stop(c, p) 838 839 // try to create a machine 840 m, err := s.addMachine() 841 c.Assert(err, jc.ErrorIsNil) 842 843 // the PA should not create it 844 s.checkNoOperations(c) 845 846 err = s.fixEnvironment(c) 847 c.Assert(err, jc.ErrorIsNil) 848 849 s.checkStartInstanceNoSecureConnection(c, m) 850 } 851 852 func (s *ProvisionerSuite) TestProvisioningDoesOccurAfterInvalidEnvironmentPublished(c *gc.C) { 853 s.PatchValue(provisioner.GetToolsFinder, func(*apiprovisioner.State) provisioner.ToolsFinder { 854 return mockToolsFinder{} 855 }) 856 p := s.newEnvironProvisioner(c) 857 defer stop(c, p) 858 859 // place a new machine into the state 860 m, err := s.addMachine() 861 c.Assert(err, jc.ErrorIsNil) 862 863 s.checkStartInstanceNoSecureConnection(c, m) 864 865 s.invalidateEnvironment(c) 866 867 // create a second machine 868 m, err = s.addMachine() 869 c.Assert(err, jc.ErrorIsNil) 870 871 // the PA should create it using the old environment 872 s.checkStartInstanceNoSecureConnection(c, m) 873 } 874 875 func (s *ProvisionerSuite) TestProvisioningDoesNotProvisionTheSameMachineAfterRestart(c *gc.C) { 876 p := s.newEnvironProvisioner(c) 877 defer stop(c, p) 878 879 // create a machine 880 m, err := s.addMachine() 881 c.Assert(err, jc.ErrorIsNil) 882 s.checkStartInstanceNoSecureConnection(c, m) 883 884 // restart the PA 885 stop(c, p) 886 p = s.newEnvironProvisioner(c) 887 defer stop(c, p) 888 889 // check that there is only one machine provisioned. 890 machines, err := s.State.AllMachines() 891 c.Assert(err, jc.ErrorIsNil) 892 c.Check(len(machines), gc.Equals, 2) 893 c.Check(machines[0].Id(), gc.Equals, "0") 894 c.Check(machines[1].CheckProvisioned("fake_nonce"), jc.IsFalse) 895 896 // the PA should not create it a second time 897 s.checkNoOperations(c) 898 } 899 900 func (s *ProvisionerSuite) TestDyingMachines(c *gc.C) { 901 p := s.newEnvironProvisioner(c) 902 defer stop(c, p) 903 904 // provision a machine 905 m0, err := s.addMachine() 906 c.Assert(err, jc.ErrorIsNil) 907 s.checkStartInstanceNoSecureConnection(c, m0) 908 909 // stop the provisioner and make the machine dying 910 stop(c, p) 911 err = m0.Destroy() 912 c.Assert(err, jc.ErrorIsNil) 913 914 // add a new, dying, unprovisioned machine 915 m1, err := s.addMachine() 916 c.Assert(err, jc.ErrorIsNil) 917 err = m1.Destroy() 918 c.Assert(err, jc.ErrorIsNil) 919 920 // start the provisioner and wait for it to reap the useless machine 921 p = s.newEnvironProvisioner(c) 922 defer stop(c, p) 923 s.checkNoOperations(c) 924 s.waitRemoved(c, m1) 925 926 // verify the other one's still fine 927 err = m0.Refresh() 928 c.Assert(err, jc.ErrorIsNil) 929 c.Assert(m0.Life(), gc.Equals, state.Dying) 930 } 931 932 func (s *ProvisionerSuite) TestProvisioningRecoversAfterInvalidEnvironmentPublished(c *gc.C) { 933 s.PatchValue(provisioner.GetToolsFinder, func(*apiprovisioner.State) provisioner.ToolsFinder { 934 return mockToolsFinder{} 935 }) 936 p := s.newEnvironProvisioner(c) 937 defer stop(c, p) 938 939 // place a new machine into the state 940 m, err := s.addMachine() 941 c.Assert(err, jc.ErrorIsNil) 942 s.checkStartInstanceNoSecureConnection(c, m) 943 944 s.invalidateEnvironment(c) 945 s.BackingState.StartSync() 946 947 // create a second machine 948 m, err = s.addMachine() 949 c.Assert(err, jc.ErrorIsNil) 950 951 // the PA should create it using the old environment 952 s.checkStartInstanceNoSecureConnection(c, m) 953 954 err = s.fixEnvironment(c) 955 c.Assert(err, jc.ErrorIsNil) 956 957 // insert our observer 958 cfgObserver := make(chan *config.Config, 1) 959 provisioner.SetObserver(p, cfgObserver) 960 961 err = s.State.UpdateEnvironConfig(map[string]interface{}{"secret": "beef"}, nil, nil) 962 c.Assert(err, jc.ErrorIsNil) 963 964 s.BackingState.StartSync() 965 966 // wait for the PA to load the new configuration 967 select { 968 case <-cfgObserver: 969 case <-time.After(coretesting.LongWait): 970 c.Fatalf("PA did not action config change") 971 } 972 973 // create a third machine 974 m, err = s.addMachine() 975 c.Assert(err, jc.ErrorIsNil) 976 977 // the PA should create it using the new environment 978 s.checkStartInstanceCustom(c, m, "beef", s.defaultConstraints, nil, nil, false, nil, true) 979 } 980 981 type mockMachineGetter struct{} 982 983 func (*mockMachineGetter) Machine(names.MachineTag) (*apiprovisioner.Machine, error) { 984 return nil, fmt.Errorf("error") 985 } 986 987 func (*mockMachineGetter) MachinesWithTransientErrors() ([]*apiprovisioner.Machine, []params.StatusResult, error) { 988 return nil, nil, fmt.Errorf("error") 989 } 990 991 func (s *ProvisionerSuite) TestMachineErrorsRetainInstances(c *gc.C) { 992 task := s.newProvisionerTask(c, config.HarvestAll, s.Environ, s.provisioner, mockToolsFinder{}) 993 defer stop(c, task) 994 995 // create a machine 996 m0, err := s.addMachine() 997 c.Assert(err, jc.ErrorIsNil) 998 s.checkStartInstance(c, m0) 999 1000 // create an instance out of band 1001 s.startUnknownInstance(c, "999") 1002 1003 // start the provisioner and ensure it doesn't kill any instances if there are error getting machines 1004 task = s.newProvisionerTask( 1005 c, 1006 config.HarvestAll, 1007 s.Environ, 1008 &mockMachineGetter{}, 1009 &mockToolsFinder{}, 1010 ) 1011 defer func() { 1012 err := task.Stop() 1013 c.Assert(err, gc.ErrorMatches, ".*failed to get machine.*") 1014 }() 1015 s.checkNoOperations(c) 1016 } 1017 1018 func (s *ProvisionerSuite) TestProvisionerObservesConfigChanges(c *gc.C) { 1019 p := s.newEnvironProvisioner(c) 1020 defer stop(c, p) 1021 1022 // Inject our observer into the provisioner 1023 cfgObserver := make(chan *config.Config, 1) 1024 provisioner.SetObserver(p, cfgObserver) 1025 1026 // Switch to reaping on All machines. 1027 attrs := map[string]interface{}{ 1028 config.ProvisionerHarvestModeKey: config.HarvestAll.String(), 1029 } 1030 err := s.State.UpdateEnvironConfig(attrs, nil, nil) 1031 c.Assert(err, jc.ErrorIsNil) 1032 1033 s.BackingState.StartSync() 1034 1035 // Wait for the PA to load the new configuration. 1036 select { 1037 case newCfg := <-cfgObserver: 1038 c.Assert( 1039 newCfg.ProvisionerHarvestMode().String(), 1040 gc.Equals, 1041 config.HarvestAll.String(), 1042 ) 1043 case <-time.After(coretesting.LongWait): 1044 c.Fatalf("PA did not action config change") 1045 } 1046 } 1047 1048 func (s *ProvisionerSuite) newProvisionerTask( 1049 c *gc.C, 1050 harvestingMethod config.HarvestMode, 1051 broker environs.InstanceBroker, 1052 machineGetter provisioner.MachineGetter, 1053 toolsFinder provisioner.ToolsFinder, 1054 ) provisioner.ProvisionerTask { 1055 1056 machineWatcher, err := s.provisioner.WatchEnvironMachines() 1057 c.Assert(err, jc.ErrorIsNil) 1058 retryWatcher, err := s.provisioner.WatchMachineErrorRetry() 1059 c.Assert(err, jc.ErrorIsNil) 1060 auth, err := authentication.NewAPIAuthenticator(s.provisioner) 1061 c.Assert(err, jc.ErrorIsNil) 1062 1063 return provisioner.NewProvisionerTask( 1064 names.NewMachineTag("0"), 1065 harvestingMethod, 1066 machineGetter, 1067 toolsFinder, 1068 machineWatcher, 1069 retryWatcher, 1070 broker, 1071 auth, 1072 imagemetadata.ReleasedStream, 1073 true, 1074 ) 1075 } 1076 1077 func (s *ProvisionerSuite) TestHarvestNoneReapsNothing(c *gc.C) { 1078 1079 task := s.newProvisionerTask(c, config.HarvestDestroyed, s.Environ, s.provisioner, mockToolsFinder{}) 1080 defer stop(c, task) 1081 task.SetHarvestMode(config.HarvestNone) 1082 1083 // Create a machine and an unknown instance. 1084 m0, err := s.addMachine() 1085 c.Assert(err, jc.ErrorIsNil) 1086 s.checkStartInstance(c, m0) 1087 s.startUnknownInstance(c, "999") 1088 1089 // Mark the first machine as dead. 1090 c.Assert(m0.EnsureDead(), gc.IsNil) 1091 1092 // Ensure we're doing nothing. 1093 s.checkNoOperations(c) 1094 } 1095 1096 func (s *ProvisionerSuite) TestHarvestUnknownReapsOnlyUnknown(c *gc.C) { 1097 1098 task := s.newProvisionerTask(c, 1099 config.HarvestDestroyed, 1100 s.Environ, 1101 s.provisioner, 1102 mockToolsFinder{}, 1103 ) 1104 defer stop(c, task) 1105 task.SetHarvestMode(config.HarvestUnknown) 1106 1107 // Create a machine and an unknown instance. 1108 m0, err := s.addMachine() 1109 c.Assert(err, jc.ErrorIsNil) 1110 i0 := s.checkStartInstance(c, m0) 1111 i1 := s.startUnknownInstance(c, "999") 1112 1113 // Mark the first machine as dead. 1114 c.Assert(m0.EnsureDead(), gc.IsNil) 1115 1116 // When only harvesting unknown machines, only one of the machines 1117 // is stopped. 1118 s.checkStopSomeInstances(c, []instance.Instance{i1}, []instance.Instance{i0}) 1119 s.waitRemoved(c, m0) 1120 } 1121 1122 func (s *ProvisionerSuite) TestHarvestDestroyedReapsOnlyDestroyed(c *gc.C) { 1123 1124 task := s.newProvisionerTask( 1125 c, 1126 config.HarvestDestroyed, 1127 s.Environ, 1128 s.provisioner, 1129 mockToolsFinder{}, 1130 ) 1131 defer stop(c, task) 1132 1133 // Create a machine and an unknown instance. 1134 m0, err := s.addMachine() 1135 c.Assert(err, jc.ErrorIsNil) 1136 i0 := s.checkStartInstance(c, m0) 1137 i1 := s.startUnknownInstance(c, "999") 1138 1139 // Mark the first machine as dead. 1140 c.Assert(m0.EnsureDead(), gc.IsNil) 1141 1142 // When only harvesting destroyed machines, only one of the 1143 // machines is stopped. 1144 s.checkStopSomeInstances(c, []instance.Instance{i0}, []instance.Instance{i1}) 1145 s.waitRemoved(c, m0) 1146 } 1147 1148 func (s *ProvisionerSuite) TestHarvestAllReapsAllTheThings(c *gc.C) { 1149 1150 task := s.newProvisionerTask(c, 1151 config.HarvestDestroyed, 1152 s.Environ, 1153 s.provisioner, 1154 mockToolsFinder{}, 1155 ) 1156 defer stop(c, task) 1157 task.SetHarvestMode(config.HarvestAll) 1158 1159 // Create a machine and an unknown instance. 1160 m0, err := s.addMachine() 1161 c.Assert(err, jc.ErrorIsNil) 1162 i0 := s.checkStartInstance(c, m0) 1163 i1 := s.startUnknownInstance(c, "999") 1164 1165 // Mark the first machine as dead. 1166 c.Assert(m0.EnsureDead(), gc.IsNil) 1167 1168 // Everything must die! 1169 s.checkStopSomeInstances(c, []instance.Instance{i0, i1}, []instance.Instance{}) 1170 s.waitRemoved(c, m0) 1171 } 1172 1173 func (s *ProvisionerSuite) TestProvisionerRetriesTransientErrors(c *gc.C) { 1174 s.PatchValue(&apiserverprovisioner.ErrorRetryWaitDelay, 5*time.Millisecond) 1175 e := &mockBroker{Environ: s.Environ, retryCount: make(map[string]int)} 1176 task := s.newProvisionerTask(c, config.HarvestAll, e, s.provisioner, mockToolsFinder{}) 1177 defer stop(c, task) 1178 1179 // Provision some machines, some will be started first time, 1180 // another will require retries. 1181 m1, err := s.addMachine() 1182 c.Assert(err, jc.ErrorIsNil) 1183 s.checkStartInstance(c, m1) 1184 m2, err := s.addMachine() 1185 c.Assert(err, jc.ErrorIsNil) 1186 s.checkStartInstance(c, m2) 1187 m3, err := s.addMachine() 1188 c.Assert(err, jc.ErrorIsNil) 1189 m4, err := s.addMachine() 1190 c.Assert(err, jc.ErrorIsNil) 1191 1192 // mockBroker will fail to start machine-3 several times; 1193 // keep setting the transient flag to retry until the 1194 // instance has started. 1195 thatsAllFolks := make(chan struct{}) 1196 go func() { 1197 for { 1198 select { 1199 case <-thatsAllFolks: 1200 return 1201 case <-time.After(coretesting.ShortWait): 1202 err := m3.SetStatus(state.StatusError, "info", map[string]interface{}{"transient": true}) 1203 c.Assert(err, jc.ErrorIsNil) 1204 } 1205 } 1206 }() 1207 s.checkStartInstance(c, m3) 1208 close(thatsAllFolks) 1209 1210 // Machine 4 is never provisioned. 1211 status, _, _, err := m4.Status() 1212 c.Assert(err, jc.ErrorIsNil) 1213 c.Assert(status, gc.Equals, state.StatusError) 1214 _, err = m4.InstanceId() 1215 c.Assert(err, jc.Satisfies, errors.IsNotProvisioned) 1216 } 1217 1218 func (s *ProvisionerSuite) TestProvisionerObservesMachineJobs(c *gc.C) { 1219 s.PatchValue(&apiserverprovisioner.ErrorRetryWaitDelay, 5*time.Millisecond) 1220 broker := &mockBroker{Environ: s.Environ, retryCount: make(map[string]int)} 1221 task := s.newProvisionerTask(c, config.HarvestAll, broker, s.provisioner, mockToolsFinder{}) 1222 defer stop(c, task) 1223 1224 added := s.ensureAvailability(c, 3) 1225 c.Assert(added, gc.HasLen, 2) 1226 byId := make(map[string]*state.Machine) 1227 for _, m := range added { 1228 byId[m.Id()] = m 1229 } 1230 for _, id := range broker.ids { 1231 s.checkStartInstance(c, byId[id]) 1232 } 1233 } 1234 1235 type mockBroker struct { 1236 environs.Environ 1237 retryCount map[string]int 1238 ids []string 1239 } 1240 1241 func (b *mockBroker) StartInstance(args environs.StartInstanceParams) (*environs.StartInstanceResult, error) { 1242 // All machines except machines 3, 4 are provisioned successfully the first time. 1243 // Machines 3 is provisioned after some attempts have been made. 1244 // Machine 4 is never provisioned. 1245 id := args.MachineConfig.MachineId 1246 // record ids so we can call checkStartInstance in the appropriate order. 1247 b.ids = append(b.ids, id) 1248 retries := b.retryCount[id] 1249 if (id != "3" && id != "4") || retries > 2 { 1250 return b.Environ.StartInstance(args) 1251 } else { 1252 b.retryCount[id] = retries + 1 1253 } 1254 return nil, fmt.Errorf("error: some error") 1255 } 1256 1257 type mockToolsFinder struct { 1258 } 1259 1260 func (f mockToolsFinder) FindTools(number version.Number, series string, arch *string) (coretools.List, error) { 1261 v, err := version.ParseBinary(fmt.Sprintf("%s-%s-%s", number, series, version.Current.Arch)) 1262 if err != nil { 1263 return nil, err 1264 } 1265 if arch != nil { 1266 v.Arch = *arch 1267 } 1268 return coretools.List{&coretools.Tools{Version: v}}, nil 1269 }