github.com/altoros/juju-vmware@v0.0.0-20150312064031-f19ae857ccca/worker/provisioner/provisioner_test.go (about)

     1  // Copyright 2012, 2013 Canonical Ltd.
     2  // Licensed under the AGPLv3, see LICENCE file for details.
     3  
     4  package provisioner_test
     5  
     6  import (
     7  	"fmt"
     8  	"strconv"
     9  	"strings"
    10  	"time"
    11  
    12  	"github.com/juju/errors"
    13  	"github.com/juju/names"
    14  	jc "github.com/juju/testing/checkers"
    15  	"github.com/juju/utils"
    16  	"github.com/juju/utils/set"
    17  	gc "gopkg.in/check.v1"
    18  
    19  	"github.com/juju/juju/agent"
    20  	"github.com/juju/juju/api"
    21  	apiprovisioner "github.com/juju/juju/api/provisioner"
    22  	"github.com/juju/juju/apiserver/params"
    23  	apiserverprovisioner "github.com/juju/juju/apiserver/provisioner"
    24  	"github.com/juju/juju/constraints"
    25  	"github.com/juju/juju/environmentserver/authentication"
    26  	"github.com/juju/juju/environs"
    27  	"github.com/juju/juju/environs/config"
    28  	"github.com/juju/juju/environs/filestorage"
    29  	"github.com/juju/juju/environs/imagemetadata"
    30  	envtesting "github.com/juju/juju/environs/testing"
    31  	"github.com/juju/juju/environs/tools"
    32  	"github.com/juju/juju/instance"
    33  	"github.com/juju/juju/juju/testing"
    34  	"github.com/juju/juju/mongo"
    35  	"github.com/juju/juju/network"
    36  	"github.com/juju/juju/provider/dummy"
    37  	"github.com/juju/juju/state"
    38  	"github.com/juju/juju/state/multiwatcher"
    39  	coretesting "github.com/juju/juju/testing"
    40  	coretools "github.com/juju/juju/tools"
    41  	"github.com/juju/juju/version"
    42  	"github.com/juju/juju/worker/provisioner"
    43  )
    44  
    45  type CommonProvisionerSuite struct {
    46  	testing.JujuConnSuite
    47  	op  <-chan dummy.Operation
    48  	cfg *config.Config
    49  	// defaultConstraints are used when adding a machine and then later in test assertions.
    50  	defaultConstraints constraints.Value
    51  
    52  	st          *api.State
    53  	provisioner *apiprovisioner.State
    54  }
    55  
    56  type ProvisionerSuite struct {
    57  	CommonProvisionerSuite
    58  }
    59  
    60  var _ = gc.Suite(&ProvisionerSuite{})
    61  
    62  var veryShortAttempt = utils.AttemptStrategy{
    63  	Total: 1 * time.Second,
    64  	Delay: 80 * time.Millisecond,
    65  }
    66  
    67  func (s *CommonProvisionerSuite) SetUpSuite(c *gc.C) {
    68  	s.JujuConnSuite.SetUpSuite(c)
    69  	s.defaultConstraints = constraints.MustParse("arch=amd64 mem=4G cpu-cores=1 root-disk=8G")
    70  }
    71  
    72  func (s *CommonProvisionerSuite) SetUpTest(c *gc.C) {
    73  	// Disable the default state policy, because the
    74  	// provisioner needs to be able to test pathological
    75  	// scenarios where a machine exists in state with
    76  	// invalid environment config.
    77  	dummy.SetStatePolicy(nil)
    78  
    79  	s.JujuConnSuite.SetUpTest(c)
    80  
    81  	// Create the operations channel with more than enough space
    82  	// for those tests that don't listen on it.
    83  	op := make(chan dummy.Operation, 500)
    84  	dummy.Listen(op)
    85  	s.op = op
    86  
    87  	cfg, err := s.State.EnvironConfig()
    88  	c.Assert(err, jc.ErrorIsNil)
    89  	s.cfg = cfg
    90  
    91  	// Create a machine for the dummy bootstrap instance,
    92  	// so the provisioner doesn't destroy it.
    93  	insts, err := s.Environ.Instances([]instance.Id{dummy.BootstrapInstanceId})
    94  	c.Assert(err, jc.ErrorIsNil)
    95  	addrs, err := insts[0].Addresses()
    96  	c.Assert(err, jc.ErrorIsNil)
    97  	machine, err := s.State.AddOneMachine(state.MachineTemplate{
    98  		Addresses:  addrs,
    99  		Series:     "quantal",
   100  		Nonce:      agent.BootstrapNonce,
   101  		InstanceId: dummy.BootstrapInstanceId,
   102  		Jobs:       []state.MachineJob{state.JobManageEnviron},
   103  	})
   104  	c.Assert(err, jc.ErrorIsNil)
   105  	c.Assert(machine.Id(), gc.Equals, "0")
   106  
   107  	err = machine.SetAgentVersion(version.Current)
   108  	c.Assert(err, jc.ErrorIsNil)
   109  
   110  	password, err := utils.RandomPassword()
   111  	c.Assert(err, jc.ErrorIsNil)
   112  	err = machine.SetPassword(password)
   113  	c.Assert(err, jc.ErrorIsNil)
   114  
   115  	s.st = s.OpenAPIAsMachine(c, machine.Tag(), password, agent.BootstrapNonce)
   116  	c.Assert(s.st, gc.NotNil)
   117  	c.Logf("API: login as %q successful", machine.Tag())
   118  	s.provisioner = s.st.Provisioner()
   119  	c.Assert(s.provisioner, gc.NotNil)
   120  }
   121  
   122  // breakDummyProvider changes the environment config in state in a way
   123  // that causes the given environMethod of the dummy provider to return
   124  // an error, which is also returned as a message to be checked.
   125  func breakDummyProvider(c *gc.C, st *state.State, environMethod string) string {
   126  	attrs := map[string]interface{}{"broken": environMethod}
   127  	err := st.UpdateEnvironConfig(attrs, nil, nil)
   128  	c.Assert(err, jc.ErrorIsNil)
   129  	return fmt.Sprintf("dummy.%s is broken", environMethod)
   130  }
   131  
   132  // invalidateEnvironment alters the environment configuration
   133  // so the Settings returned from the watcher will not pass
   134  // validation.
   135  func (s *CommonProvisionerSuite) invalidateEnvironment(c *gc.C) {
   136  	st, err := state.Open(s.MongoInfo(c), mongo.DefaultDialOpts(), state.Policy(nil))
   137  	c.Assert(err, jc.ErrorIsNil)
   138  	defer st.Close()
   139  	attrs := map[string]interface{}{"type": "unknown"}
   140  	err = st.UpdateEnvironConfig(attrs, nil, nil)
   141  	c.Assert(err, jc.ErrorIsNil)
   142  }
   143  
   144  // fixEnvironment undoes the work of invalidateEnvironment.
   145  func (s *CommonProvisionerSuite) fixEnvironment(c *gc.C) error {
   146  	st, err := state.Open(s.MongoInfo(c), mongo.DefaultDialOpts(), state.Policy(nil))
   147  	c.Assert(err, jc.ErrorIsNil)
   148  	defer st.Close()
   149  	attrs := map[string]interface{}{"type": s.cfg.AllAttrs()["type"]}
   150  	return st.UpdateEnvironConfig(attrs, nil, nil)
   151  }
   152  
   153  // stopper is stoppable.
   154  type stopper interface {
   155  	Stop() error
   156  }
   157  
   158  // stop stops a stopper.
   159  func stop(c *gc.C, s stopper) {
   160  	c.Assert(s.Stop(), gc.IsNil)
   161  }
   162  
   163  func (s *CommonProvisionerSuite) startUnknownInstance(c *gc.C, id string) instance.Instance {
   164  	instance, _ := testing.AssertStartInstance(c, s.Environ, id)
   165  	select {
   166  	case o := <-s.op:
   167  		switch o := o.(type) {
   168  		case dummy.OpStartInstance:
   169  		default:
   170  			c.Fatalf("unexpected operation %#v", o)
   171  		}
   172  	case <-time.After(coretesting.LongWait):
   173  		c.Fatalf("timed out waiting for startinstance operation")
   174  	}
   175  	return instance
   176  }
   177  
   178  func (s *CommonProvisionerSuite) checkStartInstance(c *gc.C, m *state.Machine) instance.Instance {
   179  	return s.checkStartInstanceCustom(c, m, "pork", s.defaultConstraints, nil, nil, true, nil, true)
   180  }
   181  
   182  func (s *CommonProvisionerSuite) checkStartInstanceNoSecureConnection(c *gc.C, m *state.Machine) instance.Instance {
   183  	return s.checkStartInstanceCustom(c, m, "pork", s.defaultConstraints, nil, nil, false, nil, true)
   184  }
   185  
   186  func (s *CommonProvisionerSuite) checkStartInstanceCustom(
   187  	c *gc.C, m *state.Machine,
   188  	secret string, cons constraints.Value,
   189  	networks []string, networkInfo []network.InterfaceInfo,
   190  	secureServerConnection bool,
   191  	checkPossibleTools coretools.List,
   192  	waitInstanceId bool,
   193  ) (
   194  	inst instance.Instance,
   195  ) {
   196  	s.BackingState.StartSync()
   197  	for {
   198  		select {
   199  		case o := <-s.op:
   200  			switch o := o.(type) {
   201  			case dummy.OpStartInstance:
   202  				inst = o.Instance
   203  				if waitInstanceId {
   204  					s.waitInstanceId(c, m, inst.Id())
   205  				}
   206  
   207  				// Check the instance was started with the expected params.
   208  				c.Assert(o.MachineId, gc.Equals, m.Id())
   209  				nonceParts := strings.SplitN(o.MachineNonce, ":", 2)
   210  				c.Assert(nonceParts, gc.HasLen, 2)
   211  				c.Assert(nonceParts[0], gc.Equals, names.NewMachineTag("0").String())
   212  				c.Assert(nonceParts[1], jc.Satisfies, utils.IsValidUUIDString)
   213  				c.Assert(o.Secret, gc.Equals, secret)
   214  				c.Assert(o.Networks, jc.DeepEquals, networks)
   215  				c.Assert(o.NetworkInfo, jc.DeepEquals, networkInfo)
   216  				c.Assert(o.AgentEnvironment["SECURE_STATESERVER_CONNECTION"], gc.Equals, strconv.FormatBool(secureServerConnection))
   217  
   218  				var jobs []multiwatcher.MachineJob
   219  				for _, job := range m.Jobs() {
   220  					jobs = append(jobs, job.ToParams())
   221  				}
   222  				c.Assert(o.Jobs, jc.SameContents, jobs)
   223  
   224  				if checkPossibleTools != nil {
   225  					for _, t := range o.PossibleTools {
   226  						url := fmt.Sprintf("https://%s/environment/%s/tools/%s",
   227  							s.st.Addr(), coretesting.EnvironmentTag.Id(), t.Version)
   228  						c.Check(t.URL, gc.Equals, url)
   229  						t.URL = ""
   230  					}
   231  					for _, t := range checkPossibleTools {
   232  						t.URL = ""
   233  					}
   234  					c.Assert(o.PossibleTools, gc.DeepEquals, checkPossibleTools)
   235  				}
   236  
   237  				// All provisioned machines in this test suite have
   238  				// their hardware characteristics attributes set to
   239  				// the same values as the constraints due to the dummy
   240  				// environment being used.
   241  				if !constraints.IsEmpty(&cons) {
   242  					c.Assert(o.Constraints, gc.DeepEquals, cons)
   243  					hc, err := m.HardwareCharacteristics()
   244  					c.Assert(err, jc.ErrorIsNil)
   245  					c.Assert(*hc, gc.DeepEquals, instance.HardwareCharacteristics{
   246  						Arch:     cons.Arch,
   247  						Mem:      cons.Mem,
   248  						RootDisk: cons.RootDisk,
   249  						CpuCores: cons.CpuCores,
   250  						CpuPower: cons.CpuPower,
   251  						Tags:     cons.Tags,
   252  					})
   253  				}
   254  				return
   255  			default:
   256  				c.Logf("ignoring unexpected operation %#v", o)
   257  			}
   258  		case <-time.After(2 * time.Second):
   259  			c.Fatalf("provisioner did not start an instance")
   260  			return
   261  		}
   262  	}
   263  }
   264  
   265  // checkNoOperations checks that the environ was not operated upon.
   266  func (s *CommonProvisionerSuite) checkNoOperations(c *gc.C) {
   267  	s.BackingState.StartSync()
   268  	select {
   269  	case o := <-s.op:
   270  		c.Fatalf("unexpected operation %+v", o)
   271  	case <-time.After(coretesting.ShortWait):
   272  		return
   273  	}
   274  }
   275  
   276  // checkStopInstances checks that an instance has been stopped.
   277  func (s *CommonProvisionerSuite) checkStopInstances(c *gc.C, instances ...instance.Instance) {
   278  	s.checkStopSomeInstances(c, instances, nil)
   279  }
   280  
   281  // checkStopSomeInstances checks that instancesToStop are stopped while instancesToKeep are not.
   282  func (s *CommonProvisionerSuite) checkStopSomeInstances(c *gc.C,
   283  	instancesToStop []instance.Instance, instancesToKeep []instance.Instance) {
   284  
   285  	s.BackingState.StartSync()
   286  	instanceIdsToStop := set.NewStrings()
   287  	for _, instance := range instancesToStop {
   288  		instanceIdsToStop.Add(string(instance.Id()))
   289  	}
   290  	instanceIdsToKeep := set.NewStrings()
   291  	for _, instance := range instancesToKeep {
   292  		instanceIdsToKeep.Add(string(instance.Id()))
   293  	}
   294  	// Continue checking for stop instance calls until all the instances we
   295  	// are waiting on to finish, actually finish, or we time out.
   296  	for !instanceIdsToStop.IsEmpty() {
   297  		select {
   298  		case o := <-s.op:
   299  			switch o := o.(type) {
   300  			case dummy.OpStopInstances:
   301  				for _, id := range o.Ids {
   302  					instId := string(id)
   303  					instanceIdsToStop.Remove(instId)
   304  					if instanceIdsToKeep.Contains(instId) {
   305  						c.Errorf("provisioner unexpectedly stopped instance %s", instId)
   306  					}
   307  				}
   308  			default:
   309  				c.Fatalf("unexpected operation %#v", o)
   310  				return
   311  			}
   312  		case <-time.After(2 * time.Second):
   313  			c.Fatalf("provisioner did not stop an instance")
   314  			return
   315  		}
   316  	}
   317  }
   318  
   319  func (s *CommonProvisionerSuite) waitMachine(c *gc.C, m *state.Machine, check func() bool) {
   320  	// TODO(jam): We need to grow a new method on NotifyWatcherC
   321  	// that calls StartSync while waiting for changes, then
   322  	// waitMachine and waitHardwareCharacteristics can use that
   323  	// instead
   324  	w := m.Watch()
   325  	defer stop(c, w)
   326  	timeout := time.After(coretesting.LongWait)
   327  	resync := time.After(0)
   328  	for {
   329  		select {
   330  		case <-w.Changes():
   331  			if check() {
   332  				return
   333  			}
   334  		case <-resync:
   335  			resync = time.After(coretesting.ShortWait)
   336  			s.BackingState.StartSync()
   337  		case <-timeout:
   338  			c.Fatalf("machine %v wait timed out", m)
   339  		}
   340  	}
   341  }
   342  
   343  func (s *CommonProvisionerSuite) waitHardwareCharacteristics(c *gc.C, m *state.Machine, check func() bool) {
   344  	w := m.WatchHardwareCharacteristics()
   345  	defer stop(c, w)
   346  	timeout := time.After(coretesting.LongWait)
   347  	resync := time.After(0)
   348  	for {
   349  		select {
   350  		case <-w.Changes():
   351  			if check() {
   352  				return
   353  			}
   354  		case <-resync:
   355  			resync = time.After(coretesting.ShortWait)
   356  			s.BackingState.StartSync()
   357  		case <-timeout:
   358  			c.Fatalf("hardware characteristics for machine %v wait timed out", m)
   359  		}
   360  	}
   361  }
   362  
   363  // waitRemoved waits for the supplied machine to be removed from state.
   364  func (s *CommonProvisionerSuite) waitRemoved(c *gc.C, m *state.Machine) {
   365  	s.waitMachine(c, m, func() bool {
   366  		err := m.Refresh()
   367  		if errors.IsNotFound(err) {
   368  			return true
   369  		}
   370  		c.Assert(err, jc.ErrorIsNil)
   371  		c.Logf("machine %v is still %s", m, m.Life())
   372  		return false
   373  	})
   374  }
   375  
   376  // waitInstanceId waits until the supplied machine has an instance id, then
   377  // asserts it is as expected.
   378  func (s *CommonProvisionerSuite) waitInstanceId(c *gc.C, m *state.Machine, expect instance.Id) {
   379  	s.waitHardwareCharacteristics(c, m, func() bool {
   380  		if actual, err := m.InstanceId(); err == nil {
   381  			c.Assert(actual, gc.Equals, expect)
   382  			return true
   383  		} else if !errors.IsNotProvisioned(err) {
   384  			// We don't expect any errors.
   385  			panic(err)
   386  		}
   387  		c.Logf("machine %v is still unprovisioned", m)
   388  		return false
   389  	})
   390  }
   391  
   392  func (s *CommonProvisionerSuite) newEnvironProvisioner(c *gc.C) provisioner.Provisioner {
   393  	machineTag := names.NewMachineTag("0")
   394  	agentConfig := s.AgentConfigForTag(c, machineTag)
   395  	return provisioner.NewEnvironProvisioner(s.provisioner, agentConfig)
   396  }
   397  
   398  func (s *CommonProvisionerSuite) addMachine() (*state.Machine, error) {
   399  	return s.addMachineWithRequestedNetworks(nil, s.defaultConstraints)
   400  }
   401  
   402  func (s *CommonProvisionerSuite) addMachineWithRequestedNetworks(networks []string, cons constraints.Value) (*state.Machine, error) {
   403  	return s.BackingState.AddOneMachine(state.MachineTemplate{
   404  		Series:            coretesting.FakeDefaultSeries,
   405  		Jobs:              []state.MachineJob{state.JobHostUnits},
   406  		Constraints:       cons,
   407  		RequestedNetworks: networks,
   408  	})
   409  }
   410  
   411  func (s *CommonProvisionerSuite) ensureAvailability(c *gc.C, n int) []*state.Machine {
   412  	changes, err := s.BackingState.EnsureAvailability(n, s.defaultConstraints, coretesting.FakeDefaultSeries, nil)
   413  	c.Assert(err, jc.ErrorIsNil)
   414  	added := make([]*state.Machine, len(changes.Added))
   415  	for i, mid := range changes.Added {
   416  		m, err := s.BackingState.Machine(mid)
   417  		c.Assert(err, jc.ErrorIsNil)
   418  		added[i] = m
   419  	}
   420  	return added
   421  }
   422  
   423  func (s *ProvisionerSuite) TestProvisionerStartStop(c *gc.C) {
   424  	p := s.newEnvironProvisioner(c)
   425  	c.Assert(p.Stop(), gc.IsNil)
   426  }
   427  
   428  func (s *ProvisionerSuite) TestSimple(c *gc.C) {
   429  	p := s.newEnvironProvisioner(c)
   430  	defer stop(c, p)
   431  
   432  	// Check that an instance is provisioned when the machine is created...
   433  	m, err := s.addMachine()
   434  	c.Assert(err, jc.ErrorIsNil)
   435  	instance := s.checkStartInstanceNoSecureConnection(c, m)
   436  
   437  	// ...and removed, along with the machine, when the machine is Dead.
   438  	c.Assert(m.EnsureDead(), gc.IsNil)
   439  	s.checkStopInstances(c, instance)
   440  	s.waitRemoved(c, m)
   441  }
   442  
   443  func (s *ProvisionerSuite) TestConstraints(c *gc.C) {
   444  	// Create a machine with non-standard constraints.
   445  	m, err := s.addMachine()
   446  	c.Assert(err, jc.ErrorIsNil)
   447  	cons := constraints.MustParse("mem=8G arch=amd64 cpu-cores=2 root-disk=10G")
   448  	err = m.SetConstraints(cons)
   449  	c.Assert(err, jc.ErrorIsNil)
   450  
   451  	// Start a provisioner and check those constraints are used.
   452  	p := s.newEnvironProvisioner(c)
   453  	defer stop(c, p)
   454  	s.checkStartInstanceCustom(c, m, "pork", cons, nil, nil, false, nil, true)
   455  }
   456  
   457  func (s *ProvisionerSuite) TestPossibleTools(c *gc.C) {
   458  
   459  	storageDir := c.MkDir()
   460  	s.PatchValue(&tools.DefaultBaseURL, storageDir)
   461  	stor, err := filestorage.NewFileStorageWriter(storageDir)
   462  	c.Assert(err, jc.ErrorIsNil)
   463  
   464  	// Set a current version that does not match the
   465  	// agent-version in the environ config.
   466  	currentVersion := version.MustParseBinary("1.2.3-quantal-arm64")
   467  	s.PatchValue(&version.Current, currentVersion)
   468  
   469  	// Upload some plausible matches, and some that should be filtered out.
   470  	compatibleVersion := version.MustParseBinary("1.2.3-quantal-amd64")
   471  	ignoreVersion1 := version.MustParseBinary("1.2.4-quantal-arm64")
   472  	ignoreVersion2 := version.MustParseBinary("1.2.3-precise-arm64")
   473  	availableVersions := []version.Binary{
   474  		currentVersion, compatibleVersion, ignoreVersion1, ignoreVersion2,
   475  	}
   476  	envtesting.AssertUploadFakeToolsVersions(c, stor, s.cfg.AgentStream(), s.cfg.AgentStream(), availableVersions...)
   477  
   478  	// Extract the tools that we expect to actually match.
   479  	expectedList, err := tools.FindTools(s.Environ, -1, -1, coretools.Filter{
   480  		Number: currentVersion.Number,
   481  		Series: currentVersion.Series,
   482  	})
   483  	c.Assert(err, jc.ErrorIsNil)
   484  
   485  	// Create the machine and check the tools that get passed into StartInstance.
   486  	machine, err := s.BackingState.AddOneMachine(state.MachineTemplate{
   487  		Series: "quantal",
   488  		Jobs:   []state.MachineJob{state.JobHostUnits},
   489  	})
   490  	c.Assert(err, jc.ErrorIsNil)
   491  
   492  	provisioner := s.newEnvironProvisioner(c)
   493  	defer stop(c, provisioner)
   494  	s.checkStartInstanceCustom(
   495  		c, machine, "pork", constraints.Value{},
   496  		nil, nil, false, expectedList, true,
   497  	)
   498  }
   499  
   500  func (s *ProvisionerSuite) TestProvisionerSetsErrorStatusWhenNoToolsAreAvailable(c *gc.C) {
   501  	p := s.newEnvironProvisioner(c)
   502  	defer stop(c, p)
   503  
   504  	// Check that an instance is not provisioned when the machine is created...
   505  	m, err := s.BackingState.AddOneMachine(state.MachineTemplate{
   506  		// We need a valid series that has no tools uploaded
   507  		Series:      "raring",
   508  		Jobs:        []state.MachineJob{state.JobHostUnits},
   509  		Constraints: s.defaultConstraints,
   510  	})
   511  	c.Assert(err, jc.ErrorIsNil)
   512  	s.checkNoOperations(c)
   513  
   514  	t0 := time.Now()
   515  	for time.Since(t0) < coretesting.LongWait {
   516  		// And check the machine status is set to error.
   517  		status, info, _, err := m.Status()
   518  		c.Assert(err, jc.ErrorIsNil)
   519  		if status == state.StatusPending {
   520  			time.Sleep(coretesting.ShortWait)
   521  			continue
   522  		}
   523  		c.Assert(status, gc.Equals, state.StatusError)
   524  		c.Assert(info, gc.Equals, "no matching tools available")
   525  		break
   526  	}
   527  
   528  	// Restart the PA to make sure the machine is skipped again.
   529  	stop(c, p)
   530  	p = s.newEnvironProvisioner(c)
   531  	defer stop(c, p)
   532  	s.checkNoOperations(c)
   533  }
   534  
   535  func (s *ProvisionerSuite) TestProvisionerSetsErrorStatusWhenStartInstanceFailed(c *gc.C) {
   536  	brokenMsg := breakDummyProvider(c, s.State, "StartInstance")
   537  	p := s.newEnvironProvisioner(c)
   538  	defer stop(c, p)
   539  
   540  	// Check that an instance is not provisioned when the machine is created...
   541  	m, err := s.addMachine()
   542  	c.Assert(err, jc.ErrorIsNil)
   543  	s.checkNoOperations(c)
   544  
   545  	t0 := time.Now()
   546  	for time.Since(t0) < coretesting.LongWait {
   547  		// And check the machine status is set to error.
   548  		status, info, _, err := m.Status()
   549  		c.Assert(err, jc.ErrorIsNil)
   550  		if status == state.StatusPending {
   551  			time.Sleep(coretesting.ShortWait)
   552  			continue
   553  		}
   554  		c.Assert(status, gc.Equals, state.StatusError)
   555  		c.Assert(info, gc.Equals, brokenMsg)
   556  		break
   557  	}
   558  
   559  	// Unbreak the environ config.
   560  	err = s.fixEnvironment(c)
   561  	c.Assert(err, jc.ErrorIsNil)
   562  
   563  	// Restart the PA to make sure the machine is skipped again.
   564  	stop(c, p)
   565  	p = s.newEnvironProvisioner(c)
   566  	defer stop(c, p)
   567  	s.checkNoOperations(c)
   568  }
   569  
   570  func (s *ProvisionerSuite) TestProvisionerFailedStartInstanceWithInjectedCreationError(c *gc.C) {
   571  	// create the error injection channel
   572  	errorInjectionChannel := make(chan error, 2)
   573  
   574  	p := s.newEnvironProvisioner(c)
   575  	defer stop(c, p)
   576  
   577  	// patch the dummy provider error injection channel
   578  	cleanup := dummy.PatchTransientErrorInjectionChannel(errorInjectionChannel)
   579  	defer cleanup()
   580  
   581  	retryableError := instance.NewRetryableCreationError("container failed to start and was destroyed")
   582  	destroyError := errors.New("container failed to start and failed to destroy: manual cleanup of containers needed")
   583  	// send the error message TWICE, because the provisioner will retry only ONCE
   584  	errorInjectionChannel <- retryableError
   585  	errorInjectionChannel <- destroyError
   586  
   587  	m, err := s.addMachine()
   588  	c.Assert(err, jc.ErrorIsNil)
   589  	s.checkNoOperations(c)
   590  
   591  	t0 := time.Now()
   592  	for time.Since(t0) < coretesting.LongWait {
   593  		// And check the machine status is set to error.
   594  		status, info, _, err := m.Status()
   595  		c.Assert(err, jc.ErrorIsNil)
   596  		if status == state.StatusPending {
   597  			time.Sleep(coretesting.ShortWait)
   598  			continue
   599  		}
   600  		c.Assert(status, gc.Equals, state.StatusError)
   601  		// check that the status matches the error message
   602  		c.Assert(info, gc.Equals, destroyError.Error())
   603  		break
   604  	}
   605  
   606  }
   607  
   608  func (s *ProvisionerSuite) TestProvisionerSucceedStartInstanceWithInjectedRetryableCreationError(c *gc.C) {
   609  	// create the error injection channel
   610  	errorInjectionChannel := make(chan error, 1)
   611  	c.Assert(errorInjectionChannel, gc.NotNil)
   612  
   613  	p := s.newEnvironProvisioner(c)
   614  	defer stop(c, p)
   615  
   616  	// patch the dummy provider error injection channel
   617  	cleanup := dummy.PatchTransientErrorInjectionChannel(errorInjectionChannel)
   618  	defer cleanup()
   619  
   620  	// send the error message once
   621  	// - instance creation should succeed
   622  	retryableError := instance.NewRetryableCreationError("container failed to start and was destroyed")
   623  	errorInjectionChannel <- retryableError
   624  
   625  	m, err := s.addMachine()
   626  	c.Assert(err, jc.ErrorIsNil)
   627  	s.checkStartInstanceNoSecureConnection(c, m)
   628  }
   629  
   630  func (s *ProvisionerSuite) TestProvisionerSucceedStartInstanceWithInjectedWrappedRetryableCreationError(c *gc.C) {
   631  	// create the error injection channel
   632  	errorInjectionChannel := make(chan error, 1)
   633  	c.Assert(errorInjectionChannel, gc.NotNil)
   634  
   635  	p := s.newEnvironProvisioner(c)
   636  	defer stop(c, p)
   637  
   638  	// patch the dummy provider error injection channel
   639  	cleanup := dummy.PatchTransientErrorInjectionChannel(errorInjectionChannel)
   640  	defer cleanup()
   641  
   642  	// send the error message once
   643  	// - instance creation should succeed
   644  	retryableError := errors.Wrap(errors.New(""), instance.NewRetryableCreationError("container failed to start and was destroyed"))
   645  	errorInjectionChannel <- retryableError
   646  
   647  	m, err := s.addMachine()
   648  	c.Assert(err, jc.ErrorIsNil)
   649  	s.checkStartInstanceNoSecureConnection(c, m)
   650  }
   651  
   652  func (s *ProvisionerSuite) TestProvisionerFailStartInstanceWithInjectedNonRetryableCreationError(c *gc.C) {
   653  	// create the error injection channel
   654  	errorInjectionChannel := make(chan error, 1)
   655  	c.Assert(errorInjectionChannel, gc.NotNil)
   656  
   657  	p := s.newEnvironProvisioner(c)
   658  	defer stop(c, p)
   659  
   660  	// patch the dummy provider error injection channel
   661  	cleanup := dummy.PatchTransientErrorInjectionChannel(errorInjectionChannel)
   662  	defer cleanup()
   663  
   664  	// send the error message once
   665  	// - instance creation should succeed
   666  	nonRetryableError := errors.New("some nonretryable error")
   667  	errorInjectionChannel <- nonRetryableError
   668  
   669  	m, err := s.addMachine()
   670  	c.Assert(err, jc.ErrorIsNil)
   671  	s.checkNoOperations(c)
   672  
   673  	t0 := time.Now()
   674  	for time.Since(t0) < coretesting.LongWait {
   675  		// And check the machine status is set to error.
   676  		status, info, _, err := m.Status()
   677  		c.Assert(err, jc.ErrorIsNil)
   678  		if status == state.StatusPending {
   679  			time.Sleep(coretesting.ShortWait)
   680  			continue
   681  		}
   682  		c.Assert(status, gc.Equals, state.StatusError)
   683  		// check that the status matches the error message
   684  		c.Assert(info, gc.Equals, nonRetryableError.Error())
   685  		break
   686  	}
   687  }
   688  
   689  func (s *ProvisionerSuite) TestProvisioningDoesNotOccurForContainers(c *gc.C) {
   690  	p := s.newEnvironProvisioner(c)
   691  	defer stop(c, p)
   692  
   693  	// create a machine to host the container.
   694  	m, err := s.addMachine()
   695  	c.Assert(err, jc.ErrorIsNil)
   696  	inst := s.checkStartInstanceNoSecureConnection(c, m)
   697  
   698  	// make a container on the machine we just created
   699  	template := state.MachineTemplate{
   700  		Series: coretesting.FakeDefaultSeries,
   701  		Jobs:   []state.MachineJob{state.JobHostUnits},
   702  	}
   703  	container, err := s.State.AddMachineInsideMachine(template, m.Id(), instance.LXC)
   704  	c.Assert(err, jc.ErrorIsNil)
   705  
   706  	// the PA should not attempt to create it
   707  	s.checkNoOperations(c)
   708  
   709  	// cleanup
   710  	c.Assert(container.EnsureDead(), gc.IsNil)
   711  	c.Assert(container.Remove(), gc.IsNil)
   712  	c.Assert(m.EnsureDead(), gc.IsNil)
   713  	s.checkStopInstances(c, inst)
   714  	s.waitRemoved(c, m)
   715  }
   716  
   717  func (s *ProvisionerSuite) TestProvisioningMachinesWithRequestedNetworks(c *gc.C) {
   718  	p := s.newEnvironProvisioner(c)
   719  	defer stop(c, p)
   720  
   721  	// Add and provision a machine with networks specified.
   722  	requestedNetworks := []string{"net1", "net2"}
   723  	cons := constraints.MustParse(s.defaultConstraints.String(), "networks=^net3,^net4")
   724  	expectNetworkInfo := []network.InterfaceInfo{{
   725  		MACAddress:    "aa:bb:cc:dd:ee:f0",
   726  		InterfaceName: "eth0",
   727  		ProviderId:    "net1",
   728  		NetworkName:   "net1",
   729  		VLANTag:       0,
   730  		CIDR:          "0.1.2.0/24",
   731  	}, {
   732  		MACAddress:    "aa:bb:cc:dd:ee:f1",
   733  		InterfaceName: "eth1",
   734  		ProviderId:    "net2",
   735  		NetworkName:   "net2",
   736  		VLANTag:       1,
   737  		CIDR:          "0.2.2.0/24",
   738  	}}
   739  	m, err := s.addMachineWithRequestedNetworks(requestedNetworks, cons)
   740  	c.Assert(err, jc.ErrorIsNil)
   741  	inst := s.checkStartInstanceCustom(
   742  		c, m, "pork", cons,
   743  		requestedNetworks, expectNetworkInfo, false,
   744  		nil, true,
   745  	)
   746  
   747  	_, err = s.State.Network("net1")
   748  	c.Assert(err, jc.ErrorIsNil)
   749  	_, err = s.State.Network("net2")
   750  	c.Assert(err, jc.ErrorIsNil)
   751  	_, err = s.State.Network("net3")
   752  	c.Assert(err, jc.Satisfies, errors.IsNotFound)
   753  	_, err = s.State.Network("net4")
   754  	c.Assert(err, jc.Satisfies, errors.IsNotFound)
   755  	ifaces, err := m.NetworkInterfaces()
   756  	c.Assert(err, jc.ErrorIsNil)
   757  	c.Assert(ifaces, gc.HasLen, 2)
   758  
   759  	// Cleanup.
   760  	c.Assert(m.EnsureDead(), gc.IsNil)
   761  	s.checkStopInstances(c, inst)
   762  	s.waitRemoved(c, m)
   763  }
   764  
   765  func (s *ProvisionerSuite) TestSetInstanceInfoFailureSetsErrorStatusAndStopsInstanceButKeepsGoing(c *gc.C) {
   766  	p := s.newEnvironProvisioner(c)
   767  	defer stop(c, p)
   768  
   769  	// Add and provision a machine with networks specified.
   770  	networks := []string{"bad-net1"}
   771  	// "bad-" prefix for networks causes dummy provider to report
   772  	// invalid network.InterfaceInfo.
   773  	expectNetworkInfo := []network.InterfaceInfo{
   774  		{ProviderId: "bad-net1", NetworkName: "bad-net1", CIDR: "invalid"},
   775  	}
   776  	m, err := s.addMachineWithRequestedNetworks(networks, constraints.Value{})
   777  	c.Assert(err, jc.ErrorIsNil)
   778  	inst := s.checkStartInstanceCustom(
   779  		c, m, "pork", constraints.Value{},
   780  		networks, expectNetworkInfo, false,
   781  		nil, false,
   782  	)
   783  
   784  	// Ensure machine error status was set.
   785  	t0 := time.Now()
   786  	for time.Since(t0) < coretesting.LongWait {
   787  		// And check the machine status is set to error.
   788  		status, info, _, err := m.Status()
   789  		c.Assert(err, jc.ErrorIsNil)
   790  		if status == state.StatusPending {
   791  			time.Sleep(coretesting.ShortWait)
   792  			continue
   793  		}
   794  		c.Assert(status, gc.Equals, state.StatusError)
   795  		c.Assert(info, gc.Matches, `cannot record provisioning info for "dummyenv-0": cannot add network "bad-net1": invalid CIDR address: invalid`)
   796  		break
   797  	}
   798  	s.checkStopInstances(c, inst)
   799  
   800  	// Make sure the task didn't stop with an error
   801  	died := make(chan error)
   802  	go func() {
   803  		died <- p.Wait()
   804  	}()
   805  	select {
   806  	case <-time.After(coretesting.LongWait):
   807  	case err = <-died:
   808  		c.Fatalf("provisioner task died unexpectedly with err: %v", err)
   809  	}
   810  
   811  	// Restart the PA to make sure the machine is not retried.
   812  	stop(c, p)
   813  	p = s.newEnvironProvisioner(c)
   814  	defer stop(c, p)
   815  
   816  	s.checkNoOperations(c)
   817  }
   818  
   819  func (s *ProvisionerSuite) TestProvisioningDoesNotOccurWithAnInvalidEnvironment(c *gc.C) {
   820  	s.invalidateEnvironment(c)
   821  
   822  	p := s.newEnvironProvisioner(c)
   823  	defer stop(c, p)
   824  
   825  	// try to create a machine
   826  	_, err := s.addMachine()
   827  	c.Assert(err, jc.ErrorIsNil)
   828  
   829  	// the PA should not create it
   830  	s.checkNoOperations(c)
   831  }
   832  
   833  func (s *ProvisionerSuite) TestProvisioningOccursWithFixedEnvironment(c *gc.C) {
   834  	s.invalidateEnvironment(c)
   835  
   836  	p := s.newEnvironProvisioner(c)
   837  	defer stop(c, p)
   838  
   839  	// try to create a machine
   840  	m, err := s.addMachine()
   841  	c.Assert(err, jc.ErrorIsNil)
   842  
   843  	// the PA should not create it
   844  	s.checkNoOperations(c)
   845  
   846  	err = s.fixEnvironment(c)
   847  	c.Assert(err, jc.ErrorIsNil)
   848  
   849  	s.checkStartInstanceNoSecureConnection(c, m)
   850  }
   851  
   852  func (s *ProvisionerSuite) TestProvisioningDoesOccurAfterInvalidEnvironmentPublished(c *gc.C) {
   853  	s.PatchValue(provisioner.GetToolsFinder, func(*apiprovisioner.State) provisioner.ToolsFinder {
   854  		return mockToolsFinder{}
   855  	})
   856  	p := s.newEnvironProvisioner(c)
   857  	defer stop(c, p)
   858  
   859  	// place a new machine into the state
   860  	m, err := s.addMachine()
   861  	c.Assert(err, jc.ErrorIsNil)
   862  
   863  	s.checkStartInstanceNoSecureConnection(c, m)
   864  
   865  	s.invalidateEnvironment(c)
   866  
   867  	// create a second machine
   868  	m, err = s.addMachine()
   869  	c.Assert(err, jc.ErrorIsNil)
   870  
   871  	// the PA should create it using the old environment
   872  	s.checkStartInstanceNoSecureConnection(c, m)
   873  }
   874  
   875  func (s *ProvisionerSuite) TestProvisioningDoesNotProvisionTheSameMachineAfterRestart(c *gc.C) {
   876  	p := s.newEnvironProvisioner(c)
   877  	defer stop(c, p)
   878  
   879  	// create a machine
   880  	m, err := s.addMachine()
   881  	c.Assert(err, jc.ErrorIsNil)
   882  	s.checkStartInstanceNoSecureConnection(c, m)
   883  
   884  	// restart the PA
   885  	stop(c, p)
   886  	p = s.newEnvironProvisioner(c)
   887  	defer stop(c, p)
   888  
   889  	// check that there is only one machine provisioned.
   890  	machines, err := s.State.AllMachines()
   891  	c.Assert(err, jc.ErrorIsNil)
   892  	c.Check(len(machines), gc.Equals, 2)
   893  	c.Check(machines[0].Id(), gc.Equals, "0")
   894  	c.Check(machines[1].CheckProvisioned("fake_nonce"), jc.IsFalse)
   895  
   896  	// the PA should not create it a second time
   897  	s.checkNoOperations(c)
   898  }
   899  
   900  func (s *ProvisionerSuite) TestDyingMachines(c *gc.C) {
   901  	p := s.newEnvironProvisioner(c)
   902  	defer stop(c, p)
   903  
   904  	// provision a machine
   905  	m0, err := s.addMachine()
   906  	c.Assert(err, jc.ErrorIsNil)
   907  	s.checkStartInstanceNoSecureConnection(c, m0)
   908  
   909  	// stop the provisioner and make the machine dying
   910  	stop(c, p)
   911  	err = m0.Destroy()
   912  	c.Assert(err, jc.ErrorIsNil)
   913  
   914  	// add a new, dying, unprovisioned machine
   915  	m1, err := s.addMachine()
   916  	c.Assert(err, jc.ErrorIsNil)
   917  	err = m1.Destroy()
   918  	c.Assert(err, jc.ErrorIsNil)
   919  
   920  	// start the provisioner and wait for it to reap the useless machine
   921  	p = s.newEnvironProvisioner(c)
   922  	defer stop(c, p)
   923  	s.checkNoOperations(c)
   924  	s.waitRemoved(c, m1)
   925  
   926  	// verify the other one's still fine
   927  	err = m0.Refresh()
   928  	c.Assert(err, jc.ErrorIsNil)
   929  	c.Assert(m0.Life(), gc.Equals, state.Dying)
   930  }
   931  
   932  func (s *ProvisionerSuite) TestProvisioningRecoversAfterInvalidEnvironmentPublished(c *gc.C) {
   933  	s.PatchValue(provisioner.GetToolsFinder, func(*apiprovisioner.State) provisioner.ToolsFinder {
   934  		return mockToolsFinder{}
   935  	})
   936  	p := s.newEnvironProvisioner(c)
   937  	defer stop(c, p)
   938  
   939  	// place a new machine into the state
   940  	m, err := s.addMachine()
   941  	c.Assert(err, jc.ErrorIsNil)
   942  	s.checkStartInstanceNoSecureConnection(c, m)
   943  
   944  	s.invalidateEnvironment(c)
   945  	s.BackingState.StartSync()
   946  
   947  	// create a second machine
   948  	m, err = s.addMachine()
   949  	c.Assert(err, jc.ErrorIsNil)
   950  
   951  	// the PA should create it using the old environment
   952  	s.checkStartInstanceNoSecureConnection(c, m)
   953  
   954  	err = s.fixEnvironment(c)
   955  	c.Assert(err, jc.ErrorIsNil)
   956  
   957  	// insert our observer
   958  	cfgObserver := make(chan *config.Config, 1)
   959  	provisioner.SetObserver(p, cfgObserver)
   960  
   961  	err = s.State.UpdateEnvironConfig(map[string]interface{}{"secret": "beef"}, nil, nil)
   962  	c.Assert(err, jc.ErrorIsNil)
   963  
   964  	s.BackingState.StartSync()
   965  
   966  	// wait for the PA to load the new configuration
   967  	select {
   968  	case <-cfgObserver:
   969  	case <-time.After(coretesting.LongWait):
   970  		c.Fatalf("PA did not action config change")
   971  	}
   972  
   973  	// create a third machine
   974  	m, err = s.addMachine()
   975  	c.Assert(err, jc.ErrorIsNil)
   976  
   977  	// the PA should create it using the new environment
   978  	s.checkStartInstanceCustom(c, m, "beef", s.defaultConstraints, nil, nil, false, nil, true)
   979  }
   980  
   981  type mockMachineGetter struct{}
   982  
   983  func (*mockMachineGetter) Machine(names.MachineTag) (*apiprovisioner.Machine, error) {
   984  	return nil, fmt.Errorf("error")
   985  }
   986  
   987  func (*mockMachineGetter) MachinesWithTransientErrors() ([]*apiprovisioner.Machine, []params.StatusResult, error) {
   988  	return nil, nil, fmt.Errorf("error")
   989  }
   990  
   991  func (s *ProvisionerSuite) TestMachineErrorsRetainInstances(c *gc.C) {
   992  	task := s.newProvisionerTask(c, config.HarvestAll, s.Environ, s.provisioner, mockToolsFinder{})
   993  	defer stop(c, task)
   994  
   995  	// create a machine
   996  	m0, err := s.addMachine()
   997  	c.Assert(err, jc.ErrorIsNil)
   998  	s.checkStartInstance(c, m0)
   999  
  1000  	// create an instance out of band
  1001  	s.startUnknownInstance(c, "999")
  1002  
  1003  	// start the provisioner and ensure it doesn't kill any instances if there are error getting machines
  1004  	task = s.newProvisionerTask(
  1005  		c,
  1006  		config.HarvestAll,
  1007  		s.Environ,
  1008  		&mockMachineGetter{},
  1009  		&mockToolsFinder{},
  1010  	)
  1011  	defer func() {
  1012  		err := task.Stop()
  1013  		c.Assert(err, gc.ErrorMatches, ".*failed to get machine.*")
  1014  	}()
  1015  	s.checkNoOperations(c)
  1016  }
  1017  
  1018  func (s *ProvisionerSuite) TestProvisionerObservesConfigChanges(c *gc.C) {
  1019  	p := s.newEnvironProvisioner(c)
  1020  	defer stop(c, p)
  1021  
  1022  	// Inject our observer into the provisioner
  1023  	cfgObserver := make(chan *config.Config, 1)
  1024  	provisioner.SetObserver(p, cfgObserver)
  1025  
  1026  	// Switch to reaping on All machines.
  1027  	attrs := map[string]interface{}{
  1028  		config.ProvisionerHarvestModeKey: config.HarvestAll.String(),
  1029  	}
  1030  	err := s.State.UpdateEnvironConfig(attrs, nil, nil)
  1031  	c.Assert(err, jc.ErrorIsNil)
  1032  
  1033  	s.BackingState.StartSync()
  1034  
  1035  	// Wait for the PA to load the new configuration.
  1036  	select {
  1037  	case newCfg := <-cfgObserver:
  1038  		c.Assert(
  1039  			newCfg.ProvisionerHarvestMode().String(),
  1040  			gc.Equals,
  1041  			config.HarvestAll.String(),
  1042  		)
  1043  	case <-time.After(coretesting.LongWait):
  1044  		c.Fatalf("PA did not action config change")
  1045  	}
  1046  }
  1047  
  1048  func (s *ProvisionerSuite) newProvisionerTask(
  1049  	c *gc.C,
  1050  	harvestingMethod config.HarvestMode,
  1051  	broker environs.InstanceBroker,
  1052  	machineGetter provisioner.MachineGetter,
  1053  	toolsFinder provisioner.ToolsFinder,
  1054  ) provisioner.ProvisionerTask {
  1055  
  1056  	machineWatcher, err := s.provisioner.WatchEnvironMachines()
  1057  	c.Assert(err, jc.ErrorIsNil)
  1058  	retryWatcher, err := s.provisioner.WatchMachineErrorRetry()
  1059  	c.Assert(err, jc.ErrorIsNil)
  1060  	auth, err := authentication.NewAPIAuthenticator(s.provisioner)
  1061  	c.Assert(err, jc.ErrorIsNil)
  1062  
  1063  	return provisioner.NewProvisionerTask(
  1064  		names.NewMachineTag("0"),
  1065  		harvestingMethod,
  1066  		machineGetter,
  1067  		toolsFinder,
  1068  		machineWatcher,
  1069  		retryWatcher,
  1070  		broker,
  1071  		auth,
  1072  		imagemetadata.ReleasedStream,
  1073  		true,
  1074  	)
  1075  }
  1076  
  1077  func (s *ProvisionerSuite) TestHarvestNoneReapsNothing(c *gc.C) {
  1078  
  1079  	task := s.newProvisionerTask(c, config.HarvestDestroyed, s.Environ, s.provisioner, mockToolsFinder{})
  1080  	defer stop(c, task)
  1081  	task.SetHarvestMode(config.HarvestNone)
  1082  
  1083  	// Create a machine and an unknown instance.
  1084  	m0, err := s.addMachine()
  1085  	c.Assert(err, jc.ErrorIsNil)
  1086  	s.checkStartInstance(c, m0)
  1087  	s.startUnknownInstance(c, "999")
  1088  
  1089  	// Mark the first machine as dead.
  1090  	c.Assert(m0.EnsureDead(), gc.IsNil)
  1091  
  1092  	// Ensure we're doing nothing.
  1093  	s.checkNoOperations(c)
  1094  }
  1095  
  1096  func (s *ProvisionerSuite) TestHarvestUnknownReapsOnlyUnknown(c *gc.C) {
  1097  
  1098  	task := s.newProvisionerTask(c,
  1099  		config.HarvestDestroyed,
  1100  		s.Environ,
  1101  		s.provisioner,
  1102  		mockToolsFinder{},
  1103  	)
  1104  	defer stop(c, task)
  1105  	task.SetHarvestMode(config.HarvestUnknown)
  1106  
  1107  	// Create a machine and an unknown instance.
  1108  	m0, err := s.addMachine()
  1109  	c.Assert(err, jc.ErrorIsNil)
  1110  	i0 := s.checkStartInstance(c, m0)
  1111  	i1 := s.startUnknownInstance(c, "999")
  1112  
  1113  	// Mark the first machine as dead.
  1114  	c.Assert(m0.EnsureDead(), gc.IsNil)
  1115  
  1116  	// When only harvesting unknown machines, only one of the machines
  1117  	// is stopped.
  1118  	s.checkStopSomeInstances(c, []instance.Instance{i1}, []instance.Instance{i0})
  1119  	s.waitRemoved(c, m0)
  1120  }
  1121  
  1122  func (s *ProvisionerSuite) TestHarvestDestroyedReapsOnlyDestroyed(c *gc.C) {
  1123  
  1124  	task := s.newProvisionerTask(
  1125  		c,
  1126  		config.HarvestDestroyed,
  1127  		s.Environ,
  1128  		s.provisioner,
  1129  		mockToolsFinder{},
  1130  	)
  1131  	defer stop(c, task)
  1132  
  1133  	// Create a machine and an unknown instance.
  1134  	m0, err := s.addMachine()
  1135  	c.Assert(err, jc.ErrorIsNil)
  1136  	i0 := s.checkStartInstance(c, m0)
  1137  	i1 := s.startUnknownInstance(c, "999")
  1138  
  1139  	// Mark the first machine as dead.
  1140  	c.Assert(m0.EnsureDead(), gc.IsNil)
  1141  
  1142  	// When only harvesting destroyed machines, only one of the
  1143  	// machines is stopped.
  1144  	s.checkStopSomeInstances(c, []instance.Instance{i0}, []instance.Instance{i1})
  1145  	s.waitRemoved(c, m0)
  1146  }
  1147  
  1148  func (s *ProvisionerSuite) TestHarvestAllReapsAllTheThings(c *gc.C) {
  1149  
  1150  	task := s.newProvisionerTask(c,
  1151  		config.HarvestDestroyed,
  1152  		s.Environ,
  1153  		s.provisioner,
  1154  		mockToolsFinder{},
  1155  	)
  1156  	defer stop(c, task)
  1157  	task.SetHarvestMode(config.HarvestAll)
  1158  
  1159  	// Create a machine and an unknown instance.
  1160  	m0, err := s.addMachine()
  1161  	c.Assert(err, jc.ErrorIsNil)
  1162  	i0 := s.checkStartInstance(c, m0)
  1163  	i1 := s.startUnknownInstance(c, "999")
  1164  
  1165  	// Mark the first machine as dead.
  1166  	c.Assert(m0.EnsureDead(), gc.IsNil)
  1167  
  1168  	// Everything must die!
  1169  	s.checkStopSomeInstances(c, []instance.Instance{i0, i1}, []instance.Instance{})
  1170  	s.waitRemoved(c, m0)
  1171  }
  1172  
  1173  func (s *ProvisionerSuite) TestProvisionerRetriesTransientErrors(c *gc.C) {
  1174  	s.PatchValue(&apiserverprovisioner.ErrorRetryWaitDelay, 5*time.Millisecond)
  1175  	e := &mockBroker{Environ: s.Environ, retryCount: make(map[string]int)}
  1176  	task := s.newProvisionerTask(c, config.HarvestAll, e, s.provisioner, mockToolsFinder{})
  1177  	defer stop(c, task)
  1178  
  1179  	// Provision some machines, some will be started first time,
  1180  	// another will require retries.
  1181  	m1, err := s.addMachine()
  1182  	c.Assert(err, jc.ErrorIsNil)
  1183  	s.checkStartInstance(c, m1)
  1184  	m2, err := s.addMachine()
  1185  	c.Assert(err, jc.ErrorIsNil)
  1186  	s.checkStartInstance(c, m2)
  1187  	m3, err := s.addMachine()
  1188  	c.Assert(err, jc.ErrorIsNil)
  1189  	m4, err := s.addMachine()
  1190  	c.Assert(err, jc.ErrorIsNil)
  1191  
  1192  	// mockBroker will fail to start machine-3 several times;
  1193  	// keep setting the transient flag to retry until the
  1194  	// instance has started.
  1195  	thatsAllFolks := make(chan struct{})
  1196  	go func() {
  1197  		for {
  1198  			select {
  1199  			case <-thatsAllFolks:
  1200  				return
  1201  			case <-time.After(coretesting.ShortWait):
  1202  				err := m3.SetStatus(state.StatusError, "info", map[string]interface{}{"transient": true})
  1203  				c.Assert(err, jc.ErrorIsNil)
  1204  			}
  1205  		}
  1206  	}()
  1207  	s.checkStartInstance(c, m3)
  1208  	close(thatsAllFolks)
  1209  
  1210  	// Machine 4 is never provisioned.
  1211  	status, _, _, err := m4.Status()
  1212  	c.Assert(err, jc.ErrorIsNil)
  1213  	c.Assert(status, gc.Equals, state.StatusError)
  1214  	_, err = m4.InstanceId()
  1215  	c.Assert(err, jc.Satisfies, errors.IsNotProvisioned)
  1216  }
  1217  
  1218  func (s *ProvisionerSuite) TestProvisionerObservesMachineJobs(c *gc.C) {
  1219  	s.PatchValue(&apiserverprovisioner.ErrorRetryWaitDelay, 5*time.Millisecond)
  1220  	broker := &mockBroker{Environ: s.Environ, retryCount: make(map[string]int)}
  1221  	task := s.newProvisionerTask(c, config.HarvestAll, broker, s.provisioner, mockToolsFinder{})
  1222  	defer stop(c, task)
  1223  
  1224  	added := s.ensureAvailability(c, 3)
  1225  	c.Assert(added, gc.HasLen, 2)
  1226  	byId := make(map[string]*state.Machine)
  1227  	for _, m := range added {
  1228  		byId[m.Id()] = m
  1229  	}
  1230  	for _, id := range broker.ids {
  1231  		s.checkStartInstance(c, byId[id])
  1232  	}
  1233  }
  1234  
  1235  type mockBroker struct {
  1236  	environs.Environ
  1237  	retryCount map[string]int
  1238  	ids        []string
  1239  }
  1240  
  1241  func (b *mockBroker) StartInstance(args environs.StartInstanceParams) (*environs.StartInstanceResult, error) {
  1242  	// All machines except machines 3, 4 are provisioned successfully the first time.
  1243  	// Machines 3 is provisioned after some attempts have been made.
  1244  	// Machine 4 is never provisioned.
  1245  	id := args.MachineConfig.MachineId
  1246  	// record ids so we can call checkStartInstance in the appropriate order.
  1247  	b.ids = append(b.ids, id)
  1248  	retries := b.retryCount[id]
  1249  	if (id != "3" && id != "4") || retries > 2 {
  1250  		return b.Environ.StartInstance(args)
  1251  	} else {
  1252  		b.retryCount[id] = retries + 1
  1253  	}
  1254  	return nil, fmt.Errorf("error: some error")
  1255  }
  1256  
  1257  type mockToolsFinder struct {
  1258  }
  1259  
  1260  func (f mockToolsFinder) FindTools(number version.Number, series string, arch *string) (coretools.List, error) {
  1261  	v, err := version.ParseBinary(fmt.Sprintf("%s-%s-%s", number, series, version.Current.Arch))
  1262  	if err != nil {
  1263  		return nil, err
  1264  	}
  1265  	if arch != nil {
  1266  		v.Arch = *arch
  1267  	}
  1268  	return coretools.List{&coretools.Tools{Version: v}}, nil
  1269  }