github.com/wallyworld/juju@v0.0.0-20161013125918-6cf1bc9d917a/state/model_test.go (about)

     1  // Copyright 2013 Canonical Ltd.
     2  // Licensed under the AGPLv3, see LICENCE file for details.
     3  
     4  package state_test
     5  
     6  import (
     7  	"fmt"
     8  
     9  	"github.com/juju/errors"
    10  	gitjujutesting "github.com/juju/testing"
    11  	jc "github.com/juju/testing/checkers"
    12  	"github.com/juju/utils"
    13  	"github.com/juju/utils/clock"
    14  	gc "gopkg.in/check.v1"
    15  	"gopkg.in/juju/names.v2"
    16  
    17  	"github.com/juju/juju/cloud"
    18  	"github.com/juju/juju/environs/config"
    19  	"github.com/juju/juju/mongo/mongotest"
    20  	"github.com/juju/juju/permission"
    21  	"github.com/juju/juju/state"
    22  	statetesting "github.com/juju/juju/state/testing"
    23  	"github.com/juju/juju/storage"
    24  	"github.com/juju/juju/testing"
    25  	"github.com/juju/juju/testing/factory"
    26  )
    27  
    28  type ModelSuite struct {
    29  	ConnSuite
    30  }
    31  
    32  var _ = gc.Suite(&ModelSuite{})
    33  
    34  func (s *ModelSuite) TestModel(c *gc.C) {
    35  	model, err := s.State.Model()
    36  	c.Assert(err, jc.ErrorIsNil)
    37  
    38  	expectedTag := names.NewModelTag(model.UUID())
    39  	c.Assert(model.Tag(), gc.Equals, expectedTag)
    40  	c.Assert(model.ControllerTag(), gc.Equals, s.State.ControllerTag())
    41  	c.Assert(model.Name(), gc.Equals, "testenv")
    42  	c.Assert(model.Owner(), gc.Equals, s.Owner)
    43  	c.Assert(model.Life(), gc.Equals, state.Alive)
    44  	c.Assert(model.MigrationMode(), gc.Equals, state.MigrationModeNone)
    45  }
    46  
    47  func (s *ModelSuite) TestModelDestroy(c *gc.C) {
    48  	env, err := s.State.Model()
    49  	c.Assert(err, jc.ErrorIsNil)
    50  
    51  	err = env.Destroy()
    52  	c.Assert(err, jc.ErrorIsNil)
    53  	err = env.Refresh()
    54  	c.Assert(err, jc.ErrorIsNil)
    55  	c.Assert(env.Life(), gc.Equals, state.Dying)
    56  }
    57  
    58  func (s *ModelSuite) TestNewModelNonExistentLocalUser(c *gc.C) {
    59  	cfg, _ := s.createTestModelConfig(c)
    60  	owner := names.NewUserTag("non-existent@local")
    61  
    62  	_, _, err := s.State.NewModel(state.ModelArgs{
    63  		CloudName:   "dummy",
    64  		CloudRegion: "dummy-region",
    65  		Config:      cfg,
    66  		Owner:       owner,
    67  		StorageProviderRegistry: storage.StaticProviderRegistry{},
    68  	})
    69  	c.Assert(err, gc.ErrorMatches, `cannot create model: user "non-existent" not found`)
    70  }
    71  
    72  func (s *ModelSuite) TestNewModelSameUserSameNameFails(c *gc.C) {
    73  	cfg, _ := s.createTestModelConfig(c)
    74  	owner := s.Factory.MakeUser(c, nil).UserTag()
    75  
    76  	// Create the first model.
    77  	_, st1, err := s.State.NewModel(state.ModelArgs{
    78  		CloudName:   "dummy",
    79  		CloudRegion: "dummy-region",
    80  		Config:      cfg,
    81  		Owner:       owner,
    82  		StorageProviderRegistry: storage.StaticProviderRegistry{},
    83  	})
    84  	c.Assert(err, jc.ErrorIsNil)
    85  	defer st1.Close()
    86  
    87  	// Attempt to create another model with a different UUID but the
    88  	// same owner and name as the first.
    89  	newUUID, err := utils.NewUUID()
    90  	c.Assert(err, jc.ErrorIsNil)
    91  	cfg2 := testing.CustomModelConfig(c, testing.Attrs{
    92  		"name": cfg.Name(),
    93  		"uuid": newUUID.String(),
    94  	})
    95  	_, _, err = s.State.NewModel(state.ModelArgs{
    96  		CloudName:   "dummy",
    97  		CloudRegion: "dummy-region",
    98  		Config:      cfg2,
    99  		Owner:       owner,
   100  		StorageProviderRegistry: storage.StaticProviderRegistry{},
   101  	})
   102  	errMsg := fmt.Sprintf("model %q for %s already exists", cfg2.Name(), owner.Id())
   103  	c.Assert(err, gc.ErrorMatches, errMsg)
   104  	c.Assert(errors.IsAlreadyExists(err), jc.IsTrue)
   105  
   106  	// Remove the first model.
   107  	env1, err := st1.Model()
   108  	c.Assert(err, jc.ErrorIsNil)
   109  	err = env1.Destroy()
   110  	c.Assert(err, jc.ErrorIsNil)
   111  	// Destroy only sets the model to dying and RemoveAllModelDocs can
   112  	// only be called on a dead model. Normally, the environ's lifecycle
   113  	// would be set to dead after machines and services have been cleaned up.
   114  	err = state.SetModelLifeDead(st1, env1.ModelTag().Id())
   115  	c.Assert(err, jc.ErrorIsNil)
   116  	err = st1.RemoveAllModelDocs()
   117  	c.Assert(err, jc.ErrorIsNil)
   118  
   119  	// We should now be able to create the other model.
   120  	env2, st2, err := s.State.NewModel(state.ModelArgs{
   121  		CloudName:   "dummy",
   122  		CloudRegion: "dummy-region",
   123  		Config:      cfg2,
   124  		Owner:       owner,
   125  		StorageProviderRegistry: storage.StaticProviderRegistry{},
   126  	})
   127  	c.Assert(err, jc.ErrorIsNil)
   128  	defer st2.Close()
   129  	c.Assert(env2, gc.NotNil)
   130  	c.Assert(st2, gc.NotNil)
   131  }
   132  
   133  func (s *ModelSuite) TestNewModel(c *gc.C) {
   134  	cfg, uuid := s.createTestModelConfig(c)
   135  	owner := names.NewUserTag("test@remote")
   136  
   137  	model, st, err := s.State.NewModel(state.ModelArgs{
   138  		CloudName:   "dummy",
   139  		CloudRegion: "dummy-region",
   140  		Config:      cfg,
   141  		Owner:       owner,
   142  		StorageProviderRegistry: storage.StaticProviderRegistry{},
   143  	})
   144  	c.Assert(err, jc.ErrorIsNil)
   145  	defer st.Close()
   146  
   147  	modelTag := names.NewModelTag(uuid)
   148  	assertModelMatches := func(model *state.Model) {
   149  		c.Assert(model.UUID(), gc.Equals, modelTag.Id())
   150  		c.Assert(model.Tag(), gc.Equals, modelTag)
   151  		c.Assert(model.ControllerTag(), gc.Equals, s.State.ControllerTag())
   152  		c.Assert(model.Owner(), gc.Equals, owner)
   153  		c.Assert(model.Name(), gc.Equals, "testing")
   154  		c.Assert(model.Life(), gc.Equals, state.Alive)
   155  	}
   156  	assertModelMatches(model)
   157  
   158  	// Since the model tag for the State connection is different,
   159  	// asking for this model through FindEntity returns a not found error.
   160  	model, err = s.State.GetModel(modelTag)
   161  	c.Assert(err, jc.ErrorIsNil)
   162  	assertModelMatches(model)
   163  
   164  	model, err = st.Model()
   165  	c.Assert(err, jc.ErrorIsNil)
   166  	assertModelMatches(model)
   167  
   168  	_, err = s.State.FindEntity(modelTag)
   169  	c.Assert(err, jc.Satisfies, errors.IsNotFound)
   170  
   171  	entity, err := st.FindEntity(modelTag)
   172  	c.Assert(err, jc.ErrorIsNil)
   173  	c.Assert(entity.Tag(), gc.Equals, modelTag)
   174  
   175  	// Ensure the model is functional by adding a machine
   176  	_, err = st.AddMachine("quantal", state.JobHostUnits)
   177  	c.Assert(err, jc.ErrorIsNil)
   178  }
   179  
   180  func (s *ModelSuite) TestNewModelImportingMode(c *gc.C) {
   181  	cfg, _ := s.createTestModelConfig(c)
   182  	owner := names.NewUserTag("test@remote")
   183  
   184  	env, st, err := s.State.NewModel(state.ModelArgs{
   185  		CloudName:               "dummy",
   186  		CloudRegion:             "dummy-region",
   187  		Config:                  cfg,
   188  		Owner:                   owner,
   189  		MigrationMode:           state.MigrationModeImporting,
   190  		StorageProviderRegistry: storage.StaticProviderRegistry{},
   191  	})
   192  	c.Assert(err, jc.ErrorIsNil)
   193  	defer st.Close()
   194  
   195  	c.Assert(env.MigrationMode(), gc.Equals, state.MigrationModeImporting)
   196  }
   197  
   198  func (s *ModelSuite) TestSetMigrationMode(c *gc.C) {
   199  	cfg, _ := s.createTestModelConfig(c)
   200  	owner := names.NewUserTag("test@remote")
   201  
   202  	env, st, err := s.State.NewModel(state.ModelArgs{
   203  		CloudName:   "dummy",
   204  		CloudRegion: "dummy-region",
   205  		Config:      cfg,
   206  		Owner:       owner,
   207  		StorageProviderRegistry: storage.StaticProviderRegistry{},
   208  	})
   209  	c.Assert(err, jc.ErrorIsNil)
   210  	defer st.Close()
   211  
   212  	err = env.SetMigrationMode(state.MigrationModeExporting)
   213  	c.Assert(err, jc.ErrorIsNil)
   214  	c.Assert(env.MigrationMode(), gc.Equals, state.MigrationModeExporting)
   215  }
   216  
   217  func (s *ModelSuite) TestControllerModel(c *gc.C) {
   218  	model, err := s.State.ControllerModel()
   219  	c.Assert(err, jc.ErrorIsNil)
   220  
   221  	expectedTag := names.NewModelTag(model.UUID())
   222  	c.Assert(model.Tag(), gc.Equals, expectedTag)
   223  	c.Assert(model.ControllerTag(), gc.Equals, s.State.ControllerTag())
   224  	c.Assert(model.Name(), gc.Equals, "testenv")
   225  	c.Assert(model.Owner(), gc.Equals, s.Owner)
   226  	c.Assert(model.Life(), gc.Equals, state.Alive)
   227  }
   228  
   229  func (s *ModelSuite) TestControllerModelAccessibleFromOtherModels(c *gc.C) {
   230  	cfg, _ := s.createTestModelConfig(c)
   231  	_, st, err := s.State.NewModel(state.ModelArgs{
   232  		CloudName:   "dummy",
   233  		CloudRegion: "dummy-region",
   234  		Config:      cfg,
   235  		Owner:       names.NewUserTag("test@remote"),
   236  		StorageProviderRegistry: storage.StaticProviderRegistry{},
   237  	})
   238  	c.Assert(err, jc.ErrorIsNil)
   239  	defer st.Close()
   240  
   241  	env, err := st.ControllerModel()
   242  	c.Assert(err, jc.ErrorIsNil)
   243  	c.Assert(env.Tag(), gc.Equals, s.modelTag)
   244  	c.Assert(env.Name(), gc.Equals, "testenv")
   245  	c.Assert(env.Owner(), gc.Equals, s.Owner)
   246  	c.Assert(env.Life(), gc.Equals, state.Alive)
   247  }
   248  
   249  func (s *ModelSuite) TestConfigForControllerEnv(c *gc.C) {
   250  	otherState := s.Factory.MakeModel(c, &factory.ModelParams{Name: "other"})
   251  	defer otherState.Close()
   252  
   253  	env, err := otherState.GetModel(s.modelTag)
   254  	c.Assert(err, jc.ErrorIsNil)
   255  
   256  	conf, err := env.Config()
   257  	c.Assert(err, jc.ErrorIsNil)
   258  	c.Assert(conf.Name(), gc.Equals, "testenv")
   259  	c.Assert(conf.UUID(), gc.Equals, s.modelTag.Id())
   260  }
   261  
   262  func (s *ModelSuite) TestConfigForOtherEnv(c *gc.C) {
   263  	otherState := s.Factory.MakeModel(c, &factory.ModelParams{Name: "other"})
   264  	defer otherState.Close()
   265  	otherEnv, err := otherState.Model()
   266  	c.Assert(err, jc.ErrorIsNil)
   267  
   268  	// By getting the model through a different state connection,
   269  	// the underlying state pointer in the *state.Model struct has
   270  	// a different model tag.
   271  	env, err := s.State.GetModel(otherEnv.ModelTag())
   272  	c.Assert(err, jc.ErrorIsNil)
   273  
   274  	conf, err := env.Config()
   275  	c.Assert(err, jc.ErrorIsNil)
   276  	c.Assert(conf.Name(), gc.Equals, "other")
   277  	c.Assert(conf.UUID(), gc.Equals, otherEnv.UUID())
   278  }
   279  
   280  // createTestModelConfig returns a new model config and its UUID for testing.
   281  func (s *ModelSuite) createTestModelConfig(c *gc.C) (*config.Config, string) {
   282  	return createTestModelConfig(c, s.modelTag.Id())
   283  }
   284  
   285  func createTestModelConfig(c *gc.C, controllerUUID string) (*config.Config, string) {
   286  	uuid, err := utils.NewUUID()
   287  	c.Assert(err, jc.ErrorIsNil)
   288  	if controllerUUID == "" {
   289  		controllerUUID = uuid.String()
   290  	}
   291  	return testing.CustomModelConfig(c, testing.Attrs{
   292  		"name": "testing",
   293  		"uuid": uuid.String(),
   294  	}), uuid.String()
   295  }
   296  
   297  func (s *ModelSuite) TestModelConfigSameEnvAsState(c *gc.C) {
   298  	env, err := s.State.Model()
   299  	c.Assert(err, jc.ErrorIsNil)
   300  	cfg, err := env.Config()
   301  	c.Assert(err, jc.ErrorIsNil)
   302  	c.Assert(cfg.UUID(), gc.Equals, s.State.ModelUUID())
   303  }
   304  
   305  func (s *ModelSuite) TestModelConfigDifferentEnvThanState(c *gc.C) {
   306  	otherState := s.Factory.MakeModel(c, nil)
   307  	defer otherState.Close()
   308  	env, err := otherState.Model()
   309  	c.Assert(err, jc.ErrorIsNil)
   310  	cfg, err := env.Config()
   311  	c.Assert(err, jc.ErrorIsNil)
   312  	uuid := cfg.UUID()
   313  	c.Assert(uuid, gc.Equals, env.UUID())
   314  	c.Assert(uuid, gc.Not(gc.Equals), s.State.ModelUUID())
   315  }
   316  
   317  func (s *ModelSuite) TestDestroyControllerModel(c *gc.C) {
   318  	env, err := s.State.Model()
   319  	c.Assert(err, jc.ErrorIsNil)
   320  	err = env.Destroy()
   321  	c.Assert(err, jc.ErrorIsNil)
   322  }
   323  
   324  func (s *ModelSuite) TestDestroyOtherModel(c *gc.C) {
   325  	st2 := s.Factory.MakeModel(c, nil)
   326  	defer st2.Close()
   327  	env, err := st2.Model()
   328  	c.Assert(err, jc.ErrorIsNil)
   329  	err = env.Destroy()
   330  	c.Assert(err, jc.ErrorIsNil)
   331  }
   332  
   333  func (s *ModelSuite) TestDestroyControllerNonEmptyModelFails(c *gc.C) {
   334  	st2 := s.Factory.MakeModel(c, nil)
   335  	defer st2.Close()
   336  	factory.NewFactory(st2).MakeApplication(c, nil)
   337  
   338  	env, err := s.State.Model()
   339  	c.Assert(err, jc.ErrorIsNil)
   340  	c.Assert(env.Destroy(), gc.ErrorMatches, "failed to destroy model: hosting 1 other models")
   341  }
   342  
   343  func (s *ModelSuite) TestDestroyControllerEmptyModel(c *gc.C) {
   344  	st2 := s.Factory.MakeModel(c, nil)
   345  	defer st2.Close()
   346  
   347  	controllerModel, err := s.State.Model()
   348  	c.Assert(err, jc.ErrorIsNil)
   349  	c.Assert(controllerModel.Destroy(), jc.ErrorIsNil)
   350  	c.Assert(controllerModel.Refresh(), jc.ErrorIsNil)
   351  	c.Assert(controllerModel.Life(), gc.Equals, state.Dying)
   352  
   353  	hostedModel, err := st2.Model()
   354  	c.Assert(err, jc.ErrorIsNil)
   355  	c.Assert(hostedModel.Life(), gc.Equals, state.Dead)
   356  }
   357  
   358  func (s *ModelSuite) TestDestroyControllerAndHostedModels(c *gc.C) {
   359  	st2 := s.Factory.MakeModel(c, nil)
   360  	defer st2.Close()
   361  	factory.NewFactory(st2).MakeApplication(c, nil)
   362  
   363  	controllerEnv, err := s.State.Model()
   364  	c.Assert(err, jc.ErrorIsNil)
   365  	c.Assert(controllerEnv.DestroyIncludingHosted(), jc.ErrorIsNil)
   366  
   367  	env, err := s.State.Model()
   368  	c.Assert(err, jc.ErrorIsNil)
   369  	c.Assert(env.Life(), gc.Equals, state.Dying)
   370  
   371  	assertNeedsCleanup(c, s.State)
   372  	assertCleanupRuns(c, s.State)
   373  
   374  	// Cleanups for hosted model enqueued by controller model cleanups.
   375  	assertNeedsCleanup(c, st2)
   376  	assertCleanupRuns(c, st2)
   377  
   378  	env2, err := st2.Model()
   379  	c.Assert(err, jc.ErrorIsNil)
   380  	c.Assert(env2.Life(), gc.Equals, state.Dying)
   381  
   382  	c.Assert(st2.ProcessDyingModel(), jc.ErrorIsNil)
   383  
   384  	c.Assert(env2.Refresh(), jc.ErrorIsNil)
   385  	c.Assert(env2.Life(), gc.Equals, state.Dead)
   386  
   387  	c.Assert(s.State.ProcessDyingModel(), jc.ErrorIsNil)
   388  	c.Assert(env.Refresh(), jc.ErrorIsNil)
   389  	c.Assert(env2.Life(), gc.Equals, state.Dead)
   390  }
   391  
   392  func (s *ModelSuite) TestDestroyControllerAndHostedModelsWithResources(c *gc.C) {
   393  	otherSt := s.Factory.MakeModel(c, nil)
   394  	defer otherSt.Close()
   395  
   396  	assertEnv := func(env *state.Model, st *state.State, life state.Life, expectedMachines, expectedServices int) {
   397  		c.Assert(env.Refresh(), jc.ErrorIsNil)
   398  		c.Assert(env.Life(), gc.Equals, life)
   399  
   400  		machines, err := st.AllMachines()
   401  		c.Assert(err, jc.ErrorIsNil)
   402  		c.Assert(machines, gc.HasLen, expectedMachines)
   403  
   404  		services, err := st.AllApplications()
   405  		c.Assert(err, jc.ErrorIsNil)
   406  		c.Assert(services, gc.HasLen, expectedServices)
   407  	}
   408  
   409  	// add some machines and services
   410  	otherEnv, err := otherSt.Model()
   411  	c.Assert(err, jc.ErrorIsNil)
   412  	_, err = otherSt.AddMachine("quantal", state.JobHostUnits)
   413  	c.Assert(err, jc.ErrorIsNil)
   414  	service := s.Factory.MakeApplication(c, nil)
   415  	ch, _, err := service.Charm()
   416  	c.Assert(err, jc.ErrorIsNil)
   417  
   418  	args := state.AddApplicationArgs{
   419  		Name:  service.Name(),
   420  		Charm: ch,
   421  	}
   422  	service, err = otherSt.AddApplication(args)
   423  	c.Assert(err, jc.ErrorIsNil)
   424  
   425  	controllerEnv, err := s.State.Model()
   426  	c.Assert(err, jc.ErrorIsNil)
   427  	c.Assert(controllerEnv.DestroyIncludingHosted(), jc.ErrorIsNil)
   428  
   429  	assertCleanupCount(c, s.State, 2)
   430  	assertAllMachinesDeadAndRemove(c, s.State)
   431  	assertEnv(controllerEnv, s.State, state.Dying, 0, 0)
   432  
   433  	err = s.State.ProcessDyingModel()
   434  	c.Assert(err, gc.ErrorMatches, `one or more hosted models are not yet dead`)
   435  
   436  	assertCleanupCount(c, otherSt, 3)
   437  	assertAllMachinesDeadAndRemove(c, otherSt)
   438  	assertEnv(otherEnv, otherSt, state.Dying, 0, 0)
   439  	c.Assert(otherSt.ProcessDyingModel(), jc.ErrorIsNil)
   440  
   441  	c.Assert(otherEnv.Refresh(), jc.ErrorIsNil)
   442  	c.Assert(otherEnv.Life(), gc.Equals, state.Dead)
   443  
   444  	c.Assert(s.State.ProcessDyingModel(), jc.ErrorIsNil)
   445  	c.Assert(controllerEnv.Refresh(), jc.ErrorIsNil)
   446  	c.Assert(controllerEnv.Life(), gc.Equals, state.Dead)
   447  }
   448  
   449  func (s *ModelSuite) TestDestroyControllerEmptyModelRace(c *gc.C) {
   450  	defer s.Factory.MakeModel(c, nil).Close()
   451  
   452  	// Simulate an empty model being added just before the
   453  	// remove txn is called.
   454  	defer state.SetBeforeHooks(c, s.State, func() {
   455  		s.Factory.MakeModel(c, nil).Close()
   456  	}).Check()
   457  
   458  	env, err := s.State.Model()
   459  	c.Assert(err, jc.ErrorIsNil)
   460  	c.Assert(env.Destroy(), jc.ErrorIsNil)
   461  }
   462  
   463  func (s *ModelSuite) TestDestroyControllerRemoveEmptyAddNonEmptyModel(c *gc.C) {
   464  	st2 := s.Factory.MakeModel(c, nil)
   465  	defer st2.Close()
   466  
   467  	// Simulate an empty model being removed, and a new non-empty
   468  	// model being added, just before the remove txn is called.
   469  	defer state.SetBeforeHooks(c, s.State, func() {
   470  		// Destroy the empty model, which should move it right
   471  		// along to Dead, and then remove it.
   472  		model, err := st2.Model()
   473  		c.Assert(err, jc.ErrorIsNil)
   474  		c.Assert(model.Destroy(), jc.ErrorIsNil)
   475  		err = st2.RemoveAllModelDocs()
   476  		c.Assert(err, jc.ErrorIsNil)
   477  
   478  		// Add a new, non-empty model. This should still prevent
   479  		// the controller from being destroyed.
   480  		st3 := s.Factory.MakeModel(c, nil)
   481  		defer st3.Close()
   482  		factory.NewFactory(st3).MakeApplication(c, nil)
   483  	}).Check()
   484  
   485  	env, err := s.State.Model()
   486  	c.Assert(err, jc.ErrorIsNil)
   487  	c.Assert(env.Destroy(), gc.ErrorMatches, "failed to destroy model: hosting 1 other models")
   488  }
   489  
   490  func (s *ModelSuite) TestDestroyControllerNonEmptyModelRace(c *gc.C) {
   491  	// Simulate an empty model being added just before the
   492  	// remove txn is called.
   493  	defer state.SetBeforeHooks(c, s.State, func() {
   494  		st := s.Factory.MakeModel(c, nil)
   495  		defer st.Close()
   496  		factory.NewFactory(st).MakeApplication(c, nil)
   497  	}).Check()
   498  
   499  	env, err := s.State.Model()
   500  	c.Assert(err, jc.ErrorIsNil)
   501  	c.Assert(env.Destroy(), gc.ErrorMatches, "failed to destroy model: hosting 1 other models")
   502  }
   503  
   504  func (s *ModelSuite) TestDestroyControllerAlreadyDyingRaceNoOp(c *gc.C) {
   505  	env, err := s.State.Model()
   506  	c.Assert(err, jc.ErrorIsNil)
   507  
   508  	// Simulate an model being destroyed by another client just before
   509  	// the remove txn is called.
   510  	defer state.SetBeforeHooks(c, s.State, func() {
   511  		c.Assert(env.Destroy(), jc.ErrorIsNil)
   512  	}).Check()
   513  
   514  	c.Assert(env.Destroy(), jc.ErrorIsNil)
   515  }
   516  
   517  func (s *ModelSuite) TestDestroyControllerAlreadyDyingNoOp(c *gc.C) {
   518  	env, err := s.State.Model()
   519  	c.Assert(err, jc.ErrorIsNil)
   520  
   521  	c.Assert(env.Destroy(), jc.ErrorIsNil)
   522  	c.Assert(env.Destroy(), jc.ErrorIsNil)
   523  }
   524  
   525  func (s *ModelSuite) TestDestroyModelNonEmpty(c *gc.C) {
   526  	m, err := s.State.Model()
   527  	c.Assert(err, jc.ErrorIsNil)
   528  
   529  	// Add a service to prevent the model from transitioning directly to Dead.
   530  	s.Factory.MakeApplication(c, nil)
   531  
   532  	c.Assert(m.Destroy(), jc.ErrorIsNil)
   533  	c.Assert(m.Refresh(), jc.ErrorIsNil)
   534  	c.Assert(m.Life(), gc.Equals, state.Dying)
   535  }
   536  
   537  func (s *ModelSuite) TestDestroyModelAddServiceConcurrently(c *gc.C) {
   538  	st := s.Factory.MakeModel(c, nil)
   539  	defer st.Close()
   540  	m, err := st.Model()
   541  	c.Assert(err, jc.ErrorIsNil)
   542  
   543  	defer state.SetBeforeHooks(c, st, func() {
   544  		factory.NewFactory(st).MakeApplication(c, nil)
   545  	}).Check()
   546  
   547  	c.Assert(m.Destroy(), jc.ErrorIsNil)
   548  	c.Assert(m.Refresh(), jc.ErrorIsNil)
   549  	c.Assert(m.Life(), gc.Equals, state.Dying)
   550  }
   551  
   552  func (s *ModelSuite) TestDestroyModelAddMachineConcurrently(c *gc.C) {
   553  	st := s.Factory.MakeModel(c, nil)
   554  	defer st.Close()
   555  	m, err := st.Model()
   556  	c.Assert(err, jc.ErrorIsNil)
   557  
   558  	defer state.SetBeforeHooks(c, st, func() {
   559  		factory.NewFactory(st).MakeMachine(c, nil)
   560  	}).Check()
   561  
   562  	c.Assert(m.Destroy(), jc.ErrorIsNil)
   563  	c.Assert(m.Refresh(), jc.ErrorIsNil)
   564  	c.Assert(m.Life(), gc.Equals, state.Dying)
   565  }
   566  
   567  func (s *ModelSuite) TestDestroyModelEmpty(c *gc.C) {
   568  	st := s.Factory.MakeModel(c, nil)
   569  	defer st.Close()
   570  	m, err := st.Model()
   571  	c.Assert(err, jc.ErrorIsNil)
   572  
   573  	c.Assert(m.Destroy(), jc.ErrorIsNil)
   574  	c.Assert(m.Refresh(), jc.ErrorIsNil)
   575  	c.Assert(m.Life(), gc.Equals, state.Dead)
   576  }
   577  
   578  func (s *ModelSuite) TestProcessDyingServerEnvironTransitionDyingToDead(c *gc.C) {
   579  	s.assertDyingEnvironTransitionDyingToDead(c, s.State)
   580  }
   581  
   582  func (s *ModelSuite) TestProcessDyingHostedEnvironTransitionDyingToDead(c *gc.C) {
   583  	st := s.Factory.MakeModel(c, nil)
   584  	defer st.Close()
   585  	s.assertDyingEnvironTransitionDyingToDead(c, st)
   586  }
   587  
   588  func (s *ModelSuite) assertDyingEnvironTransitionDyingToDead(c *gc.C, st *state.State) {
   589  	// Add a service to prevent the model from transitioning directly to Dead.
   590  	// Add the service before getting the Model, otherwise we'll have to run
   591  	// the transaction twice, and hit the hook point too early.
   592  	svc := factory.NewFactory(st).MakeApplication(c, nil)
   593  	env, err := st.Model()
   594  	c.Assert(err, jc.ErrorIsNil)
   595  
   596  	// ProcessDyingModel is called by a worker after Destroy is called. To
   597  	// avoid a race, we jump the gun here and test immediately after the
   598  	// environement was set to dead.
   599  	defer state.SetAfterHooks(c, st, func() {
   600  		c.Assert(env.Refresh(), jc.ErrorIsNil)
   601  		c.Assert(env.Life(), gc.Equals, state.Dying)
   602  
   603  		err := svc.Destroy()
   604  		c.Assert(err, jc.ErrorIsNil)
   605  
   606  		c.Assert(st.ProcessDyingModel(), jc.ErrorIsNil)
   607  
   608  		c.Assert(env.Refresh(), jc.ErrorIsNil)
   609  		c.Assert(env.Life(), gc.Equals, state.Dead)
   610  	}).Check()
   611  
   612  	c.Assert(env.Destroy(), jc.ErrorIsNil)
   613  }
   614  
   615  func (s *ModelSuite) TestProcessDyingEnvironWithMachinesAndServicesNoOp(c *gc.C) {
   616  	st := s.Factory.MakeModel(c, nil)
   617  	defer st.Close()
   618  
   619  	// calling ProcessDyingModel on a live environ should fail.
   620  	err := st.ProcessDyingModel()
   621  	c.Assert(err, gc.ErrorMatches, "model is not dying")
   622  
   623  	// add some machines and services
   624  	env, err := st.Model()
   625  	c.Assert(err, jc.ErrorIsNil)
   626  	_, err = st.AddMachine("quantal", state.JobHostUnits)
   627  	c.Assert(err, jc.ErrorIsNil)
   628  	service := s.Factory.MakeApplication(c, nil)
   629  	ch, _, err := service.Charm()
   630  	c.Assert(err, jc.ErrorIsNil)
   631  	args := state.AddApplicationArgs{
   632  		Name:  service.Name(),
   633  		Charm: ch,
   634  	}
   635  	service, err = st.AddApplication(args)
   636  	c.Assert(err, jc.ErrorIsNil)
   637  
   638  	assertEnv := func(life state.Life, expectedMachines, expectedServices int) {
   639  		c.Assert(env.Refresh(), jc.ErrorIsNil)
   640  		c.Assert(env.Life(), gc.Equals, life)
   641  
   642  		machines, err := st.AllMachines()
   643  		c.Assert(err, jc.ErrorIsNil)
   644  		c.Assert(machines, gc.HasLen, expectedMachines)
   645  
   646  		services, err := st.AllApplications()
   647  		c.Assert(err, jc.ErrorIsNil)
   648  		c.Assert(services, gc.HasLen, expectedServices)
   649  	}
   650  
   651  	// Simulate processing a dying envrionment after an envrionment is set to
   652  	// dying, but before the cleanup has removed machines and services.
   653  	defer state.SetAfterHooks(c, st, func() {
   654  		assertEnv(state.Dying, 1, 1)
   655  		err := st.ProcessDyingModel()
   656  		c.Assert(err, gc.ErrorMatches, `model not empty, found 1 machine\(s\)`)
   657  		assertEnv(state.Dying, 1, 1)
   658  	}).Check()
   659  
   660  	c.Assert(env.Refresh(), jc.ErrorIsNil)
   661  	c.Assert(env.Destroy(), jc.ErrorIsNil)
   662  }
   663  
   664  func (s *ModelSuite) TestProcessDyingControllerEnvironWithHostedEnvsNoOp(c *gc.C) {
   665  	// Add a non-empty model to the controller.
   666  	st := s.Factory.MakeModel(c, nil)
   667  	defer st.Close()
   668  	factory.NewFactory(st).MakeApplication(c, nil)
   669  
   670  	controllerEnv, err := s.State.Model()
   671  	c.Assert(err, jc.ErrorIsNil)
   672  	c.Assert(controllerEnv.DestroyIncludingHosted(), jc.ErrorIsNil)
   673  
   674  	err = s.State.ProcessDyingModel()
   675  	c.Assert(err, gc.ErrorMatches, `one or more hosted models are not yet dead`)
   676  
   677  	c.Assert(controllerEnv.Refresh(), jc.ErrorIsNil)
   678  	c.Assert(controllerEnv.Life(), gc.Equals, state.Dying)
   679  }
   680  
   681  func (s *ModelSuite) TestListModelUsers(c *gc.C) {
   682  	env, err := s.State.Model()
   683  	c.Assert(err, jc.ErrorIsNil)
   684  
   685  	expected := addModelUsers(c, s.State)
   686  	obtained, err := env.Users()
   687  	c.Assert(err, gc.IsNil)
   688  
   689  	assertObtainedUsersMatchExpectedUsers(c, obtained, expected)
   690  }
   691  
   692  func (s *ModelSuite) TestMisMatchedEnvs(c *gc.C) {
   693  	// create another model
   694  	otherEnvState := s.Factory.MakeModel(c, nil)
   695  	defer otherEnvState.Close()
   696  	otherEnv, err := otherEnvState.Model()
   697  	c.Assert(err, jc.ErrorIsNil)
   698  
   699  	// get that model from State
   700  	env, err := s.State.GetModel(otherEnv.ModelTag())
   701  	c.Assert(err, jc.ErrorIsNil)
   702  
   703  	// check that the Users method errors
   704  	users, err := env.Users()
   705  	c.Assert(users, gc.IsNil)
   706  	c.Assert(err, gc.ErrorMatches, "cannot lookup model users outside the current model")
   707  }
   708  
   709  func (s *ModelSuite) TestListUsersIgnoredDeletedUsers(c *gc.C) {
   710  	model, err := s.State.Model()
   711  	c.Assert(err, jc.ErrorIsNil)
   712  
   713  	expectedUsers := addModelUsers(c, s.State)
   714  
   715  	obtainedUsers, err := model.Users()
   716  	c.Assert(err, jc.ErrorIsNil)
   717  	assertObtainedUsersMatchExpectedUsers(c, obtainedUsers, expectedUsers)
   718  
   719  	lastUser := obtainedUsers[len(obtainedUsers)-1]
   720  	err = s.State.RemoveUser(lastUser.UserTag)
   721  	c.Assert(err, jc.ErrorIsNil)
   722  	expectedAfterDeletion := obtainedUsers[:len(obtainedUsers)-1]
   723  
   724  	obtainedUsers, err = model.Users()
   725  	c.Assert(err, jc.ErrorIsNil)
   726  	assertObtainedUsersMatchExpectedUsers(c, obtainedUsers, expectedAfterDeletion)
   727  }
   728  
   729  func (s *ModelSuite) TestListUsersTwoModels(c *gc.C) {
   730  	env, err := s.State.Model()
   731  	c.Assert(err, jc.ErrorIsNil)
   732  
   733  	otherEnvState := s.Factory.MakeModel(c, nil)
   734  	defer otherEnvState.Close()
   735  	otherEnv, err := otherEnvState.Model()
   736  	c.Assert(err, jc.ErrorIsNil)
   737  
   738  	// Add users to both models
   739  	expectedUsers := addModelUsers(c, s.State)
   740  	expectedUsersOtherEnv := addModelUsers(c, otherEnvState)
   741  
   742  	// test that only the expected users are listed for each model
   743  	obtainedUsers, err := env.Users()
   744  	c.Assert(err, jc.ErrorIsNil)
   745  	assertObtainedUsersMatchExpectedUsers(c, obtainedUsers, expectedUsers)
   746  
   747  	obtainedUsersOtherEnv, err := otherEnv.Users()
   748  	c.Assert(err, jc.ErrorIsNil)
   749  	assertObtainedUsersMatchExpectedUsers(c, obtainedUsersOtherEnv, expectedUsersOtherEnv)
   750  }
   751  
   752  func addModelUsers(c *gc.C, st *state.State) (expected []permission.UserAccess) {
   753  	// get the model owner
   754  	testAdmin := names.NewUserTag("test-admin")
   755  	owner, err := st.UserAccess(testAdmin, st.ModelTag())
   756  	c.Assert(err, jc.ErrorIsNil)
   757  
   758  	f := factory.NewFactory(st)
   759  	return []permission.UserAccess{
   760  		// we expect the owner to be an existing model user
   761  		owner,
   762  		// add new users to the model
   763  		f.MakeModelUser(c, nil),
   764  		f.MakeModelUser(c, nil),
   765  		f.MakeModelUser(c, nil),
   766  	}
   767  }
   768  
   769  func assertObtainedUsersMatchExpectedUsers(c *gc.C, obtainedUsers, expectedUsers []permission.UserAccess) {
   770  	c.Assert(len(obtainedUsers), gc.Equals, len(expectedUsers))
   771  	for i, obtained := range obtainedUsers {
   772  		c.Assert(obtained.Object.Id(), gc.Equals, expectedUsers[i].Object.Id())
   773  		c.Assert(obtained.UserTag, gc.Equals, expectedUsers[i].UserTag)
   774  		c.Assert(obtained.DisplayName, gc.Equals, expectedUsers[i].DisplayName)
   775  		c.Assert(obtained.CreatedBy, gc.Equals, expectedUsers[i].CreatedBy)
   776  	}
   777  }
   778  
   779  func (s *ModelSuite) TestAllModels(c *gc.C) {
   780  	s.Factory.MakeModel(c, &factory.ModelParams{
   781  		Name: "test", Owner: names.NewUserTag("bob@remote")}).Close()
   782  	s.Factory.MakeModel(c, &factory.ModelParams{
   783  		Name: "test", Owner: names.NewUserTag("mary@remote")}).Close()
   784  	envs, err := s.State.AllModels()
   785  	c.Assert(err, jc.ErrorIsNil)
   786  	c.Assert(envs, gc.HasLen, 3)
   787  	var obtained []string
   788  	for _, env := range envs {
   789  		obtained = append(obtained, fmt.Sprintf("%s/%s", env.Owner().Id(), env.Name()))
   790  	}
   791  	expected := []string{
   792  		"bob@remote/test",
   793  		"mary@remote/test",
   794  		"test-admin/testenv",
   795  	}
   796  	c.Assert(obtained, jc.DeepEquals, expected)
   797  }
   798  
   799  func (s *ModelSuite) TestHostedModelCount(c *gc.C) {
   800  	c.Assert(state.HostedModelCount(c, s.State), gc.Equals, 0)
   801  
   802  	st1 := s.Factory.MakeModel(c, nil)
   803  	defer st1.Close()
   804  	c.Assert(state.HostedModelCount(c, s.State), gc.Equals, 1)
   805  
   806  	st2 := s.Factory.MakeModel(c, nil)
   807  	defer st2.Close()
   808  	c.Assert(state.HostedModelCount(c, s.State), gc.Equals, 2)
   809  
   810  	env1, err := st1.Model()
   811  	c.Assert(err, jc.ErrorIsNil)
   812  	c.Assert(env1.Destroy(), jc.ErrorIsNil)
   813  	c.Assert(st1.RemoveAllModelDocs(), jc.ErrorIsNil)
   814  	c.Assert(state.HostedModelCount(c, s.State), gc.Equals, 1)
   815  
   816  	env2, err := st2.Model()
   817  	c.Assert(err, jc.ErrorIsNil)
   818  	c.Assert(env2.Destroy(), jc.ErrorIsNil)
   819  	c.Assert(st2.RemoveAllModelDocs(), jc.ErrorIsNil)
   820  	c.Assert(state.HostedModelCount(c, s.State), gc.Equals, 0)
   821  }
   822  
   823  type ModelCloudValidationSuite struct {
   824  	gitjujutesting.MgoSuite
   825  }
   826  
   827  var _ = gc.Suite(&ModelCloudValidationSuite{})
   828  
   829  // TODO(axw) concurrency tests when we can modify the cloud definition,
   830  // and update/remove credentials.
   831  
   832  func (s *ModelCloudValidationSuite) TestNewModelCloudNameMismatch(c *gc.C) {
   833  	st, owner := s.initializeState(c, []cloud.Region{{Name: "some-region"}}, []cloud.AuthType{cloud.EmptyAuthType}, nil)
   834  	defer st.Close()
   835  	cfg, _ := createTestModelConfig(c, st.ModelUUID())
   836  	_, _, err := st.NewModel(state.ModelArgs{
   837  		CloudName: "another",
   838  		Config:    cfg,
   839  		Owner:     owner,
   840  		StorageProviderRegistry: storage.StaticProviderRegistry{},
   841  	})
   842  	c.Assert(err, gc.ErrorMatches, "controller cloud dummy does not match model cloud another")
   843  }
   844  
   845  func (s *ModelCloudValidationSuite) TestNewModelUnknownCloudRegion(c *gc.C) {
   846  	st, owner := s.initializeState(c, []cloud.Region{{Name: "some-region"}}, []cloud.AuthType{cloud.EmptyAuthType}, nil)
   847  	defer st.Close()
   848  	cfg, _ := createTestModelConfig(c, st.ModelUUID())
   849  	_, _, err := st.NewModel(state.ModelArgs{
   850  		CloudName:   "dummy",
   851  		CloudRegion: "dummy-region",
   852  		Config:      cfg,
   853  		Owner:       owner,
   854  		StorageProviderRegistry: storage.StaticProviderRegistry{},
   855  	})
   856  	c.Assert(err, gc.ErrorMatches, `region "dummy-region" not found \(expected one of \["some-region"\]\)`)
   857  }
   858  
   859  func (s *ModelCloudValidationSuite) TestNewModelMissingCloudRegion(c *gc.C) {
   860  	st, owner := s.initializeState(c, []cloud.Region{{Name: "dummy-region"}}, []cloud.AuthType{cloud.EmptyAuthType}, nil)
   861  	defer st.Close()
   862  	cfg, _ := createTestModelConfig(c, st.ModelUUID())
   863  	_, _, err := st.NewModel(state.ModelArgs{
   864  		CloudName: "dummy",
   865  		Config:    cfg,
   866  		Owner:     owner,
   867  		StorageProviderRegistry: storage.StaticProviderRegistry{},
   868  	})
   869  	c.Assert(err, gc.ErrorMatches, "missing CloudRegion not valid")
   870  }
   871  
   872  func (s *ModelCloudValidationSuite) TestNewModelUnknownCloudCredential(c *gc.C) {
   873  	regions := []cloud.Region{cloud.Region{Name: "dummy-region"}}
   874  	controllerCredentialTag := names.NewCloudCredentialTag("dummy/test@remote/controller-credential")
   875  	st, owner := s.initializeState(
   876  		c, regions, []cloud.AuthType{cloud.UserPassAuthType}, map[names.CloudCredentialTag]cloud.Credential{
   877  			controllerCredentialTag: cloud.NewCredential(cloud.UserPassAuthType, nil),
   878  		},
   879  	)
   880  	defer st.Close()
   881  	unknownCredentialTag := names.NewCloudCredentialTag("dummy/" + owner.Id() + "/unknown-credential")
   882  	cfg, _ := createTestModelConfig(c, st.ModelUUID())
   883  	_, _, err := st.NewModel(state.ModelArgs{
   884  		CloudName:               "dummy",
   885  		CloudRegion:             "dummy-region",
   886  		Config:                  cfg,
   887  		Owner:                   owner,
   888  		CloudCredential:         unknownCredentialTag,
   889  		StorageProviderRegistry: storage.StaticProviderRegistry{},
   890  	})
   891  	c.Assert(err, gc.ErrorMatches, `credential "dummy/test@remote/unknown-credential" not found`)
   892  }
   893  
   894  func (s *ModelCloudValidationSuite) TestNewModelMissingCloudCredential(c *gc.C) {
   895  	regions := []cloud.Region{cloud.Region{Name: "dummy-region"}}
   896  	controllerCredentialTag := names.NewCloudCredentialTag("dummy/test@remote/controller-credential")
   897  	st, owner := s.initializeState(
   898  		c, regions, []cloud.AuthType{cloud.UserPassAuthType}, map[names.CloudCredentialTag]cloud.Credential{
   899  			controllerCredentialTag: cloud.NewCredential(cloud.UserPassAuthType, nil),
   900  		},
   901  	)
   902  	defer st.Close()
   903  	cfg, _ := createTestModelConfig(c, st.ModelUUID())
   904  	_, _, err := st.NewModel(state.ModelArgs{
   905  		CloudName:   "dummy",
   906  		CloudRegion: "dummy-region",
   907  		Config:      cfg,
   908  		Owner:       owner,
   909  		StorageProviderRegistry: storage.StaticProviderRegistry{},
   910  	})
   911  	c.Assert(err, gc.ErrorMatches, "missing CloudCredential not valid")
   912  }
   913  
   914  func (s *ModelCloudValidationSuite) TestNewModelMissingCloudCredentialSupportsEmptyAuth(c *gc.C) {
   915  	regions := []cloud.Region{
   916  		cloud.Region{
   917  			Name:             "dummy-region",
   918  			Endpoint:         "dummy-endpoint",
   919  			IdentityEndpoint: "dummy-identity-endpoint",
   920  			StorageEndpoint:  "dummy-storage-endpoint",
   921  		},
   922  	}
   923  	st, owner := s.initializeState(c, regions, []cloud.AuthType{cloud.EmptyAuthType}, nil)
   924  	defer st.Close()
   925  	cfg, _ := createTestModelConfig(c, st.ModelUUID())
   926  	cfg, err := cfg.Apply(map[string]interface{}{"name": "whatever"})
   927  	c.Assert(err, jc.ErrorIsNil)
   928  	_, newSt, err := st.NewModel(state.ModelArgs{
   929  		CloudName: "dummy", CloudRegion: "dummy-region", Config: cfg, Owner: owner,
   930  		StorageProviderRegistry: storage.StaticProviderRegistry{},
   931  	})
   932  	c.Assert(err, jc.ErrorIsNil)
   933  	newSt.Close()
   934  }
   935  
   936  func (s *ModelCloudValidationSuite) TestNewModelOtherUserCloudCredential(c *gc.C) {
   937  	controllerCredentialTag := names.NewCloudCredentialTag("dummy/test@remote/controller-credential")
   938  	st, _ := s.initializeState(
   939  		c, nil, []cloud.AuthType{cloud.UserPassAuthType}, map[names.CloudCredentialTag]cloud.Credential{
   940  			controllerCredentialTag: cloud.NewCredential(cloud.UserPassAuthType, nil),
   941  		},
   942  	)
   943  	defer st.Close()
   944  	owner := factory.NewFactory(st).MakeUser(c, nil).UserTag()
   945  	cfg, _ := createTestModelConfig(c, st.ModelUUID())
   946  	_, _, err := st.NewModel(state.ModelArgs{
   947  		CloudName:               "dummy",
   948  		Config:                  cfg,
   949  		Owner:                   owner,
   950  		CloudCredential:         controllerCredentialTag,
   951  		StorageProviderRegistry: storage.StaticProviderRegistry{},
   952  	})
   953  	c.Assert(err, gc.ErrorMatches, `credential "dummy/test@remote/controller-credential" not found`)
   954  }
   955  
   956  func (s *ModelCloudValidationSuite) initializeState(
   957  	c *gc.C,
   958  	regions []cloud.Region,
   959  	authTypes []cloud.AuthType,
   960  	credentials map[names.CloudCredentialTag]cloud.Credential,
   961  ) (*state.State, names.UserTag) {
   962  	owner := names.NewUserTag("test@remote")
   963  	cfg, _ := createTestModelConfig(c, "")
   964  	var controllerRegion string
   965  	var controllerCredential names.CloudCredentialTag
   966  	if len(regions) > 0 {
   967  		controllerRegion = regions[0].Name
   968  	}
   969  	if len(credentials) > 0 {
   970  		// pick an arbitrary credential
   971  		for controllerCredential = range credentials {
   972  		}
   973  	}
   974  	controllerCfg := testing.FakeControllerConfig()
   975  	st, err := state.Initialize(state.InitializeParams{
   976  		Clock:            clock.WallClock,
   977  		ControllerConfig: controllerCfg,
   978  		ControllerModelArgs: state.ModelArgs{
   979  			Owner:                   owner,
   980  			Config:                  cfg,
   981  			CloudName:               "dummy",
   982  			CloudRegion:             controllerRegion,
   983  			CloudCredential:         controllerCredential,
   984  			StorageProviderRegistry: storage.StaticProviderRegistry{},
   985  		},
   986  		CloudName: "dummy",
   987  		Cloud: cloud.Cloud{
   988  			Type:      "dummy",
   989  			AuthTypes: authTypes,
   990  			Regions:   regions,
   991  		},
   992  		CloudCredentials: credentials,
   993  		MongoInfo:        statetesting.NewMongoInfo(),
   994  		MongoDialOpts:    mongotest.DialOpts(),
   995  	})
   996  	c.Assert(err, jc.ErrorIsNil)
   997  	return st, owner
   998  }
   999  
  1000  func assertCleanupRuns(c *gc.C, st *state.State) {
  1001  	err := st.Cleanup()
  1002  	c.Assert(err, jc.ErrorIsNil)
  1003  }
  1004  
  1005  func assertNeedsCleanup(c *gc.C, st *state.State) {
  1006  	actual, err := st.NeedsCleanup()
  1007  	c.Assert(err, jc.ErrorIsNil)
  1008  	c.Assert(actual, jc.IsTrue)
  1009  }
  1010  
  1011  func assertDoesNotNeedCleanup(c *gc.C, st *state.State) {
  1012  	actual, err := st.NeedsCleanup()
  1013  	c.Assert(err, jc.ErrorIsNil)
  1014  	c.Assert(actual, jc.IsFalse)
  1015  }
  1016  
  1017  // assertCleanupCount is useful because certain cleanups cause other cleanups
  1018  // to be queued; it makes more sense to just run cleanup again than to unpick
  1019  // object destruction so that we run the cleanups inline while running cleanups.
  1020  func assertCleanupCount(c *gc.C, st *state.State, count int) {
  1021  	for i := 0; i < count; i++ {
  1022  		c.Logf("checking cleanups %d", i)
  1023  		assertNeedsCleanup(c, st)
  1024  		assertCleanupRuns(c, st)
  1025  	}
  1026  	assertDoesNotNeedCleanup(c, st)
  1027  }
  1028  
  1029  // The provisioner will remove dead machines once their backing instances are
  1030  // stopped. For the tests, we remove them directly.
  1031  func assertAllMachinesDeadAndRemove(c *gc.C, st *state.State) {
  1032  	machines, err := st.AllMachines()
  1033  	c.Assert(err, jc.ErrorIsNil)
  1034  	for _, m := range machines {
  1035  		if m.IsManager() {
  1036  			continue
  1037  		}
  1038  		if _, isContainer := m.ParentId(); isContainer {
  1039  			continue
  1040  		}
  1041  		manual, err := m.IsManual()
  1042  		c.Assert(err, jc.ErrorIsNil)
  1043  		if manual {
  1044  			continue
  1045  		}
  1046  
  1047  		c.Assert(m.Life(), gc.Equals, state.Dead)
  1048  		c.Assert(m.Remove(), jc.ErrorIsNil)
  1049  	}
  1050  }