github.com/juju/juju@v0.0.0-20240327075706-a90865de2538/worker/instancemutater/worker_test.go (about)

     1  // Copyright 2019 Canonical Ltd.
     2  // Licensed under the AGPLv3, see LICENCE file for details.
     3  
     4  package instancemutater_test
     5  
     6  import (
     7  	"strconv"
     8  	"sync"
     9  	"time"
    10  
    11  	"github.com/juju/errors"
    12  	"github.com/juju/loggo"
    13  	"github.com/juju/names/v5"
    14  	"github.com/juju/testing"
    15  	jc "github.com/juju/testing/checkers"
    16  	"github.com/juju/worker/v3"
    17  	"github.com/juju/worker/v3/workertest"
    18  	"go.uber.org/mock/gomock"
    19  	gc "gopkg.in/check.v1"
    20  
    21  	apiinstancemutater "github.com/juju/juju/api/agent/instancemutater"
    22  	"github.com/juju/juju/core/instance"
    23  	"github.com/juju/juju/core/life"
    24  	"github.com/juju/juju/core/lxdprofile"
    25  	"github.com/juju/juju/core/status"
    26  	"github.com/juju/juju/core/watcher"
    27  	"github.com/juju/juju/rpc/params"
    28  	"github.com/juju/juju/worker/instancemutater"
    29  	"github.com/juju/juju/worker/instancemutater/mocks"
    30  	workermocks "github.com/juju/juju/worker/mocks"
    31  )
    32  
    33  type workerConfigSuite struct {
    34  	testing.IsolationSuite
    35  }
    36  
    37  var _ = gc.Suite(&workerConfigSuite{})
    38  
    39  func (s *workerConfigSuite) SetUpTest(c *gc.C) {
    40  	s.IsolationSuite.SetUpTest(c)
    41  }
    42  
    43  func (s *workerConfigSuite) TestInvalidConfigValidate(c *gc.C) {
    44  	ctrl := gomock.NewController(c)
    45  	defer ctrl.Finish()
    46  
    47  	testcases := []struct {
    48  		description string
    49  		config      instancemutater.Config
    50  		err         string
    51  	}{
    52  		{
    53  			description: "Test empty configuration",
    54  			config:      instancemutater.Config{},
    55  			err:         "nil Logger not valid",
    56  		},
    57  		{
    58  			description: "Test no Logger",
    59  			config:      instancemutater.Config{},
    60  			err:         "nil Logger not valid",
    61  		},
    62  		{
    63  			description: "Test no api",
    64  			config: instancemutater.Config{
    65  				Logger: loggo.GetLogger("test"),
    66  			},
    67  			err: "nil Facade not valid",
    68  		},
    69  		{
    70  			description: "Test no environ",
    71  			config: instancemutater.Config{
    72  				Logger: loggo.GetLogger("test"),
    73  				Facade: mocks.NewMockInstanceMutaterAPI(ctrl),
    74  			},
    75  			err: "nil Broker not valid",
    76  		},
    77  		{
    78  			description: "Test no agent",
    79  			config: instancemutater.Config{
    80  				Logger: loggo.GetLogger("test"),
    81  				Facade: mocks.NewMockInstanceMutaterAPI(ctrl),
    82  				Broker: mocks.NewMockLXDProfiler(ctrl),
    83  			},
    84  			err: "nil AgentConfig not valid",
    85  		},
    86  		{
    87  			description: "Test no tag",
    88  			config: instancemutater.Config{
    89  				Logger:      loggo.GetLogger("test"),
    90  				Facade:      mocks.NewMockInstanceMutaterAPI(ctrl),
    91  				Broker:      mocks.NewMockLXDProfiler(ctrl),
    92  				AgentConfig: mocks.NewMockConfig(ctrl),
    93  			},
    94  			err: "nil Tag not valid",
    95  		},
    96  		{
    97  			description: "Test no GetMachineWatcher",
    98  			config: instancemutater.Config{
    99  				Logger:      loggo.GetLogger("test"),
   100  				Facade:      mocks.NewMockInstanceMutaterAPI(ctrl),
   101  				Broker:      mocks.NewMockLXDProfiler(ctrl),
   102  				AgentConfig: mocks.NewMockConfig(ctrl),
   103  				Tag:         names.NewMachineTag("3"),
   104  			},
   105  			err: "nil GetMachineWatcher not valid",
   106  		},
   107  		{
   108  			description: "Test no GetRequiredLXDProfiles",
   109  			config: instancemutater.Config{
   110  				Logger:            loggo.GetLogger("test"),
   111  				Facade:            mocks.NewMockInstanceMutaterAPI(ctrl),
   112  				Broker:            mocks.NewMockLXDProfiler(ctrl),
   113  				AgentConfig:       mocks.NewMockConfig(ctrl),
   114  				Tag:               names.NewMachineTag("3"),
   115  				GetMachineWatcher: getMachineWatcher,
   116  			},
   117  			err: "nil GetRequiredLXDProfiles not valid",
   118  		},
   119  	}
   120  	for i, test := range testcases {
   121  		c.Logf("%d %s", i, test.description)
   122  		err := test.config.Validate()
   123  		c.Assert(err, gc.ErrorMatches, test.err)
   124  	}
   125  }
   126  
   127  var getMachineWatcher = func() (watcher.StringsWatcher, error) {
   128  	return &fakeStringsWatcher{}, nil
   129  }
   130  
   131  func (s *workerConfigSuite) TestValidConfigValidate(c *gc.C) {
   132  	ctrl := gomock.NewController(c)
   133  	defer ctrl.Finish()
   134  
   135  	config := instancemutater.Config{
   136  		Facade:                 mocks.NewMockInstanceMutaterAPI(ctrl),
   137  		Logger:                 loggo.GetLogger("test"),
   138  		Broker:                 mocks.NewMockLXDProfiler(ctrl),
   139  		AgentConfig:            mocks.NewMockConfig(ctrl),
   140  		Tag:                    names.MachineTag{},
   141  		GetMachineWatcher:      getMachineWatcher,
   142  		GetRequiredLXDProfiles: func(_ string) []string { return []string{} },
   143  		GetRequiredContext: func(w instancemutater.MutaterContext) instancemutater.MutaterContext {
   144  			return w
   145  		},
   146  	}
   147  	err := config.Validate()
   148  	c.Assert(err, gc.IsNil)
   149  }
   150  
   151  type workerSuite struct {
   152  	testing.IsolationSuite
   153  
   154  	logger                 loggo.Logger
   155  	facade                 *mocks.MockInstanceMutaterAPI
   156  	broker                 *mocks.MockLXDProfiler
   157  	agentConfig            *mocks.MockConfig
   158  	machine                map[int]*mocks.MockMutaterMachine
   159  	machineTag             names.MachineTag
   160  	machinesWorker         *workermocks.MockWorker
   161  	context                *mocks.MockMutaterContext
   162  	appLXDProfileWorker    map[int]*workermocks.MockWorker
   163  	getRequiredLXDProfiles instancemutater.RequiredLXDProfilesFunc
   164  
   165  	// doneWG is a collection of things each test needs to wait to
   166  	// be completed within the test.
   167  	doneWG *sync.WaitGroup
   168  
   169  	newWorkerFunc func(instancemutater.Config, instancemutater.RequiredMutaterContextFunc) (worker.Worker, error)
   170  }
   171  
   172  var _ = gc.Suite(&workerSuite{})
   173  
   174  func (s *workerSuite) SetUpTest(c *gc.C) {
   175  	s.IsolationSuite.SetUpTest(c)
   176  
   177  	s.logger = loggo.GetLogger("workerSuite")
   178  	s.logger.SetLogLevel(loggo.TRACE)
   179  
   180  	s.newWorkerFunc = instancemutater.NewEnvironTestWorker
   181  	s.machineTag = names.NewMachineTag("0")
   182  	s.getRequiredLXDProfiles = func(modelName string) []string {
   183  		return []string{"default", "juju-testing"}
   184  	}
   185  	s.doneWG = new(sync.WaitGroup)
   186  }
   187  
   188  type workerEnvironSuite struct {
   189  	workerSuite
   190  }
   191  
   192  var _ = gc.Suite(&workerEnvironSuite{})
   193  
   194  // TestFullWorkflow uses the the expectation scenarios from each of the tests
   195  // below to compose a test of the whole instance mutator scenario, from start
   196  // to finish for an EnvironWorker.
   197  func (s *workerEnvironSuite) TestFullWorkflow(c *gc.C) {
   198  	defer s.setup(c, 1).Finish()
   199  
   200  	s.notifyMachines([][]string{{"0"}})
   201  	s.expectFacadeMachineTag(0)
   202  	s.notifyMachineAppLXDProfile(0, 1)
   203  	s.expectMachineCharmProfilingInfo(0, 3)
   204  	s.expectLXDProfileNamesTrue()
   205  	s.expectSetCharmProfiles(0)
   206  	s.expectAssignLXDProfiles()
   207  	s.expectAliveAndSetModificationStatusIdle(0)
   208  	s.expectModificationStatusApplied(0)
   209  
   210  	s.cleanKill(c, s.workerForScenario(c))
   211  }
   212  
   213  func (s *workerEnvironSuite) TestVerifyCurrentProfilesTrue(c *gc.C) {
   214  	defer s.setup(c, 1).Finish()
   215  
   216  	s.notifyMachines([][]string{{"0"}})
   217  	s.expectFacadeMachineTag(0)
   218  	s.notifyMachineAppLXDProfile(0, 1)
   219  	s.expectAliveAndSetModificationStatusIdle(0)
   220  	s.expectMachineCharmProfilingInfo(0, 2)
   221  	s.expectLXDProfileNamesTrue()
   222  	s.expectModificationStatusApplied(0)
   223  
   224  	s.cleanKill(c, s.workerForScenario(c))
   225  }
   226  
   227  func (s *workerEnvironSuite) TestRemoveAllCharmProfiles(c *gc.C) {
   228  	defer s.setup(c, 1).Finish()
   229  
   230  	s.notifyMachines([][]string{{"0"}})
   231  	s.expectFacadeMachineTag(0)
   232  	s.notifyMachineAppLXDProfile(0, 1)
   233  	s.expectAliveAndSetModificationStatusIdle(0)
   234  	s.expectCharmProfilingInfoRemove(0)
   235  	s.expectLXDProfileNamesTrue()
   236  	s.expectRemoveAllCharmProfiles(0)
   237  	s.expectModificationStatusApplied(0)
   238  
   239  	s.cleanKill(c, s.workerForScenario(c))
   240  }
   241  
   242  func (s *workerEnvironSuite) TestMachineNotifyTwice(c *gc.C) {
   243  	defer s.setup(c, 2).Finish()
   244  
   245  	// A WaitGroup for this test to synchronize when the
   246  	// machine notifications are sent.  The 2nd group must
   247  	// be after machine 0 gets Life() == Alive.
   248  	var group sync.WaitGroup
   249  	s.notifyMachinesWaitGroup([][]string{{"0", "1"}, {"0"}}, &group)
   250  	s.expectFacadeMachineTag(0)
   251  	s.expectFacadeMachineTag(1)
   252  	s.notifyMachineAppLXDProfile(0, 1)
   253  	s.notifyMachineAppLXDProfile(1, 1)
   254  	s.expectAliveAndSetModificationStatusIdle(1)
   255  	s.expectMachineCharmProfilingInfo(0, 2)
   256  	s.expectMachineCharmProfilingInfo(1, 2)
   257  	s.expectLXDProfileNamesTrue()
   258  	s.expectLXDProfileNamesTrue()
   259  	s.expectMachineAliveStatusIdleMachineDead(0, &group)
   260  
   261  	s.cleanKill(c, s.workerForScenario(c))
   262  }
   263  
   264  func (s *workerEnvironSuite) TestNoChangeFoundOne(c *gc.C) {
   265  	defer s.setup(c, 1).Finish()
   266  
   267  	s.notifyMachines([][]string{{"0"}})
   268  	s.expectFacadeMachineTag(0)
   269  	s.notifyMachineAppLXDProfile(0, 1)
   270  	s.expectCharmProfilingInfoSimpleNoChange(0)
   271  
   272  	s.cleanKill(c, s.workerForScenario(c))
   273  }
   274  
   275  func (s *workerEnvironSuite) TestNoMachineFound(c *gc.C) {
   276  	defer s.setup(c, 1).Finish()
   277  
   278  	s.notifyMachines([][]string{{"0"}})
   279  	s.expectFacadeReturnsNoMachine()
   280  
   281  	err := s.errorKill(c, s.workerForScenario(c))
   282  	c.Assert(err, jc.Satisfies, errors.IsNotFound)
   283  }
   284  
   285  func (s *workerEnvironSuite) TestCharmProfilingInfoNotProvisioned(c *gc.C) {
   286  	defer s.setup(c, 1).Finish()
   287  
   288  	s.notifyMachines([][]string{{"0"}})
   289  	s.expectFacadeMachineTag(0)
   290  	s.notifyMachineAppLXDProfile(0, 1)
   291  	s.expectCharmProfileInfoNotProvisioned(0)
   292  
   293  	s.cleanKill(c, s.workerForScenario(c))
   294  }
   295  
   296  func (s *workerEnvironSuite) TestCharmProfilingInfoError(c *gc.C) {
   297  	defer s.setup(c, 1).Finish()
   298  
   299  	s.notifyMachines([][]string{{"0"}})
   300  	s.expectFacadeMachineTag(0)
   301  	s.notifyMachineAppLXDProfile(0, 1)
   302  	s.expectCharmProfileInfoError(0)
   303  	s.expectContextKillError()
   304  
   305  	err := s.errorKill(c, s.workerForScenarioWithContext(c))
   306  	c.Assert(err, jc.Satisfies, params.IsCodeNotSupported)
   307  }
   308  
   309  func (s *workerEnvironSuite) TestMachineNotSupported(c *gc.C) {
   310  	defer s.setup(c, 1).Finish()
   311  
   312  	s.notifyMachines([][]string{{"0"}})
   313  	s.expectFacadeMachineTag(0)
   314  
   315  	// We need another sync point here, because the worker can be killed
   316  	// before this method is called.
   317  	s.doneWG.Add(1)
   318  	s.machine[0].EXPECT().WatchLXDProfileVerificationNeeded().DoAndReturn(
   319  		func() (watcher.NotifyWatcher, error) {
   320  			s.doneWG.Done()
   321  			return nil, errors.NotSupportedf("")
   322  		},
   323  	)
   324  
   325  	s.cleanKill(c, s.workerForScenario(c))
   326  }
   327  
   328  func (s *workerSuite) setup(c *gc.C, machineCount int) *gomock.Controller {
   329  	ctrl := gomock.NewController(c)
   330  
   331  	s.facade = mocks.NewMockInstanceMutaterAPI(ctrl)
   332  	s.broker = mocks.NewMockLXDProfiler(ctrl)
   333  	s.agentConfig = mocks.NewMockConfig(ctrl)
   334  	s.machinesWorker = workermocks.NewMockWorker(ctrl)
   335  	s.context = mocks.NewMockMutaterContext(ctrl)
   336  
   337  	s.machine = make(map[int]*mocks.MockMutaterMachine, machineCount)
   338  	s.appLXDProfileWorker = make(map[int]*workermocks.MockWorker)
   339  	for i := 0; i < machineCount; i += 1 {
   340  		s.machine[i] = mocks.NewMockMutaterMachine(ctrl)
   341  		s.appLXDProfileWorker[i] = workermocks.NewMockWorker(ctrl)
   342  	}
   343  
   344  	s.expectContainerTypeNone()
   345  	return ctrl
   346  }
   347  
   348  // workerForScenario creates worker config based on the suite's mocks.
   349  // Any supplied behaviour functions are executed, then a new worker
   350  // is started successfully and returned.
   351  func (s *workerSuite) workerForScenario(c *gc.C) worker.Worker {
   352  	config := instancemutater.Config{
   353  		Facade:                 s.facade,
   354  		Logger:                 s.logger,
   355  		Broker:                 s.broker,
   356  		AgentConfig:            s.agentConfig,
   357  		Tag:                    s.machineTag,
   358  		GetRequiredLXDProfiles: s.getRequiredLXDProfiles,
   359  	}
   360  
   361  	w, err := s.newWorkerFunc(config, func(ctx instancemutater.MutaterContext) instancemutater.MutaterContext {
   362  		return ctx
   363  	})
   364  	c.Assert(err, jc.ErrorIsNil)
   365  	return w
   366  }
   367  
   368  func (s *workerSuite) workerForScenarioWithContext(c *gc.C) worker.Worker {
   369  	config := instancemutater.Config{
   370  		Facade:                 s.facade,
   371  		Logger:                 s.logger,
   372  		Broker:                 s.broker,
   373  		AgentConfig:            s.agentConfig,
   374  		Tag:                    s.machineTag,
   375  		GetRequiredLXDProfiles: s.getRequiredLXDProfiles,
   376  	}
   377  
   378  	w, err := s.newWorkerFunc(config, func(ctx instancemutater.MutaterContext) instancemutater.MutaterContext {
   379  		c := mutaterContextShim{
   380  			MutaterContext: ctx,
   381  			mockContext:    s.context,
   382  		}
   383  		return c
   384  	})
   385  	c.Assert(err, jc.ErrorIsNil)
   386  	return w
   387  }
   388  
   389  func (s *workerSuite) expectFacadeMachineTag(machine int) {
   390  	tag := names.NewMachineTag(strconv.Itoa(machine))
   391  	s.facade.EXPECT().Machine(tag).Return(s.machine[machine], nil).AnyTimes()
   392  	s.machine[machine].EXPECT().Tag().Return(tag).AnyTimes()
   393  }
   394  
   395  func (s *workerSuite) expectFacadeReturnsNoMachine() {
   396  	do := s.workGroupAddGetDoneWithMachineFunc()
   397  	s.facade.EXPECT().Machine(s.machineTag).Return(nil, errors.NewNotFound(nil, "machine")).Do(do)
   398  }
   399  
   400  func (s *workerSuite) expectContainerTypeNone() {
   401  	for _, m := range s.machine {
   402  		m.EXPECT().ContainerType().Return(instance.NONE, nil).AnyTimes()
   403  	}
   404  }
   405  
   406  func (s *workerSuite) expectCharmProfilingInfoSimpleNoChange(machine int) {
   407  	do := s.workGroupAddGetDoneFunc()
   408  	s.machine[machine].EXPECT().CharmProfilingInfo().Return(&apiinstancemutater.UnitProfileInfo{}, nil).Do(do)
   409  }
   410  
   411  func (s *workerSuite) workGroupAddGetDoneFunc() func() {
   412  	s.doneWG.Add(1)
   413  	return func() { s.doneWG.Done() }
   414  }
   415  
   416  func (s *workerSuite) workGroupAddGetDoneWithErrorFunc() func(error) {
   417  	s.doneWG.Add(1)
   418  	return func(error) { s.doneWG.Done() }
   419  }
   420  
   421  func (s *workerSuite) workGroupAddGetDoneWithMachineFunc() func(tag names.MachineTag) {
   422  	s.doneWG.Add(1)
   423  	return func(tag names.MachineTag) { s.doneWG.Done() }
   424  }
   425  
   426  func (s *workerSuite) workGroupAddGetDoneWithStatusFunc() func(status.Status, string, map[string]interface{}) {
   427  	s.doneWG.Add(1)
   428  	return func(status.Status, string, map[string]interface{}) { s.doneWG.Done() }
   429  }
   430  
   431  func (s *workerSuite) expectLXDProfileNamesTrue() {
   432  	s.broker.EXPECT().LXDProfileNames("juju-23423-0").Return([]string{"default", "juju-testing", "juju-testing-one-2"}, nil)
   433  }
   434  
   435  func (s *workerSuite) expectMachineCharmProfilingInfo(machine, rev int) {
   436  	s.expectCharmProfilingInfo(s.machine[machine], rev)
   437  }
   438  
   439  func (s *workerSuite) expectCharmProfilingInfo(mock *mocks.MockMutaterMachine, rev int) {
   440  	mock.EXPECT().CharmProfilingInfo().Return(&apiinstancemutater.UnitProfileInfo{
   441  		CurrentProfiles: []string{"default", "juju-testing", "juju-testing-one-2"},
   442  		InstanceId:      "juju-23423-0",
   443  		ModelName:       "testing",
   444  		ProfileChanges: []apiinstancemutater.UnitProfileChanges{
   445  			{
   446  				ApplicationName: "one",
   447  				Revision:        rev,
   448  				Profile: lxdprofile.Profile{
   449  					Config: map[string]string{"hi": "bye"},
   450  				},
   451  			},
   452  		},
   453  	}, nil)
   454  }
   455  
   456  func (s *workerSuite) expectCharmProfilingInfoRemove(machine int) {
   457  	s.machine[machine].EXPECT().CharmProfilingInfo().Return(&apiinstancemutater.UnitProfileInfo{
   458  		CurrentProfiles: []string{"default", "juju-testing", "juju-testing-one-2"},
   459  		InstanceId:      "juju-23423-0",
   460  		ModelName:       "testing",
   461  		ProfileChanges:  []apiinstancemutater.UnitProfileChanges{},
   462  	}, nil)
   463  }
   464  
   465  func (s *workerSuite) expectCharmProfileInfoNotProvisioned(machine int) {
   466  	do := s.workGroupAddGetDoneFunc()
   467  	err := params.Error{
   468  		Message: "machine 0 not provisioned",
   469  		Code:    params.CodeNotProvisioned,
   470  	}
   471  	s.machine[machine].EXPECT().CharmProfilingInfo().Return(&apiinstancemutater.UnitProfileInfo{}, err).Do(do)
   472  }
   473  
   474  func (s *workerSuite) expectCharmProfileInfoError(machine int) {
   475  	do := s.workGroupAddGetDoneFunc()
   476  	err := params.Error{
   477  		Message: "machine 0 not supported",
   478  		Code:    params.CodeNotSupported,
   479  	}
   480  	s.machine[machine].EXPECT().CharmProfilingInfo().Return(&apiinstancemutater.UnitProfileInfo{}, err).Do(do)
   481  }
   482  
   483  func (s *workerSuite) expectAliveAndSetModificationStatusIdle(machine int) {
   484  	mExp := s.machine[machine].EXPECT()
   485  	mExp.Refresh().Return(nil)
   486  	mExp.Life().Return(life.Alive)
   487  	mExp.SetModificationStatus(status.Idle, "", nil).Return(nil)
   488  }
   489  
   490  func (s *workerSuite) expectMachineAliveStatusIdleMachineDead(machine int, group *sync.WaitGroup) {
   491  	mExp := s.machine[machine].EXPECT()
   492  
   493  	group.Add(1)
   494  	notificationSync := func() { group.Done() }
   495  
   496  	mExp.Refresh().Return(nil).Times(2)
   497  	o1 := mExp.Life().Return(life.Alive).Do(notificationSync)
   498  
   499  	mExp.SetModificationStatus(status.Idle, "", nil).Return(nil)
   500  
   501  	s.machine[0].EXPECT().SetModificationStatus(status.Applied, "", nil).Return(nil)
   502  	doWithStatus := s.workGroupAddGetDoneWithStatusFunc()
   503  	s.machine[1].EXPECT().SetModificationStatus(status.Applied, "", nil).Return(nil).Do(doWithStatus)
   504  
   505  	do := s.workGroupAddGetDoneFunc()
   506  	mExp.Life().Return(life.Dead).After(o1).Do(do)
   507  }
   508  
   509  func (s *workerSuite) expectModificationStatusApplied(machine int) {
   510  	do := s.workGroupAddGetDoneWithStatusFunc()
   511  	s.machine[machine].EXPECT().SetModificationStatus(status.Applied, "", nil).Return(nil).Do(do)
   512  }
   513  
   514  func (s *workerSuite) expectAssignLXDProfiles() {
   515  	profiles := []string{"default", "juju-testing", "juju-testing-one-3"}
   516  	s.broker.EXPECT().AssignLXDProfiles("juju-23423-0", profiles, gomock.Any()).Return(profiles, nil)
   517  }
   518  
   519  func (s *workerSuite) expectSetCharmProfiles(machine int) {
   520  	s.machine[machine].EXPECT().SetCharmProfiles([]string{"default", "juju-testing", "juju-testing-one-3"})
   521  }
   522  
   523  func (s *workerSuite) expectRemoveAllCharmProfiles(machine int) {
   524  	profiles := []string{"default", "juju-testing"}
   525  	s.machine[machine].EXPECT().SetCharmProfiles(profiles)
   526  	s.broker.EXPECT().AssignLXDProfiles("juju-23423-0", profiles, gomock.Any()).Return(profiles, nil)
   527  }
   528  
   529  // notifyMachines returns a suite behaviour that will cause the instance mutator
   530  // watcher to send a number of notifications equal to the supplied argument.
   531  // Once notifications have been consumed, we notify via the suite's channel.
   532  func (s *workerSuite) notifyMachines(values [][]string) {
   533  	ch := make(chan []string)
   534  	s.doneWG.Add(1)
   535  	go func() {
   536  		for _, v := range values {
   537  			ch <- v
   538  		}
   539  		s.doneWG.Done()
   540  	}()
   541  
   542  	s.machinesWorker.EXPECT().Kill().AnyTimes()
   543  	s.machinesWorker.EXPECT().Wait().Return(nil).AnyTimes()
   544  
   545  	s.facade.EXPECT().WatchModelMachines().Return(
   546  		&fakeStringsWatcher{
   547  			Worker: s.machinesWorker,
   548  			ch:     ch,
   549  		}, nil)
   550  }
   551  
   552  func (s *workerSuite) notifyMachinesWaitGroup(values [][]string, group *sync.WaitGroup) {
   553  	ch := make(chan []string)
   554  	s.doneWG.Add(1)
   555  	go func() {
   556  		for _, v := range values {
   557  			ch <- v
   558  			group.Wait()
   559  		}
   560  		s.doneWG.Done()
   561  	}()
   562  
   563  	s.machinesWorker.EXPECT().Kill().AnyTimes()
   564  	s.machinesWorker.EXPECT().Wait().Return(nil).AnyTimes()
   565  
   566  	s.facade.EXPECT().WatchModelMachines().Return(
   567  		&fakeStringsWatcher{
   568  			Worker: s.machinesWorker,
   569  			ch:     ch,
   570  		}, nil)
   571  }
   572  
   573  // notifyAppLXDProfile returns a suite behaviour that will cause the instance mutator
   574  // watcher to send a number of notifications equal to the supplied argument.
   575  // Once notifications have been consumed, we notify via the suite's channel.
   576  func (s *workerSuite) notifyMachineAppLXDProfile(machine, times int) {
   577  	s.notifyAppLXDProfile(s.machine[machine], machine, times)
   578  }
   579  
   580  func (s *workerContainerSuite) notifyContainerAppLXDProfile(times int) {
   581  	s.notifyAppLXDProfile(s.lxdContainer, 0, times)
   582  }
   583  
   584  func (s *workerSuite) notifyAppLXDProfile(mock *mocks.MockMutaterMachine, which, times int) {
   585  	ch := make(chan struct{})
   586  	s.doneWG.Add(1)
   587  	go func() {
   588  		for i := 0; i < times; i += 1 {
   589  			ch <- struct{}{}
   590  		}
   591  		s.doneWG.Done()
   592  	}()
   593  
   594  	w := s.appLXDProfileWorker[which]
   595  	w.EXPECT().Kill().AnyTimes()
   596  	w.EXPECT().Wait().Return(nil).AnyTimes()
   597  
   598  	mock.EXPECT().WatchLXDProfileVerificationNeeded().Return(
   599  		&fakeNotifyWatcher{
   600  			Worker: w,
   601  			ch:     ch,
   602  		}, nil)
   603  }
   604  
   605  // mutaterContextShim is required to override the KillWithError context. We
   606  // can't mock out the whole thing as their are private methods, so we just
   607  // compose it and send it back with a new KillWithError method.
   608  type mutaterContextShim struct {
   609  	instancemutater.MutaterContext
   610  	mockContext *mocks.MockMutaterContext
   611  }
   612  
   613  func (c mutaterContextShim) KillWithError(err error) {
   614  	if c.mockContext != nil {
   615  		c.mockContext.KillWithError(err)
   616  	}
   617  	// We still want to call the original context to ensure that errorKill
   618  	// still passes.
   619  	c.MutaterContext.KillWithError(err)
   620  }
   621  
   622  func (s *workerSuite) expectContextKillError() {
   623  	do := s.workGroupAddGetDoneWithErrorFunc()
   624  	s.context.EXPECT().KillWithError(gomock.Any()).Do(do)
   625  }
   626  
   627  // cleanKill waits for notifications to be processed, then waits for the input
   628  // worker to be killed cleanly. If either ops time out, the test fails.
   629  func (s *workerSuite) cleanKill(c *gc.C, w worker.Worker) {
   630  	s.waitDone(c)
   631  	workertest.CleanKill(c, w)
   632  }
   633  
   634  // errorKill waits for notifications to be processed, then waits for the input
   635  // worker to be killed.  Any error is returned to the caller. If either ops
   636  // time out, the test fails.
   637  func (s *workerSuite) errorKill(c *gc.C, w worker.Worker) error {
   638  	s.waitDone(c)
   639  	return workertest.CheckKill(c, w)
   640  }
   641  
   642  func (s *workerSuite) waitDone(c *gc.C) {
   643  	ch := make(chan struct{})
   644  	go func() {
   645  		s.doneWG.Wait()
   646  		close(ch)
   647  	}()
   648  
   649  	select {
   650  	case <-ch:
   651  	case <-time.After(testing.LongWait):
   652  		c.Errorf("timed out waiting for notifications to be consumed")
   653  	}
   654  }
   655  
   656  type workerContainerSuite struct {
   657  	workerSuite
   658  
   659  	lxdContainerTag names.Tag
   660  	kvmContainerTag names.Tag
   661  	lxdContainer    *mocks.MockMutaterMachine
   662  	kvmContainer    *mocks.MockMutaterMachine
   663  }
   664  
   665  var _ = gc.Suite(&workerContainerSuite{})
   666  
   667  func (s *workerContainerSuite) SetUpTest(c *gc.C) {
   668  	s.workerSuite.SetUpTest(c)
   669  
   670  	s.lxdContainerTag = names.NewMachineTag("0/lxd/0")
   671  	s.kvmContainerTag = names.NewMachineTag("0/kvm/0")
   672  	s.newWorkerFunc = instancemutater.NewContainerTestWorker
   673  	s.getRequiredLXDProfiles = func(modelName string) []string {
   674  		return []string{"default"}
   675  	}
   676  }
   677  
   678  // TestFullWorkflow uses the the expectation scenarios from each of the tests
   679  // below to compose a test of the whole instance mutator scenario, from start
   680  // to finish for a ContainerWorker.
   681  func (s *workerContainerSuite) TestFullWorkflow(c *gc.C) {
   682  	defer s.setup(c).Finish()
   683  
   684  	s.notifyContainers(0, [][]string{{"0/lxd/0", "0/kvm/0"}})
   685  	s.expectFacadeMachineTag(0)
   686  	s.expectFacadeContainerTags()
   687  	s.expectContainerTypes()
   688  	s.notifyContainerAppLXDProfile(1)
   689  	s.expectContainerCharmProfilingInfo(3)
   690  	s.expectLXDProfileNamesTrue()
   691  	s.expectContainerSetCharmProfiles()
   692  	s.expectAssignLXDProfiles()
   693  	s.expectContainerAliveAndSetModificationStatusIdle()
   694  	s.expectContainerModificationStatusApplied()
   695  
   696  	s.cleanKill(c, s.workerForScenario(c))
   697  }
   698  
   699  func (s *workerContainerSuite) setup(c *gc.C) *gomock.Controller {
   700  	ctrl := s.workerSuite.setup(c, 1)
   701  	s.lxdContainer = mocks.NewMockMutaterMachine(ctrl)
   702  	s.kvmContainer = mocks.NewMockMutaterMachine(ctrl)
   703  	return ctrl
   704  }
   705  
   706  func (s *workerContainerSuite) expectFacadeContainerTags() {
   707  	s.facade.EXPECT().Machine(s.lxdContainerTag).Return(s.lxdContainer, nil).AnyTimes()
   708  	s.lxdContainer.EXPECT().Tag().Return(s.lxdContainerTag).AnyTimes()
   709  	s.facade.EXPECT().Machine(s.kvmContainerTag).Return(s.kvmContainer, nil).AnyTimes()
   710  	s.kvmContainer.EXPECT().Tag().Return(s.kvmContainerTag).AnyTimes()
   711  }
   712  
   713  func (s *workerContainerSuite) expectContainerTypes() {
   714  	s.lxdContainer.EXPECT().ContainerType().Return(instance.LXD, nil).AnyTimes()
   715  	s.kvmContainer.EXPECT().ContainerType().Return(instance.KVM, nil).AnyTimes()
   716  }
   717  
   718  func (s *workerContainerSuite) expectContainerCharmProfilingInfo(rev int) {
   719  	s.expectCharmProfilingInfo(s.lxdContainer, rev)
   720  }
   721  
   722  func (s *workerContainerSuite) expectContainerAliveAndSetModificationStatusIdle() {
   723  	cExp := s.lxdContainer.EXPECT()
   724  	cExp.Refresh().Return(nil)
   725  	cExp.Life().Return(life.Alive)
   726  	cExp.SetModificationStatus(status.Idle, gomock.Any(), gomock.Any()).Return(nil)
   727  }
   728  
   729  func (s *workerContainerSuite) expectContainerModificationStatusApplied() {
   730  	do := s.workGroupAddGetDoneWithStatusFunc()
   731  	s.lxdContainer.EXPECT().SetModificationStatus(status.Applied, "", nil).Return(nil).Do(do)
   732  }
   733  
   734  func (s *workerContainerSuite) expectAssignLXDProfiles() {
   735  	profiles := []string{"default", "juju-testing-one-3"}
   736  	s.broker.EXPECT().AssignLXDProfiles("juju-23423-0", profiles, gomock.Any()).Return(profiles, nil)
   737  }
   738  
   739  func (s *workerContainerSuite) expectContainerSetCharmProfiles() {
   740  	s.lxdContainer.EXPECT().SetCharmProfiles([]string{"default", "juju-testing-one-3"})
   741  }
   742  
   743  // notifyContainers returns a suite behaviour that will cause the instance mutator
   744  // watcher to send a number of notifications equal to the supplied argument.
   745  // Once notifications have been consumed, we notify via the suite's channel.
   746  func (s *workerContainerSuite) notifyContainers(machine int, values [][]string) {
   747  	ch := make(chan []string)
   748  	s.doneWG.Add(1)
   749  	go func() {
   750  		for _, v := range values {
   751  			ch <- v
   752  		}
   753  		s.doneWG.Done()
   754  	}()
   755  
   756  	s.machinesWorker.EXPECT().Kill().AnyTimes()
   757  	s.machinesWorker.EXPECT().Wait().Return(nil).AnyTimes()
   758  
   759  	s.machine[machine].EXPECT().WatchContainers().Return(
   760  		&fakeStringsWatcher{
   761  			Worker: s.machinesWorker,
   762  			ch:     ch,
   763  		}, nil)
   764  }
   765  
   766  type fakeStringsWatcher struct {
   767  	worker.Worker
   768  	ch <-chan []string
   769  }
   770  
   771  func (w *fakeStringsWatcher) Changes() watcher.StringsChannel {
   772  	return w.ch
   773  }
   774  
   775  type fakeNotifyWatcher struct {
   776  	worker.Worker
   777  	ch <-chan struct{}
   778  }
   779  
   780  func (w *fakeNotifyWatcher) Changes() watcher.NotifyChannel {
   781  	return w.ch
   782  }