github.com/axw/juju@v0.0.0-20161005053422-4bd6544d08d4/worker/upgradesteps/worker_test.go (about)

     1  // Copyright 2015 Canonical Ltd.
     2  // Licensed under the AGPLv3, see LICENCE file for details.
     3  
     4  package upgradesteps
     5  
     6  import (
     7  	"fmt"
     8  	"time"
     9  
    10  	"github.com/juju/errors"
    11  	"github.com/juju/loggo"
    12  	jc "github.com/juju/testing/checkers"
    13  	"github.com/juju/utils"
    14  	"github.com/juju/utils/arch"
    15  	"github.com/juju/utils/series"
    16  	gc "gopkg.in/check.v1"
    17  	"gopkg.in/juju/names.v2"
    18  
    19  	"github.com/juju/juju/agent"
    20  	cmdutil "github.com/juju/juju/cmd/jujud/util"
    21  	"github.com/juju/juju/constraints"
    22  	"github.com/juju/juju/environs"
    23  	"github.com/juju/juju/instance"
    24  	"github.com/juju/juju/mongo/mongotest"
    25  	"github.com/juju/juju/state"
    26  	"github.com/juju/juju/state/multiwatcher"
    27  	"github.com/juju/juju/state/stateenvirons"
    28  	statetesting "github.com/juju/juju/state/testing"
    29  	"github.com/juju/juju/status"
    30  	coretesting "github.com/juju/juju/testing"
    31  	"github.com/juju/juju/testing/factory"
    32  	"github.com/juju/juju/upgrades"
    33  	jujuversion "github.com/juju/juju/version"
    34  	"github.com/juju/juju/worker"
    35  	"github.com/juju/juju/worker/gate"
    36  	"github.com/juju/version"
    37  )
    38  
    39  // TODO(mjs) - these tests are too tightly coupled to the
    40  // implementation. They needn't be internal tests.
    41  
    42  type UpgradeSuite struct {
    43  	statetesting.StateSuite
    44  
    45  	oldVersion      version.Binary
    46  	logWriter       loggo.TestWriter
    47  	connectionDead  bool
    48  	machineIsMaster bool
    49  	preUpgradeError bool
    50  }
    51  
    52  var _ = gc.Suite(&UpgradeSuite{})
    53  
    54  const fails = true
    55  const succeeds = false
    56  
    57  func (s *UpgradeSuite) SetUpTest(c *gc.C) {
    58  	s.StateSuite.SetUpTest(c)
    59  
    60  	s.preUpgradeError = false
    61  	// Most of these tests normally finish sub-second on a fast machine.
    62  	// If any given test hits a minute, we have almost certainly become
    63  	// wedged, so dump the logs.
    64  	coretesting.DumpTestLogsAfter(time.Minute, c, s)
    65  
    66  	s.oldVersion = version.Binary{
    67  		Number: jujuversion.Current,
    68  		Arch:   arch.HostArch(),
    69  		Series: series.HostSeries(),
    70  	}
    71  	s.oldVersion.Major = 1
    72  	s.oldVersion.Minor = 16
    73  
    74  	// Don't wait so long in tests.
    75  	s.PatchValue(&UpgradeStartTimeoutMaster, time.Duration(time.Millisecond*50))
    76  	s.PatchValue(&UpgradeStartTimeoutSecondary, time.Duration(time.Millisecond*60))
    77  
    78  	// Allow tests to make the API connection appear to be dead.
    79  	s.connectionDead = false
    80  	s.PatchValue(&cmdutil.ConnectionIsDead, func(loggo.Logger, cmdutil.Pinger) bool {
    81  		return s.connectionDead
    82  	})
    83  
    84  	s.machineIsMaster = true
    85  	fakeIsMachineMaster := func(*state.State, string) (bool, error) {
    86  		return s.machineIsMaster, nil
    87  	}
    88  	s.PatchValue(&IsMachineMaster, fakeIsMachineMaster)
    89  
    90  }
    91  
    92  func (s *UpgradeSuite) captureLogs(c *gc.C) {
    93  	c.Assert(loggo.RegisterWriter("upgrade-tests", &s.logWriter), gc.IsNil)
    94  	s.AddCleanup(func(*gc.C) {
    95  		loggo.RemoveWriter("upgrade-tests")
    96  		s.logWriter.Clear()
    97  	})
    98  }
    99  
   100  func (s *UpgradeSuite) countUpgradeAttempts(upgradeErr error) *int {
   101  	count := 0
   102  	s.PatchValue(&PerformUpgrade, func(version.Number, []upgrades.Target, upgrades.Context) error {
   103  		count++
   104  		return upgradeErr
   105  	})
   106  	return &count
   107  }
   108  
   109  func (s *UpgradeSuite) TestNewChannelWhenNoUpgradeRequired(c *gc.C) {
   110  	// Set the agent's initial upgradedToVersion to almost the same as
   111  	// the current version. We want it to be different to
   112  	// jujuversion.Current (so that we can see it change) but not to
   113  	// trigger upgrade steps.
   114  	config := NewFakeConfigSetter(names.NewMachineTag("0"), makeBumpedCurrentVersion().Number)
   115  	agent := NewFakeAgent(config)
   116  
   117  	lock, err := NewLock(agent)
   118  	c.Assert(err, jc.ErrorIsNil)
   119  
   120  	c.Assert(lock.IsUnlocked(), jc.IsTrue)
   121  	// The agent's version should have been updated.
   122  	c.Assert(config.Version, gc.Equals, jujuversion.Current)
   123  
   124  }
   125  
   126  func (s *UpgradeSuite) TestNewChannelWhenUpgradeRequired(c *gc.C) {
   127  	// Set the agent's upgradedToVersion so that upgrade steps are required.
   128  	initialVersion := version.MustParse("1.16.0")
   129  	config := NewFakeConfigSetter(names.NewMachineTag("0"), initialVersion)
   130  	agent := NewFakeAgent(config)
   131  
   132  	lock, err := NewLock(agent)
   133  	c.Assert(err, jc.ErrorIsNil)
   134  
   135  	c.Assert(lock.IsUnlocked(), jc.IsFalse)
   136  	// The agent's version should NOT have been updated.
   137  	c.Assert(config.Version, gc.Equals, initialVersion)
   138  }
   139  
   140  func (s *UpgradeSuite) TestRetryStrategy(c *gc.C) {
   141  	retries := getUpgradeRetryStrategy()
   142  	c.Assert(retries.Delay, gc.Equals, 2*time.Minute)
   143  	c.Assert(retries.Min, gc.Equals, 5)
   144  }
   145  
   146  func (s *UpgradeSuite) TestNoUpgradeNecessary(c *gc.C) {
   147  	attemptsP := s.countUpgradeAttempts(nil)
   148  	s.captureLogs(c)
   149  	s.oldVersion.Number = jujuversion.Current // nothing to do
   150  
   151  	workerErr, config, _, doneLock := s.runUpgradeWorker(c, multiwatcher.JobHostUnits)
   152  
   153  	c.Check(workerErr, gc.IsNil)
   154  	c.Check(*attemptsP, gc.Equals, 0)
   155  	c.Check(config.Version, gc.Equals, jujuversion.Current)
   156  	c.Check(doneLock.IsUnlocked(), jc.IsTrue)
   157  }
   158  
   159  func (s *UpgradeSuite) TestUpgradeStepsFailure(c *gc.C) {
   160  	// This test checks what happens when every upgrade attempt fails.
   161  	// A number of retries should be observed and the agent should end
   162  	// up in a state where it is is still running but is reporting an
   163  	// error and the upgrade is not flagged as having completed (which
   164  	// prevents most of the agent's workers from running and keeps the
   165  	// API in restricted mode).
   166  
   167  	attemptsP := s.countUpgradeAttempts(errors.New("boom"))
   168  	s.captureLogs(c)
   169  
   170  	workerErr, config, statusCalls, doneLock := s.runUpgradeWorker(c, multiwatcher.JobHostUnits)
   171  
   172  	// The worker shouldn't return an error so that the worker and
   173  	// agent keep running.
   174  	c.Check(workerErr, gc.IsNil)
   175  
   176  	c.Check(*attemptsP, gc.Equals, maxUpgradeRetries)
   177  	c.Check(config.Version, gc.Equals, s.oldVersion.Number) // Upgrade didn't finish
   178  	c.Assert(statusCalls, jc.DeepEquals,
   179  		s.makeExpectedStatusCalls(maxUpgradeRetries-1, fails, "boom"))
   180  	c.Assert(s.logWriter.Log(), jc.LogMatches,
   181  		s.makeExpectedUpgradeLogs(maxUpgradeRetries-1, "hostMachine", fails, "boom"))
   182  	c.Assert(doneLock.IsUnlocked(), jc.IsFalse)
   183  }
   184  
   185  func (s *UpgradeSuite) TestUpgradeStepsRetries(c *gc.C) {
   186  	// This test checks what happens when the first upgrade attempt
   187  	// fails but the following on succeeds. The final state should be
   188  	// the same as a successful upgrade which worked first go.
   189  	attempts := 0
   190  	fail := true
   191  	fakePerformUpgrade := func(version.Number, []upgrades.Target, upgrades.Context) error {
   192  		attempts++
   193  		if fail {
   194  			fail = false
   195  			return errors.New("boom")
   196  		} else {
   197  			return nil
   198  		}
   199  	}
   200  	s.PatchValue(&PerformUpgrade, fakePerformUpgrade)
   201  	s.captureLogs(c)
   202  
   203  	workerErr, config, statusCalls, doneLock := s.runUpgradeWorker(c, multiwatcher.JobHostUnits)
   204  
   205  	c.Check(workerErr, gc.IsNil)
   206  	c.Check(attempts, gc.Equals, 2)
   207  	c.Check(config.Version, gc.Equals, jujuversion.Current) // Upgrade finished
   208  	c.Assert(statusCalls, jc.DeepEquals, s.makeExpectedStatusCalls(1, succeeds, "boom"))
   209  	c.Assert(s.logWriter.Log(), jc.LogMatches, s.makeExpectedUpgradeLogs(1, "hostMachine", succeeds, "boom"))
   210  	c.Check(doneLock.IsUnlocked(), jc.IsTrue)
   211  }
   212  
   213  func (s *UpgradeSuite) TestOtherUpgradeRunFailure(c *gc.C) {
   214  	// This test checks what happens something other than the upgrade
   215  	// steps themselves fails, ensuring the something is logged and
   216  	// the agent status is updated.
   217  
   218  	fakePerformUpgrade := func(version.Number, []upgrades.Target, upgrades.Context) error {
   219  		// Delete UpgradeInfo for the upgrade so that finaliseUpgrade() will fail
   220  		s.State.ClearUpgradeInfo()
   221  		return nil
   222  	}
   223  	s.PatchValue(&PerformUpgrade, fakePerformUpgrade)
   224  	s.Factory.MakeMachine(c, &factory.MachineParams{
   225  		Jobs: []state.MachineJob{state.JobManageModel},
   226  	})
   227  	s.captureLogs(c)
   228  
   229  	workerErr, config, statusCalls, doneLock := s.runUpgradeWorker(c, multiwatcher.JobManageModel)
   230  
   231  	c.Check(workerErr, gc.IsNil)
   232  	c.Check(config.Version, gc.Equals, jujuversion.Current) // Upgrade almost finished
   233  	failReason := `upgrade done but: cannot set upgrade status to "finishing": ` +
   234  		`Another status change may have occurred concurrently`
   235  	c.Assert(statusCalls, jc.DeepEquals,
   236  		s.makeExpectedStatusCalls(0, fails, failReason))
   237  	c.Assert(s.logWriter.Log(), jc.LogMatches,
   238  		s.makeExpectedUpgradeLogs(0, "databaseMaster", fails, failReason))
   239  	c.Assert(doneLock.IsUnlocked(), jc.IsFalse)
   240  }
   241  
   242  func (s *UpgradeSuite) TestAPIConnectionFailure(c *gc.C) {
   243  	// This test checks what happens when an upgrade fails because the
   244  	// connection to mongo has gone away. This will happen when the
   245  	// mongo master changes. In this case we want the upgrade worker
   246  	// to return immediately without further retries. The error should
   247  	// be returned by the worker so that the agent will restart.
   248  
   249  	attemptsP := s.countUpgradeAttempts(errors.New("boom"))
   250  	s.connectionDead = true // Make the connection to state appear to be dead
   251  	s.captureLogs(c)
   252  
   253  	workerErr, config, _, doneLock := s.runUpgradeWorker(c, multiwatcher.JobHostUnits)
   254  
   255  	c.Check(workerErr, gc.ErrorMatches, "API connection lost during upgrade: boom")
   256  	c.Check(*attemptsP, gc.Equals, 1)
   257  	c.Check(config.Version, gc.Equals, s.oldVersion.Number) // Upgrade didn't finish
   258  	c.Assert(doneLock.IsUnlocked(), jc.IsFalse)
   259  }
   260  
   261  func (s *UpgradeSuite) TestAbortWhenOtherControllerDoesntStartUpgrade(c *gc.C) {
   262  	// This test checks when a controller is upgrading and one of
   263  	// the other controllers doesn't signal it is ready in time.
   264  
   265  	err := s.State.SetModelAgentVersion(jujuversion.Current)
   266  	c.Assert(err, jc.ErrorIsNil)
   267  
   268  	// The master controller in this scenario is functionally tested
   269  	// elsewhere.
   270  	s.machineIsMaster = false
   271  
   272  	s.create3Controllers(c)
   273  	s.captureLogs(c)
   274  	attemptsP := s.countUpgradeAttempts(nil)
   275  
   276  	workerErr, config, statusCalls, doneLock := s.runUpgradeWorker(c, multiwatcher.JobManageModel)
   277  
   278  	c.Check(workerErr, gc.IsNil)
   279  	c.Check(*attemptsP, gc.Equals, 0)
   280  	c.Check(config.Version, gc.Equals, s.oldVersion.Number) // Upgrade didn't happen
   281  	c.Assert(doneLock.IsUnlocked(), jc.IsFalse)
   282  
   283  	// The environment agent-version should still be the new version.
   284  	// It's up to the master to trigger the rollback.
   285  	s.assertEnvironAgentVersion(c, jujuversion.Current)
   286  
   287  	causeMsg := " timed out after 60ms"
   288  	c.Assert(s.logWriter.Log(), jc.LogMatches, []jc.SimpleMessage{
   289  		{loggo.INFO, "waiting for other controllers to be ready for upgrade"},
   290  		{loggo.ERROR, "aborted wait for other controllers: timed out after 60ms"},
   291  		{loggo.ERROR, `upgrade from .+ to .+ for "machine-0" failed \(giving up\): ` +
   292  			"aborted wait for other controllers:" + causeMsg},
   293  	})
   294  	c.Assert(statusCalls, jc.DeepEquals, []StatusCall{{
   295  		status.Error,
   296  		fmt.Sprintf(
   297  			"upgrade to %s failed (giving up): aborted wait for other controllers:"+causeMsg,
   298  			jujuversion.Current),
   299  	}})
   300  }
   301  
   302  func (s *UpgradeSuite) TestSuccessMaster(c *gc.C) {
   303  	// This test checks what happens when an upgrade works on the
   304  	// first attempt on a master controller.
   305  	s.machineIsMaster = true
   306  	info := s.checkSuccess(c, "databaseMaster", func(*state.UpgradeInfo) {})
   307  	c.Assert(info.Status(), gc.Equals, state.UpgradeFinishing)
   308  }
   309  
   310  func (s *UpgradeSuite) TestSuccessSecondary(c *gc.C) {
   311  	// This test checks what happens when an upgrade works on the
   312  	// first attempt on a secondary controller.
   313  	s.machineIsMaster = false
   314  	mungeInfo := func(info *state.UpgradeInfo) {
   315  		// Indicate that the master is done
   316  		err := info.SetStatus(state.UpgradeRunning)
   317  		c.Assert(err, jc.ErrorIsNil)
   318  		err = info.SetStatus(state.UpgradeFinishing)
   319  		c.Assert(err, jc.ErrorIsNil)
   320  	}
   321  	s.checkSuccess(c, "controller", mungeInfo)
   322  }
   323  
   324  func (s *UpgradeSuite) checkSuccess(c *gc.C, target string, mungeInfo func(*state.UpgradeInfo)) *state.UpgradeInfo {
   325  	_, machineIdB, machineIdC := s.create3Controllers(c)
   326  
   327  	// Indicate that machine B and C are ready to upgrade
   328  	vPrevious := s.oldVersion.Number
   329  	vNext := jujuversion.Current
   330  	info, err := s.State.EnsureUpgradeInfo(machineIdB, vPrevious, vNext)
   331  	c.Assert(err, jc.ErrorIsNil)
   332  	_, err = s.State.EnsureUpgradeInfo(machineIdC, vPrevious, vNext)
   333  	c.Assert(err, jc.ErrorIsNil)
   334  
   335  	mungeInfo(info)
   336  
   337  	attemptsP := s.countUpgradeAttempts(nil)
   338  	s.captureLogs(c)
   339  
   340  	workerErr, config, statusCalls, doneLock := s.runUpgradeWorker(c, multiwatcher.JobManageModel)
   341  
   342  	c.Check(workerErr, gc.IsNil)
   343  	c.Check(*attemptsP, gc.Equals, 1)
   344  	c.Check(config.Version, gc.Equals, jujuversion.Current) // Upgrade finished
   345  	c.Assert(statusCalls, jc.DeepEquals, s.makeExpectedStatusCalls(0, succeeds, ""))
   346  	c.Assert(s.logWriter.Log(), jc.LogMatches, s.makeExpectedUpgradeLogs(0, target, succeeds, ""))
   347  	c.Check(doneLock.IsUnlocked(), jc.IsTrue)
   348  
   349  	err = info.Refresh()
   350  	c.Assert(err, jc.ErrorIsNil)
   351  	c.Assert(info.ControllersDone(), jc.DeepEquals, []string{"0"})
   352  	return info
   353  }
   354  
   355  func (s *UpgradeSuite) TestJobsToTargets(c *gc.C) {
   356  	check := func(jobs []multiwatcher.MachineJob, isMaster bool, expectedTargets ...upgrades.Target) {
   357  		c.Assert(jobsToTargets(jobs, isMaster), jc.SameContents, expectedTargets)
   358  	}
   359  
   360  	check([]multiwatcher.MachineJob{multiwatcher.JobHostUnits}, false, upgrades.HostMachine)
   361  	check([]multiwatcher.MachineJob{multiwatcher.JobManageModel}, false, upgrades.Controller)
   362  	check([]multiwatcher.MachineJob{multiwatcher.JobManageModel}, true,
   363  		upgrades.Controller, upgrades.DatabaseMaster)
   364  	check([]multiwatcher.MachineJob{multiwatcher.JobManageModel, multiwatcher.JobHostUnits}, false,
   365  		upgrades.Controller, upgrades.HostMachine)
   366  	check([]multiwatcher.MachineJob{multiwatcher.JobManageModel, multiwatcher.JobHostUnits}, true,
   367  		upgrades.Controller, upgrades.DatabaseMaster, upgrades.HostMachine)
   368  }
   369  
   370  func (s *UpgradeSuite) TestPreUpgradeFail(c *gc.C) {
   371  	s.preUpgradeError = true
   372  	s.captureLogs(c)
   373  
   374  	workerErr, config, statusCalls, doneLock := s.runUpgradeWorker(c, multiwatcher.JobHostUnits)
   375  
   376  	c.Check(workerErr, jc.ErrorIsNil)
   377  	c.Check(config.Version, gc.Equals, s.oldVersion.Number) // Upgrade didn't finish
   378  	c.Assert(doneLock.IsUnlocked(), jc.IsFalse)
   379  
   380  	causeMessage := `machine 0 cannot be upgraded: preupgrade error`
   381  	failMessage := fmt.Sprintf(
   382  		`upgrade from %s to %s for "machine-0" failed \(giving up\): %s`,
   383  		s.oldVersion.Number, jujuversion.Current, causeMessage)
   384  	c.Assert(s.logWriter.Log(), jc.LogMatches, []jc.SimpleMessage{
   385  		{loggo.INFO, "checking that upgrade can proceed"},
   386  		{loggo.ERROR, failMessage},
   387  	})
   388  
   389  	statusMessage := fmt.Sprintf(
   390  		`upgrade to %s failed (giving up): %s`, jujuversion.Current, causeMessage)
   391  	c.Assert(statusCalls, jc.DeepEquals, []StatusCall{{
   392  		status.Error, statusMessage,
   393  	}})
   394  }
   395  
   396  // Run just the upgradesteps worker with a fake machine agent and
   397  // fake agent config.
   398  func (s *UpgradeSuite) runUpgradeWorker(c *gc.C, jobs ...multiwatcher.MachineJob) (
   399  	error, *fakeConfigSetter, []StatusCall, gate.Lock,
   400  ) {
   401  	s.setInstantRetryStrategy(c)
   402  	config := s.makeFakeConfig()
   403  	agent := NewFakeAgent(config)
   404  	doneLock, err := NewLock(agent)
   405  	c.Assert(err, jc.ErrorIsNil)
   406  	machineStatus := &testStatusSetter{}
   407  	worker, err := NewWorker(doneLock, agent, nil, jobs, s.openStateForUpgrade, s.preUpgradeSteps, machineStatus)
   408  	c.Assert(err, jc.ErrorIsNil)
   409  	return worker.Wait(), config, machineStatus.Calls, doneLock
   410  }
   411  
   412  func (s *UpgradeSuite) openStateForUpgrade() (*state.State, error) {
   413  	mongoInfo := s.State.MongoConnectionInfo()
   414  	newPolicy := stateenvirons.GetNewPolicyFunc(
   415  		stateenvirons.GetNewEnvironFunc(environs.New),
   416  	)
   417  	st, err := state.Open(s.State.ModelTag(), s.State.ControllerTag(), mongoInfo, mongotest.DialOpts(), newPolicy)
   418  	if err != nil {
   419  		return nil, err
   420  	}
   421  	return st, nil
   422  }
   423  
   424  func (s *UpgradeSuite) preUpgradeSteps(st *state.State, agentConf agent.Config, isController, isMasterController bool) error {
   425  	if s.preUpgradeError {
   426  		return errors.New("preupgrade error")
   427  	}
   428  	return nil
   429  }
   430  
   431  func (s *UpgradeSuite) makeFakeConfig() *fakeConfigSetter {
   432  	return NewFakeConfigSetter(names.NewMachineTag("0"), s.oldVersion.Number)
   433  }
   434  
   435  func (s *UpgradeSuite) create3Controllers(c *gc.C) (machineIdA, machineIdB, machineIdC string) {
   436  	machine0 := s.Factory.MakeMachine(c, &factory.MachineParams{
   437  		Jobs: []state.MachineJob{state.JobManageModel},
   438  	})
   439  	machineIdA = machine0.Id()
   440  	s.setMachineAlive(c, machineIdA)
   441  
   442  	changes, err := s.State.EnableHA(3, constraints.Value{}, "quantal", nil)
   443  	c.Assert(err, jc.ErrorIsNil)
   444  	c.Assert(len(changes.Added), gc.Equals, 2)
   445  
   446  	machineIdB = changes.Added[0]
   447  	s.setMachineProvisioned(c, machineIdB)
   448  	s.setMachineAlive(c, machineIdB)
   449  
   450  	machineIdC = changes.Added[1]
   451  	s.setMachineProvisioned(c, machineIdC)
   452  	s.setMachineAlive(c, machineIdC)
   453  
   454  	return
   455  }
   456  
   457  func (s *UpgradeSuite) setMachineProvisioned(c *gc.C, id string) {
   458  	machine, err := s.State.Machine(id)
   459  	c.Assert(err, jc.ErrorIsNil)
   460  	err = machine.SetProvisioned(instance.Id(id+"-inst"), "nonce", nil)
   461  	c.Assert(err, jc.ErrorIsNil)
   462  }
   463  
   464  func (s *UpgradeSuite) setMachineAlive(c *gc.C, id string) {
   465  	machine, err := s.State.Machine(id)
   466  	c.Assert(err, jc.ErrorIsNil)
   467  	pinger, err := machine.SetAgentPresence()
   468  	c.Assert(err, jc.ErrorIsNil)
   469  	s.AddCleanup(func(c *gc.C) {
   470  		c.Assert(worker.Stop(pinger), jc.ErrorIsNil)
   471  	})
   472  }
   473  
   474  // Return a version the same as the current software version, but with
   475  // the build number bumped.
   476  //
   477  // The version Tag is also cleared so that upgrades.PerformUpgrade
   478  // doesn't think it needs to run upgrade steps unnecessarily.
   479  func makeBumpedCurrentVersion() version.Binary {
   480  	v := version.Binary{
   481  		Number: jujuversion.Current,
   482  		Arch:   arch.HostArch(),
   483  		Series: series.HostSeries(),
   484  	}
   485  	v.Build++
   486  	v.Tag = ""
   487  	return v
   488  }
   489  
   490  const maxUpgradeRetries = 3
   491  
   492  func (s *UpgradeSuite) setInstantRetryStrategy(c *gc.C) {
   493  	// TODO(katco): 2016-08-09: lp:1611427
   494  	s.PatchValue(&getUpgradeRetryStrategy, func() utils.AttemptStrategy {
   495  		c.Logf("setting instant retry strategy for upgrade: retries=%d", maxUpgradeRetries)
   496  		return utils.AttemptStrategy{
   497  			Delay: 0,
   498  			Min:   maxUpgradeRetries,
   499  		}
   500  	})
   501  }
   502  
   503  func (s *UpgradeSuite) makeExpectedStatusCalls(retryCount int, expectFail bool, failReason string) []StatusCall {
   504  	calls := []StatusCall{{
   505  		status.Started,
   506  		fmt.Sprintf("upgrading to %s", jujuversion.Current),
   507  	}}
   508  	for i := 0; i < retryCount; i++ {
   509  		calls = append(calls, StatusCall{
   510  			status.Error,
   511  			fmt.Sprintf("upgrade to %s failed (will retry): %s", jujuversion.Current, failReason),
   512  		})
   513  	}
   514  	if expectFail {
   515  		calls = append(calls, StatusCall{
   516  			status.Error,
   517  			fmt.Sprintf("upgrade to %s failed (giving up): %s", jujuversion.Current, failReason),
   518  		})
   519  	} else {
   520  		calls = append(calls, StatusCall{status.Started, ""})
   521  	}
   522  	return calls
   523  }
   524  
   525  func (s *UpgradeSuite) makeExpectedUpgradeLogs(retryCount int, target string, expectFail bool, failReason string) []jc.SimpleMessage {
   526  	outLogs := []jc.SimpleMessage{}
   527  
   528  	if target == "databaseMaster" || target == "controller" {
   529  		outLogs = append(outLogs, jc.SimpleMessage{
   530  			loggo.INFO, "waiting for other controllers to be ready for upgrade",
   531  		})
   532  		var waitMsg string
   533  		switch target {
   534  		case "databaseMaster":
   535  			waitMsg = "all controllers are ready to run upgrade steps"
   536  		case "controller":
   537  			waitMsg = "the master has completed its upgrade steps"
   538  		}
   539  		outLogs = append(outLogs, jc.SimpleMessage{loggo.INFO, "finished waiting - " + waitMsg})
   540  	}
   541  
   542  	outLogs = append(outLogs, jc.SimpleMessage{
   543  		loggo.INFO, fmt.Sprintf(
   544  			`starting upgrade from %s to %s for "machine-0"`,
   545  			s.oldVersion.Number, jujuversion.Current),
   546  	})
   547  
   548  	failMessage := fmt.Sprintf(
   549  		`upgrade from %s to %s for "machine-0" failed \(%%s\): %s`,
   550  		s.oldVersion.Number, jujuversion.Current, failReason)
   551  
   552  	for i := 0; i < retryCount; i++ {
   553  		outLogs = append(outLogs, jc.SimpleMessage{loggo.ERROR, fmt.Sprintf(failMessage, "will retry")})
   554  	}
   555  	if expectFail {
   556  		outLogs = append(outLogs, jc.SimpleMessage{loggo.ERROR, fmt.Sprintf(failMessage, "giving up")})
   557  	} else {
   558  		outLogs = append(outLogs, jc.SimpleMessage{loggo.INFO,
   559  			fmt.Sprintf(`upgrade to %s completed successfully.`, jujuversion.Current)})
   560  	}
   561  	return outLogs
   562  }
   563  
   564  func (s *UpgradeSuite) assertEnvironAgentVersion(c *gc.C, expected version.Number) {
   565  	envConfig, err := s.State.ModelConfig()
   566  	c.Assert(err, jc.ErrorIsNil)
   567  	agentVersion, ok := envConfig.AgentVersion()
   568  	c.Assert(ok, jc.IsTrue)
   569  	c.Assert(agentVersion, gc.Equals, expected)
   570  }
   571  
   572  // NewFakeConfigSetter returns a fakeConfigSetter which implements
   573  // just enough of the agent.ConfigSetter interface to keep the upgrade
   574  // steps worker happy.
   575  func NewFakeConfigSetter(agentTag names.Tag, initialVersion version.Number) *fakeConfigSetter {
   576  	return &fakeConfigSetter{
   577  		AgentTag: agentTag,
   578  		Version:  initialVersion,
   579  	}
   580  }
   581  
   582  type fakeConfigSetter struct {
   583  	agent.ConfigSetter
   584  	AgentTag names.Tag
   585  	Version  version.Number
   586  }
   587  
   588  func (s *fakeConfigSetter) Tag() names.Tag {
   589  	return s.AgentTag
   590  }
   591  
   592  func (s *fakeConfigSetter) UpgradedToVersion() version.Number {
   593  	return s.Version
   594  }
   595  
   596  func (s *fakeConfigSetter) SetUpgradedToVersion(newVersion version.Number) {
   597  	s.Version = newVersion
   598  }
   599  
   600  // NewFakeAgent returns a fakeAgent which implements the agent.Agent
   601  // interface. This provides enough MachineAgent functionality to
   602  // support upgrades.
   603  func NewFakeAgent(confSetter agent.ConfigSetter) *fakeAgent {
   604  	return &fakeAgent{
   605  		config: confSetter,
   606  	}
   607  }
   608  
   609  type fakeAgent struct {
   610  	config agent.ConfigSetter
   611  }
   612  
   613  func (a *fakeAgent) CurrentConfig() agent.Config {
   614  	return a.config
   615  }
   616  
   617  func (a *fakeAgent) ChangeConfig(mutate agent.ConfigMutator) error {
   618  	return mutate(a.config)
   619  }
   620  
   621  type StatusCall struct {
   622  	Status status.Status
   623  	Info   string
   624  }
   625  
   626  type testStatusSetter struct {
   627  	Calls []StatusCall
   628  }
   629  
   630  func (s *testStatusSetter) SetStatus(status status.Status, info string, _ map[string]interface{}) error {
   631  	s.Calls = append(s.Calls, StatusCall{status, info})
   632  	return nil
   633  }