github.com/axw/juju@v0.0.0-20161005053422-4bd6544d08d4/worker/upgradesteps/worker_test.go (about) 1 // Copyright 2015 Canonical Ltd. 2 // Licensed under the AGPLv3, see LICENCE file for details. 3 4 package upgradesteps 5 6 import ( 7 "fmt" 8 "time" 9 10 "github.com/juju/errors" 11 "github.com/juju/loggo" 12 jc "github.com/juju/testing/checkers" 13 "github.com/juju/utils" 14 "github.com/juju/utils/arch" 15 "github.com/juju/utils/series" 16 gc "gopkg.in/check.v1" 17 "gopkg.in/juju/names.v2" 18 19 "github.com/juju/juju/agent" 20 cmdutil "github.com/juju/juju/cmd/jujud/util" 21 "github.com/juju/juju/constraints" 22 "github.com/juju/juju/environs" 23 "github.com/juju/juju/instance" 24 "github.com/juju/juju/mongo/mongotest" 25 "github.com/juju/juju/state" 26 "github.com/juju/juju/state/multiwatcher" 27 "github.com/juju/juju/state/stateenvirons" 28 statetesting "github.com/juju/juju/state/testing" 29 "github.com/juju/juju/status" 30 coretesting "github.com/juju/juju/testing" 31 "github.com/juju/juju/testing/factory" 32 "github.com/juju/juju/upgrades" 33 jujuversion "github.com/juju/juju/version" 34 "github.com/juju/juju/worker" 35 "github.com/juju/juju/worker/gate" 36 "github.com/juju/version" 37 ) 38 39 // TODO(mjs) - these tests are too tightly coupled to the 40 // implementation. They needn't be internal tests. 41 42 type UpgradeSuite struct { 43 statetesting.StateSuite 44 45 oldVersion version.Binary 46 logWriter loggo.TestWriter 47 connectionDead bool 48 machineIsMaster bool 49 preUpgradeError bool 50 } 51 52 var _ = gc.Suite(&UpgradeSuite{}) 53 54 const fails = true 55 const succeeds = false 56 57 func (s *UpgradeSuite) SetUpTest(c *gc.C) { 58 s.StateSuite.SetUpTest(c) 59 60 s.preUpgradeError = false 61 // Most of these tests normally finish sub-second on a fast machine. 62 // If any given test hits a minute, we have almost certainly become 63 // wedged, so dump the logs. 64 coretesting.DumpTestLogsAfter(time.Minute, c, s) 65 66 s.oldVersion = version.Binary{ 67 Number: jujuversion.Current, 68 Arch: arch.HostArch(), 69 Series: series.HostSeries(), 70 } 71 s.oldVersion.Major = 1 72 s.oldVersion.Minor = 16 73 74 // Don't wait so long in tests. 75 s.PatchValue(&UpgradeStartTimeoutMaster, time.Duration(time.Millisecond*50)) 76 s.PatchValue(&UpgradeStartTimeoutSecondary, time.Duration(time.Millisecond*60)) 77 78 // Allow tests to make the API connection appear to be dead. 79 s.connectionDead = false 80 s.PatchValue(&cmdutil.ConnectionIsDead, func(loggo.Logger, cmdutil.Pinger) bool { 81 return s.connectionDead 82 }) 83 84 s.machineIsMaster = true 85 fakeIsMachineMaster := func(*state.State, string) (bool, error) { 86 return s.machineIsMaster, nil 87 } 88 s.PatchValue(&IsMachineMaster, fakeIsMachineMaster) 89 90 } 91 92 func (s *UpgradeSuite) captureLogs(c *gc.C) { 93 c.Assert(loggo.RegisterWriter("upgrade-tests", &s.logWriter), gc.IsNil) 94 s.AddCleanup(func(*gc.C) { 95 loggo.RemoveWriter("upgrade-tests") 96 s.logWriter.Clear() 97 }) 98 } 99 100 func (s *UpgradeSuite) countUpgradeAttempts(upgradeErr error) *int { 101 count := 0 102 s.PatchValue(&PerformUpgrade, func(version.Number, []upgrades.Target, upgrades.Context) error { 103 count++ 104 return upgradeErr 105 }) 106 return &count 107 } 108 109 func (s *UpgradeSuite) TestNewChannelWhenNoUpgradeRequired(c *gc.C) { 110 // Set the agent's initial upgradedToVersion to almost the same as 111 // the current version. We want it to be different to 112 // jujuversion.Current (so that we can see it change) but not to 113 // trigger upgrade steps. 114 config := NewFakeConfigSetter(names.NewMachineTag("0"), makeBumpedCurrentVersion().Number) 115 agent := NewFakeAgent(config) 116 117 lock, err := NewLock(agent) 118 c.Assert(err, jc.ErrorIsNil) 119 120 c.Assert(lock.IsUnlocked(), jc.IsTrue) 121 // The agent's version should have been updated. 122 c.Assert(config.Version, gc.Equals, jujuversion.Current) 123 124 } 125 126 func (s *UpgradeSuite) TestNewChannelWhenUpgradeRequired(c *gc.C) { 127 // Set the agent's upgradedToVersion so that upgrade steps are required. 128 initialVersion := version.MustParse("1.16.0") 129 config := NewFakeConfigSetter(names.NewMachineTag("0"), initialVersion) 130 agent := NewFakeAgent(config) 131 132 lock, err := NewLock(agent) 133 c.Assert(err, jc.ErrorIsNil) 134 135 c.Assert(lock.IsUnlocked(), jc.IsFalse) 136 // The agent's version should NOT have been updated. 137 c.Assert(config.Version, gc.Equals, initialVersion) 138 } 139 140 func (s *UpgradeSuite) TestRetryStrategy(c *gc.C) { 141 retries := getUpgradeRetryStrategy() 142 c.Assert(retries.Delay, gc.Equals, 2*time.Minute) 143 c.Assert(retries.Min, gc.Equals, 5) 144 } 145 146 func (s *UpgradeSuite) TestNoUpgradeNecessary(c *gc.C) { 147 attemptsP := s.countUpgradeAttempts(nil) 148 s.captureLogs(c) 149 s.oldVersion.Number = jujuversion.Current // nothing to do 150 151 workerErr, config, _, doneLock := s.runUpgradeWorker(c, multiwatcher.JobHostUnits) 152 153 c.Check(workerErr, gc.IsNil) 154 c.Check(*attemptsP, gc.Equals, 0) 155 c.Check(config.Version, gc.Equals, jujuversion.Current) 156 c.Check(doneLock.IsUnlocked(), jc.IsTrue) 157 } 158 159 func (s *UpgradeSuite) TestUpgradeStepsFailure(c *gc.C) { 160 // This test checks what happens when every upgrade attempt fails. 161 // A number of retries should be observed and the agent should end 162 // up in a state where it is is still running but is reporting an 163 // error and the upgrade is not flagged as having completed (which 164 // prevents most of the agent's workers from running and keeps the 165 // API in restricted mode). 166 167 attemptsP := s.countUpgradeAttempts(errors.New("boom")) 168 s.captureLogs(c) 169 170 workerErr, config, statusCalls, doneLock := s.runUpgradeWorker(c, multiwatcher.JobHostUnits) 171 172 // The worker shouldn't return an error so that the worker and 173 // agent keep running. 174 c.Check(workerErr, gc.IsNil) 175 176 c.Check(*attemptsP, gc.Equals, maxUpgradeRetries) 177 c.Check(config.Version, gc.Equals, s.oldVersion.Number) // Upgrade didn't finish 178 c.Assert(statusCalls, jc.DeepEquals, 179 s.makeExpectedStatusCalls(maxUpgradeRetries-1, fails, "boom")) 180 c.Assert(s.logWriter.Log(), jc.LogMatches, 181 s.makeExpectedUpgradeLogs(maxUpgradeRetries-1, "hostMachine", fails, "boom")) 182 c.Assert(doneLock.IsUnlocked(), jc.IsFalse) 183 } 184 185 func (s *UpgradeSuite) TestUpgradeStepsRetries(c *gc.C) { 186 // This test checks what happens when the first upgrade attempt 187 // fails but the following on succeeds. The final state should be 188 // the same as a successful upgrade which worked first go. 189 attempts := 0 190 fail := true 191 fakePerformUpgrade := func(version.Number, []upgrades.Target, upgrades.Context) error { 192 attempts++ 193 if fail { 194 fail = false 195 return errors.New("boom") 196 } else { 197 return nil 198 } 199 } 200 s.PatchValue(&PerformUpgrade, fakePerformUpgrade) 201 s.captureLogs(c) 202 203 workerErr, config, statusCalls, doneLock := s.runUpgradeWorker(c, multiwatcher.JobHostUnits) 204 205 c.Check(workerErr, gc.IsNil) 206 c.Check(attempts, gc.Equals, 2) 207 c.Check(config.Version, gc.Equals, jujuversion.Current) // Upgrade finished 208 c.Assert(statusCalls, jc.DeepEquals, s.makeExpectedStatusCalls(1, succeeds, "boom")) 209 c.Assert(s.logWriter.Log(), jc.LogMatches, s.makeExpectedUpgradeLogs(1, "hostMachine", succeeds, "boom")) 210 c.Check(doneLock.IsUnlocked(), jc.IsTrue) 211 } 212 213 func (s *UpgradeSuite) TestOtherUpgradeRunFailure(c *gc.C) { 214 // This test checks what happens something other than the upgrade 215 // steps themselves fails, ensuring the something is logged and 216 // the agent status is updated. 217 218 fakePerformUpgrade := func(version.Number, []upgrades.Target, upgrades.Context) error { 219 // Delete UpgradeInfo for the upgrade so that finaliseUpgrade() will fail 220 s.State.ClearUpgradeInfo() 221 return nil 222 } 223 s.PatchValue(&PerformUpgrade, fakePerformUpgrade) 224 s.Factory.MakeMachine(c, &factory.MachineParams{ 225 Jobs: []state.MachineJob{state.JobManageModel}, 226 }) 227 s.captureLogs(c) 228 229 workerErr, config, statusCalls, doneLock := s.runUpgradeWorker(c, multiwatcher.JobManageModel) 230 231 c.Check(workerErr, gc.IsNil) 232 c.Check(config.Version, gc.Equals, jujuversion.Current) // Upgrade almost finished 233 failReason := `upgrade done but: cannot set upgrade status to "finishing": ` + 234 `Another status change may have occurred concurrently` 235 c.Assert(statusCalls, jc.DeepEquals, 236 s.makeExpectedStatusCalls(0, fails, failReason)) 237 c.Assert(s.logWriter.Log(), jc.LogMatches, 238 s.makeExpectedUpgradeLogs(0, "databaseMaster", fails, failReason)) 239 c.Assert(doneLock.IsUnlocked(), jc.IsFalse) 240 } 241 242 func (s *UpgradeSuite) TestAPIConnectionFailure(c *gc.C) { 243 // This test checks what happens when an upgrade fails because the 244 // connection to mongo has gone away. This will happen when the 245 // mongo master changes. In this case we want the upgrade worker 246 // to return immediately without further retries. The error should 247 // be returned by the worker so that the agent will restart. 248 249 attemptsP := s.countUpgradeAttempts(errors.New("boom")) 250 s.connectionDead = true // Make the connection to state appear to be dead 251 s.captureLogs(c) 252 253 workerErr, config, _, doneLock := s.runUpgradeWorker(c, multiwatcher.JobHostUnits) 254 255 c.Check(workerErr, gc.ErrorMatches, "API connection lost during upgrade: boom") 256 c.Check(*attemptsP, gc.Equals, 1) 257 c.Check(config.Version, gc.Equals, s.oldVersion.Number) // Upgrade didn't finish 258 c.Assert(doneLock.IsUnlocked(), jc.IsFalse) 259 } 260 261 func (s *UpgradeSuite) TestAbortWhenOtherControllerDoesntStartUpgrade(c *gc.C) { 262 // This test checks when a controller is upgrading and one of 263 // the other controllers doesn't signal it is ready in time. 264 265 err := s.State.SetModelAgentVersion(jujuversion.Current) 266 c.Assert(err, jc.ErrorIsNil) 267 268 // The master controller in this scenario is functionally tested 269 // elsewhere. 270 s.machineIsMaster = false 271 272 s.create3Controllers(c) 273 s.captureLogs(c) 274 attemptsP := s.countUpgradeAttempts(nil) 275 276 workerErr, config, statusCalls, doneLock := s.runUpgradeWorker(c, multiwatcher.JobManageModel) 277 278 c.Check(workerErr, gc.IsNil) 279 c.Check(*attemptsP, gc.Equals, 0) 280 c.Check(config.Version, gc.Equals, s.oldVersion.Number) // Upgrade didn't happen 281 c.Assert(doneLock.IsUnlocked(), jc.IsFalse) 282 283 // The environment agent-version should still be the new version. 284 // It's up to the master to trigger the rollback. 285 s.assertEnvironAgentVersion(c, jujuversion.Current) 286 287 causeMsg := " timed out after 60ms" 288 c.Assert(s.logWriter.Log(), jc.LogMatches, []jc.SimpleMessage{ 289 {loggo.INFO, "waiting for other controllers to be ready for upgrade"}, 290 {loggo.ERROR, "aborted wait for other controllers: timed out after 60ms"}, 291 {loggo.ERROR, `upgrade from .+ to .+ for "machine-0" failed \(giving up\): ` + 292 "aborted wait for other controllers:" + causeMsg}, 293 }) 294 c.Assert(statusCalls, jc.DeepEquals, []StatusCall{{ 295 status.Error, 296 fmt.Sprintf( 297 "upgrade to %s failed (giving up): aborted wait for other controllers:"+causeMsg, 298 jujuversion.Current), 299 }}) 300 } 301 302 func (s *UpgradeSuite) TestSuccessMaster(c *gc.C) { 303 // This test checks what happens when an upgrade works on the 304 // first attempt on a master controller. 305 s.machineIsMaster = true 306 info := s.checkSuccess(c, "databaseMaster", func(*state.UpgradeInfo) {}) 307 c.Assert(info.Status(), gc.Equals, state.UpgradeFinishing) 308 } 309 310 func (s *UpgradeSuite) TestSuccessSecondary(c *gc.C) { 311 // This test checks what happens when an upgrade works on the 312 // first attempt on a secondary controller. 313 s.machineIsMaster = false 314 mungeInfo := func(info *state.UpgradeInfo) { 315 // Indicate that the master is done 316 err := info.SetStatus(state.UpgradeRunning) 317 c.Assert(err, jc.ErrorIsNil) 318 err = info.SetStatus(state.UpgradeFinishing) 319 c.Assert(err, jc.ErrorIsNil) 320 } 321 s.checkSuccess(c, "controller", mungeInfo) 322 } 323 324 func (s *UpgradeSuite) checkSuccess(c *gc.C, target string, mungeInfo func(*state.UpgradeInfo)) *state.UpgradeInfo { 325 _, machineIdB, machineIdC := s.create3Controllers(c) 326 327 // Indicate that machine B and C are ready to upgrade 328 vPrevious := s.oldVersion.Number 329 vNext := jujuversion.Current 330 info, err := s.State.EnsureUpgradeInfo(machineIdB, vPrevious, vNext) 331 c.Assert(err, jc.ErrorIsNil) 332 _, err = s.State.EnsureUpgradeInfo(machineIdC, vPrevious, vNext) 333 c.Assert(err, jc.ErrorIsNil) 334 335 mungeInfo(info) 336 337 attemptsP := s.countUpgradeAttempts(nil) 338 s.captureLogs(c) 339 340 workerErr, config, statusCalls, doneLock := s.runUpgradeWorker(c, multiwatcher.JobManageModel) 341 342 c.Check(workerErr, gc.IsNil) 343 c.Check(*attemptsP, gc.Equals, 1) 344 c.Check(config.Version, gc.Equals, jujuversion.Current) // Upgrade finished 345 c.Assert(statusCalls, jc.DeepEquals, s.makeExpectedStatusCalls(0, succeeds, "")) 346 c.Assert(s.logWriter.Log(), jc.LogMatches, s.makeExpectedUpgradeLogs(0, target, succeeds, "")) 347 c.Check(doneLock.IsUnlocked(), jc.IsTrue) 348 349 err = info.Refresh() 350 c.Assert(err, jc.ErrorIsNil) 351 c.Assert(info.ControllersDone(), jc.DeepEquals, []string{"0"}) 352 return info 353 } 354 355 func (s *UpgradeSuite) TestJobsToTargets(c *gc.C) { 356 check := func(jobs []multiwatcher.MachineJob, isMaster bool, expectedTargets ...upgrades.Target) { 357 c.Assert(jobsToTargets(jobs, isMaster), jc.SameContents, expectedTargets) 358 } 359 360 check([]multiwatcher.MachineJob{multiwatcher.JobHostUnits}, false, upgrades.HostMachine) 361 check([]multiwatcher.MachineJob{multiwatcher.JobManageModel}, false, upgrades.Controller) 362 check([]multiwatcher.MachineJob{multiwatcher.JobManageModel}, true, 363 upgrades.Controller, upgrades.DatabaseMaster) 364 check([]multiwatcher.MachineJob{multiwatcher.JobManageModel, multiwatcher.JobHostUnits}, false, 365 upgrades.Controller, upgrades.HostMachine) 366 check([]multiwatcher.MachineJob{multiwatcher.JobManageModel, multiwatcher.JobHostUnits}, true, 367 upgrades.Controller, upgrades.DatabaseMaster, upgrades.HostMachine) 368 } 369 370 func (s *UpgradeSuite) TestPreUpgradeFail(c *gc.C) { 371 s.preUpgradeError = true 372 s.captureLogs(c) 373 374 workerErr, config, statusCalls, doneLock := s.runUpgradeWorker(c, multiwatcher.JobHostUnits) 375 376 c.Check(workerErr, jc.ErrorIsNil) 377 c.Check(config.Version, gc.Equals, s.oldVersion.Number) // Upgrade didn't finish 378 c.Assert(doneLock.IsUnlocked(), jc.IsFalse) 379 380 causeMessage := `machine 0 cannot be upgraded: preupgrade error` 381 failMessage := fmt.Sprintf( 382 `upgrade from %s to %s for "machine-0" failed \(giving up\): %s`, 383 s.oldVersion.Number, jujuversion.Current, causeMessage) 384 c.Assert(s.logWriter.Log(), jc.LogMatches, []jc.SimpleMessage{ 385 {loggo.INFO, "checking that upgrade can proceed"}, 386 {loggo.ERROR, failMessage}, 387 }) 388 389 statusMessage := fmt.Sprintf( 390 `upgrade to %s failed (giving up): %s`, jujuversion.Current, causeMessage) 391 c.Assert(statusCalls, jc.DeepEquals, []StatusCall{{ 392 status.Error, statusMessage, 393 }}) 394 } 395 396 // Run just the upgradesteps worker with a fake machine agent and 397 // fake agent config. 398 func (s *UpgradeSuite) runUpgradeWorker(c *gc.C, jobs ...multiwatcher.MachineJob) ( 399 error, *fakeConfigSetter, []StatusCall, gate.Lock, 400 ) { 401 s.setInstantRetryStrategy(c) 402 config := s.makeFakeConfig() 403 agent := NewFakeAgent(config) 404 doneLock, err := NewLock(agent) 405 c.Assert(err, jc.ErrorIsNil) 406 machineStatus := &testStatusSetter{} 407 worker, err := NewWorker(doneLock, agent, nil, jobs, s.openStateForUpgrade, s.preUpgradeSteps, machineStatus) 408 c.Assert(err, jc.ErrorIsNil) 409 return worker.Wait(), config, machineStatus.Calls, doneLock 410 } 411 412 func (s *UpgradeSuite) openStateForUpgrade() (*state.State, error) { 413 mongoInfo := s.State.MongoConnectionInfo() 414 newPolicy := stateenvirons.GetNewPolicyFunc( 415 stateenvirons.GetNewEnvironFunc(environs.New), 416 ) 417 st, err := state.Open(s.State.ModelTag(), s.State.ControllerTag(), mongoInfo, mongotest.DialOpts(), newPolicy) 418 if err != nil { 419 return nil, err 420 } 421 return st, nil 422 } 423 424 func (s *UpgradeSuite) preUpgradeSteps(st *state.State, agentConf agent.Config, isController, isMasterController bool) error { 425 if s.preUpgradeError { 426 return errors.New("preupgrade error") 427 } 428 return nil 429 } 430 431 func (s *UpgradeSuite) makeFakeConfig() *fakeConfigSetter { 432 return NewFakeConfigSetter(names.NewMachineTag("0"), s.oldVersion.Number) 433 } 434 435 func (s *UpgradeSuite) create3Controllers(c *gc.C) (machineIdA, machineIdB, machineIdC string) { 436 machine0 := s.Factory.MakeMachine(c, &factory.MachineParams{ 437 Jobs: []state.MachineJob{state.JobManageModel}, 438 }) 439 machineIdA = machine0.Id() 440 s.setMachineAlive(c, machineIdA) 441 442 changes, err := s.State.EnableHA(3, constraints.Value{}, "quantal", nil) 443 c.Assert(err, jc.ErrorIsNil) 444 c.Assert(len(changes.Added), gc.Equals, 2) 445 446 machineIdB = changes.Added[0] 447 s.setMachineProvisioned(c, machineIdB) 448 s.setMachineAlive(c, machineIdB) 449 450 machineIdC = changes.Added[1] 451 s.setMachineProvisioned(c, machineIdC) 452 s.setMachineAlive(c, machineIdC) 453 454 return 455 } 456 457 func (s *UpgradeSuite) setMachineProvisioned(c *gc.C, id string) { 458 machine, err := s.State.Machine(id) 459 c.Assert(err, jc.ErrorIsNil) 460 err = machine.SetProvisioned(instance.Id(id+"-inst"), "nonce", nil) 461 c.Assert(err, jc.ErrorIsNil) 462 } 463 464 func (s *UpgradeSuite) setMachineAlive(c *gc.C, id string) { 465 machine, err := s.State.Machine(id) 466 c.Assert(err, jc.ErrorIsNil) 467 pinger, err := machine.SetAgentPresence() 468 c.Assert(err, jc.ErrorIsNil) 469 s.AddCleanup(func(c *gc.C) { 470 c.Assert(worker.Stop(pinger), jc.ErrorIsNil) 471 }) 472 } 473 474 // Return a version the same as the current software version, but with 475 // the build number bumped. 476 // 477 // The version Tag is also cleared so that upgrades.PerformUpgrade 478 // doesn't think it needs to run upgrade steps unnecessarily. 479 func makeBumpedCurrentVersion() version.Binary { 480 v := version.Binary{ 481 Number: jujuversion.Current, 482 Arch: arch.HostArch(), 483 Series: series.HostSeries(), 484 } 485 v.Build++ 486 v.Tag = "" 487 return v 488 } 489 490 const maxUpgradeRetries = 3 491 492 func (s *UpgradeSuite) setInstantRetryStrategy(c *gc.C) { 493 // TODO(katco): 2016-08-09: lp:1611427 494 s.PatchValue(&getUpgradeRetryStrategy, func() utils.AttemptStrategy { 495 c.Logf("setting instant retry strategy for upgrade: retries=%d", maxUpgradeRetries) 496 return utils.AttemptStrategy{ 497 Delay: 0, 498 Min: maxUpgradeRetries, 499 } 500 }) 501 } 502 503 func (s *UpgradeSuite) makeExpectedStatusCalls(retryCount int, expectFail bool, failReason string) []StatusCall { 504 calls := []StatusCall{{ 505 status.Started, 506 fmt.Sprintf("upgrading to %s", jujuversion.Current), 507 }} 508 for i := 0; i < retryCount; i++ { 509 calls = append(calls, StatusCall{ 510 status.Error, 511 fmt.Sprintf("upgrade to %s failed (will retry): %s", jujuversion.Current, failReason), 512 }) 513 } 514 if expectFail { 515 calls = append(calls, StatusCall{ 516 status.Error, 517 fmt.Sprintf("upgrade to %s failed (giving up): %s", jujuversion.Current, failReason), 518 }) 519 } else { 520 calls = append(calls, StatusCall{status.Started, ""}) 521 } 522 return calls 523 } 524 525 func (s *UpgradeSuite) makeExpectedUpgradeLogs(retryCount int, target string, expectFail bool, failReason string) []jc.SimpleMessage { 526 outLogs := []jc.SimpleMessage{} 527 528 if target == "databaseMaster" || target == "controller" { 529 outLogs = append(outLogs, jc.SimpleMessage{ 530 loggo.INFO, "waiting for other controllers to be ready for upgrade", 531 }) 532 var waitMsg string 533 switch target { 534 case "databaseMaster": 535 waitMsg = "all controllers are ready to run upgrade steps" 536 case "controller": 537 waitMsg = "the master has completed its upgrade steps" 538 } 539 outLogs = append(outLogs, jc.SimpleMessage{loggo.INFO, "finished waiting - " + waitMsg}) 540 } 541 542 outLogs = append(outLogs, jc.SimpleMessage{ 543 loggo.INFO, fmt.Sprintf( 544 `starting upgrade from %s to %s for "machine-0"`, 545 s.oldVersion.Number, jujuversion.Current), 546 }) 547 548 failMessage := fmt.Sprintf( 549 `upgrade from %s to %s for "machine-0" failed \(%%s\): %s`, 550 s.oldVersion.Number, jujuversion.Current, failReason) 551 552 for i := 0; i < retryCount; i++ { 553 outLogs = append(outLogs, jc.SimpleMessage{loggo.ERROR, fmt.Sprintf(failMessage, "will retry")}) 554 } 555 if expectFail { 556 outLogs = append(outLogs, jc.SimpleMessage{loggo.ERROR, fmt.Sprintf(failMessage, "giving up")}) 557 } else { 558 outLogs = append(outLogs, jc.SimpleMessage{loggo.INFO, 559 fmt.Sprintf(`upgrade to %s completed successfully.`, jujuversion.Current)}) 560 } 561 return outLogs 562 } 563 564 func (s *UpgradeSuite) assertEnvironAgentVersion(c *gc.C, expected version.Number) { 565 envConfig, err := s.State.ModelConfig() 566 c.Assert(err, jc.ErrorIsNil) 567 agentVersion, ok := envConfig.AgentVersion() 568 c.Assert(ok, jc.IsTrue) 569 c.Assert(agentVersion, gc.Equals, expected) 570 } 571 572 // NewFakeConfigSetter returns a fakeConfigSetter which implements 573 // just enough of the agent.ConfigSetter interface to keep the upgrade 574 // steps worker happy. 575 func NewFakeConfigSetter(agentTag names.Tag, initialVersion version.Number) *fakeConfigSetter { 576 return &fakeConfigSetter{ 577 AgentTag: agentTag, 578 Version: initialVersion, 579 } 580 } 581 582 type fakeConfigSetter struct { 583 agent.ConfigSetter 584 AgentTag names.Tag 585 Version version.Number 586 } 587 588 func (s *fakeConfigSetter) Tag() names.Tag { 589 return s.AgentTag 590 } 591 592 func (s *fakeConfigSetter) UpgradedToVersion() version.Number { 593 return s.Version 594 } 595 596 func (s *fakeConfigSetter) SetUpgradedToVersion(newVersion version.Number) { 597 s.Version = newVersion 598 } 599 600 // NewFakeAgent returns a fakeAgent which implements the agent.Agent 601 // interface. This provides enough MachineAgent functionality to 602 // support upgrades. 603 func NewFakeAgent(confSetter agent.ConfigSetter) *fakeAgent { 604 return &fakeAgent{ 605 config: confSetter, 606 } 607 } 608 609 type fakeAgent struct { 610 config agent.ConfigSetter 611 } 612 613 func (a *fakeAgent) CurrentConfig() agent.Config { 614 return a.config 615 } 616 617 func (a *fakeAgent) ChangeConfig(mutate agent.ConfigMutator) error { 618 return mutate(a.config) 619 } 620 621 type StatusCall struct { 622 Status status.Status 623 Info string 624 } 625 626 type testStatusSetter struct { 627 Calls []StatusCall 628 } 629 630 func (s *testStatusSetter) SetStatus(status status.Status, info string, _ map[string]interface{}) error { 631 s.Calls = append(s.Calls, StatusCall{status, info}) 632 return nil 633 }