github.com/bugraaydogar/snapd@v0.0.0-20210315170335-8c70bb858939/overlord/overlord_test.go (about) 1 // -*- Mode: Go; indent-tabs-mode: t -*- 2 3 /* 4 * Copyright (C) 2016-2017 Canonical Ltd 5 * 6 * This program is free software: you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 3 as 8 * published by the Free Software Foundation. 9 * 10 * This program is distributed in the hope that it will be useful, 11 * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 * GNU General Public License for more details. 14 * 15 * You should have received a copy of the GNU General Public License 16 * along with this program. If not, see <http://www.gnu.org/licenses/>. 17 * 18 */ 19 20 package overlord_test 21 22 import ( 23 "encoding/json" 24 "errors" 25 "fmt" 26 "io/ioutil" 27 "os" 28 "path/filepath" 29 "syscall" 30 "testing" 31 "time" 32 33 . "gopkg.in/check.v1" 34 "gopkg.in/tomb.v2" 35 36 "github.com/snapcore/snapd/dirs" 37 "github.com/snapcore/snapd/osutil" 38 "github.com/snapcore/snapd/overlord" 39 "github.com/snapcore/snapd/overlord/auth" 40 "github.com/snapcore/snapd/overlord/devicestate/devicestatetest" 41 "github.com/snapcore/snapd/overlord/hookstate" 42 "github.com/snapcore/snapd/overlord/ifacestate" 43 "github.com/snapcore/snapd/overlord/patch" 44 "github.com/snapcore/snapd/overlord/snapstate" 45 "github.com/snapcore/snapd/overlord/state" 46 "github.com/snapcore/snapd/snap" 47 "github.com/snapcore/snapd/snapdenv" 48 "github.com/snapcore/snapd/snapdtool" 49 "github.com/snapcore/snapd/store" 50 "github.com/snapcore/snapd/testutil" 51 "github.com/snapcore/snapd/timings" 52 ) 53 54 func TestOverlord(t *testing.T) { TestingT(t) } 55 56 type overlordSuite struct { 57 testutil.BaseTest 58 } 59 60 var _ = Suite(&overlordSuite{}) 61 62 type ticker struct { 63 tickerChannel chan time.Time 64 } 65 66 func (w *ticker) tick(n int) { 67 for i := 0; i < n; i++ { 68 w.tickerChannel <- time.Now() 69 } 70 } 71 72 func fakePruneTicker() (w *ticker, restore func()) { 73 w = &ticker{ 74 tickerChannel: make(chan time.Time), 75 } 76 restore = overlord.MockPruneTicker(func(t *time.Ticker) <-chan time.Time { 77 return w.tickerChannel 78 }) 79 return w, restore 80 } 81 82 func (ovs *overlordSuite) SetUpTest(c *C) { 83 tmpdir := c.MkDir() 84 dirs.SetRootDir(tmpdir) 85 ovs.AddCleanup(func() { dirs.SetRootDir("") }) 86 ovs.AddCleanup(osutil.MockMountInfo("")) 87 88 dirs.SnapStateFile = filepath.Join(tmpdir, "test.json") 89 snapstate.CanAutoRefresh = nil 90 ovs.AddCleanup(func() { ifacestate.MockSecurityBackends(nil) }) 91 } 92 93 func (ovs *overlordSuite) TestNew(c *C) { 94 restore := patch.Mock(42, 2, nil) 95 defer restore() 96 97 var configstateInitCalled bool 98 overlord.MockConfigstateInit(func(*state.State, *hookstate.HookManager) error { 99 configstateInitCalled = true 100 return nil 101 }) 102 103 o, err := overlord.New(nil) 104 c.Assert(err, IsNil) 105 c.Check(o, NotNil) 106 107 c.Check(o.StateEngine(), NotNil) 108 c.Check(o.TaskRunner(), NotNil) 109 c.Check(o.SnapManager(), NotNil) 110 c.Check(o.AssertManager(), NotNil) 111 c.Check(o.InterfaceManager(), NotNil) 112 c.Check(o.HookManager(), NotNil) 113 c.Check(o.DeviceManager(), NotNil) 114 c.Check(o.CommandManager(), NotNil) 115 c.Check(o.SnapshotManager(), NotNil) 116 c.Check(configstateInitCalled, Equals, true) 117 118 o.InterfaceManager().DisableUDevMonitor() 119 120 s := o.State() 121 c.Check(s, NotNil) 122 c.Check(o.Engine().State(), Equals, s) 123 124 s.Lock() 125 defer s.Unlock() 126 var patchLevel, patchSublevel int 127 s.Get("patch-level", &patchLevel) 128 c.Check(patchLevel, Equals, 42) 129 s.Get("patch-sublevel", &patchSublevel) 130 c.Check(patchSublevel, Equals, 2) 131 var refreshPrivacyKey string 132 s.Get("refresh-privacy-key", &refreshPrivacyKey) 133 c.Check(refreshPrivacyKey, HasLen, 16) 134 135 // store is setup 136 sto := snapstate.Store(s, nil) 137 c.Check(sto, FitsTypeOf, &store.Store{}) 138 c.Check(sto.(*store.Store).CacheDownloads(), Equals, 5) 139 } 140 141 func (ovs *overlordSuite) TestNewStore(c *C) { 142 // this is a shallow test, the deep testing happens in the 143 // remodeling tests in managers_test.go 144 o, err := overlord.New(nil) 145 c.Assert(err, IsNil) 146 147 devBE := o.DeviceManager().StoreContextBackend() 148 149 sto := o.NewStore(devBE) 150 c.Check(sto, FitsTypeOf, &store.Store{}) 151 c.Check(sto.(*store.Store).CacheDownloads(), Equals, 5) 152 } 153 154 func (ovs *overlordSuite) TestNewWithGoodState(c *C) { 155 // ensure we don't write state load timing in the state on really 156 // slow architectures (e.g. risc-v) 157 oldDurationThreshold := timings.DurationThreshold 158 timings.DurationThreshold = time.Second * 30 159 defer func() { timings.DurationThreshold = oldDurationThreshold }() 160 161 fakeState := []byte(fmt.Sprintf(`{"data":{"patch-level":%d,"patch-sublevel":%d,"patch-sublevel-last-version":%q,"some":"data","refresh-privacy-key":"0123456789ABCDEF"},"changes":null,"tasks":null,"last-change-id":0,"last-task-id":0,"last-lane-id":0}`, patch.Level, patch.Sublevel, snapdtool.Version)) 162 err := ioutil.WriteFile(dirs.SnapStateFile, fakeState, 0600) 163 c.Assert(err, IsNil) 164 165 o, err := overlord.New(nil) 166 c.Assert(err, IsNil) 167 168 state := o.State() 169 c.Assert(err, IsNil) 170 state.Lock() 171 defer state.Unlock() 172 173 d, err := state.MarshalJSON() 174 c.Assert(err, IsNil) 175 176 var got, expected map[string]interface{} 177 err = json.Unmarshal(d, &got) 178 c.Assert(err, IsNil) 179 err = json.Unmarshal(fakeState, &expected) 180 c.Assert(err, IsNil) 181 182 data, _ := got["data"].(map[string]interface{}) 183 c.Assert(data, NotNil) 184 185 c.Check(got, DeepEquals, expected) 186 } 187 188 func (ovs *overlordSuite) TestNewWithStateSnapmgrUpdate(c *C) { 189 fakeState := []byte(fmt.Sprintf(`{"data":{"patch-level":%d,"some":"data"},"changes":null,"tasks":null,"last-change-id":0,"last-task-id":0,"last-lane-id":0}`, patch.Level)) 190 err := ioutil.WriteFile(dirs.SnapStateFile, fakeState, 0600) 191 c.Assert(err, IsNil) 192 193 o, err := overlord.New(nil) 194 c.Assert(err, IsNil) 195 196 state := o.State() 197 c.Assert(err, IsNil) 198 state.Lock() 199 defer state.Unlock() 200 201 var refreshPrivacyKey string 202 state.Get("refresh-privacy-key", &refreshPrivacyKey) 203 c.Check(refreshPrivacyKey, HasLen, 16) 204 } 205 206 func (ovs *overlordSuite) TestNewWithInvalidState(c *C) { 207 fakeState := []byte(``) 208 err := ioutil.WriteFile(dirs.SnapStateFile, fakeState, 0600) 209 c.Assert(err, IsNil) 210 211 _, err = overlord.New(nil) 212 c.Assert(err, ErrorMatches, "cannot read state: EOF") 213 } 214 215 func (ovs *overlordSuite) TestNewWithPatches(c *C) { 216 p := func(s *state.State) error { 217 s.Set("patched", true) 218 return nil 219 } 220 sp := func(s *state.State) error { 221 s.Set("patched2", true) 222 return nil 223 } 224 patch.Mock(1, 1, map[int][]patch.PatchFunc{1: {p, sp}}) 225 226 fakeState := []byte(fmt.Sprintf(`{"data":{"patch-level":0, "patch-sublevel":0}}`)) 227 err := ioutil.WriteFile(dirs.SnapStateFile, fakeState, 0600) 228 c.Assert(err, IsNil) 229 230 o, err := overlord.New(nil) 231 c.Assert(err, IsNil) 232 233 state := o.State() 234 c.Assert(err, IsNil) 235 state.Lock() 236 defer state.Unlock() 237 238 var level int 239 err = state.Get("patch-level", &level) 240 c.Assert(err, IsNil) 241 c.Check(level, Equals, 1) 242 243 var sublevel int 244 c.Assert(state.Get("patch-sublevel", &sublevel), IsNil) 245 c.Check(sublevel, Equals, 1) 246 247 var b bool 248 err = state.Get("patched", &b) 249 c.Assert(err, IsNil) 250 c.Check(b, Equals, true) 251 252 c.Assert(state.Get("patched2", &b), IsNil) 253 c.Check(b, Equals, true) 254 } 255 256 func (ovs *overlordSuite) TestNewFailedConfigstate(c *C) { 257 restore := patch.Mock(42, 2, nil) 258 defer restore() 259 260 var configstateInitCalled bool 261 restore = overlord.MockConfigstateInit(func(*state.State, *hookstate.HookManager) error { 262 configstateInitCalled = true 263 return fmt.Errorf("bad bad") 264 }) 265 defer restore() 266 267 o, err := overlord.New(nil) 268 c.Assert(err, ErrorMatches, "bad bad") 269 c.Check(o, IsNil) 270 c.Check(configstateInitCalled, Equals, true) 271 } 272 273 type witnessManager struct { 274 state *state.State 275 expectedEnsure int 276 ensureCalled chan struct{} 277 ensureCallback func(s *state.State) error 278 startedUp int 279 } 280 281 func (wm *witnessManager) StartUp() error { 282 wm.startedUp++ 283 return nil 284 } 285 286 func (wm *witnessManager) Ensure() error { 287 if wm.expectedEnsure--; wm.expectedEnsure == 0 { 288 close(wm.ensureCalled) 289 return nil 290 } 291 if wm.ensureCallback != nil { 292 return wm.ensureCallback(wm.state) 293 } 294 return nil 295 } 296 297 // markSeeded flags the state under the overlord as seeded to avoid running the seeding code in these tests 298 func markSeeded(o *overlord.Overlord) { 299 st := o.State() 300 st.Lock() 301 st.Set("seeded", true) 302 devicestatetest.SetDevice(st, &auth.DeviceState{ 303 Brand: "canonical", 304 Model: "pc", 305 Serial: "serialserial", 306 }) 307 st.Unlock() 308 } 309 310 func (ovs *overlordSuite) TestTrivialRunAndStop(c *C) { 311 o, err := overlord.New(nil) 312 c.Assert(err, IsNil) 313 314 markSeeded(o) 315 // make sure we don't try to talk to the store 316 snapstate.CanAutoRefresh = nil 317 318 err = o.StartUp() 319 c.Assert(err, IsNil) 320 321 o.Loop() 322 323 err = o.Stop() 324 c.Assert(err, IsNil) 325 } 326 327 func (ovs *overlordSuite) TestUnknownTasks(c *C) { 328 o, err := overlord.New(nil) 329 c.Assert(err, IsNil) 330 o.InterfaceManager().DisableUDevMonitor() 331 332 markSeeded(o) 333 // make sure we don't try to talk to the store 334 snapstate.CanAutoRefresh = nil 335 336 // unknown tasks are ignored and succeed 337 st := o.State() 338 st.Lock() 339 defer st.Unlock() 340 t := st.NewTask("unknown", "...") 341 chg := st.NewChange("change-w-unknown", "...") 342 chg.AddTask(t) 343 344 st.Unlock() 345 err = o.Settle(1 * time.Second) 346 st.Lock() 347 c.Assert(err, IsNil) 348 349 c.Check(chg.Status(), Equals, state.DoneStatus) 350 } 351 352 func (ovs *overlordSuite) TestEnsureLoopRunAndStop(c *C) { 353 restoreIntv := overlord.MockEnsureInterval(10 * time.Millisecond) 354 defer restoreIntv() 355 o := overlord.Mock() 356 357 witness := &witnessManager{ 358 state: o.State(), 359 expectedEnsure: 3, 360 ensureCalled: make(chan struct{}), 361 } 362 o.AddManager(witness) 363 364 err := o.StartUp() 365 c.Assert(err, IsNil) 366 367 o.Loop() 368 defer o.Stop() 369 370 t0 := time.Now() 371 select { 372 case <-witness.ensureCalled: 373 case <-time.After(2 * time.Second): 374 c.Fatal("Ensure calls not happening") 375 } 376 c.Check(time.Since(t0) >= 10*time.Millisecond, Equals, true) 377 378 err = o.Stop() 379 c.Assert(err, IsNil) 380 381 c.Check(witness.startedUp, Equals, 1) 382 } 383 384 func (ovs *overlordSuite) TestEnsureLoopMediatedEnsureBeforeImmediate(c *C) { 385 restoreIntv := overlord.MockEnsureInterval(10 * time.Minute) 386 defer restoreIntv() 387 o := overlord.Mock() 388 389 ensure := func(s *state.State) error { 390 s.EnsureBefore(0) 391 return nil 392 } 393 394 witness := &witnessManager{ 395 state: o.State(), 396 expectedEnsure: 2, 397 ensureCalled: make(chan struct{}), 398 ensureCallback: ensure, 399 } 400 o.AddManager(witness) 401 402 c.Assert(o.StartUp(), IsNil) 403 404 o.Loop() 405 defer o.Stop() 406 407 select { 408 case <-witness.ensureCalled: 409 case <-time.After(2 * time.Second): 410 c.Fatal("Ensure calls not happening") 411 } 412 } 413 414 func (ovs *overlordSuite) TestEnsureLoopMediatedEnsureBefore(c *C) { 415 restoreIntv := overlord.MockEnsureInterval(10 * time.Minute) 416 defer restoreIntv() 417 o := overlord.Mock() 418 419 ensure := func(s *state.State) error { 420 s.EnsureBefore(10 * time.Millisecond) 421 return nil 422 } 423 424 witness := &witnessManager{ 425 state: o.State(), 426 expectedEnsure: 2, 427 ensureCalled: make(chan struct{}), 428 ensureCallback: ensure, 429 } 430 o.AddManager(witness) 431 432 c.Assert(o.StartUp(), IsNil) 433 434 o.Loop() 435 defer o.Stop() 436 437 select { 438 case <-witness.ensureCalled: 439 case <-time.After(2 * time.Second): 440 c.Fatal("Ensure calls not happening") 441 } 442 } 443 444 func (ovs *overlordSuite) TestEnsureBeforeSleepy(c *C) { 445 restoreIntv := overlord.MockEnsureInterval(10 * time.Minute) 446 defer restoreIntv() 447 o := overlord.Mock() 448 449 ensure := func(s *state.State) error { 450 overlord.MockEnsureNext(o, time.Now().Add(-10*time.Hour)) 451 s.EnsureBefore(0) 452 return nil 453 } 454 455 witness := &witnessManager{ 456 state: o.State(), 457 expectedEnsure: 2, 458 ensureCalled: make(chan struct{}), 459 ensureCallback: ensure, 460 } 461 o.AddManager(witness) 462 463 c.Assert(o.StartUp(), IsNil) 464 465 o.Loop() 466 defer o.Stop() 467 468 select { 469 case <-witness.ensureCalled: 470 case <-time.After(2 * time.Second): 471 c.Fatal("Ensure calls not happening") 472 } 473 } 474 475 func (ovs *overlordSuite) TestEnsureBeforeLater(c *C) { 476 restoreIntv := overlord.MockEnsureInterval(10 * time.Minute) 477 defer restoreIntv() 478 o := overlord.Mock() 479 480 ensure := func(s *state.State) error { 481 overlord.MockEnsureNext(o, time.Now().Add(-10*time.Hour)) 482 s.EnsureBefore(time.Second * 5) 483 return nil 484 } 485 486 witness := &witnessManager{ 487 state: o.State(), 488 expectedEnsure: 2, 489 ensureCalled: make(chan struct{}), 490 ensureCallback: ensure, 491 } 492 o.AddManager(witness) 493 494 c.Assert(o.StartUp(), IsNil) 495 496 o.Loop() 497 defer o.Stop() 498 499 select { 500 case <-witness.ensureCalled: 501 case <-time.After(2 * time.Second): 502 c.Fatal("Ensure calls not happening") 503 } 504 } 505 506 func (ovs *overlordSuite) TestEnsureLoopMediatedEnsureBeforeOutsideEnsure(c *C) { 507 restoreIntv := overlord.MockEnsureInterval(10 * time.Minute) 508 defer restoreIntv() 509 o := overlord.Mock() 510 511 ch := make(chan struct{}) 512 ensure := func(s *state.State) error { 513 close(ch) 514 return nil 515 } 516 517 witness := &witnessManager{ 518 state: o.State(), 519 expectedEnsure: 2, 520 ensureCalled: make(chan struct{}), 521 ensureCallback: ensure, 522 } 523 o.AddManager(witness) 524 525 c.Assert(o.StartUp(), IsNil) 526 527 o.Loop() 528 defer o.Stop() 529 530 select { 531 case <-ch: 532 case <-time.After(2 * time.Second): 533 c.Fatal("Ensure calls not happening") 534 } 535 536 o.State().EnsureBefore(0) 537 538 select { 539 case <-witness.ensureCalled: 540 case <-time.After(2 * time.Second): 541 c.Fatal("Ensure calls not happening") 542 } 543 } 544 545 func (ovs *overlordSuite) TestEnsureLoopPrune(c *C) { 546 restoreIntv := overlord.MockPruneInterval(200*time.Millisecond, 1000*time.Millisecond, 1000*time.Millisecond) 547 defer restoreIntv() 548 o := overlord.Mock() 549 550 st := o.State() 551 st.Lock() 552 t1 := st.NewTask("foo", "...") 553 chg1 := st.NewChange("abort", "...") 554 chg1.AddTask(t1) 555 chg2 := st.NewChange("prune", "...") 556 chg2.SetStatus(state.DoneStatus) 557 t0 := chg2.ReadyTime() 558 st.Unlock() 559 560 // observe the loop cycles to detect when prune should have happened 561 pruneHappened := make(chan struct{}) 562 cycles := -1 563 waitForPrune := func(_ *state.State) error { 564 if cycles == -1 { 565 if time.Since(t0) > 1000*time.Millisecond { 566 cycles = 2 // wait a couple more loop cycles 567 } 568 return nil 569 } 570 if cycles > 0 { 571 cycles-- 572 if cycles == 0 { 573 close(pruneHappened) 574 } 575 } 576 return nil 577 } 578 witness := &witnessManager{ 579 ensureCallback: waitForPrune, 580 } 581 o.AddManager(witness) 582 583 c.Assert(o.StartUp(), IsNil) 584 585 o.Loop() 586 587 select { 588 case <-pruneHappened: 589 case <-time.After(2 * time.Second): 590 c.Fatal("Pruning should have happened by now") 591 } 592 593 err := o.Stop() 594 c.Assert(err, IsNil) 595 596 st.Lock() 597 defer st.Unlock() 598 599 c.Assert(st.Change(chg1.ID()), Equals, chg1) 600 c.Assert(st.Change(chg2.ID()), IsNil) 601 602 c.Assert(t1.Status(), Equals, state.HoldStatus) 603 } 604 605 func (ovs *overlordSuite) TestEnsureLoopPruneRunsMultipleTimes(c *C) { 606 restoreIntv := overlord.MockPruneInterval(100*time.Millisecond, 5*time.Millisecond, 1*time.Hour) 607 defer restoreIntv() 608 o := overlord.Mock() 609 610 // create two changes, one that can be pruned now, one in progress 611 st := o.State() 612 st.Lock() 613 t1 := st.NewTask("foo", "...") 614 chg1 := st.NewChange("pruneNow", "...") 615 chg1.AddTask(t1) 616 t1.SetStatus(state.DoneStatus) 617 t2 := st.NewTask("foo", "...") 618 chg2 := st.NewChange("pruneNext", "...") 619 chg2.AddTask(t2) 620 t2.SetStatus(state.DoStatus) 621 c.Check(st.Changes(), HasLen, 2) 622 st.Unlock() 623 624 w, restoreTicker := fakePruneTicker() 625 defer restoreTicker() 626 627 // start the loop that runs the prune ticker 628 o.Loop() 629 630 // this needs to be more than pruneWait=5ms mocked above 631 time.Sleep(10 * time.Millisecond) 632 w.tick(2) 633 634 st.Lock() 635 c.Check(st.Changes(), HasLen, 1) 636 chg2.SetStatus(state.DoneStatus) 637 st.Unlock() 638 639 // this needs to be more than pruneWait=5ms mocked above 640 time.Sleep(10 * time.Millisecond) 641 // tick twice for extra Ensure 642 w.tick(2) 643 644 st.Lock() 645 c.Check(st.Changes(), HasLen, 0) 646 st.Unlock() 647 648 // cleanup loop ticker 649 err := o.Stop() 650 c.Assert(err, IsNil) 651 } 652 653 func (ovs *overlordSuite) TestOverlordStartUpSetsStartOfOperation(c *C) { 654 restoreIntv := overlord.MockPruneInterval(100*time.Millisecond, 1000*time.Millisecond, 1*time.Hour) 655 defer restoreIntv() 656 657 // use real overlord, we need device manager to be there 658 o, err := overlord.New(nil) 659 c.Assert(err, IsNil) 660 661 st := o.State() 662 st.Lock() 663 defer st.Unlock() 664 665 // sanity check, not set 666 var opTime time.Time 667 c.Assert(st.Get("start-of-operation-time", &opTime), Equals, state.ErrNoState) 668 st.Unlock() 669 670 c.Assert(o.StartUp(), IsNil) 671 672 st.Lock() 673 c.Assert(st.Get("start-of-operation-time", &opTime), IsNil) 674 } 675 676 func (ovs *overlordSuite) TestEnsureLoopPruneDoesntAbortShortlyAfterStartOfOperation(c *C) { 677 w, restoreTicker := fakePruneTicker() 678 defer restoreTicker() 679 680 // use real overlord, we need device manager to be there 681 o, err := overlord.New(nil) 682 c.Assert(err, IsNil) 683 684 markSeeded(o) 685 686 // avoid immediate transition to Done due to unknown kind 687 o.TaskRunner().AddHandler("bar", func(t *state.Task, _ *tomb.Tomb) error { 688 return &state.Retry{} 689 }, nil) 690 691 st := o.State() 692 st.Lock() 693 694 // start of operation time is 50min ago, this is less then abort limit 695 opTime := time.Now().Add(-50 * time.Minute) 696 st.Set("start-of-operation-time", opTime) 697 698 // spawn time one month ago 699 spawnTime := time.Now().AddDate(0, -1, 0) 700 restoreTimeNow := state.MockTime(spawnTime) 701 702 t := st.NewTask("bar", "...") 703 chg := st.NewChange("other-change", "...") 704 chg.AddTask(t) 705 706 restoreTimeNow() 707 708 // sanity 709 c.Check(st.Changes(), HasLen, 1) 710 711 st.Unlock() 712 c.Assert(o.StartUp(), IsNil) 713 714 // start the loop that runs the prune ticker 715 o.Loop() 716 w.tick(2) 717 718 c.Assert(o.Stop(), IsNil) 719 720 st.Lock() 721 defer st.Unlock() 722 c.Assert(st.Changes(), HasLen, 1) 723 c.Check(chg.Status(), Equals, state.DoingStatus) 724 } 725 726 func (ovs *overlordSuite) TestEnsureLoopPruneAbortsOld(c *C) { 727 // Ensure interval is not relevant for this test 728 restoreEnsureIntv := overlord.MockEnsureInterval(10 * time.Hour) 729 defer restoreEnsureIntv() 730 731 w, restoreTicker := fakePruneTicker() 732 defer restoreTicker() 733 734 // use real overlord, we need device manager to be there 735 o, err := overlord.New(nil) 736 c.Assert(err, IsNil) 737 738 // avoid immediate transition to Done due to having unknown kind 739 o.TaskRunner().AddHandler("bar", func(t *state.Task, _ *tomb.Tomb) error { 740 return &state.Retry{} 741 }, nil) 742 743 markSeeded(o) 744 745 st := o.State() 746 st.Lock() 747 748 // start of operation time is a year ago 749 opTime := time.Now().AddDate(-1, 0, 0) 750 st.Set("start-of-operation-time", opTime) 751 752 st.Unlock() 753 c.Assert(o.StartUp(), IsNil) 754 st.Lock() 755 756 // spawn time one month ago 757 spawnTime := time.Now().AddDate(0, -1, 0) 758 restoreTimeNow := state.MockTime(spawnTime) 759 t := st.NewTask("bar", "...") 760 chg := st.NewChange("other-change", "...") 761 chg.AddTask(t) 762 763 restoreTimeNow() 764 765 // sanity 766 c.Check(st.Changes(), HasLen, 1) 767 st.Unlock() 768 769 // start the loop that runs the prune ticker 770 o.Loop() 771 w.tick(2) 772 773 c.Assert(o.Stop(), IsNil) 774 775 st.Lock() 776 defer st.Unlock() 777 778 // sanity 779 op, err := o.DeviceManager().StartOfOperationTime() 780 c.Assert(err, IsNil) 781 c.Check(op.Equal(opTime), Equals, true) 782 783 c.Assert(st.Changes(), HasLen, 1) 784 // change was aborted 785 c.Check(chg.Status(), Equals, state.HoldStatus) 786 } 787 788 func (ovs *overlordSuite) TestEnsureLoopNoPruneWhenPreseed(c *C) { 789 w, restoreTicker := fakePruneTicker() 790 defer restoreTicker() 791 792 restore := snapdenv.MockPreseeding(true) 793 defer restore() 794 795 restorePreseedExitWithErr := overlord.MockPreseedExitWithError(func(err error) {}) 796 defer restorePreseedExitWithErr() 797 798 o, err := overlord.New(nil) 799 c.Assert(err, IsNil) 800 markSeeded(o) 801 802 // avoid immediate transition to Done due to unknown kind 803 o.TaskRunner().AddHandler("bar", func(t *state.Task, _ *tomb.Tomb) error { 804 return &state.Retry{} 805 }, nil) 806 807 // sanity 808 _, err = o.DeviceManager().StartOfOperationTime() 809 c.Assert(err, ErrorMatches, `internal error: unexpected call to StartOfOperationTime in preseed mode`) 810 811 c.Assert(o.StartUp(), IsNil) 812 813 st := o.State() 814 st.Lock() 815 816 // spawn time one month ago 817 spawnTime := time.Now().AddDate(0, -1, 0) 818 restoreTimeNow := state.MockTime(spawnTime) 819 t := st.NewTask("bar", "...") 820 chg := st.NewChange("change", "...") 821 chg.AddTask(t) 822 restoreTimeNow() 823 824 st.Unlock() 825 // start the loop that runs the prune ticker 826 o.Loop() 827 w.tick(2) 828 829 c.Assert(o.Stop(), IsNil) 830 831 st.Lock() 832 defer st.Unlock() 833 834 var opTime time.Time 835 c.Assert(st.Get("start-of-operation-time", &opTime), Equals, state.ErrNoState) 836 c.Check(chg.Status(), Equals, state.DoingStatus) 837 } 838 839 func (ovs *overlordSuite) TestCheckpoint(c *C) { 840 oldUmask := syscall.Umask(0) 841 defer syscall.Umask(oldUmask) 842 843 o, err := overlord.New(nil) 844 c.Assert(err, IsNil) 845 846 s := o.State() 847 s.Lock() 848 s.Set("mark", 1) 849 s.Unlock() 850 851 st, err := os.Stat(dirs.SnapStateFile) 852 c.Assert(err, IsNil) 853 c.Assert(st.Mode(), Equals, os.FileMode(0600)) 854 855 c.Check(dirs.SnapStateFile, testutil.FileContains, `"mark":1`) 856 } 857 858 type sampleManager struct { 859 ensureCallback func() 860 } 861 862 func newSampleManager(s *state.State, runner *state.TaskRunner) *sampleManager { 863 sm := &sampleManager{} 864 865 runner.AddHandler("runMgr1", func(t *state.Task, _ *tomb.Tomb) error { 866 s := t.State() 867 s.Lock() 868 defer s.Unlock() 869 s.Set("runMgr1Mark", 1) 870 return nil 871 }, nil) 872 runner.AddHandler("runMgr2", func(t *state.Task, _ *tomb.Tomb) error { 873 s := t.State() 874 s.Lock() 875 defer s.Unlock() 876 s.Set("runMgr2Mark", 1) 877 return nil 878 }, nil) 879 runner.AddHandler("runMgrEnsureBefore", func(t *state.Task, _ *tomb.Tomb) error { 880 s := t.State() 881 s.Lock() 882 defer s.Unlock() 883 s.EnsureBefore(20 * time.Millisecond) 884 return nil 885 }, nil) 886 runner.AddHandler("runMgrForever", func(t *state.Task, _ *tomb.Tomb) error { 887 s := t.State() 888 s.Lock() 889 defer s.Unlock() 890 s.EnsureBefore(20 * time.Millisecond) 891 return &state.Retry{} 892 }, nil) 893 runner.AddHandler("runMgrWCleanup", func(t *state.Task, _ *tomb.Tomb) error { 894 s := t.State() 895 s.Lock() 896 defer s.Unlock() 897 s.Set("runMgrWCleanupMark", 1) 898 return nil 899 }, nil) 900 runner.AddCleanup("runMgrWCleanup", func(t *state.Task, _ *tomb.Tomb) error { 901 s := t.State() 902 s.Lock() 903 defer s.Unlock() 904 s.Set("runMgrWCleanupCleanedUp", 1) 905 return nil 906 }) 907 908 return sm 909 } 910 911 func (sm *sampleManager) Ensure() error { 912 if sm.ensureCallback != nil { 913 sm.ensureCallback() 914 } 915 return nil 916 } 917 918 func (ovs *overlordSuite) TestTrivialSettle(c *C) { 919 restoreIntv := overlord.MockEnsureInterval(1 * time.Minute) 920 defer restoreIntv() 921 o := overlord.Mock() 922 923 s := o.State() 924 sm1 := newSampleManager(s, o.TaskRunner()) 925 o.AddManager(sm1) 926 o.AddManager(o.TaskRunner()) 927 928 defer o.Engine().Stop() 929 930 s.Lock() 931 defer s.Unlock() 932 933 chg := s.NewChange("chg", "...") 934 t1 := s.NewTask("runMgr1", "1...") 935 chg.AddTask(t1) 936 937 s.Unlock() 938 o.Settle(5 * time.Second) 939 s.Lock() 940 c.Check(t1.Status(), Equals, state.DoneStatus) 941 942 var v int 943 err := s.Get("runMgr1Mark", &v) 944 c.Check(err, IsNil) 945 } 946 947 func (ovs *overlordSuite) TestSettleNotConverging(c *C) { 948 restoreIntv := overlord.MockEnsureInterval(1 * time.Minute) 949 defer restoreIntv() 950 o := overlord.Mock() 951 952 s := o.State() 953 sm1 := newSampleManager(s, o.TaskRunner()) 954 o.AddManager(sm1) 955 o.AddManager(o.TaskRunner()) 956 957 defer o.Engine().Stop() 958 959 s.Lock() 960 defer s.Unlock() 961 962 chg := s.NewChange("chg", "...") 963 t1 := s.NewTask("runMgrForever", "1...") 964 chg.AddTask(t1) 965 966 s.Unlock() 967 err := o.Settle(250 * time.Millisecond) 968 s.Lock() 969 970 c.Check(err, ErrorMatches, `Settle is not converging`) 971 972 } 973 974 func (ovs *overlordSuite) TestSettleChain(c *C) { 975 restoreIntv := overlord.MockEnsureInterval(1 * time.Minute) 976 defer restoreIntv() 977 o := overlord.Mock() 978 979 s := o.State() 980 sm1 := newSampleManager(s, o.TaskRunner()) 981 o.AddManager(sm1) 982 o.AddManager(o.TaskRunner()) 983 984 defer o.Engine().Stop() 985 986 s.Lock() 987 defer s.Unlock() 988 989 chg := s.NewChange("chg", "...") 990 t1 := s.NewTask("runMgr1", "1...") 991 t2 := s.NewTask("runMgr2", "2...") 992 t2.WaitFor(t1) 993 chg.AddAll(state.NewTaskSet(t1, t2)) 994 995 s.Unlock() 996 o.Settle(5 * time.Second) 997 s.Lock() 998 c.Check(t1.Status(), Equals, state.DoneStatus) 999 c.Check(t2.Status(), Equals, state.DoneStatus) 1000 1001 var v int 1002 err := s.Get("runMgr1Mark", &v) 1003 c.Check(err, IsNil) 1004 err = s.Get("runMgr2Mark", &v) 1005 c.Check(err, IsNil) 1006 } 1007 1008 func (ovs *overlordSuite) TestSettleChainWCleanup(c *C) { 1009 restoreIntv := overlord.MockEnsureInterval(1 * time.Minute) 1010 defer restoreIntv() 1011 o := overlord.Mock() 1012 1013 s := o.State() 1014 sm1 := newSampleManager(s, o.TaskRunner()) 1015 o.AddManager(sm1) 1016 o.AddManager(o.TaskRunner()) 1017 1018 defer o.Engine().Stop() 1019 1020 s.Lock() 1021 defer s.Unlock() 1022 1023 chg := s.NewChange("chg", "...") 1024 t1 := s.NewTask("runMgrWCleanup", "1...") 1025 t2 := s.NewTask("runMgr2", "2...") 1026 t2.WaitFor(t1) 1027 chg.AddAll(state.NewTaskSet(t1, t2)) 1028 1029 s.Unlock() 1030 o.Settle(5 * time.Second) 1031 s.Lock() 1032 c.Check(t1.Status(), Equals, state.DoneStatus) 1033 c.Check(t2.Status(), Equals, state.DoneStatus) 1034 1035 var v int 1036 err := s.Get("runMgrWCleanupMark", &v) 1037 c.Check(err, IsNil) 1038 err = s.Get("runMgr2Mark", &v) 1039 c.Check(err, IsNil) 1040 1041 err = s.Get("runMgrWCleanupCleanedUp", &v) 1042 c.Check(err, IsNil) 1043 } 1044 1045 func (ovs *overlordSuite) TestSettleExplicitEnsureBefore(c *C) { 1046 restoreIntv := overlord.MockEnsureInterval(1 * time.Minute) 1047 defer restoreIntv() 1048 o := overlord.Mock() 1049 1050 s := o.State() 1051 sm1 := newSampleManager(s, o.TaskRunner()) 1052 sm1.ensureCallback = func() { 1053 s.Lock() 1054 defer s.Unlock() 1055 v := 0 1056 s.Get("ensureCount", &v) 1057 s.Set("ensureCount", v+1) 1058 } 1059 1060 o.AddManager(sm1) 1061 o.AddManager(o.TaskRunner()) 1062 1063 defer o.Engine().Stop() 1064 1065 s.Lock() 1066 defer s.Unlock() 1067 1068 chg := s.NewChange("chg", "...") 1069 t := s.NewTask("runMgrEnsureBefore", "...") 1070 chg.AddTask(t) 1071 1072 s.Unlock() 1073 o.Settle(5 * time.Second) 1074 s.Lock() 1075 c.Check(t.Status(), Equals, state.DoneStatus) 1076 1077 var v int 1078 err := s.Get("ensureCount", &v) 1079 c.Check(err, IsNil) 1080 c.Check(v, Equals, 2) 1081 } 1082 1083 func (ovs *overlordSuite) TestEnsureErrorWhenPreseeding(c *C) { 1084 restore := snapdenv.MockPreseeding(true) 1085 defer restore() 1086 1087 restoreIntv := overlord.MockEnsureInterval(1 * time.Millisecond) 1088 defer restoreIntv() 1089 1090 var errorCallbackError error 1091 restoreExitWithErr := overlord.MockPreseedExitWithError(func(err error) { 1092 errorCallbackError = err 1093 }) 1094 defer restoreExitWithErr() 1095 1096 o, err := overlord.New(nil) 1097 c.Assert(err, IsNil) 1098 markSeeded(o) 1099 1100 err = o.StartUp() 1101 c.Assert(err, IsNil) 1102 1103 o.TaskRunner().AddHandler("bar", func(t *state.Task, _ *tomb.Tomb) error { 1104 return fmt.Errorf("bar error") 1105 }, nil) 1106 1107 s := o.State() 1108 s.Lock() 1109 defer s.Unlock() 1110 1111 chg := s.NewChange("chg", "...") 1112 t := s.NewTask("bar", "...") 1113 chg.AddTask(t) 1114 1115 s.Unlock() 1116 o.Loop() 1117 time.Sleep(1 * time.Second) 1118 o.Stop() 1119 1120 s.Lock() 1121 c.Check(t.Status(), Equals, state.ErrorStatus) 1122 c.Check(errorCallbackError, ErrorMatches, "bar error") 1123 } 1124 1125 func (ovs *overlordSuite) TestRequestRestartNoHandler(c *C) { 1126 o, err := overlord.New(nil) 1127 c.Assert(err, IsNil) 1128 1129 o.State().RequestRestart(state.RestartDaemon) 1130 } 1131 1132 type testRestartBehavior struct { 1133 restartRequested state.RestartType 1134 rebootState string 1135 rebootVerifiedErr error 1136 } 1137 1138 func (rb *testRestartBehavior) HandleRestart(t state.RestartType) { 1139 rb.restartRequested = t 1140 } 1141 1142 func (rb *testRestartBehavior) RebootAsExpected(_ *state.State) error { 1143 rb.rebootState = "as-expected" 1144 return rb.rebootVerifiedErr 1145 } 1146 1147 func (rb *testRestartBehavior) RebootDidNotHappen(_ *state.State) error { 1148 rb.rebootState = "did-not-happen" 1149 return rb.rebootVerifiedErr 1150 } 1151 1152 func (ovs *overlordSuite) TestRequestRestartHandler(c *C) { 1153 rb := &testRestartBehavior{} 1154 1155 o, err := overlord.New(rb) 1156 c.Assert(err, IsNil) 1157 1158 o.State().RequestRestart(state.RestartDaemon) 1159 1160 c.Check(rb.restartRequested, Equals, state.RestartDaemon) 1161 } 1162 1163 func (ovs *overlordSuite) TestVerifyRebootNoPendingReboot(c *C) { 1164 fakeState := []byte(fmt.Sprintf(`{"data":{"patch-level":%d,"patch-sublevel":%d,"some":"data","refresh-privacy-key":"0123456789ABCDEF"},"changes":null,"tasks":null,"last-change-id":0,"last-task-id":0,"last-lane-id":0}`, patch.Level, patch.Sublevel)) 1165 err := ioutil.WriteFile(dirs.SnapStateFile, fakeState, 0600) 1166 c.Assert(err, IsNil) 1167 1168 rb := &testRestartBehavior{} 1169 1170 _, err = overlord.New(rb) 1171 c.Assert(err, IsNil) 1172 1173 c.Check(rb.rebootState, Equals, "as-expected") 1174 } 1175 1176 func (ovs *overlordSuite) TestVerifyRebootOK(c *C) { 1177 fakeState := []byte(fmt.Sprintf(`{"data":{"patch-level":%d,"patch-sublevel":%d,"some":"data","refresh-privacy-key":"0123456789ABCDEF","system-restart-from-boot-id":%q},"changes":null,"tasks":null,"last-change-id":0,"last-task-id":0,"last-lane-id":0}`, patch.Level, patch.Sublevel, "boot-id-prev")) 1178 err := ioutil.WriteFile(dirs.SnapStateFile, fakeState, 0600) 1179 c.Assert(err, IsNil) 1180 1181 rb := &testRestartBehavior{} 1182 1183 _, err = overlord.New(rb) 1184 c.Assert(err, IsNil) 1185 1186 c.Check(rb.rebootState, Equals, "as-expected") 1187 } 1188 1189 func (ovs *overlordSuite) TestVerifyRebootOKButError(c *C) { 1190 fakeState := []byte(fmt.Sprintf(`{"data":{"patch-level":%d,"patch-sublevel":%d,"some":"data","refresh-privacy-key":"0123456789ABCDEF","system-restart-from-boot-id":%q},"changes":null,"tasks":null,"last-change-id":0,"last-task-id":0,"last-lane-id":0}`, patch.Level, patch.Sublevel, "boot-id-prev")) 1191 err := ioutil.WriteFile(dirs.SnapStateFile, fakeState, 0600) 1192 c.Assert(err, IsNil) 1193 1194 e := errors.New("boom") 1195 rb := &testRestartBehavior{rebootVerifiedErr: e} 1196 1197 _, err = overlord.New(rb) 1198 c.Assert(err, Equals, e) 1199 1200 c.Check(rb.rebootState, Equals, "as-expected") 1201 } 1202 1203 func (ovs *overlordSuite) TestVerifyRebootDidNotHappen(c *C) { 1204 curBootID, err := osutil.BootID() 1205 c.Assert(err, IsNil) 1206 1207 fakeState := []byte(fmt.Sprintf(`{"data":{"patch-level":%d,"patch-sublevel":%d,"some":"data","refresh-privacy-key":"0123456789ABCDEF","system-restart-from-boot-id":%q},"changes":null,"tasks":null,"last-change-id":0,"last-task-id":0,"last-lane-id":0}`, patch.Level, patch.Sublevel, curBootID)) 1208 err = ioutil.WriteFile(dirs.SnapStateFile, fakeState, 0600) 1209 c.Assert(err, IsNil) 1210 1211 rb := &testRestartBehavior{} 1212 1213 _, err = overlord.New(rb) 1214 c.Assert(err, IsNil) 1215 1216 c.Check(rb.rebootState, Equals, "did-not-happen") 1217 } 1218 1219 func (ovs *overlordSuite) TestVerifyRebootDidNotHappenError(c *C) { 1220 curBootID, err := osutil.BootID() 1221 c.Assert(err, IsNil) 1222 1223 fakeState := []byte(fmt.Sprintf(`{"data":{"patch-level":%d,"patch-sublevel":%d,"some":"data","refresh-privacy-key":"0123456789ABCDEF","system-restart-from-boot-id":%q},"changes":null,"tasks":null,"last-change-id":0,"last-task-id":0,"last-lane-id":0}`, patch.Level, patch.Sublevel, curBootID)) 1224 err = ioutil.WriteFile(dirs.SnapStateFile, fakeState, 0600) 1225 c.Assert(err, IsNil) 1226 1227 e := errors.New("boom") 1228 rb := &testRestartBehavior{rebootVerifiedErr: e} 1229 1230 _, err = overlord.New(rb) 1231 c.Assert(err, Equals, e) 1232 1233 c.Check(rb.rebootState, Equals, "did-not-happen") 1234 } 1235 1236 func (ovs *overlordSuite) TestOverlordCanStandby(c *C) { 1237 restoreIntv := overlord.MockEnsureInterval(10 * time.Millisecond) 1238 defer restoreIntv() 1239 o := overlord.Mock() 1240 witness := &witnessManager{ 1241 state: o.State(), 1242 expectedEnsure: 3, 1243 ensureCalled: make(chan struct{}), 1244 } 1245 o.AddManager(witness) 1246 1247 c.Assert(o.StartUp(), IsNil) 1248 1249 // can only standby after loop ran once 1250 c.Assert(o.CanStandby(), Equals, false) 1251 1252 o.Loop() 1253 defer o.Stop() 1254 1255 select { 1256 case <-witness.ensureCalled: 1257 case <-time.After(2 * time.Second): 1258 c.Fatal("Ensure calls not happening") 1259 } 1260 1261 c.Assert(o.CanStandby(), Equals, true) 1262 } 1263 1264 func (ovs *overlordSuite) TestStartupTimeout(c *C) { 1265 o, err := overlord.New(nil) 1266 c.Assert(err, IsNil) 1267 1268 to, _, err := o.StartupTimeout() 1269 c.Assert(err, IsNil) 1270 c.Check(to, Equals, 30*time.Second) 1271 1272 st := o.State() 1273 st.Lock() 1274 defer st.Unlock() 1275 1276 // have two snaps 1277 snapstate.Set(st, "core18", &snapstate.SnapState{ 1278 Active: true, 1279 Sequence: []*snap.SideInfo{ 1280 {RealName: "core18", Revision: snap.R(1)}, 1281 }, 1282 Current: snap.R(1), 1283 SnapType: "base", 1284 }) 1285 snapstate.Set(st, "foo", &snapstate.SnapState{ 1286 Active: true, 1287 Sequence: []*snap.SideInfo{ 1288 {RealName: "foo", Revision: snap.R(1)}, 1289 }, 1290 Current: snap.R(1), 1291 SnapType: "app", 1292 }) 1293 1294 st.Unlock() 1295 to, reasoning, err := o.StartupTimeout() 1296 st.Lock() 1297 c.Assert(err, IsNil) 1298 1299 c.Check(to, Equals, (30+5+5)*time.Second) 1300 c.Check(reasoning, Equals, "pessimistic estimate of 30s plus 5s per snap") 1301 }