github.com/rigado/snapd@v2.42.5-go-mod+incompatible/overlord/overlord_test.go (about)

     1  // -*- Mode: Go; indent-tabs-mode: t -*-
     2  
     3  /*
     4   * Copyright (C) 2016-2017 Canonical Ltd
     5   *
     6   * This program is free software: you can redistribute it and/or modify
     7   * it under the terms of the GNU General Public License version 3 as
     8   * published by the Free Software Foundation.
     9   *
    10   * This program is distributed in the hope that it will be useful,
    11   * but WITHOUT ANY WARRANTY; without even the implied warranty of
    12   * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    13   * GNU General Public License for more details.
    14   *
    15   * You should have received a copy of the GNU General Public License
    16   * along with this program.  If not, see <http://www.gnu.org/licenses/>.
    17   *
    18   */
    19  
    20  package overlord_test
    21  
    22  import (
    23  	"encoding/json"
    24  	"errors"
    25  	"fmt"
    26  	"io/ioutil"
    27  	"os"
    28  	"path/filepath"
    29  	"syscall"
    30  	"testing"
    31  	"time"
    32  
    33  	. "gopkg.in/check.v1"
    34  	"gopkg.in/tomb.v2"
    35  
    36  	"github.com/snapcore/snapd/cmd"
    37  	"github.com/snapcore/snapd/dirs"
    38  	"github.com/snapcore/snapd/osutil"
    39  	"github.com/snapcore/snapd/overlord"
    40  	"github.com/snapcore/snapd/overlord/auth"
    41  	"github.com/snapcore/snapd/overlord/devicestate/devicestatetest"
    42  	"github.com/snapcore/snapd/overlord/hookstate"
    43  	"github.com/snapcore/snapd/overlord/ifacestate"
    44  	"github.com/snapcore/snapd/overlord/patch"
    45  	"github.com/snapcore/snapd/overlord/snapstate"
    46  	"github.com/snapcore/snapd/overlord/state"
    47  	"github.com/snapcore/snapd/snap"
    48  	"github.com/snapcore/snapd/store"
    49  	"github.com/snapcore/snapd/testutil"
    50  )
    51  
    52  func TestOverlord(t *testing.T) { TestingT(t) }
    53  
    54  type overlordSuite struct {
    55  	restoreBackends func()
    56  }
    57  
    58  var _ = Suite(&overlordSuite{})
    59  
    60  func (ovs *overlordSuite) SetUpTest(c *C) {
    61  	tmpdir := c.MkDir()
    62  	dirs.SetRootDir(tmpdir)
    63  	dirs.SnapStateFile = filepath.Join(tmpdir, "test.json")
    64  	snapstate.CanAutoRefresh = nil
    65  	ovs.restoreBackends = ifacestate.MockSecurityBackends(nil)
    66  }
    67  
    68  func (ovs *overlordSuite) TearDownTest(c *C) {
    69  	dirs.SetRootDir("/")
    70  	ovs.restoreBackends()
    71  }
    72  
    73  func (ovs *overlordSuite) TestNew(c *C) {
    74  	restore := patch.Mock(42, 2, nil)
    75  	defer restore()
    76  
    77  	var configstateInitCalled bool
    78  	overlord.MockConfigstateInit(func(*state.State, *hookstate.HookManager) error {
    79  		configstateInitCalled = true
    80  		return nil
    81  	})
    82  
    83  	o, err := overlord.New(nil)
    84  	c.Assert(err, IsNil)
    85  	c.Check(o, NotNil)
    86  
    87  	c.Check(o.StateEngine(), NotNil)
    88  	c.Check(o.TaskRunner(), NotNil)
    89  	c.Check(o.SnapManager(), NotNil)
    90  	c.Check(o.AssertManager(), NotNil)
    91  	c.Check(o.InterfaceManager(), NotNil)
    92  	c.Check(o.HookManager(), NotNil)
    93  	c.Check(o.DeviceManager(), NotNil)
    94  	c.Check(o.CommandManager(), NotNil)
    95  	c.Check(o.SnapshotManager(), NotNil)
    96  	c.Check(configstateInitCalled, Equals, true)
    97  
    98  	o.InterfaceManager().DisableUDevMonitor()
    99  
   100  	s := o.State()
   101  	c.Check(s, NotNil)
   102  	c.Check(o.Engine().State(), Equals, s)
   103  
   104  	s.Lock()
   105  	defer s.Unlock()
   106  	var patchLevel, patchSublevel int
   107  	s.Get("patch-level", &patchLevel)
   108  	c.Check(patchLevel, Equals, 42)
   109  	s.Get("patch-sublevel", &patchSublevel)
   110  	c.Check(patchSublevel, Equals, 2)
   111  	var refreshPrivacyKey string
   112  	s.Get("refresh-privacy-key", &refreshPrivacyKey)
   113  	c.Check(refreshPrivacyKey, HasLen, 16)
   114  
   115  	// store is setup
   116  	sto := snapstate.Store(s, nil)
   117  	c.Check(sto, FitsTypeOf, &store.Store{})
   118  	c.Check(sto.(*store.Store).CacheDownloads(), Equals, 5)
   119  }
   120  
   121  func (ovs *overlordSuite) TestNewStore(c *C) {
   122  	// this is a shallow test, the deep testing happens in the
   123  	// remodeling tests in managers_test.go
   124  	o, err := overlord.New(nil)
   125  	c.Assert(err, IsNil)
   126  
   127  	devBE := o.DeviceManager().StoreContextBackend()
   128  
   129  	sto := o.NewStore(devBE)
   130  	c.Check(sto, FitsTypeOf, &store.Store{})
   131  	c.Check(sto.(*store.Store).CacheDownloads(), Equals, 5)
   132  }
   133  
   134  func (ovs *overlordSuite) TestNewWithGoodState(c *C) {
   135  	fakeState := []byte(fmt.Sprintf(`{"data":{"patch-level":%d,"patch-sublevel":%d,"patch-sublevel-last-version":%q,"some":"data","refresh-privacy-key":"0123456789ABCDEF"},"changes":null,"tasks":null,"last-change-id":0,"last-task-id":0,"last-lane-id":0}`, patch.Level, patch.Sublevel, cmd.Version))
   136  	err := ioutil.WriteFile(dirs.SnapStateFile, fakeState, 0600)
   137  	c.Assert(err, IsNil)
   138  
   139  	o, err := overlord.New(nil)
   140  	c.Assert(err, IsNil)
   141  
   142  	state := o.State()
   143  	c.Assert(err, IsNil)
   144  	state.Lock()
   145  	defer state.Unlock()
   146  
   147  	d, err := state.MarshalJSON()
   148  	c.Assert(err, IsNil)
   149  
   150  	var got, expected map[string]interface{}
   151  	err = json.Unmarshal(d, &got)
   152  	c.Assert(err, IsNil)
   153  	err = json.Unmarshal(fakeState, &expected)
   154  	c.Assert(err, IsNil)
   155  
   156  	data, _ := got["data"].(map[string]interface{})
   157  	c.Assert(data, NotNil)
   158  
   159  	c.Check(got, DeepEquals, expected)
   160  }
   161  
   162  func (ovs *overlordSuite) TestNewWithStateSnapmgrUpdate(c *C) {
   163  	fakeState := []byte(fmt.Sprintf(`{"data":{"patch-level":%d,"some":"data"},"changes":null,"tasks":null,"last-change-id":0,"last-task-id":0,"last-lane-id":0}`, patch.Level))
   164  	err := ioutil.WriteFile(dirs.SnapStateFile, fakeState, 0600)
   165  	c.Assert(err, IsNil)
   166  
   167  	o, err := overlord.New(nil)
   168  	c.Assert(err, IsNil)
   169  
   170  	state := o.State()
   171  	c.Assert(err, IsNil)
   172  	state.Lock()
   173  	defer state.Unlock()
   174  
   175  	var refreshPrivacyKey string
   176  	state.Get("refresh-privacy-key", &refreshPrivacyKey)
   177  	c.Check(refreshPrivacyKey, HasLen, 16)
   178  }
   179  
   180  func (ovs *overlordSuite) TestNewWithInvalidState(c *C) {
   181  	fakeState := []byte(``)
   182  	err := ioutil.WriteFile(dirs.SnapStateFile, fakeState, 0600)
   183  	c.Assert(err, IsNil)
   184  
   185  	_, err = overlord.New(nil)
   186  	c.Assert(err, ErrorMatches, "cannot read state: EOF")
   187  }
   188  
   189  func (ovs *overlordSuite) TestNewWithPatches(c *C) {
   190  	p := func(s *state.State) error {
   191  		s.Set("patched", true)
   192  		return nil
   193  	}
   194  	sp := func(s *state.State) error {
   195  		s.Set("patched2", true)
   196  		return nil
   197  	}
   198  	patch.Mock(1, 1, map[int][]patch.PatchFunc{1: {p, sp}})
   199  
   200  	fakeState := []byte(fmt.Sprintf(`{"data":{"patch-level":0, "patch-sublevel":0}}`))
   201  	err := ioutil.WriteFile(dirs.SnapStateFile, fakeState, 0600)
   202  	c.Assert(err, IsNil)
   203  
   204  	o, err := overlord.New(nil)
   205  	c.Assert(err, IsNil)
   206  
   207  	state := o.State()
   208  	c.Assert(err, IsNil)
   209  	state.Lock()
   210  	defer state.Unlock()
   211  
   212  	var level int
   213  	err = state.Get("patch-level", &level)
   214  	c.Assert(err, IsNil)
   215  	c.Check(level, Equals, 1)
   216  
   217  	var sublevel int
   218  	c.Assert(state.Get("patch-sublevel", &sublevel), IsNil)
   219  	c.Check(sublevel, Equals, 1)
   220  
   221  	var b bool
   222  	err = state.Get("patched", &b)
   223  	c.Assert(err, IsNil)
   224  	c.Check(b, Equals, true)
   225  
   226  	c.Assert(state.Get("patched2", &b), IsNil)
   227  	c.Check(b, Equals, true)
   228  }
   229  
   230  func (ovs *overlordSuite) TestNewFailedConfigstate(c *C) {
   231  	restore := patch.Mock(42, 2, nil)
   232  	defer restore()
   233  
   234  	var configstateInitCalled bool
   235  	restore = overlord.MockConfigstateInit(func(*state.State, *hookstate.HookManager) error {
   236  		configstateInitCalled = true
   237  		return fmt.Errorf("bad bad")
   238  	})
   239  	defer restore()
   240  
   241  	o, err := overlord.New(nil)
   242  	c.Assert(err, ErrorMatches, "bad bad")
   243  	c.Check(o, IsNil)
   244  	c.Check(configstateInitCalled, Equals, true)
   245  }
   246  
   247  type witnessManager struct {
   248  	state          *state.State
   249  	expectedEnsure int
   250  	ensureCalled   chan struct{}
   251  	ensureCallback func(s *state.State) error
   252  	startedUp      int
   253  }
   254  
   255  func (wm *witnessManager) StartUp() error {
   256  	wm.startedUp++
   257  	return nil
   258  }
   259  
   260  func (wm *witnessManager) Ensure() error {
   261  	if wm.expectedEnsure--; wm.expectedEnsure == 0 {
   262  		close(wm.ensureCalled)
   263  		return nil
   264  	}
   265  	if wm.ensureCallback != nil {
   266  		return wm.ensureCallback(wm.state)
   267  	}
   268  	return nil
   269  }
   270  
   271  // markSeeded flags the state under the overlord as seeded to avoid running the seeding code in these tests
   272  func markSeeded(o *overlord.Overlord) {
   273  	st := o.State()
   274  	st.Lock()
   275  	st.Set("seeded", true)
   276  	devicestatetest.SetDevice(st, &auth.DeviceState{
   277  		Brand:  "canonical",
   278  		Model:  "pc",
   279  		Serial: "serialserial",
   280  	})
   281  	st.Unlock()
   282  }
   283  
   284  func (ovs *overlordSuite) TestTrivialRunAndStop(c *C) {
   285  	o, err := overlord.New(nil)
   286  	c.Assert(err, IsNil)
   287  
   288  	markSeeded(o)
   289  	// make sure we don't try to talk to the store
   290  	snapstate.CanAutoRefresh = nil
   291  
   292  	err = o.StartUp()
   293  	c.Assert(err, IsNil)
   294  
   295  	o.Loop()
   296  
   297  	err = o.Stop()
   298  	c.Assert(err, IsNil)
   299  }
   300  
   301  func (ovs *overlordSuite) TestUnknownTasks(c *C) {
   302  	o, err := overlord.New(nil)
   303  	c.Assert(err, IsNil)
   304  	o.InterfaceManager().DisableUDevMonitor()
   305  
   306  	markSeeded(o)
   307  	// make sure we don't try to talk to the store
   308  	snapstate.CanAutoRefresh = nil
   309  
   310  	// unknown tasks are ignored and succeed
   311  	st := o.State()
   312  	st.Lock()
   313  	defer st.Unlock()
   314  	t := st.NewTask("unknown", "...")
   315  	chg := st.NewChange("change-w-unknown", "...")
   316  	chg.AddTask(t)
   317  
   318  	st.Unlock()
   319  	err = o.Settle(1 * time.Second)
   320  	st.Lock()
   321  	c.Assert(err, IsNil)
   322  
   323  	c.Check(chg.Status(), Equals, state.DoneStatus)
   324  }
   325  
   326  func (ovs *overlordSuite) TestEnsureLoopRunAndStop(c *C) {
   327  	restoreIntv := overlord.MockEnsureInterval(10 * time.Millisecond)
   328  	defer restoreIntv()
   329  	o := overlord.Mock()
   330  
   331  	witness := &witnessManager{
   332  		state:          o.State(),
   333  		expectedEnsure: 3,
   334  		ensureCalled:   make(chan struct{}),
   335  	}
   336  	o.AddManager(witness)
   337  
   338  	err := o.StartUp()
   339  	c.Assert(err, IsNil)
   340  
   341  	o.Loop()
   342  	defer o.Stop()
   343  
   344  	t0 := time.Now()
   345  	select {
   346  	case <-witness.ensureCalled:
   347  	case <-time.After(2 * time.Second):
   348  		c.Fatal("Ensure calls not happening")
   349  	}
   350  	c.Check(time.Since(t0) >= 10*time.Millisecond, Equals, true)
   351  
   352  	err = o.Stop()
   353  	c.Assert(err, IsNil)
   354  
   355  	c.Check(witness.startedUp, Equals, 1)
   356  }
   357  
   358  func (ovs *overlordSuite) TestEnsureLoopMediatedEnsureBeforeImmediate(c *C) {
   359  	restoreIntv := overlord.MockEnsureInterval(10 * time.Minute)
   360  	defer restoreIntv()
   361  	o := overlord.Mock()
   362  
   363  	ensure := func(s *state.State) error {
   364  		s.EnsureBefore(0)
   365  		return nil
   366  	}
   367  
   368  	witness := &witnessManager{
   369  		state:          o.State(),
   370  		expectedEnsure: 2,
   371  		ensureCalled:   make(chan struct{}),
   372  		ensureCallback: ensure,
   373  	}
   374  	o.AddManager(witness)
   375  
   376  	c.Assert(o.StartUp(), IsNil)
   377  
   378  	o.Loop()
   379  	defer o.Stop()
   380  
   381  	select {
   382  	case <-witness.ensureCalled:
   383  	case <-time.After(2 * time.Second):
   384  		c.Fatal("Ensure calls not happening")
   385  	}
   386  }
   387  
   388  func (ovs *overlordSuite) TestEnsureLoopMediatedEnsureBefore(c *C) {
   389  	restoreIntv := overlord.MockEnsureInterval(10 * time.Minute)
   390  	defer restoreIntv()
   391  	o := overlord.Mock()
   392  
   393  	ensure := func(s *state.State) error {
   394  		s.EnsureBefore(10 * time.Millisecond)
   395  		return nil
   396  	}
   397  
   398  	witness := &witnessManager{
   399  		state:          o.State(),
   400  		expectedEnsure: 2,
   401  		ensureCalled:   make(chan struct{}),
   402  		ensureCallback: ensure,
   403  	}
   404  	o.AddManager(witness)
   405  
   406  	c.Assert(o.StartUp(), IsNil)
   407  
   408  	o.Loop()
   409  	defer o.Stop()
   410  
   411  	select {
   412  	case <-witness.ensureCalled:
   413  	case <-time.After(2 * time.Second):
   414  		c.Fatal("Ensure calls not happening")
   415  	}
   416  }
   417  
   418  func (ovs *overlordSuite) TestEnsureBeforeSleepy(c *C) {
   419  	restoreIntv := overlord.MockEnsureInterval(10 * time.Minute)
   420  	defer restoreIntv()
   421  	o := overlord.Mock()
   422  
   423  	ensure := func(s *state.State) error {
   424  		overlord.MockEnsureNext(o, time.Now().Add(-10*time.Hour))
   425  		s.EnsureBefore(0)
   426  		return nil
   427  	}
   428  
   429  	witness := &witnessManager{
   430  		state:          o.State(),
   431  		expectedEnsure: 2,
   432  		ensureCalled:   make(chan struct{}),
   433  		ensureCallback: ensure,
   434  	}
   435  	o.AddManager(witness)
   436  
   437  	c.Assert(o.StartUp(), IsNil)
   438  
   439  	o.Loop()
   440  	defer o.Stop()
   441  
   442  	select {
   443  	case <-witness.ensureCalled:
   444  	case <-time.After(2 * time.Second):
   445  		c.Fatal("Ensure calls not happening")
   446  	}
   447  }
   448  
   449  func (ovs *overlordSuite) TestEnsureBeforeLater(c *C) {
   450  	restoreIntv := overlord.MockEnsureInterval(10 * time.Minute)
   451  	defer restoreIntv()
   452  	o := overlord.Mock()
   453  
   454  	ensure := func(s *state.State) error {
   455  		overlord.MockEnsureNext(o, time.Now().Add(-10*time.Hour))
   456  		s.EnsureBefore(time.Second * 5)
   457  		return nil
   458  	}
   459  
   460  	witness := &witnessManager{
   461  		state:          o.State(),
   462  		expectedEnsure: 2,
   463  		ensureCalled:   make(chan struct{}),
   464  		ensureCallback: ensure,
   465  	}
   466  	o.AddManager(witness)
   467  
   468  	c.Assert(o.StartUp(), IsNil)
   469  
   470  	o.Loop()
   471  	defer o.Stop()
   472  
   473  	select {
   474  	case <-witness.ensureCalled:
   475  	case <-time.After(2 * time.Second):
   476  		c.Fatal("Ensure calls not happening")
   477  	}
   478  }
   479  
   480  func (ovs *overlordSuite) TestEnsureLoopMediatedEnsureBeforeOutsideEnsure(c *C) {
   481  	restoreIntv := overlord.MockEnsureInterval(10 * time.Minute)
   482  	defer restoreIntv()
   483  	o := overlord.Mock()
   484  
   485  	ch := make(chan struct{})
   486  	ensure := func(s *state.State) error {
   487  		close(ch)
   488  		return nil
   489  	}
   490  
   491  	witness := &witnessManager{
   492  		state:          o.State(),
   493  		expectedEnsure: 2,
   494  		ensureCalled:   make(chan struct{}),
   495  		ensureCallback: ensure,
   496  	}
   497  	o.AddManager(witness)
   498  
   499  	c.Assert(o.StartUp(), IsNil)
   500  
   501  	o.Loop()
   502  	defer o.Stop()
   503  
   504  	select {
   505  	case <-ch:
   506  	case <-time.After(2 * time.Second):
   507  		c.Fatal("Ensure calls not happening")
   508  	}
   509  
   510  	o.State().EnsureBefore(0)
   511  
   512  	select {
   513  	case <-witness.ensureCalled:
   514  	case <-time.After(2 * time.Second):
   515  		c.Fatal("Ensure calls not happening")
   516  	}
   517  }
   518  
   519  func (ovs *overlordSuite) TestEnsureLoopPrune(c *C) {
   520  	restoreIntv := overlord.MockPruneInterval(200*time.Millisecond, 1000*time.Millisecond, 1000*time.Millisecond)
   521  	defer restoreIntv()
   522  	o := overlord.Mock()
   523  
   524  	st := o.State()
   525  	st.Lock()
   526  	t1 := st.NewTask("foo", "...")
   527  	chg1 := st.NewChange("abort", "...")
   528  	chg1.AddTask(t1)
   529  	chg2 := st.NewChange("prune", "...")
   530  	chg2.SetStatus(state.DoneStatus)
   531  	t0 := chg2.ReadyTime()
   532  	st.Unlock()
   533  
   534  	// observe the loop cycles to detect when prune should have happened
   535  	pruneHappened := make(chan struct{})
   536  	cycles := -1
   537  	waitForPrune := func(_ *state.State) error {
   538  		if cycles == -1 {
   539  			if time.Since(t0) > 1000*time.Millisecond {
   540  				cycles = 2 // wait a couple more loop cycles
   541  			}
   542  			return nil
   543  		}
   544  		if cycles > 0 {
   545  			cycles--
   546  			if cycles == 0 {
   547  				close(pruneHappened)
   548  			}
   549  		}
   550  		return nil
   551  	}
   552  	witness := &witnessManager{
   553  		ensureCallback: waitForPrune,
   554  	}
   555  	o.AddManager(witness)
   556  
   557  	c.Assert(o.StartUp(), IsNil)
   558  
   559  	o.Loop()
   560  
   561  	select {
   562  	case <-pruneHappened:
   563  	case <-time.After(2 * time.Second):
   564  		c.Fatal("Pruning should have happened by now")
   565  	}
   566  
   567  	err := o.Stop()
   568  	c.Assert(err, IsNil)
   569  
   570  	st.Lock()
   571  	defer st.Unlock()
   572  
   573  	c.Assert(st.Change(chg1.ID()), Equals, chg1)
   574  	c.Assert(st.Change(chg2.ID()), IsNil)
   575  
   576  	c.Assert(t1.Status(), Equals, state.HoldStatus)
   577  }
   578  
   579  func (ovs *overlordSuite) TestEnsureLoopPruneRunsMultipleTimes(c *C) {
   580  	restoreIntv := overlord.MockPruneInterval(100*time.Millisecond, 1000*time.Millisecond, 1*time.Hour)
   581  	defer restoreIntv()
   582  	o := overlord.Mock()
   583  
   584  	// create two changes, one that can be pruned now, one in progress
   585  	st := o.State()
   586  	st.Lock()
   587  	t1 := st.NewTask("foo", "...")
   588  	chg1 := st.NewChange("pruneNow", "...")
   589  	chg1.AddTask(t1)
   590  	t1.SetStatus(state.DoneStatus)
   591  	t2 := st.NewTask("foo", "...")
   592  	chg2 := st.NewChange("pruneNext", "...")
   593  	chg2.AddTask(t2)
   594  	t2.SetStatus(state.DoStatus)
   595  	c.Check(st.Changes(), HasLen, 2)
   596  	st.Unlock()
   597  
   598  	// start the loop that runs the prune ticker
   599  	o.Loop()
   600  
   601  	// ensure the first change is pruned
   602  	time.Sleep(1500 * time.Millisecond)
   603  	st.Lock()
   604  	c.Check(st.Changes(), HasLen, 1)
   605  	st.Unlock()
   606  
   607  	// ensure the second is also purged after it is ready
   608  	st.Lock()
   609  	chg2.SetStatus(state.DoneStatus)
   610  	st.Unlock()
   611  	time.Sleep(1500 * time.Millisecond)
   612  	st.Lock()
   613  	c.Check(st.Changes(), HasLen, 0)
   614  	st.Unlock()
   615  
   616  	// cleanup loop ticker
   617  	err := o.Stop()
   618  	c.Assert(err, IsNil)
   619  }
   620  
   621  func (ovs *overlordSuite) TestCheckpoint(c *C) {
   622  	oldUmask := syscall.Umask(0)
   623  	defer syscall.Umask(oldUmask)
   624  
   625  	o, err := overlord.New(nil)
   626  	c.Assert(err, IsNil)
   627  
   628  	s := o.State()
   629  	s.Lock()
   630  	s.Set("mark", 1)
   631  	s.Unlock()
   632  
   633  	st, err := os.Stat(dirs.SnapStateFile)
   634  	c.Assert(err, IsNil)
   635  	c.Assert(st.Mode(), Equals, os.FileMode(0600))
   636  
   637  	c.Check(dirs.SnapStateFile, testutil.FileContains, `"mark":1`)
   638  }
   639  
   640  type sampleManager struct {
   641  	ensureCallback func()
   642  }
   643  
   644  func newSampleManager(s *state.State, runner *state.TaskRunner) *sampleManager {
   645  	sm := &sampleManager{}
   646  
   647  	runner.AddHandler("runMgr1", func(t *state.Task, _ *tomb.Tomb) error {
   648  		s := t.State()
   649  		s.Lock()
   650  		defer s.Unlock()
   651  		s.Set("runMgr1Mark", 1)
   652  		return nil
   653  	}, nil)
   654  	runner.AddHandler("runMgr2", func(t *state.Task, _ *tomb.Tomb) error {
   655  		s := t.State()
   656  		s.Lock()
   657  		defer s.Unlock()
   658  		s.Set("runMgr2Mark", 1)
   659  		return nil
   660  	}, nil)
   661  	runner.AddHandler("runMgrEnsureBefore", func(t *state.Task, _ *tomb.Tomb) error {
   662  		s := t.State()
   663  		s.Lock()
   664  		defer s.Unlock()
   665  		s.EnsureBefore(20 * time.Millisecond)
   666  		return nil
   667  	}, nil)
   668  	runner.AddHandler("runMgrForever", func(t *state.Task, _ *tomb.Tomb) error {
   669  		s := t.State()
   670  		s.Lock()
   671  		defer s.Unlock()
   672  		s.EnsureBefore(20 * time.Millisecond)
   673  		return &state.Retry{}
   674  	}, nil)
   675  	runner.AddHandler("runMgrWCleanup", func(t *state.Task, _ *tomb.Tomb) error {
   676  		s := t.State()
   677  		s.Lock()
   678  		defer s.Unlock()
   679  		s.Set("runMgrWCleanupMark", 1)
   680  		return nil
   681  	}, nil)
   682  	runner.AddCleanup("runMgrWCleanup", func(t *state.Task, _ *tomb.Tomb) error {
   683  		s := t.State()
   684  		s.Lock()
   685  		defer s.Unlock()
   686  		s.Set("runMgrWCleanupCleanedUp", 1)
   687  		return nil
   688  	})
   689  
   690  	return sm
   691  }
   692  
   693  func (sm *sampleManager) Ensure() error {
   694  	if sm.ensureCallback != nil {
   695  		sm.ensureCallback()
   696  	}
   697  	return nil
   698  }
   699  
   700  func (ovs *overlordSuite) TestTrivialSettle(c *C) {
   701  	restoreIntv := overlord.MockEnsureInterval(1 * time.Minute)
   702  	defer restoreIntv()
   703  	o := overlord.Mock()
   704  
   705  	s := o.State()
   706  	sm1 := newSampleManager(s, o.TaskRunner())
   707  	o.AddManager(sm1)
   708  	o.AddManager(o.TaskRunner())
   709  
   710  	defer o.Engine().Stop()
   711  
   712  	s.Lock()
   713  	defer s.Unlock()
   714  
   715  	chg := s.NewChange("chg", "...")
   716  	t1 := s.NewTask("runMgr1", "1...")
   717  	chg.AddTask(t1)
   718  
   719  	s.Unlock()
   720  	o.Settle(5 * time.Second)
   721  	s.Lock()
   722  	c.Check(t1.Status(), Equals, state.DoneStatus)
   723  
   724  	var v int
   725  	err := s.Get("runMgr1Mark", &v)
   726  	c.Check(err, IsNil)
   727  }
   728  
   729  func (ovs *overlordSuite) TestSettleNotConverging(c *C) {
   730  	restoreIntv := overlord.MockEnsureInterval(1 * time.Minute)
   731  	defer restoreIntv()
   732  	o := overlord.Mock()
   733  
   734  	s := o.State()
   735  	sm1 := newSampleManager(s, o.TaskRunner())
   736  	o.AddManager(sm1)
   737  	o.AddManager(o.TaskRunner())
   738  
   739  	defer o.Engine().Stop()
   740  
   741  	s.Lock()
   742  	defer s.Unlock()
   743  
   744  	chg := s.NewChange("chg", "...")
   745  	t1 := s.NewTask("runMgrForever", "1...")
   746  	chg.AddTask(t1)
   747  
   748  	s.Unlock()
   749  	err := o.Settle(250 * time.Millisecond)
   750  	s.Lock()
   751  
   752  	c.Check(err, ErrorMatches, `Settle is not converging`)
   753  
   754  }
   755  
   756  func (ovs *overlordSuite) TestSettleChain(c *C) {
   757  	restoreIntv := overlord.MockEnsureInterval(1 * time.Minute)
   758  	defer restoreIntv()
   759  	o := overlord.Mock()
   760  
   761  	s := o.State()
   762  	sm1 := newSampleManager(s, o.TaskRunner())
   763  	o.AddManager(sm1)
   764  	o.AddManager(o.TaskRunner())
   765  
   766  	defer o.Engine().Stop()
   767  
   768  	s.Lock()
   769  	defer s.Unlock()
   770  
   771  	chg := s.NewChange("chg", "...")
   772  	t1 := s.NewTask("runMgr1", "1...")
   773  	t2 := s.NewTask("runMgr2", "2...")
   774  	t2.WaitFor(t1)
   775  	chg.AddAll(state.NewTaskSet(t1, t2))
   776  
   777  	s.Unlock()
   778  	o.Settle(5 * time.Second)
   779  	s.Lock()
   780  	c.Check(t1.Status(), Equals, state.DoneStatus)
   781  	c.Check(t2.Status(), Equals, state.DoneStatus)
   782  
   783  	var v int
   784  	err := s.Get("runMgr1Mark", &v)
   785  	c.Check(err, IsNil)
   786  	err = s.Get("runMgr2Mark", &v)
   787  	c.Check(err, IsNil)
   788  }
   789  
   790  func (ovs *overlordSuite) TestSettleChainWCleanup(c *C) {
   791  	restoreIntv := overlord.MockEnsureInterval(1 * time.Minute)
   792  	defer restoreIntv()
   793  	o := overlord.Mock()
   794  
   795  	s := o.State()
   796  	sm1 := newSampleManager(s, o.TaskRunner())
   797  	o.AddManager(sm1)
   798  	o.AddManager(o.TaskRunner())
   799  
   800  	defer o.Engine().Stop()
   801  
   802  	s.Lock()
   803  	defer s.Unlock()
   804  
   805  	chg := s.NewChange("chg", "...")
   806  	t1 := s.NewTask("runMgrWCleanup", "1...")
   807  	t2 := s.NewTask("runMgr2", "2...")
   808  	t2.WaitFor(t1)
   809  	chg.AddAll(state.NewTaskSet(t1, t2))
   810  
   811  	s.Unlock()
   812  	o.Settle(5 * time.Second)
   813  	s.Lock()
   814  	c.Check(t1.Status(), Equals, state.DoneStatus)
   815  	c.Check(t2.Status(), Equals, state.DoneStatus)
   816  
   817  	var v int
   818  	err := s.Get("runMgrWCleanupMark", &v)
   819  	c.Check(err, IsNil)
   820  	err = s.Get("runMgr2Mark", &v)
   821  	c.Check(err, IsNil)
   822  
   823  	err = s.Get("runMgrWCleanupCleanedUp", &v)
   824  	c.Check(err, IsNil)
   825  }
   826  
   827  func (ovs *overlordSuite) TestSettleExplicitEnsureBefore(c *C) {
   828  	restoreIntv := overlord.MockEnsureInterval(1 * time.Minute)
   829  	defer restoreIntv()
   830  	o := overlord.Mock()
   831  
   832  	s := o.State()
   833  	sm1 := newSampleManager(s, o.TaskRunner())
   834  	sm1.ensureCallback = func() {
   835  		s.Lock()
   836  		defer s.Unlock()
   837  		v := 0
   838  		s.Get("ensureCount", &v)
   839  		s.Set("ensureCount", v+1)
   840  	}
   841  
   842  	o.AddManager(sm1)
   843  	o.AddManager(o.TaskRunner())
   844  
   845  	defer o.Engine().Stop()
   846  
   847  	s.Lock()
   848  	defer s.Unlock()
   849  
   850  	chg := s.NewChange("chg", "...")
   851  	t := s.NewTask("runMgrEnsureBefore", "...")
   852  	chg.AddTask(t)
   853  
   854  	s.Unlock()
   855  	o.Settle(5 * time.Second)
   856  	s.Lock()
   857  	c.Check(t.Status(), Equals, state.DoneStatus)
   858  
   859  	var v int
   860  	err := s.Get("ensureCount", &v)
   861  	c.Check(err, IsNil)
   862  	c.Check(v, Equals, 2)
   863  }
   864  
   865  func (ovs *overlordSuite) TestRequestRestartNoHandler(c *C) {
   866  	o, err := overlord.New(nil)
   867  	c.Assert(err, IsNil)
   868  
   869  	o.State().RequestRestart(state.RestartDaemon)
   870  }
   871  
   872  type testRestartBehavior struct {
   873  	restartRequested  state.RestartType
   874  	rebootState       string
   875  	rebootVerifiedErr error
   876  }
   877  
   878  func (rb *testRestartBehavior) HandleRestart(t state.RestartType) {
   879  	rb.restartRequested = t
   880  }
   881  
   882  func (rb *testRestartBehavior) RebootAsExpected(_ *state.State) error {
   883  	rb.rebootState = "as-expected"
   884  	return rb.rebootVerifiedErr
   885  }
   886  
   887  func (rb *testRestartBehavior) RebootDidNotHappen(_ *state.State) error {
   888  	rb.rebootState = "did-not-happen"
   889  	return rb.rebootVerifiedErr
   890  }
   891  
   892  func (ovs *overlordSuite) TestRequestRestartHandler(c *C) {
   893  	rb := &testRestartBehavior{}
   894  
   895  	o, err := overlord.New(rb)
   896  	c.Assert(err, IsNil)
   897  
   898  	o.State().RequestRestart(state.RestartDaemon)
   899  
   900  	c.Check(rb.restartRequested, Equals, state.RestartDaemon)
   901  }
   902  
   903  func (ovs *overlordSuite) TestVerifyRebootNoPendingReboot(c *C) {
   904  	fakeState := []byte(fmt.Sprintf(`{"data":{"patch-level":%d,"patch-sublevel":%d,"some":"data","refresh-privacy-key":"0123456789ABCDEF"},"changes":null,"tasks":null,"last-change-id":0,"last-task-id":0,"last-lane-id":0}`, patch.Level, patch.Sublevel))
   905  	err := ioutil.WriteFile(dirs.SnapStateFile, fakeState, 0600)
   906  	c.Assert(err, IsNil)
   907  
   908  	rb := &testRestartBehavior{}
   909  
   910  	_, err = overlord.New(rb)
   911  	c.Assert(err, IsNil)
   912  
   913  	c.Check(rb.rebootState, Equals, "as-expected")
   914  }
   915  
   916  func (ovs *overlordSuite) TestVerifyRebootOK(c *C) {
   917  	fakeState := []byte(fmt.Sprintf(`{"data":{"patch-level":%d,"patch-sublevel":%d,"some":"data","refresh-privacy-key":"0123456789ABCDEF","system-restart-from-boot-id":%q},"changes":null,"tasks":null,"last-change-id":0,"last-task-id":0,"last-lane-id":0}`, patch.Level, patch.Sublevel, "boot-id-prev"))
   918  	err := ioutil.WriteFile(dirs.SnapStateFile, fakeState, 0600)
   919  	c.Assert(err, IsNil)
   920  
   921  	rb := &testRestartBehavior{}
   922  
   923  	_, err = overlord.New(rb)
   924  	c.Assert(err, IsNil)
   925  
   926  	c.Check(rb.rebootState, Equals, "as-expected")
   927  }
   928  
   929  func (ovs *overlordSuite) TestVerifyRebootOKButError(c *C) {
   930  	fakeState := []byte(fmt.Sprintf(`{"data":{"patch-level":%d,"patch-sublevel":%d,"some":"data","refresh-privacy-key":"0123456789ABCDEF","system-restart-from-boot-id":%q},"changes":null,"tasks":null,"last-change-id":0,"last-task-id":0,"last-lane-id":0}`, patch.Level, patch.Sublevel, "boot-id-prev"))
   931  	err := ioutil.WriteFile(dirs.SnapStateFile, fakeState, 0600)
   932  	c.Assert(err, IsNil)
   933  
   934  	e := errors.New("boom")
   935  	rb := &testRestartBehavior{rebootVerifiedErr: e}
   936  
   937  	_, err = overlord.New(rb)
   938  	c.Assert(err, Equals, e)
   939  
   940  	c.Check(rb.rebootState, Equals, "as-expected")
   941  }
   942  
   943  func (ovs *overlordSuite) TestVerifyRebootDidNotHappen(c *C) {
   944  	curBootID, err := osutil.BootID()
   945  	c.Assert(err, IsNil)
   946  
   947  	fakeState := []byte(fmt.Sprintf(`{"data":{"patch-level":%d,"patch-sublevel":%d,"some":"data","refresh-privacy-key":"0123456789ABCDEF","system-restart-from-boot-id":%q},"changes":null,"tasks":null,"last-change-id":0,"last-task-id":0,"last-lane-id":0}`, patch.Level, patch.Sublevel, curBootID))
   948  	err = ioutil.WriteFile(dirs.SnapStateFile, fakeState, 0600)
   949  	c.Assert(err, IsNil)
   950  
   951  	rb := &testRestartBehavior{}
   952  
   953  	_, err = overlord.New(rb)
   954  	c.Assert(err, IsNil)
   955  
   956  	c.Check(rb.rebootState, Equals, "did-not-happen")
   957  }
   958  
   959  func (ovs *overlordSuite) TestVerifyRebootDidNotHappenError(c *C) {
   960  	curBootID, err := osutil.BootID()
   961  	c.Assert(err, IsNil)
   962  
   963  	fakeState := []byte(fmt.Sprintf(`{"data":{"patch-level":%d,"patch-sublevel":%d,"some":"data","refresh-privacy-key":"0123456789ABCDEF","system-restart-from-boot-id":%q},"changes":null,"tasks":null,"last-change-id":0,"last-task-id":0,"last-lane-id":0}`, patch.Level, patch.Sublevel, curBootID))
   964  	err = ioutil.WriteFile(dirs.SnapStateFile, fakeState, 0600)
   965  	c.Assert(err, IsNil)
   966  
   967  	e := errors.New("boom")
   968  	rb := &testRestartBehavior{rebootVerifiedErr: e}
   969  
   970  	_, err = overlord.New(rb)
   971  	c.Assert(err, Equals, e)
   972  
   973  	c.Check(rb.rebootState, Equals, "did-not-happen")
   974  }
   975  
   976  func (ovs *overlordSuite) TestOverlordCanStandby(c *C) {
   977  	restoreIntv := overlord.MockEnsureInterval(10 * time.Millisecond)
   978  	defer restoreIntv()
   979  	o := overlord.Mock()
   980  	witness := &witnessManager{
   981  		state:          o.State(),
   982  		expectedEnsure: 3,
   983  		ensureCalled:   make(chan struct{}),
   984  	}
   985  	o.AddManager(witness)
   986  
   987  	c.Assert(o.StartUp(), IsNil)
   988  
   989  	// can only standby after loop ran once
   990  	c.Assert(o.CanStandby(), Equals, false)
   991  
   992  	o.Loop()
   993  	defer o.Stop()
   994  
   995  	select {
   996  	case <-witness.ensureCalled:
   997  	case <-time.After(2 * time.Second):
   998  		c.Fatal("Ensure calls not happening")
   999  	}
  1000  
  1001  	c.Assert(o.CanStandby(), Equals, true)
  1002  }
  1003  
  1004  func (ovs *overlordSuite) TestStartupTimeout(c *C) {
  1005  	o, err := overlord.New(nil)
  1006  	c.Assert(err, IsNil)
  1007  
  1008  	to, _, err := o.StartupTimeout()
  1009  	c.Assert(err, IsNil)
  1010  	c.Check(to, Equals, 30*time.Second)
  1011  
  1012  	st := o.State()
  1013  	st.Lock()
  1014  	defer st.Unlock()
  1015  
  1016  	// have two snaps
  1017  	snapstate.Set(st, "core18", &snapstate.SnapState{
  1018  		Active: true,
  1019  		Sequence: []*snap.SideInfo{
  1020  			{RealName: "core18", Revision: snap.R(1)},
  1021  		},
  1022  		Current:  snap.R(1),
  1023  		SnapType: "base",
  1024  	})
  1025  	snapstate.Set(st, "foo", &snapstate.SnapState{
  1026  		Active: true,
  1027  		Sequence: []*snap.SideInfo{
  1028  			{RealName: "foo", Revision: snap.R(1)},
  1029  		},
  1030  		Current:  snap.R(1),
  1031  		SnapType: "app",
  1032  	})
  1033  
  1034  	st.Unlock()
  1035  	to, reasoning, err := o.StartupTimeout()
  1036  	st.Lock()
  1037  	c.Assert(err, IsNil)
  1038  
  1039  	c.Check(to, Equals, (30+5+5)*time.Second)
  1040  	c.Check(reasoning, Equals, "pessimistic estimate of 30s plus 5s per snap")
  1041  }