github.com/david-imola/snapd@v0.0.0-20210611180407-2de8ddeece6d/bootloader/lk_test.go (about)

     1  // -*- Mode: Go; indent-tabs-mode: t -*-
     2  
     3  /*
     4   * Copyright (C) 2019 Canonical Ltd
     5   *
     6   * This program is free software: you can redistribute it and/or modify
     7   * it under the terms of the GNU General Public License version 3 as
     8   * published by the Free Software Foundation.
     9   *
    10   * This program is distributed in the hope that it will be useful,
    11   * but WITHOUT ANY WARRANTY; without even the implied warranty of
    12   * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    13   * GNU General Public License for more details.
    14   *
    15   * You should have received a copy of the GNU General Public License
    16   * along with this program.  If not, see <http://www.gnu.org/licenses/>.
    17   *
    18   */
    19  
    20  package bootloader_test
    21  
    22  import (
    23  	"fmt"
    24  	"io/ioutil"
    25  	"os"
    26  	"path/filepath"
    27  	"sort"
    28  
    29  	. "gopkg.in/check.v1"
    30  
    31  	"github.com/snapcore/snapd/boot"
    32  	"github.com/snapcore/snapd/bootloader"
    33  	"github.com/snapcore/snapd/bootloader/lkenv"
    34  	"github.com/snapcore/snapd/logger"
    35  	"github.com/snapcore/snapd/osutil"
    36  	"github.com/snapcore/snapd/osutil/disks"
    37  	"github.com/snapcore/snapd/snap"
    38  	"github.com/snapcore/snapd/snap/snapfile"
    39  	"github.com/snapcore/snapd/snap/snaptest"
    40  )
    41  
    42  type lkTestSuite struct {
    43  	baseBootenvTestSuite
    44  }
    45  
    46  var _ = Suite(&lkTestSuite{})
    47  
    48  func (s *lkTestSuite) TestNewLk(c *C) {
    49  	// TODO: update this test when v1 lk uses the kernel command line parameter
    50  	//       too
    51  
    52  	// no files means bl is not present, but we can still create the bl object
    53  	l := bootloader.NewLk(s.rootdir, nil)
    54  	c.Assert(l, NotNil)
    55  	c.Assert(l.Name(), Equals, "lk")
    56  
    57  	present, err := l.Present()
    58  	c.Assert(err, IsNil)
    59  	c.Assert(present, Equals, false)
    60  
    61  	// now with files present, the bl is present
    62  	bootloader.MockLkFiles(c, s.rootdir, nil)
    63  	present, err = l.Present()
    64  	c.Assert(err, IsNil)
    65  	c.Assert(present, Equals, true)
    66  	c.Check(bootloader.LkRuntimeMode(l), Equals, true)
    67  	f, err := bootloader.LkConfigFile(l)
    68  	c.Assert(err, IsNil)
    69  	c.Check(f, Equals, filepath.Join(s.rootdir, "/dev/disk/by-partlabel", "snapbootsel"))
    70  }
    71  
    72  func (s *lkTestSuite) TestNewLkPresentChecksBackupStorageToo(c *C) {
    73  	// no files means bl is not present, but we can still create the bl object
    74  	l := bootloader.NewLk(s.rootdir, &bootloader.Options{
    75  		Role: bootloader.RoleSole,
    76  	})
    77  	c.Assert(l, NotNil)
    78  	c.Assert(l.Name(), Equals, "lk")
    79  
    80  	present, err := l.Present()
    81  	c.Assert(err, IsNil)
    82  	c.Assert(present, Equals, false)
    83  
    84  	// now mock just the backup env file
    85  	f, err := bootloader.LkConfigFile(l)
    86  	c.Assert(err, IsNil)
    87  	c.Check(f, Equals, filepath.Join(s.rootdir, "/dev/disk/by-partlabel", "snapbootsel"))
    88  
    89  	err = os.MkdirAll(filepath.Dir(f), 0755)
    90  	c.Assert(err, IsNil)
    91  
    92  	err = ioutil.WriteFile(f+"bak", nil, 0644)
    93  	c.Assert(err, IsNil)
    94  
    95  	// now the bootloader is present because the backup exists
    96  	present, err = l.Present()
    97  	c.Assert(err, IsNil)
    98  	c.Assert(present, Equals, true)
    99  }
   100  
   101  func (s *lkTestSuite) TestNewLkUC20Run(c *C) {
   102  	// no files means bl is not present, but we can still create the bl object
   103  	opts := &bootloader.Options{
   104  		Role: bootloader.RoleRunMode,
   105  	}
   106  	// use ubuntu-boot as the root dir
   107  	l := bootloader.NewLk(boot.InitramfsUbuntuBootDir, opts)
   108  	c.Assert(l, NotNil)
   109  	c.Assert(l.Name(), Equals, "lk")
   110  
   111  	present, err := l.Present()
   112  	c.Assert(err, IsNil)
   113  	c.Assert(present, Equals, false)
   114  
   115  	// now with files present, the bl is present
   116  	r := bootloader.MockLkFiles(c, s.rootdir, opts)
   117  	defer r()
   118  	present, err = l.Present()
   119  	c.Assert(err, IsNil)
   120  	c.Assert(present, Equals, true)
   121  	c.Check(bootloader.LkRuntimeMode(l), Equals, true)
   122  	f, err := bootloader.LkConfigFile(l)
   123  	c.Assert(err, IsNil)
   124  	// note that the config file here is not relative to ubuntu-boot dir we used
   125  	// when creating the bootloader, it is relative to the rootdir
   126  	c.Check(f, Equals, filepath.Join(s.rootdir, "/dev/disk/by-partuuid", "snapbootsel-partuuid"))
   127  }
   128  
   129  func (s *lkTestSuite) TestNewLkUC20Recovery(c *C) {
   130  	// no files means bl is not present, but we can still create the bl object
   131  	opts := &bootloader.Options{
   132  		Role: bootloader.RoleRecovery,
   133  	}
   134  	// use ubuntu-seed as the root dir
   135  	l := bootloader.NewLk(boot.InitramfsUbuntuSeedDir, opts)
   136  	c.Assert(l, NotNil)
   137  	c.Assert(l.Name(), Equals, "lk")
   138  
   139  	present, err := l.Present()
   140  	c.Assert(err, IsNil)
   141  	c.Assert(present, Equals, false)
   142  
   143  	// now with files present, the bl is present
   144  	r := bootloader.MockLkFiles(c, s.rootdir, opts)
   145  	defer r()
   146  	present, err = l.Present()
   147  	c.Assert(err, IsNil)
   148  	c.Assert(present, Equals, true)
   149  	c.Check(bootloader.LkRuntimeMode(l), Equals, true)
   150  	f, err := bootloader.LkConfigFile(l)
   151  	c.Assert(err, IsNil)
   152  	// note that the config file here is not relative to ubuntu-boot dir we used
   153  	// when creating the bootloader, it is relative to the rootdir
   154  	c.Check(f, Equals, filepath.Join(s.rootdir, "/dev/disk/by-partuuid", "snaprecoverysel-partuuid"))
   155  }
   156  
   157  func (s *lkTestSuite) TestNewLkImageBuildingTime(c *C) {
   158  	for _, role := range []bootloader.Role{bootloader.RoleSole, bootloader.RoleRecovery} {
   159  		opts := &bootloader.Options{
   160  			PrepareImageTime: true,
   161  			Role:             role,
   162  		}
   163  		r := bootloader.MockLkFiles(c, s.rootdir, opts)
   164  		defer r()
   165  		l := bootloader.NewLk(s.rootdir, opts)
   166  		c.Assert(l, NotNil)
   167  		c.Check(bootloader.LkRuntimeMode(l), Equals, false)
   168  		f, err := bootloader.LkConfigFile(l)
   169  		c.Assert(err, IsNil)
   170  		switch role {
   171  		case bootloader.RoleSole:
   172  			c.Check(f, Equals, filepath.Join(s.rootdir, "/boot/lk", "snapbootsel.bin"))
   173  		case bootloader.RoleRecovery:
   174  			c.Check(f, Equals, filepath.Join(s.rootdir, "/boot/lk", "snaprecoverysel.bin"))
   175  		}
   176  	}
   177  }
   178  
   179  func (s *lkTestSuite) TestSetGetBootVar(c *C) {
   180  	tt := []struct {
   181  		role  bootloader.Role
   182  		key   string
   183  		value string
   184  	}{
   185  		{
   186  			bootloader.RoleSole,
   187  			"snap_mode",
   188  			boot.TryingStatus,
   189  		},
   190  		{
   191  			bootloader.RoleRecovery,
   192  			"snapd_recovery_mode",
   193  			boot.ModeRecover,
   194  		},
   195  		{
   196  			bootloader.RoleRunMode,
   197  			"kernel_status",
   198  			boot.TryStatus,
   199  		},
   200  	}
   201  	for _, t := range tt {
   202  		opts := &bootloader.Options{
   203  			Role: t.role,
   204  		}
   205  		r := bootloader.MockLkFiles(c, s.rootdir, opts)
   206  		defer r()
   207  		l := bootloader.NewLk(s.rootdir, opts)
   208  		bootVars := map[string]string{t.key: t.value}
   209  		l.SetBootVars(bootVars)
   210  
   211  		v, err := l.GetBootVars(t.key)
   212  		c.Assert(err, IsNil)
   213  		c.Check(v, HasLen, 1)
   214  		c.Check(v[t.key], Equals, t.value)
   215  	}
   216  }
   217  
   218  func (s *lkTestSuite) TestExtractKernelAssetsUnpacksBootimgImageBuilding(c *C) {
   219  	for _, role := range []bootloader.Role{bootloader.RoleSole, bootloader.RoleRecovery} {
   220  		opts := &bootloader.Options{
   221  			PrepareImageTime: true,
   222  			Role:             role,
   223  		}
   224  		r := bootloader.MockLkFiles(c, s.rootdir, opts)
   225  		defer r()
   226  		l := bootloader.NewLk(s.rootdir, opts)
   227  
   228  		c.Assert(l, NotNil)
   229  
   230  		files := [][]string{
   231  			{"kernel.img", "I'm a kernel"},
   232  			{"initrd.img", "...and I'm an initrd"},
   233  			{"boot.img", "...and I'm an boot image"},
   234  			{"dtbs/foo.dtb", "g'day, I'm foo.dtb"},
   235  			{"dtbs/bar.dtb", "hello, I'm bar.dtb"},
   236  			// must be last
   237  			{"meta/kernel.yaml", "version: 4.2"},
   238  		}
   239  		si := &snap.SideInfo{
   240  			RealName: "ubuntu-kernel",
   241  			Revision: snap.R(42),
   242  		}
   243  		fn := snaptest.MakeTestSnapWithFiles(c, packageKernel, files)
   244  		snapf, err := snapfile.Open(fn)
   245  		c.Assert(err, IsNil)
   246  
   247  		info, err := snap.ReadInfoFromSnapFile(snapf, si)
   248  		c.Assert(err, IsNil)
   249  
   250  		if role == bootloader.RoleSole {
   251  			err = l.ExtractKernelAssets(info, snapf)
   252  		} else {
   253  			// this isn't quite how ExtractRecoveryKernel is typically called,
   254  			// typically it will be called with an actual recovery system dir,
   255  			// but for our purposes this is close enough, we just extract files
   256  			// to some directory
   257  			err = l.ExtractRecoveryKernelAssets(s.rootdir, info, snapf)
   258  		}
   259  		c.Assert(err, IsNil)
   260  
   261  		// just boot.img and snapbootsel.bin are there, no kernel.img
   262  		infos, err := ioutil.ReadDir(filepath.Join(s.rootdir, "boot", "lk", ""))
   263  		c.Assert(err, IsNil)
   264  		var fnames []string
   265  		for _, info := range infos {
   266  			fnames = append(fnames, info.Name())
   267  		}
   268  		sort.Strings(fnames)
   269  		c.Assert(fnames, HasLen, 2)
   270  		expFiles := []string{"boot.img"}
   271  		if role == bootloader.RoleSole {
   272  			expFiles = append(expFiles, "snapbootsel.bin")
   273  		} else {
   274  			expFiles = append(expFiles, "snaprecoverysel.bin")
   275  		}
   276  		c.Assert(fnames, DeepEquals, expFiles)
   277  
   278  		// clean up the rootdir for the next iteration
   279  		c.Assert(os.RemoveAll(s.rootdir), IsNil)
   280  	}
   281  }
   282  
   283  func (s *lkTestSuite) TestExtractKernelAssetsUnpacksCustomBootimgImageBuilding(c *C) {
   284  	opts := &bootloader.Options{
   285  		PrepareImageTime: true,
   286  		Role:             bootloader.RoleSole,
   287  	}
   288  	bootloader.MockLkFiles(c, s.rootdir, opts)
   289  	l := bootloader.NewLk(s.rootdir, opts)
   290  
   291  	c.Assert(l, NotNil)
   292  
   293  	// first configure custom boot image file name
   294  	f, err := bootloader.LkConfigFile(l)
   295  	c.Assert(err, IsNil)
   296  	env := lkenv.NewEnv(f, "", lkenv.V1)
   297  	env.Load()
   298  	env.Set("bootimg_file_name", "boot-2.img")
   299  	err = env.Save()
   300  	c.Assert(err, IsNil)
   301  
   302  	files := [][]string{
   303  		{"kernel.img", "I'm a kernel"},
   304  		{"initrd.img", "...and I'm an initrd"},
   305  		{"boot-2.img", "...and I'm an boot image"},
   306  		{"dtbs/foo.dtb", "g'day, I'm foo.dtb"},
   307  		{"dtbs/bar.dtb", "hello, I'm bar.dtb"},
   308  		// must be last
   309  		{"meta/kernel.yaml", "version: 4.2"},
   310  	}
   311  	si := &snap.SideInfo{
   312  		RealName: "ubuntu-kernel",
   313  		Revision: snap.R(42),
   314  	}
   315  	fn := snaptest.MakeTestSnapWithFiles(c, packageKernel, files)
   316  	snapf, err := snapfile.Open(fn)
   317  	c.Assert(err, IsNil)
   318  
   319  	info, err := snap.ReadInfoFromSnapFile(snapf, si)
   320  	c.Assert(err, IsNil)
   321  
   322  	err = l.ExtractKernelAssets(info, snapf)
   323  	c.Assert(err, IsNil)
   324  
   325  	// boot-2.img is there
   326  	bootimg := filepath.Join(s.rootdir, "boot", "lk", "boot-2.img")
   327  	c.Assert(osutil.FileExists(bootimg), Equals, true)
   328  }
   329  
   330  func (s *lkTestSuite) TestExtractKernelAssetsUnpacksAndRemoveInRuntimeMode(c *C) {
   331  	logbuf, r := logger.MockLogger()
   332  	defer r()
   333  	opts := &bootloader.Options{
   334  		Role: bootloader.RoleSole,
   335  	}
   336  	r = bootloader.MockLkFiles(c, s.rootdir, opts)
   337  	defer r()
   338  	lk := bootloader.NewLk(s.rootdir, opts)
   339  	c.Assert(lk, NotNil)
   340  
   341  	// ensure we have a valid boot env
   342  	// TODO: this will follow the same logic as RoleRunMode eventually
   343  	bootselPartition := filepath.Join(s.rootdir, "/dev/disk/by-partlabel/snapbootsel")
   344  	lkenv := lkenv.NewEnv(bootselPartition, "", lkenv.V1)
   345  
   346  	// don't need to initialize this env, the same file will already have been
   347  	// setup by MockLkFiles()
   348  
   349  	// mock a kernel snap that has a boot.img
   350  	files := [][]string{
   351  		{"boot.img", "I'm the default boot image name"},
   352  	}
   353  	si := &snap.SideInfo{
   354  		RealName: "ubuntu-kernel",
   355  		Revision: snap.R(42),
   356  	}
   357  	fn := snaptest.MakeTestSnapWithFiles(c, packageKernel, files)
   358  	snapf, err := snapfile.Open(fn)
   359  	c.Assert(err, IsNil)
   360  
   361  	info, err := snap.ReadInfoFromSnapFile(snapf, si)
   362  	c.Assert(err, IsNil)
   363  
   364  	// now extract
   365  	err = lk.ExtractKernelAssets(info, snapf)
   366  	c.Assert(err, IsNil)
   367  
   368  	// and validate it went to the "boot_a" partition
   369  	bootA := filepath.Join(s.rootdir, "/dev/disk/by-partlabel/boot_a")
   370  	content, err := ioutil.ReadFile(bootA)
   371  	c.Assert(err, IsNil)
   372  	c.Assert(string(content), Equals, "I'm the default boot image name")
   373  
   374  	// also validate that bootB is empty
   375  	bootB := filepath.Join(s.rootdir, "/dev/disk/by-partlabel/boot_b")
   376  	content, err = ioutil.ReadFile(bootB)
   377  	c.Assert(err, IsNil)
   378  	c.Assert(content, HasLen, 0)
   379  
   380  	// test that boot partition got set
   381  	err = lkenv.Load()
   382  	c.Assert(err, IsNil)
   383  	bootPart, err := lkenv.GetKernelBootPartition("ubuntu-kernel_42.snap")
   384  	c.Assert(err, IsNil)
   385  	c.Assert(bootPart, Equals, "boot_a")
   386  
   387  	// now remove the kernel
   388  	err = lk.RemoveKernelAssets(info)
   389  	c.Assert(err, IsNil)
   390  	// and ensure its no longer available in the boot partitions
   391  	err = lkenv.Load()
   392  	c.Assert(err, IsNil)
   393  	bootPart, err = lkenv.GetKernelBootPartition("ubuntu-kernel_42.snap")
   394  	c.Assert(err, ErrorMatches, fmt.Sprintf("cannot find kernel %[1]q: no boot image partition has value %[1]q", "ubuntu-kernel_42.snap"))
   395  	c.Assert(bootPart, Equals, "")
   396  
   397  	c.Assert(logbuf.String(), Equals, "")
   398  }
   399  
   400  func (s *lkTestSuite) TestExtractKernelAssetsUnpacksAndRemoveInRuntimeModeUC20(c *C) {
   401  	logbuf, r := logger.MockLogger()
   402  	defer r()
   403  
   404  	opts := &bootloader.Options{
   405  		Role: bootloader.RoleRunMode,
   406  	}
   407  	r = bootloader.MockLkFiles(c, s.rootdir, opts)
   408  	defer r()
   409  	lk := bootloader.NewLk(s.rootdir, opts)
   410  	c.Assert(lk, NotNil)
   411  
   412  	// all expected files are created for RoleRunMode bootloader in
   413  	// MockLkFiles
   414  
   415  	// ensure we have a valid boot env
   416  	disk, err := disks.DiskFromDeviceName("lk-boot-disk")
   417  	c.Assert(err, IsNil)
   418  
   419  	partuuid, err := disk.FindMatchingPartitionUUIDWithPartLabel("snapbootsel")
   420  	c.Assert(err, IsNil)
   421  
   422  	// also confirm that we can load the backup file partition too
   423  	backupPartuuid, err := disk.FindMatchingPartitionUUIDWithPartLabel("snapbootselbak")
   424  	c.Assert(err, IsNil)
   425  
   426  	bootselPartition := filepath.Join(s.rootdir, "/dev/disk/by-partuuid", partuuid)
   427  	bootselPartitionBackup := filepath.Join(s.rootdir, "/dev/disk/by-partuuid", backupPartuuid)
   428  	env := lkenv.NewEnv(bootselPartition, "", lkenv.V2Run)
   429  	backupEnv := lkenv.NewEnv(bootselPartitionBackup, "", lkenv.V2Run)
   430  
   431  	// mock a kernel snap that has a boot.img
   432  	files := [][]string{
   433  		{"boot.img", "I'm the default boot image name"},
   434  	}
   435  	si := &snap.SideInfo{
   436  		RealName: "ubuntu-kernel",
   437  		Revision: snap.R(42),
   438  	}
   439  	fn := snaptest.MakeTestSnapWithFiles(c, packageKernel, files)
   440  	snapf, err := snapfile.Open(fn)
   441  	c.Assert(err, IsNil)
   442  
   443  	info, err := snap.ReadInfoFromSnapFile(snapf, si)
   444  	c.Assert(err, IsNil)
   445  
   446  	// now extract
   447  	err = lk.ExtractKernelAssets(info, snapf)
   448  	c.Assert(err, IsNil)
   449  
   450  	// and validate it went to the "boot_a" partition
   451  	bootAPartUUID, err := disk.FindMatchingPartitionUUIDWithPartLabel("boot_a")
   452  	c.Assert(err, IsNil)
   453  	bootA := filepath.Join(s.rootdir, "/dev/disk/by-partuuid", bootAPartUUID)
   454  	content, err := ioutil.ReadFile(bootA)
   455  	c.Assert(err, IsNil)
   456  	c.Assert(string(content), Equals, "I'm the default boot image name")
   457  
   458  	// also validate that bootB is empty
   459  	bootBPartUUID, err := disk.FindMatchingPartitionUUIDWithPartLabel("boot_b")
   460  	c.Assert(err, IsNil)
   461  	bootB := filepath.Join(s.rootdir, "/dev/disk/by-partuuid", bootBPartUUID)
   462  	content, err = ioutil.ReadFile(bootB)
   463  	c.Assert(err, IsNil)
   464  	c.Assert(content, HasLen, 0)
   465  
   466  	// test that boot partition got set
   467  	err = env.Load()
   468  	c.Assert(err, IsNil)
   469  	bootPart, err := env.GetKernelBootPartition("ubuntu-kernel_42.snap")
   470  	c.Assert(err, IsNil)
   471  	c.Assert(bootPart, Equals, "boot_a")
   472  
   473  	// in the backup too
   474  	err = backupEnv.Load()
   475  	c.Assert(logbuf.String(), Equals, "")
   476  	c.Assert(err, IsNil)
   477  
   478  	bootPart, err = backupEnv.GetKernelBootPartition("ubuntu-kernel_42.snap")
   479  	c.Assert(err, IsNil)
   480  	c.Assert(bootPart, Equals, "boot_a")
   481  
   482  	// now remove the kernel
   483  	err = lk.RemoveKernelAssets(info)
   484  	c.Assert(err, IsNil)
   485  	// and ensure its no longer available in the boot partitions
   486  	err = env.Load()
   487  	c.Assert(err, IsNil)
   488  	_, err = env.GetKernelBootPartition("ubuntu-kernel_42.snap")
   489  	c.Assert(err, ErrorMatches, fmt.Sprintf("cannot find kernel %[1]q: no boot image partition has value %[1]q", "ubuntu-kernel_42.snap"))
   490  	err = backupEnv.Load()
   491  	c.Assert(err, IsNil)
   492  	// in the backup too
   493  	_, err = backupEnv.GetKernelBootPartition("ubuntu-kernel_42.snap")
   494  	c.Assert(err, ErrorMatches, fmt.Sprintf("cannot find kernel %[1]q: no boot image partition has value %[1]q", "ubuntu-kernel_42.snap"))
   495  
   496  	c.Assert(logbuf.String(), Equals, "")
   497  }
   498  
   499  func (s *lkTestSuite) TestExtractRecoveryKernelAssetsAtRuntime(c *C) {
   500  	opts := &bootloader.Options{
   501  		// as called when creating a recovery system at runtime
   502  		PrepareImageTime: false,
   503  		Role:             bootloader.RoleRecovery,
   504  	}
   505  	r := bootloader.MockLkFiles(c, s.rootdir, opts)
   506  	defer r()
   507  	l := bootloader.NewLk(s.rootdir, opts)
   508  
   509  	c.Assert(l, NotNil)
   510  
   511  	files := [][]string{
   512  		{"kernel.img", "I'm a kernel"},
   513  		{"initrd.img", "...and I'm an initrd"},
   514  		{"boot.img", "...and I'm an boot image"},
   515  		{"meta/kernel.yaml", "version: 4.2"},
   516  	}
   517  	si := &snap.SideInfo{
   518  		RealName: "ubuntu-kernel",
   519  		Revision: snap.R(42),
   520  	}
   521  	fn := snaptest.MakeTestSnapWithFiles(c, packageKernel, files)
   522  	snapf, err := snapfile.Open(fn)
   523  	c.Assert(err, IsNil)
   524  
   525  	info, err := snap.ReadInfoFromSnapFile(snapf, si)
   526  	c.Assert(err, IsNil)
   527  
   528  	relativeRecoverySystemDir := "systems/1234"
   529  	c.Assert(os.MkdirAll(filepath.Join(s.rootdir, relativeRecoverySystemDir), 0755), IsNil)
   530  	err = l.ExtractRecoveryKernelAssets(relativeRecoverySystemDir, info, snapf)
   531  	c.Assert(err, ErrorMatches, "internal error: extracting recovery kernel assets is not supported for a runtime lk bootloader")
   532  }
   533  
   534  // TODO:UC20: when runtime addition (and deletion) of recovery systems is
   535  //            implemented, add tests for that here with lkenv