github.com/blixtra/rkt@v0.8.1-0.20160204105720-ab0d1add1a43/stage0/run.go (about)

     1  // Copyright 2014 The rkt Authors
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  //+build linux
    16  
    17  package stage0
    18  
    19  //
    20  // rkt is a reference implementation of the app container specification.
    21  //
    22  // Execution on rkt is divided into a number of stages, and the `rkt`
    23  // binary implements the first stage (stage 0)
    24  //
    25  
    26  import (
    27  	"encoding/json"
    28  	"errors"
    29  	"fmt"
    30  	"io/ioutil"
    31  	"net"
    32  	"os"
    33  	"path"
    34  	"path/filepath"
    35  	"runtime"
    36  	"strconv"
    37  	"strings"
    38  	"syscall"
    39  	"time"
    40  
    41  	"github.com/appc/spec/schema"
    42  	"github.com/appc/spec/schema/types"
    43  	"github.com/coreos/rkt/common"
    44  	"github.com/coreos/rkt/common/apps"
    45  	"github.com/coreos/rkt/pkg/aci"
    46  	"github.com/coreos/rkt/pkg/fileutil"
    47  	"github.com/coreos/rkt/pkg/label"
    48  	"github.com/coreos/rkt/pkg/sys"
    49  	"github.com/coreos/rkt/pkg/tpm"
    50  	"github.com/coreos/rkt/pkg/uid"
    51  	"github.com/coreos/rkt/store"
    52  	"github.com/coreos/rkt/version"
    53  	"github.com/hashicorp/errwrap"
    54  )
    55  
    56  const (
    57  	// Default perm bits for the regular files
    58  	// within the stage1 directory. (e.g. image manifest,
    59  	// pod manifest, stage1ID, etc).
    60  	defaultRegularFilePerm = os.FileMode(0640)
    61  
    62  	// Default perm bits for the regular directories
    63  	// within the stage1 directory.
    64  	defaultRegularDirPerm = os.FileMode(0750)
    65  )
    66  
    67  var debugEnabled bool
    68  
    69  // configuration parameters required by Prepare
    70  type PrepareConfig struct {
    71  	*CommonConfig
    72  	Apps               *apps.Apps          // apps to prepare
    73  	InheritEnv         bool                // inherit parent environment into apps
    74  	ExplicitEnv        []string            // always set these environment variables for all the apps
    75  	Ports              []types.ExposedPort // list of ports that rkt will expose on the host
    76  	UseOverlay         bool                // prepare pod with overlay fs
    77  	SkipTreeStoreCheck bool                // skip checking the treestore before rendering
    78  	PodManifest        string              // use the pod manifest specified by the user, this will ignore flags such as '--volume', '--port', etc.
    79  	PrivateUsers       *uid.UidRange       // User namespaces
    80  }
    81  
    82  // configuration parameters needed by Run
    83  type RunConfig struct {
    84  	*CommonConfig
    85  	Net         common.NetList // pod should have its own network stack
    86  	LockFd      int            // lock file descriptor
    87  	Interactive bool           // whether the pod is interactive or not
    88  	MDSRegister bool           // whether to register with metadata service or not
    89  	Apps        schema.AppList // applications (prepare gets them via Apps)
    90  	LocalConfig string         // Path to local configuration
    91  	RktGid      int            // group id of the 'rkt' group, -1 if there's no rkt group.
    92  	DNS         []string       // DNS name servers to write in /etc/resolv.conf
    93  	DNSSearch   []string       // DNS search domains to write in /etc/resolv.conf
    94  	DNSOpt      []string       // DNS options to write in /etc/resolv.conf
    95  }
    96  
    97  // configuration shared by both Run and Prepare
    98  type CommonConfig struct {
    99  	Store        *store.Store // store containing all of the configured application images
   100  	Stage1Image  types.Hash   // stage1 image containing usable /init and /enter entrypoints
   101  	UUID         *types.UUID  // UUID of the pod
   102  	RootHash     string       // Hash of the root filesystem
   103  	ManifestData string       // The pod manifest data
   104  	Debug        bool
   105  	MountLabel   string // selinux label to use for fs
   106  	ProcessLabel string // selinux label to use for process
   107  }
   108  
   109  func init() {
   110  	// this ensures that main runs only on main thread (thread group leader).
   111  	// since namespace ops (unshare, setns) are done for a single thread, we
   112  	// must ensure that the goroutine does not jump from OS thread to thread
   113  	runtime.LockOSThread()
   114  }
   115  
   116  func InitDebug() {
   117  	debugEnabled = true
   118  	log.SetDebug(true)
   119  }
   120  
   121  func debug(format string, i ...interface{}) {
   122  	if debugEnabled {
   123  		log.Printf(format, i...)
   124  	}
   125  }
   126  
   127  // MergeEnvs amends appEnv setting variables in setEnv before setting anything new from os.Environ if inheritEnv = true
   128  // setEnv is expected to be in the os.Environ() key=value format
   129  func MergeEnvs(appEnv *types.Environment, inheritEnv bool, setEnv []string) {
   130  	for _, ev := range setEnv {
   131  		pair := strings.SplitN(ev, "=", 2)
   132  		appEnv.Set(pair[0], pair[1])
   133  	}
   134  
   135  	if inheritEnv {
   136  		for _, ev := range os.Environ() {
   137  			pair := strings.SplitN(ev, "=", 2)
   138  			if _, exists := appEnv.Get(pair[0]); !exists {
   139  				appEnv.Set(pair[0], pair[1])
   140  			}
   141  		}
   142  	}
   143  }
   144  
   145  func imageNameToAppName(name types.ACIdentifier) (*types.ACName, error) {
   146  	parts := strings.Split(name.String(), "/")
   147  	last := parts[len(parts)-1]
   148  
   149  	sn, err := types.SanitizeACName(last)
   150  	if err != nil {
   151  		return nil, err
   152  	}
   153  
   154  	return types.MustACName(sn), nil
   155  }
   156  
   157  // deduplicateMPs removes Mounts with duplicated paths. If there's more than
   158  // one Mount with the same path, it keeps the first one encountered.
   159  func deduplicateMPs(mounts []schema.Mount) []schema.Mount {
   160  	var res []schema.Mount
   161  	seen := make(map[string]struct{})
   162  	for _, m := range mounts {
   163  		if _, ok := seen[m.Path]; !ok {
   164  			res = append(res, m)
   165  			seen[m.Path] = struct{}{}
   166  		}
   167  	}
   168  	return res
   169  }
   170  
   171  // MergeMounts combines the global and per-app mount slices
   172  func MergeMounts(mounts []schema.Mount, appMounts []schema.Mount) []schema.Mount {
   173  	ml := append(appMounts, mounts...)
   174  	return deduplicateMPs(ml)
   175  }
   176  
   177  // generatePodManifest creates the pod manifest from the command line input.
   178  // It returns the pod manifest as []byte on success.
   179  // This is invoked if no pod manifest is specified at the command line.
   180  func generatePodManifest(cfg PrepareConfig, dir string) ([]byte, error) {
   181  	pm := schema.PodManifest{
   182  		ACKind: "PodManifest",
   183  		Apps:   make(schema.AppList, 0),
   184  	}
   185  
   186  	v, err := types.NewSemVer(version.Version)
   187  	if err != nil {
   188  		return nil, errwrap.Wrap(errors.New("error creating version"), err)
   189  	}
   190  	pm.ACVersion = *v
   191  
   192  	if err := cfg.Apps.Walk(func(app *apps.App) error {
   193  		img := app.ImageID
   194  
   195  		am, err := cfg.Store.GetImageManifest(img.String())
   196  		if err != nil {
   197  			return errwrap.Wrap(errors.New("error getting the manifest"), err)
   198  		}
   199  		appName, err := imageNameToAppName(am.Name)
   200  		if err != nil {
   201  			return errwrap.Wrap(errors.New("error converting image name to app name"), err)
   202  		}
   203  		if err := prepareAppImage(cfg, *appName, img, dir, cfg.UseOverlay); err != nil {
   204  			return errwrap.Wrap(fmt.Errorf("error setting up image %s", img), err)
   205  		}
   206  		if pm.Apps.Get(*appName) != nil {
   207  			return fmt.Errorf("error: multiple apps with name %s", am.Name)
   208  		}
   209  		if am.App == nil && app.Exec == "" {
   210  			return fmt.Errorf("error: image %s has no app section and --exec argument is not provided", img)
   211  		}
   212  		ra := schema.RuntimeApp{
   213  			// TODO(vc): leverage RuntimeApp.Name for disambiguating the apps
   214  			Name: *appName,
   215  			App:  am.App,
   216  			Image: schema.RuntimeImage{
   217  				Name:   &am.Name,
   218  				ID:     img,
   219  				Labels: am.Labels,
   220  			},
   221  			Annotations: am.Annotations,
   222  			Mounts:      MergeMounts(cfg.Apps.Mounts, app.Mounts),
   223  		}
   224  
   225  		if execOverride := app.Exec; execOverride != "" {
   226  			// Create a minimal App section if not present
   227  			if am.App == nil {
   228  				ra.App = &types.App{
   229  					User:  strconv.Itoa(os.Getuid()),
   230  					Group: strconv.Itoa(os.Getgid()),
   231  				}
   232  			}
   233  			ra.App.Exec = []string{execOverride}
   234  		}
   235  
   236  		if execAppends := app.Args; execAppends != nil {
   237  			ra.App.Exec = append(ra.App.Exec, execAppends...)
   238  		}
   239  
   240  		if memoryOverride := app.MemoryLimit; memoryOverride != nil {
   241  			isolator := memoryOverride.AsIsolator()
   242  			ra.App.Isolators = append(ra.App.Isolators, isolator)
   243  		}
   244  
   245  		if cpuOverride := app.CPULimit; cpuOverride != nil {
   246  			isolator := cpuOverride.AsIsolator()
   247  			ra.App.Isolators = append(ra.App.Isolators, isolator)
   248  		}
   249  
   250  		if cfg.InheritEnv || len(cfg.ExplicitEnv) > 0 {
   251  			MergeEnvs(&ra.App.Environment, cfg.InheritEnv, cfg.ExplicitEnv)
   252  		}
   253  		pm.Apps = append(pm.Apps, ra)
   254  		return nil
   255  	}); err != nil {
   256  		return nil, err
   257  	}
   258  
   259  	// TODO(jonboulle): check that app mountpoint expectations are
   260  	// satisfied here, rather than waiting for stage1
   261  	pm.Volumes = cfg.Apps.Volumes
   262  	pm.Ports = cfg.Ports
   263  
   264  	pmb, err := json.Marshal(pm)
   265  	if err != nil {
   266  		return nil, errwrap.Wrap(errors.New("error marshalling pod manifest"), err)
   267  	}
   268  	return pmb, nil
   269  }
   270  
   271  // validatePodManifest reads the user-specified pod manifest, prepares the app images
   272  // and validates the pod manifest. If the pod manifest passes validation, it returns
   273  // the manifest as []byte.
   274  // TODO(yifan): More validation in the future.
   275  func validatePodManifest(cfg PrepareConfig, dir string) ([]byte, error) {
   276  	pmb, err := ioutil.ReadFile(cfg.PodManifest)
   277  	if err != nil {
   278  		return nil, errwrap.Wrap(errors.New("error reading pod manifest"), err)
   279  	}
   280  	var pm schema.PodManifest
   281  	if err := json.Unmarshal(pmb, &pm); err != nil {
   282  		return nil, errwrap.Wrap(errors.New("error unmarshaling pod manifest"), err)
   283  	}
   284  
   285  	appNames := make(map[types.ACName]struct{})
   286  	for _, ra := range pm.Apps {
   287  		img := ra.Image
   288  
   289  		if img.ID.Empty() {
   290  			return nil, fmt.Errorf("no image ID for app %q", ra.Name)
   291  		}
   292  		am, err := cfg.Store.GetImageManifest(img.ID.String())
   293  		if err != nil {
   294  			return nil, errwrap.Wrap(errors.New("error getting the image manifest from store"), err)
   295  		}
   296  		if err := prepareAppImage(cfg, ra.Name, img.ID, dir, cfg.UseOverlay); err != nil {
   297  			return nil, errwrap.Wrap(fmt.Errorf("error setting up image %s", img), err)
   298  		}
   299  		if _, ok := appNames[ra.Name]; ok {
   300  			return nil, fmt.Errorf("multiple apps with same name %s", ra.Name)
   301  		}
   302  		appNames[ra.Name] = struct{}{}
   303  		if ra.App == nil && am.App == nil {
   304  			return nil, fmt.Errorf("no app section in the pod manifest or the image manifest")
   305  		}
   306  	}
   307  	return pmb, nil
   308  }
   309  
   310  // Prepare sets up a pod based on the given config.
   311  func Prepare(cfg PrepareConfig, dir string, uuid *types.UUID) error {
   312  	if err := os.MkdirAll(common.AppsInfoPath(dir), defaultRegularDirPerm); err != nil {
   313  		return errwrap.Wrap(errors.New("error creating apps info directory"), err)
   314  	}
   315  	debug("Preparing stage1")
   316  	if err := prepareStage1Image(cfg, cfg.Stage1Image, dir, cfg.UseOverlay); err != nil {
   317  		return errwrap.Wrap(errors.New("error preparing stage1"), err)
   318  	}
   319  
   320  	var pmb []byte
   321  	var err error
   322  	if len(cfg.PodManifest) > 0 {
   323  		pmb, err = validatePodManifest(cfg, dir)
   324  	} else {
   325  		pmb, err = generatePodManifest(cfg, dir)
   326  	}
   327  	if err != nil {
   328  		return err
   329  	}
   330  
   331  	cfg.CommonConfig.ManifestData = string(pmb)
   332  
   333  	debug("Writing pod manifest")
   334  	fn := common.PodManifestPath(dir)
   335  	if err := ioutil.WriteFile(fn, pmb, defaultRegularFilePerm); err != nil {
   336  		return errwrap.Wrap(errors.New("error writing pod manifest"), err)
   337  	}
   338  
   339  	if cfg.UseOverlay {
   340  		// mark the pod as prepared with overlay
   341  		f, err := os.Create(filepath.Join(dir, common.OverlayPreparedFilename))
   342  		if err != nil {
   343  			return errwrap.Wrap(errors.New("error writing overlay marker file"), err)
   344  		}
   345  		defer f.Close()
   346  	}
   347  
   348  	if cfg.PrivateUsers.Shift > 0 {
   349  		// mark the pod as prepared for user namespaces
   350  		uidrangeBytes := cfg.PrivateUsers.Serialize()
   351  
   352  		if err := ioutil.WriteFile(filepath.Join(dir, common.PrivateUsersPreparedFilename), uidrangeBytes, defaultRegularFilePerm); err != nil {
   353  			return errwrap.Wrap(errors.New("error writing userns marker file"), err)
   354  		}
   355  	}
   356  
   357  	return nil
   358  }
   359  
   360  func preparedWithOverlay(dir string) (bool, error) {
   361  	_, err := os.Stat(filepath.Join(dir, common.OverlayPreparedFilename))
   362  	if os.IsNotExist(err) {
   363  		return false, nil
   364  	}
   365  	if err != nil {
   366  		return false, err
   367  	}
   368  
   369  	if !common.SupportsOverlay() {
   370  		return false, fmt.Errorf("the pod was prepared with overlay but overlay is not supported")
   371  	}
   372  
   373  	return true, nil
   374  }
   375  
   376  func preparedWithPrivateUsers(dir string) (string, error) {
   377  	bytes, err := ioutil.ReadFile(filepath.Join(dir, common.PrivateUsersPreparedFilename))
   378  	if os.IsNotExist(err) {
   379  		return "", nil
   380  	}
   381  	if err != nil {
   382  		return "", err
   383  	}
   384  
   385  	return string(bytes), nil
   386  }
   387  
   388  func addResolvConf(cfg RunConfig, rootfs string) {
   389  	content := "# Generated by rkt\n\n"
   390  	if len(cfg.DNSSearch) > 0 {
   391  		content += fmt.Sprintf("search %s\n", strings.Join(cfg.DNSSearch, " "))
   392  	}
   393  	for _, server := range cfg.DNS {
   394  		// skip empty entries
   395  		if server == "" {
   396  			continue
   397  		}
   398  		// comment invalid entries
   399  		if net.ParseIP(server) == nil {
   400  			content += "# "
   401  		}
   402  		content += "nameserver " + server + "\n"
   403  	}
   404  	if len(cfg.DNSOpt) > 0 {
   405  		content += fmt.Sprintf("options %s\n", strings.Join(cfg.DNSOpt, " "))
   406  	}
   407  	content += "\n"
   408  
   409  	if err := ioutil.WriteFile(filepath.Join(rootfs, "etc/rkt-resolv.conf"), []byte(content), 0644); err != nil {
   410  		log.Fatalf("error writing /etc/rkt-resolv.conf: %v\n", err)
   411  	}
   412  }
   413  
   414  // Run mounts the right overlay filesystems and actually runs the prepared
   415  // pod by exec()ing the stage1 init inside the pod filesystem.
   416  func Run(cfg RunConfig, dir string, dataDir string) {
   417  	useOverlay, err := preparedWithOverlay(dir)
   418  	if err != nil {
   419  		log.FatalE("error preparing overlay", err)
   420  	}
   421  
   422  	privateUsers, err := preparedWithPrivateUsers(dir)
   423  	if err != nil {
   424  		log.FatalE("error preparing private users", err)
   425  	}
   426  
   427  	debug("Setting up stage1")
   428  	if err := setupStage1Image(cfg, dir, useOverlay); err != nil {
   429  		log.FatalE("error setting up stage1", err)
   430  	}
   431  	debug("Wrote filesystem to %s\n", dir)
   432  
   433  	for _, app := range cfg.Apps {
   434  		if err := setupAppImage(cfg, app.Name, app.Image.ID, dir, useOverlay); err != nil {
   435  			log.FatalE("error setting up app image", err)
   436  		}
   437  	}
   438  
   439  	destRootfs := common.Stage1RootfsPath(dir)
   440  
   441  	if len(cfg.DNS) > 0 || len(cfg.DNSSearch) > 0 || len(cfg.DNSOpt) > 0 {
   442  		addResolvConf(cfg, destRootfs)
   443  	}
   444  
   445  	if err := os.Setenv(common.EnvLockFd, fmt.Sprintf("%v", cfg.LockFd)); err != nil {
   446  		log.FatalE("setting lock fd environment", err)
   447  	}
   448  
   449  	if err := os.Setenv(common.EnvSELinuxContext, fmt.Sprintf("%v", cfg.ProcessLabel)); err != nil {
   450  		log.FatalE("setting SELinux context environment", err)
   451  	}
   452  
   453  	debug("Pivoting to filesystem %s", dir)
   454  	if err := os.Chdir(dir); err != nil {
   455  		log.FatalE("failed changing to dir", err)
   456  	}
   457  
   458  	ep, err := getStage1Entrypoint(dir, runEntrypoint)
   459  	if err != nil {
   460  		log.FatalE("error determining 'run' entrypoint", err)
   461  	}
   462  	args := []string{filepath.Join(destRootfs, ep)}
   463  	debug("Execing %s", ep)
   464  
   465  	if cfg.Debug {
   466  		args = append(args, "--debug")
   467  	}
   468  
   469  	args = append(args, "--net="+cfg.Net.String())
   470  
   471  	if cfg.Interactive {
   472  		args = append(args, "--interactive")
   473  	}
   474  	if len(privateUsers) > 0 {
   475  		args = append(args, "--private-users="+privateUsers)
   476  	}
   477  	if cfg.MDSRegister {
   478  		mdsToken, err := registerPod(".", cfg.UUID, cfg.Apps)
   479  		if err != nil {
   480  			log.FatalE("failed to register the pod", err)
   481  		}
   482  
   483  		args = append(args, "--mds-token="+mdsToken)
   484  	}
   485  
   486  	if cfg.LocalConfig != "" {
   487  		args = append(args, "--local-config="+cfg.LocalConfig)
   488  	}
   489  
   490  	args = append(args, cfg.UUID.String())
   491  
   492  	// make sure the lock fd stays open across exec
   493  	if err := sys.CloseOnExec(cfg.LockFd, false); err != nil {
   494  		log.Fatalf("error clearing FD_CLOEXEC on lock fd")
   495  	}
   496  
   497  	tpmEvent := fmt.Sprintf("rkt: Rootfs: %s Manifest: %s Stage 1 args: %s", cfg.CommonConfig.RootHash, cfg.CommonConfig.ManifestData, strings.Join(args, " "))
   498  	// If there's no TPM available or there's a failure for some other
   499  	// reason, ignore it and continue anyway. Long term we'll want policy
   500  	// that enforces TPM behaviour, but we don't have any infrastructure
   501  	// around that yet.
   502  	_ = tpm.Extend(tpmEvent)
   503  	if err := syscall.Exec(args[0], args, os.Environ()); err != nil {
   504  		log.FatalE("error execing init", err)
   505  	}
   506  }
   507  
   508  // prepareAppImage renders and verifies the tree cache of the app image that
   509  // corresponds to the given app name.
   510  // When useOverlay is false, it attempts to render and expand the app image
   511  func prepareAppImage(cfg PrepareConfig, appName types.ACName, img types.Hash, cdir string, useOverlay bool) error {
   512  	debug("Loading image %s", img.String())
   513  
   514  	am, err := cfg.Store.GetImageManifest(img.String())
   515  	if err != nil {
   516  		return errwrap.Wrap(errors.New("error getting the manifest"), err)
   517  	}
   518  
   519  	if _, hasOS := am.Labels.Get("os"); !hasOS {
   520  		return fmt.Errorf("missing os label in the image manifest")
   521  	}
   522  	if _, hasArch := am.Labels.Get("arch"); !hasArch {
   523  		return fmt.Errorf("missing arch label in the image manifest")
   524  	}
   525  
   526  	if err := types.IsValidOSArch(am.Labels.ToMap(), ValidOSArch); err != nil {
   527  		return err
   528  	}
   529  
   530  	appInfoDir := common.AppInfoPath(cdir, appName)
   531  	if err := os.MkdirAll(appInfoDir, defaultRegularDirPerm); err != nil {
   532  		return errwrap.Wrap(errors.New("error creating apps info directory"), err)
   533  	}
   534  
   535  	if useOverlay {
   536  		if cfg.PrivateUsers.Shift > 0 {
   537  			return fmt.Errorf("cannot use both overlay and user namespace: not implemented yet. (Try --no-overlay)")
   538  		}
   539  		treeStoreID, _, err := cfg.Store.RenderTreeStore(img.String(), false)
   540  		if err != nil {
   541  			return errwrap.Wrap(errors.New("error rendering tree image"), err)
   542  		}
   543  
   544  		if !cfg.SkipTreeStoreCheck {
   545  			hash, err := cfg.Store.CheckTreeStore(treeStoreID)
   546  			if err != nil {
   547  				log.PrintE("warning: tree cache is in a bad state: %v. Rebuilding...", err)
   548  				var err error
   549  				treeStoreID, hash, err = cfg.Store.RenderTreeStore(img.String(), true)
   550  				if err != nil {
   551  					return errwrap.Wrap(errors.New("error rendering tree image"), err)
   552  				}
   553  			}
   554  			cfg.CommonConfig.RootHash = hash
   555  		}
   556  
   557  		if err := ioutil.WriteFile(common.AppTreeStoreIDPath(cdir, appName), []byte(treeStoreID), defaultRegularFilePerm); err != nil {
   558  			return errwrap.Wrap(errors.New("error writing app treeStoreID"), err)
   559  		}
   560  	} else {
   561  		ad := common.AppPath(cdir, appName)
   562  		err := os.MkdirAll(ad, defaultRegularDirPerm)
   563  		if err != nil {
   564  			return errwrap.Wrap(errors.New("error creating image directory"), err)
   565  		}
   566  
   567  		shiftedUid, shiftedGid, err := cfg.PrivateUsers.ShiftRange(uint32(os.Getuid()), uint32(os.Getgid()))
   568  		if err != nil {
   569  			return errwrap.Wrap(errors.New("error getting uid, gid"), err)
   570  		}
   571  
   572  		if err := os.Chown(ad, int(shiftedUid), int(shiftedGid)); err != nil {
   573  			return errwrap.Wrap(fmt.Errorf("error shifting app %q's stage2 dir", appName), err)
   574  		}
   575  
   576  		if err := aci.RenderACIWithImageID(img, ad, cfg.Store, cfg.PrivateUsers); err != nil {
   577  			return errwrap.Wrap(errors.New("error rendering ACI"), err)
   578  		}
   579  	}
   580  	if err := writeManifest(*cfg.CommonConfig, img, appInfoDir); err != nil {
   581  		return err
   582  	}
   583  	return nil
   584  }
   585  
   586  // setupAppImage mounts the overlay filesystem for the app image that
   587  // corresponds to the given hash. Then, it creates the tmp directory.
   588  // When useOverlay is false it just creates the tmp directory for this app.
   589  func setupAppImage(cfg RunConfig, appName types.ACName, img types.Hash, cdir string, useOverlay bool) error {
   590  	ad := common.AppPath(cdir, appName)
   591  	if useOverlay {
   592  		err := os.MkdirAll(ad, defaultRegularDirPerm)
   593  		if err != nil {
   594  			return errwrap.Wrap(errors.New("error creating image directory"), err)
   595  		}
   596  		treeStoreID, err := ioutil.ReadFile(common.AppTreeStoreIDPath(cdir, appName))
   597  		if err != nil {
   598  			return err
   599  		}
   600  		if err := copyAppManifest(cdir, appName, ad); err != nil {
   601  			return err
   602  		}
   603  		if err := overlayRender(cfg, string(treeStoreID), cdir, ad, appName.String()); err != nil {
   604  			return errwrap.Wrap(errors.New("error rendering overlay filesystem"), err)
   605  		}
   606  	}
   607  
   608  	return nil
   609  }
   610  
   611  // prepareStage1Image renders and verifies tree cache of the given hash
   612  // when using overlay.
   613  // When useOverlay is false, it attempts to render and expand the stage1.
   614  func prepareStage1Image(cfg PrepareConfig, img types.Hash, cdir string, useOverlay bool) error {
   615  	s1 := common.Stage1ImagePath(cdir)
   616  	if err := os.MkdirAll(s1, defaultRegularDirPerm); err != nil {
   617  		return errwrap.Wrap(errors.New("error creating stage1 directory"), err)
   618  	}
   619  
   620  	treeStoreID, _, err := cfg.Store.RenderTreeStore(img.String(), false)
   621  	if err != nil {
   622  		return errwrap.Wrap(errors.New("error rendering tree image"), err)
   623  	}
   624  
   625  	if !cfg.SkipTreeStoreCheck {
   626  		hash, err := cfg.Store.CheckTreeStore(treeStoreID)
   627  		if err != nil {
   628  			log.Printf("warning: tree cache is in a bad state: %v. Rebuilding...", err)
   629  			var err error
   630  			treeStoreID, hash, err = cfg.Store.RenderTreeStore(img.String(), true)
   631  			if err != nil {
   632  				return errwrap.Wrap(errors.New("error rendering tree image"), err)
   633  			}
   634  		}
   635  		cfg.CommonConfig.RootHash = hash
   636  	}
   637  
   638  	if err := writeManifest(*cfg.CommonConfig, img, s1); err != nil {
   639  		return errwrap.Wrap(errors.New("error writing manifest"), err)
   640  	}
   641  
   642  	if !useOverlay {
   643  		destRootfs := filepath.Join(s1, "rootfs")
   644  		cachedTreePath := cfg.Store.GetTreeStoreRootFS(treeStoreID)
   645  		if err := fileutil.CopyTree(cachedTreePath, destRootfs, cfg.PrivateUsers); err != nil {
   646  			return errwrap.Wrap(errors.New("error rendering ACI"), err)
   647  		}
   648  	}
   649  
   650  	fn := path.Join(cdir, common.Stage1TreeStoreIDFilename)
   651  	if err := ioutil.WriteFile(fn, []byte(treeStoreID), defaultRegularFilePerm); err != nil {
   652  		return errwrap.Wrap(errors.New("error writing stage1 treeStoreID"), err)
   653  	}
   654  	return nil
   655  }
   656  
   657  // setupStage1Image mounts the overlay filesystem for stage1.
   658  // When useOverlay is false it is a noop
   659  func setupStage1Image(cfg RunConfig, cdir string, useOverlay bool) error {
   660  	s1 := common.Stage1ImagePath(cdir)
   661  	if useOverlay {
   662  		treeStoreID, err := ioutil.ReadFile(filepath.Join(cdir, common.Stage1TreeStoreIDFilename))
   663  		if err != nil {
   664  			return err
   665  		}
   666  
   667  		// pass an empty appName: make sure it remains consistent with
   668  		// overlayStatusDirTemplate
   669  		if err := overlayRender(cfg, string(treeStoreID), cdir, s1, ""); err != nil {
   670  			return errwrap.Wrap(errors.New("error rendering overlay filesystem"), err)
   671  		}
   672  
   673  		// we will later read the status from the upper layer of the overlay fs
   674  		// force the status directory to be there by touching it
   675  		statusPath := filepath.Join(s1, "rootfs", "rkt", "status")
   676  		if err := os.Chtimes(statusPath, time.Now(), time.Now()); err != nil {
   677  			return errwrap.Wrap(errors.New("error touching status dir"), err)
   678  		}
   679  	}
   680  
   681  	return nil
   682  }
   683  
   684  // writeManifest takes an img ID and writes the corresponding manifest in dest
   685  func writeManifest(cfg CommonConfig, img types.Hash, dest string) error {
   686  	mb, err := cfg.Store.GetImageManifestJSON(img.String())
   687  	if err != nil {
   688  		return err
   689  	}
   690  
   691  	debug("Writing image manifest")
   692  	if err := ioutil.WriteFile(filepath.Join(dest, "manifest"), mb, defaultRegularFilePerm); err != nil {
   693  		return errwrap.Wrap(errors.New("error writing image manifest"), err)
   694  	}
   695  
   696  	return nil
   697  }
   698  
   699  // copyAppManifest copies to saved image manifest for the given appName and
   700  // writes it in the dest directory.
   701  func copyAppManifest(cdir string, appName types.ACName, dest string) error {
   702  	appInfoDir := common.AppInfoPath(cdir, appName)
   703  	sourceFn := filepath.Join(appInfoDir, "manifest")
   704  	destFn := filepath.Join(dest, "manifest")
   705  	if err := fileutil.CopyRegularFile(sourceFn, destFn); err != nil {
   706  		return errwrap.Wrap(errors.New("error copying image manifest"), err)
   707  	}
   708  	return nil
   709  }
   710  
   711  // overlayRender renders the image that corresponds to the given hash using the
   712  // overlay filesystem.
   713  // It mounts an overlay filesystem from the cached tree of the image as rootfs.
   714  func overlayRender(cfg RunConfig, treeStoreID string, cdir string, dest string, appName string) error {
   715  	cachedTreePath := cfg.Store.GetTreeStoreRootFS(treeStoreID)
   716  	fi, err := os.Stat(cachedTreePath)
   717  	if err != nil {
   718  		return err
   719  	}
   720  	imgMode := fi.Mode()
   721  
   722  	destRootfs := path.Join(dest, "rootfs")
   723  	if err := os.MkdirAll(destRootfs, imgMode); err != nil {
   724  		return err
   725  	}
   726  
   727  	overlayDir := path.Join(cdir, "overlay")
   728  	if err := os.MkdirAll(overlayDir, defaultRegularDirPerm); err != nil {
   729  		return err
   730  	}
   731  
   732  	// Since the parent directory (rkt/pods/$STATE/$POD_UUID) has the 'S_ISGID' bit, here
   733  	// we need to explicitly turn the bit off when creating this overlay
   734  	// directory so that it won't inherit the bit. Otherwise the files
   735  	// created by users within the pod will inherit the 'S_ISGID' bit
   736  	// as well.
   737  	if err := os.Chmod(overlayDir, defaultRegularDirPerm); err != nil {
   738  		return err
   739  	}
   740  
   741  	imgDir := path.Join(overlayDir, treeStoreID)
   742  	if err := os.MkdirAll(imgDir, defaultRegularDirPerm); err != nil {
   743  		return err
   744  	}
   745  
   746  	// Also make 'rkt/pods/$STATE/$POD_UUID/overlay/$IMAGE_ID' to be readable by 'rkt' group
   747  	// As 'rkt' status will read the 'rkt/pods/$STATE/$POD_UUID/overlay/$IMAGE_ID/upper/rkt/status/$APP'
   748  	// to get exit status.
   749  	if err := os.Chown(imgDir, -1, cfg.RktGid); err != nil {
   750  		return err
   751  	}
   752  
   753  	upperDir := path.Join(imgDir, "upper", appName)
   754  	if err := os.MkdirAll(upperDir, imgMode); err != nil {
   755  		return err
   756  	}
   757  	if err := label.SetFileLabel(upperDir, cfg.MountLabel); err != nil {
   758  		return err
   759  	}
   760  
   761  	workDir := path.Join(imgDir, "work", appName)
   762  	if err := os.MkdirAll(workDir, defaultRegularDirPerm); err != nil {
   763  		return err
   764  	}
   765  	if err := label.SetFileLabel(workDir, cfg.MountLabel); err != nil {
   766  		return err
   767  	}
   768  
   769  	opts := fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s", cachedTreePath, upperDir, workDir)
   770  	opts = label.FormatMountLabel(opts, cfg.MountLabel)
   771  	if err := syscall.Mount("overlay", destRootfs, "overlay", 0, opts); err != nil {
   772  		return errwrap.Wrap(errors.New("error mounting"), err)
   773  	}
   774  
   775  	return nil
   776  }