github.com/shishir-a412ed/docker@v1.3.2-0.20180103180333-fda904911d87/builder/dockerfile/internals.go (about)

     1  package dockerfile
     2  
     3  // internals for handling commands. Covers many areas and a lot of
     4  // non-contiguous functionality. Please read the comments.
     5  
     6  import (
     7  	"crypto/sha256"
     8  	"encoding/hex"
     9  	"fmt"
    10  	"io"
    11  	"os"
    12  	"path"
    13  	"path/filepath"
    14  	"strconv"
    15  	"strings"
    16  
    17  	"github.com/docker/docker/api/types"
    18  	"github.com/docker/docker/api/types/backend"
    19  	"github.com/docker/docker/api/types/container"
    20  	"github.com/docker/docker/image"
    21  	"github.com/docker/docker/pkg/archive"
    22  	"github.com/docker/docker/pkg/chrootarchive"
    23  	"github.com/docker/docker/pkg/containerfs"
    24  	"github.com/docker/docker/pkg/idtools"
    25  	"github.com/docker/docker/pkg/stringid"
    26  	"github.com/docker/docker/pkg/symlink"
    27  	"github.com/docker/docker/pkg/system"
    28  	"github.com/docker/go-connections/nat"
    29  	lcUser "github.com/opencontainers/runc/libcontainer/user"
    30  	"github.com/pkg/errors"
    31  )
    32  
    33  // Archiver defines an interface for copying files from one destination to
    34  // another using Tar/Untar.
    35  type Archiver interface {
    36  	TarUntar(src, dst string) error
    37  	UntarPath(src, dst string) error
    38  	CopyWithTar(src, dst string) error
    39  	CopyFileWithTar(src, dst string) error
    40  	IDMappings() *idtools.IDMappings
    41  }
    42  
    43  // The builder will use the following interfaces if the container fs implements
    44  // these for optimized copies to and from the container.
    45  type extractor interface {
    46  	ExtractArchive(src io.Reader, dst string, opts *archive.TarOptions) error
    47  }
    48  
    49  type archiver interface {
    50  	ArchivePath(src string, opts *archive.TarOptions) (io.ReadCloser, error)
    51  }
    52  
    53  // helper functions to get tar/untar func
    54  func untarFunc(i interface{}) containerfs.UntarFunc {
    55  	if ea, ok := i.(extractor); ok {
    56  		return ea.ExtractArchive
    57  	}
    58  	return chrootarchive.Untar
    59  }
    60  
    61  func tarFunc(i interface{}) containerfs.TarFunc {
    62  	if ap, ok := i.(archiver); ok {
    63  		return ap.ArchivePath
    64  	}
    65  	return archive.TarWithOptions
    66  }
    67  
    68  func (b *Builder) getArchiver(src, dst containerfs.Driver) Archiver {
    69  	t, u := tarFunc(src), untarFunc(dst)
    70  	return &containerfs.Archiver{
    71  		SrcDriver:     src,
    72  		DstDriver:     dst,
    73  		Tar:           t,
    74  		Untar:         u,
    75  		IDMappingsVar: b.idMappings,
    76  	}
    77  }
    78  
    79  func (b *Builder) commit(dispatchState *dispatchState, comment string) error {
    80  	if b.disableCommit {
    81  		return nil
    82  	}
    83  	if !dispatchState.hasFromImage() {
    84  		return errors.New("Please provide a source image with `from` prior to commit")
    85  	}
    86  
    87  	optionsPlatform := system.ParsePlatform(b.options.Platform)
    88  	runConfigWithCommentCmd := copyRunConfig(dispatchState.runConfig, withCmdComment(comment, optionsPlatform.OS))
    89  	hit, err := b.probeCache(dispatchState, runConfigWithCommentCmd)
    90  	if err != nil || hit {
    91  		return err
    92  	}
    93  	id, err := b.create(runConfigWithCommentCmd)
    94  	if err != nil {
    95  		return err
    96  	}
    97  
    98  	return b.commitContainer(dispatchState, id, runConfigWithCommentCmd)
    99  }
   100  
   101  func (b *Builder) commitContainer(dispatchState *dispatchState, id string, containerConfig *container.Config) error {
   102  	if b.disableCommit {
   103  		return nil
   104  	}
   105  
   106  	commitCfg := &backend.ContainerCommitConfig{
   107  		ContainerCommitConfig: types.ContainerCommitConfig{
   108  			Author: dispatchState.maintainer,
   109  			Pause:  true,
   110  			// TODO: this should be done by Commit()
   111  			Config: copyRunConfig(dispatchState.runConfig),
   112  		},
   113  		ContainerConfig: containerConfig,
   114  	}
   115  
   116  	// Commit the container
   117  	imageID, err := b.docker.Commit(id, commitCfg)
   118  	if err != nil {
   119  		return err
   120  	}
   121  
   122  	dispatchState.imageID = imageID
   123  	return nil
   124  }
   125  
   126  func (b *Builder) exportImage(state *dispatchState, imageMount *imageMount, runConfig *container.Config) error {
   127  	optionsPlatform := system.ParsePlatform(b.options.Platform)
   128  	newLayer, err := imageMount.Layer().Commit(optionsPlatform.OS)
   129  	if err != nil {
   130  		return err
   131  	}
   132  
   133  	// add an image mount without an image so the layer is properly unmounted
   134  	// if there is an error before we can add the full mount with image
   135  	b.imageSources.Add(newImageMount(nil, newLayer))
   136  
   137  	parentImage, ok := imageMount.Image().(*image.Image)
   138  	if !ok {
   139  		return errors.Errorf("unexpected image type")
   140  	}
   141  
   142  	newImage := image.NewChildImage(parentImage, image.ChildConfig{
   143  		Author:          state.maintainer,
   144  		ContainerConfig: runConfig,
   145  		DiffID:          newLayer.DiffID(),
   146  		Config:          copyRunConfig(state.runConfig),
   147  	}, parentImage.OS)
   148  
   149  	// TODO: it seems strange to marshal this here instead of just passing in the
   150  	// image struct
   151  	config, err := newImage.MarshalJSON()
   152  	if err != nil {
   153  		return errors.Wrap(err, "failed to encode image config")
   154  	}
   155  
   156  	exportedImage, err := b.docker.CreateImage(config, state.imageID, parentImage.OS)
   157  	if err != nil {
   158  		return errors.Wrapf(err, "failed to export image")
   159  	}
   160  
   161  	state.imageID = exportedImage.ImageID()
   162  	b.imageSources.Add(newImageMount(exportedImage, newLayer))
   163  	return nil
   164  }
   165  
   166  func (b *Builder) performCopy(state *dispatchState, inst copyInstruction) error {
   167  	srcHash := getSourceHashFromInfos(inst.infos)
   168  
   169  	var chownComment string
   170  	if inst.chownStr != "" {
   171  		chownComment = fmt.Sprintf("--chown=%s", inst.chownStr)
   172  	}
   173  	commentStr := fmt.Sprintf("%s %s%s in %s ", inst.cmdName, chownComment, srcHash, inst.dest)
   174  
   175  	// TODO: should this have been using origPaths instead of srcHash in the comment?
   176  	optionsPlatform := system.ParsePlatform(b.options.Platform)
   177  	runConfigWithCommentCmd := copyRunConfig(
   178  		state.runConfig,
   179  		withCmdCommentString(commentStr, optionsPlatform.OS))
   180  	hit, err := b.probeCache(state, runConfigWithCommentCmd)
   181  	if err != nil || hit {
   182  		return err
   183  	}
   184  
   185  	imageMount, err := b.imageSources.Get(state.imageID, true)
   186  	if err != nil {
   187  		return errors.Wrapf(err, "failed to get destination image %q", state.imageID)
   188  	}
   189  
   190  	destInfo, err := createDestInfo(state.runConfig.WorkingDir, inst, imageMount, b.options.Platform)
   191  	if err != nil {
   192  		return err
   193  	}
   194  
   195  	chownPair := b.idMappings.RootPair()
   196  	// if a chown was requested, perform the steps to get the uid, gid
   197  	// translated (if necessary because of user namespaces), and replace
   198  	// the root pair with the chown pair for copy operations
   199  	if inst.chownStr != "" {
   200  		chownPair, err = parseChownFlag(inst.chownStr, destInfo.root.Path(), b.idMappings)
   201  		if err != nil {
   202  			return errors.Wrapf(err, "unable to convert uid/gid chown string to host mapping")
   203  		}
   204  	}
   205  
   206  	for _, info := range inst.infos {
   207  		opts := copyFileOptions{
   208  			decompress: inst.allowLocalDecompression,
   209  			archiver:   b.getArchiver(info.root, destInfo.root),
   210  			chownPair:  chownPair,
   211  		}
   212  		if err := performCopyForInfo(destInfo, info, opts); err != nil {
   213  			return errors.Wrapf(err, "failed to copy files")
   214  		}
   215  	}
   216  	return b.exportImage(state, imageMount, runConfigWithCommentCmd)
   217  }
   218  
   219  func parseChownFlag(chown, ctrRootPath string, idMappings *idtools.IDMappings) (idtools.IDPair, error) {
   220  	var userStr, grpStr string
   221  	parts := strings.Split(chown, ":")
   222  	if len(parts) > 2 {
   223  		return idtools.IDPair{}, errors.New("invalid chown string format: " + chown)
   224  	}
   225  	if len(parts) == 1 {
   226  		// if no group specified, use the user spec as group as well
   227  		userStr, grpStr = parts[0], parts[0]
   228  	} else {
   229  		userStr, grpStr = parts[0], parts[1]
   230  	}
   231  
   232  	passwdPath, err := symlink.FollowSymlinkInScope(filepath.Join(ctrRootPath, "etc", "passwd"), ctrRootPath)
   233  	if err != nil {
   234  		return idtools.IDPair{}, errors.Wrapf(err, "can't resolve /etc/passwd path in container rootfs")
   235  	}
   236  	groupPath, err := symlink.FollowSymlinkInScope(filepath.Join(ctrRootPath, "etc", "group"), ctrRootPath)
   237  	if err != nil {
   238  		return idtools.IDPair{}, errors.Wrapf(err, "can't resolve /etc/group path in container rootfs")
   239  	}
   240  	uid, err := lookupUser(userStr, passwdPath)
   241  	if err != nil {
   242  		return idtools.IDPair{}, errors.Wrapf(err, "can't find uid for user "+userStr)
   243  	}
   244  	gid, err := lookupGroup(grpStr, groupPath)
   245  	if err != nil {
   246  		return idtools.IDPair{}, errors.Wrapf(err, "can't find gid for group "+grpStr)
   247  	}
   248  
   249  	// convert as necessary because of user namespaces
   250  	chownPair, err := idMappings.ToHost(idtools.IDPair{UID: uid, GID: gid})
   251  	if err != nil {
   252  		return idtools.IDPair{}, errors.Wrapf(err, "unable to convert uid/gid to host mapping")
   253  	}
   254  	return chownPair, nil
   255  }
   256  
   257  func lookupUser(userStr, filepath string) (int, error) {
   258  	// if the string is actually a uid integer, parse to int and return
   259  	// as we don't need to translate with the help of files
   260  	uid, err := strconv.Atoi(userStr)
   261  	if err == nil {
   262  		return uid, nil
   263  	}
   264  	users, err := lcUser.ParsePasswdFileFilter(filepath, func(u lcUser.User) bool {
   265  		return u.Name == userStr
   266  	})
   267  	if err != nil {
   268  		return 0, err
   269  	}
   270  	if len(users) == 0 {
   271  		return 0, errors.New("no such user: " + userStr)
   272  	}
   273  	return users[0].Uid, nil
   274  }
   275  
   276  func lookupGroup(groupStr, filepath string) (int, error) {
   277  	// if the string is actually a gid integer, parse to int and return
   278  	// as we don't need to translate with the help of files
   279  	gid, err := strconv.Atoi(groupStr)
   280  	if err == nil {
   281  		return gid, nil
   282  	}
   283  	groups, err := lcUser.ParseGroupFileFilter(filepath, func(g lcUser.Group) bool {
   284  		return g.Name == groupStr
   285  	})
   286  	if err != nil {
   287  		return 0, err
   288  	}
   289  	if len(groups) == 0 {
   290  		return 0, errors.New("no such group: " + groupStr)
   291  	}
   292  	return groups[0].Gid, nil
   293  }
   294  
   295  func createDestInfo(workingDir string, inst copyInstruction, imageMount *imageMount, platform string) (copyInfo, error) {
   296  	// Twiddle the destination when it's a relative path - meaning, make it
   297  	// relative to the WORKINGDIR
   298  	dest, err := normalizeDest(workingDir, inst.dest, platform)
   299  	if err != nil {
   300  		return copyInfo{}, errors.Wrapf(err, "invalid %s", inst.cmdName)
   301  	}
   302  
   303  	destMount, err := imageMount.Source()
   304  	if err != nil {
   305  		return copyInfo{}, errors.Wrapf(err, "failed to mount copy source")
   306  	}
   307  
   308  	return newCopyInfoFromSource(destMount, dest, ""), nil
   309  }
   310  
   311  // normalizeDest normalises the destination of a COPY/ADD command in a
   312  // platform semantically consistent way.
   313  func normalizeDest(workingDir, requested string, platform string) (string, error) {
   314  	dest := fromSlash(requested, platform)
   315  	endsInSlash := strings.HasSuffix(dest, string(separator(platform)))
   316  
   317  	if platform != "windows" {
   318  		if !path.IsAbs(requested) {
   319  			dest = path.Join("/", filepath.ToSlash(workingDir), dest)
   320  			// Make sure we preserve any trailing slash
   321  			if endsInSlash {
   322  				dest += "/"
   323  			}
   324  		}
   325  		return dest, nil
   326  	}
   327  
   328  	// We are guaranteed that the working directory is already consistent,
   329  	// However, Windows also has, for now, the limitation that ADD/COPY can
   330  	// only be done to the system drive, not any drives that might be present
   331  	// as a result of a bind mount.
   332  	//
   333  	// So... if the path requested is Linux-style absolute (/foo or \\foo),
   334  	// we assume it is the system drive. If it is a Windows-style absolute
   335  	// (DRIVE:\\foo), error if DRIVE is not C. And finally, ensure we
   336  	// strip any configured working directories drive letter so that it
   337  	// can be subsequently legitimately converted to a Windows volume-style
   338  	// pathname.
   339  
   340  	// Not a typo - filepath.IsAbs, not system.IsAbs on this next check as
   341  	// we only want to validate where the DriveColon part has been supplied.
   342  	if filepath.IsAbs(dest) {
   343  		if strings.ToUpper(string(dest[0])) != "C" {
   344  			return "", fmt.Errorf("Windows does not support destinations not on the system drive (C:)")
   345  		}
   346  		dest = dest[2:] // Strip the drive letter
   347  	}
   348  
   349  	// Cannot handle relative where WorkingDir is not the system drive.
   350  	if len(workingDir) > 0 {
   351  		if ((len(workingDir) > 1) && !system.IsAbs(workingDir[2:])) || (len(workingDir) == 1) {
   352  			return "", fmt.Errorf("Current WorkingDir %s is not platform consistent", workingDir)
   353  		}
   354  		if !system.IsAbs(dest) {
   355  			if string(workingDir[0]) != "C" {
   356  				return "", fmt.Errorf("Windows does not support relative paths when WORKDIR is not the system drive")
   357  			}
   358  			dest = filepath.Join(string(os.PathSeparator), workingDir[2:], dest)
   359  			// Make sure we preserve any trailing slash
   360  			if endsInSlash {
   361  				dest += string(os.PathSeparator)
   362  			}
   363  		}
   364  	}
   365  	return dest, nil
   366  }
   367  
   368  // For backwards compat, if there's just one info then use it as the
   369  // cache look-up string, otherwise hash 'em all into one
   370  func getSourceHashFromInfos(infos []copyInfo) string {
   371  	if len(infos) == 1 {
   372  		return infos[0].hash
   373  	}
   374  	var hashs []string
   375  	for _, info := range infos {
   376  		hashs = append(hashs, info.hash)
   377  	}
   378  	return hashStringSlice("multi", hashs)
   379  }
   380  
   381  func hashStringSlice(prefix string, slice []string) string {
   382  	hasher := sha256.New()
   383  	hasher.Write([]byte(strings.Join(slice, ",")))
   384  	return prefix + ":" + hex.EncodeToString(hasher.Sum(nil))
   385  }
   386  
   387  type runConfigModifier func(*container.Config)
   388  
   389  func withCmd(cmd []string) runConfigModifier {
   390  	return func(runConfig *container.Config) {
   391  		runConfig.Cmd = cmd
   392  	}
   393  }
   394  
   395  // withCmdComment sets Cmd to a nop comment string. See withCmdCommentString for
   396  // why there are two almost identical versions of this.
   397  func withCmdComment(comment string, platform string) runConfigModifier {
   398  	return func(runConfig *container.Config) {
   399  		runConfig.Cmd = append(getShell(runConfig, platform), "#(nop) ", comment)
   400  	}
   401  }
   402  
   403  // withCmdCommentString exists to maintain compatibility with older versions.
   404  // A few instructions (workdir, copy, add) used a nop comment that is a single arg
   405  // where as all the other instructions used a two arg comment string. This
   406  // function implements the single arg version.
   407  func withCmdCommentString(comment string, platform string) runConfigModifier {
   408  	return func(runConfig *container.Config) {
   409  		runConfig.Cmd = append(getShell(runConfig, platform), "#(nop) "+comment)
   410  	}
   411  }
   412  
   413  func withEnv(env []string) runConfigModifier {
   414  	return func(runConfig *container.Config) {
   415  		runConfig.Env = env
   416  	}
   417  }
   418  
   419  // withEntrypointOverride sets an entrypoint on runConfig if the command is
   420  // not empty. The entrypoint is left unmodified if command is empty.
   421  //
   422  // The dockerfile RUN instruction expect to run without an entrypoint
   423  // so the runConfig entrypoint needs to be modified accordingly. ContainerCreate
   424  // will change a []string{""} entrypoint to nil, so we probe the cache with the
   425  // nil entrypoint.
   426  func withEntrypointOverride(cmd []string, entrypoint []string) runConfigModifier {
   427  	return func(runConfig *container.Config) {
   428  		if len(cmd) > 0 {
   429  			runConfig.Entrypoint = entrypoint
   430  		}
   431  	}
   432  }
   433  
   434  func copyRunConfig(runConfig *container.Config, modifiers ...runConfigModifier) *container.Config {
   435  	copy := *runConfig
   436  	copy.Cmd = copyStringSlice(runConfig.Cmd)
   437  	copy.Env = copyStringSlice(runConfig.Env)
   438  	copy.Entrypoint = copyStringSlice(runConfig.Entrypoint)
   439  	copy.OnBuild = copyStringSlice(runConfig.OnBuild)
   440  	copy.Shell = copyStringSlice(runConfig.Shell)
   441  
   442  	if copy.Volumes != nil {
   443  		copy.Volumes = make(map[string]struct{}, len(runConfig.Volumes))
   444  		for k, v := range runConfig.Volumes {
   445  			copy.Volumes[k] = v
   446  		}
   447  	}
   448  
   449  	if copy.ExposedPorts != nil {
   450  		copy.ExposedPorts = make(nat.PortSet, len(runConfig.ExposedPorts))
   451  		for k, v := range runConfig.ExposedPorts {
   452  			copy.ExposedPorts[k] = v
   453  		}
   454  	}
   455  
   456  	if copy.Labels != nil {
   457  		copy.Labels = make(map[string]string, len(runConfig.Labels))
   458  		for k, v := range runConfig.Labels {
   459  			copy.Labels[k] = v
   460  		}
   461  	}
   462  
   463  	for _, modifier := range modifiers {
   464  		modifier(&copy)
   465  	}
   466  	return &copy
   467  }
   468  
   469  func copyStringSlice(orig []string) []string {
   470  	if orig == nil {
   471  		return nil
   472  	}
   473  	return append([]string{}, orig...)
   474  }
   475  
   476  // getShell is a helper function which gets the right shell for prefixing the
   477  // shell-form of RUN, ENTRYPOINT and CMD instructions
   478  func getShell(c *container.Config, os string) []string {
   479  	if 0 == len(c.Shell) {
   480  		return append([]string{}, defaultShellForOS(os)[:]...)
   481  	}
   482  	return append([]string{}, c.Shell[:]...)
   483  }
   484  
   485  func (b *Builder) probeCache(dispatchState *dispatchState, runConfig *container.Config) (bool, error) {
   486  	cachedID, err := b.imageProber.Probe(dispatchState.imageID, runConfig)
   487  	if cachedID == "" || err != nil {
   488  		return false, err
   489  	}
   490  	fmt.Fprint(b.Stdout, " ---> Using cache\n")
   491  
   492  	dispatchState.imageID = cachedID
   493  	return true, nil
   494  }
   495  
   496  var defaultLogConfig = container.LogConfig{Type: "none"}
   497  
   498  func (b *Builder) probeAndCreate(dispatchState *dispatchState, runConfig *container.Config) (string, error) {
   499  	if hit, err := b.probeCache(dispatchState, runConfig); err != nil || hit {
   500  		return "", err
   501  	}
   502  	// Set a log config to override any default value set on the daemon
   503  	hostConfig := &container.HostConfig{LogConfig: defaultLogConfig}
   504  	optionsPlatform := system.ParsePlatform(b.options.Platform)
   505  	container, err := b.containerManager.Create(runConfig, hostConfig, optionsPlatform.OS)
   506  	return container.ID, err
   507  }
   508  
   509  func (b *Builder) create(runConfig *container.Config) (string, error) {
   510  	hostConfig := hostConfigFromOptions(b.options)
   511  	optionsPlatform := system.ParsePlatform(b.options.Platform)
   512  	container, err := b.containerManager.Create(runConfig, hostConfig, optionsPlatform.OS)
   513  	if err != nil {
   514  		return "", err
   515  	}
   516  	// TODO: could this be moved into containerManager.Create() ?
   517  	for _, warning := range container.Warnings {
   518  		fmt.Fprintf(b.Stdout, " ---> [Warning] %s\n", warning)
   519  	}
   520  	fmt.Fprintf(b.Stdout, " ---> Running in %s\n", stringid.TruncateID(container.ID))
   521  	return container.ID, nil
   522  }
   523  
   524  func hostConfigFromOptions(options *types.ImageBuildOptions) *container.HostConfig {
   525  	resources := container.Resources{
   526  		CgroupParent: options.CgroupParent,
   527  		CPUShares:    options.CPUShares,
   528  		CPUPeriod:    options.CPUPeriod,
   529  		CPUQuota:     options.CPUQuota,
   530  		CpusetCpus:   options.CPUSetCPUs,
   531  		CpusetMems:   options.CPUSetMems,
   532  		Memory:       options.Memory,
   533  		MemorySwap:   options.MemorySwap,
   534  		Ulimits:      options.Ulimits,
   535  	}
   536  
   537  	return &container.HostConfig{
   538  		SecurityOpt: options.SecurityOpt,
   539  		Isolation:   options.Isolation,
   540  		ShmSize:     options.ShmSize,
   541  		Resources:   resources,
   542  		NetworkMode: container.NetworkMode(options.NetworkMode),
   543  		// Set a log config to override any default value set on the daemon
   544  		LogConfig:  defaultLogConfig,
   545  		ExtraHosts: options.ExtraHosts,
   546  	}
   547  }
   548  
   549  // fromSlash works like filepath.FromSlash but with a given OS platform field
   550  func fromSlash(path, platform string) string {
   551  	if platform == "windows" {
   552  		return strings.Replace(path, "/", "\\", -1)
   553  	}
   554  	return path
   555  }
   556  
   557  // separator returns a OS path separator for the given OS platform
   558  func separator(platform string) byte {
   559  	if platform == "windows" {
   560  		return '\\'
   561  	}
   562  	return '/'
   563  }