github.com/noxiouz/docker@v0.7.3-0.20160629055221-3d231c78e8c5/pkg/archive/archive.go (about)

     1  package archive
     2  
     3  import (
     4  	"archive/tar"
     5  	"bufio"
     6  	"bytes"
     7  	"compress/bzip2"
     8  	"compress/gzip"
     9  	"errors"
    10  	"fmt"
    11  	"io"
    12  	"io/ioutil"
    13  	"os"
    14  	"os/exec"
    15  	"path/filepath"
    16  	"runtime"
    17  	"strings"
    18  	"syscall"
    19  
    20  	"github.com/Sirupsen/logrus"
    21  	"github.com/docker/docker/pkg/fileutils"
    22  	"github.com/docker/docker/pkg/idtools"
    23  	"github.com/docker/docker/pkg/ioutils"
    24  	"github.com/docker/docker/pkg/pools"
    25  	"github.com/docker/docker/pkg/promise"
    26  	"github.com/docker/docker/pkg/system"
    27  )
    28  
    29  type (
    30  	// Archive is a type of io.ReadCloser which has two interfaces Read and Closer.
    31  	Archive io.ReadCloser
    32  	// Reader is a type of io.Reader.
    33  	Reader io.Reader
    34  	// Compression is the state represents if compressed or not.
    35  	Compression int
    36  	// WhiteoutFormat is the format of whiteouts unpacked
    37  	WhiteoutFormat int
    38  	// TarChownOptions wraps the chown options UID and GID.
    39  	TarChownOptions struct {
    40  		UID, GID int
    41  	}
    42  	// TarOptions wraps the tar options.
    43  	TarOptions struct {
    44  		IncludeFiles     []string
    45  		ExcludePatterns  []string
    46  		Compression      Compression
    47  		NoLchown         bool
    48  		UIDMaps          []idtools.IDMap
    49  		GIDMaps          []idtools.IDMap
    50  		ChownOpts        *TarChownOptions
    51  		IncludeSourceDir bool
    52  		// WhiteoutFormat is the expected on disk format for whiteout files.
    53  		// This format will be converted to the standard format on pack
    54  		// and from the standard format on unpack.
    55  		WhiteoutFormat WhiteoutFormat
    56  		// When unpacking, specifies whether overwriting a directory with a
    57  		// non-directory is allowed and vice versa.
    58  		NoOverwriteDirNonDir bool
    59  		// For each include when creating an archive, the included name will be
    60  		// replaced with the matching name from this map.
    61  		RebaseNames map[string]string
    62  	}
    63  
    64  	// Archiver allows the reuse of most utility functions of this package
    65  	// with a pluggable Untar function. Also, to facilitate the passing of
    66  	// specific id mappings for untar, an archiver can be created with maps
    67  	// which will then be passed to Untar operations
    68  	Archiver struct {
    69  		Untar   func(io.Reader, string, *TarOptions) error
    70  		UIDMaps []idtools.IDMap
    71  		GIDMaps []idtools.IDMap
    72  	}
    73  
    74  	// breakoutError is used to differentiate errors related to breaking out
    75  	// When testing archive breakout in the unit tests, this error is expected
    76  	// in order for the test to pass.
    77  	breakoutError error
    78  )
    79  
    80  var (
    81  	// ErrNotImplemented is the error message of function not implemented.
    82  	ErrNotImplemented = errors.New("Function not implemented")
    83  	defaultArchiver   = &Archiver{Untar: Untar, UIDMaps: nil, GIDMaps: nil}
    84  )
    85  
    86  const (
    87  	// HeaderSize is the size in bytes of a tar header
    88  	HeaderSize = 512
    89  )
    90  
    91  const (
    92  	// Uncompressed represents the uncompressed.
    93  	Uncompressed Compression = iota
    94  	// Bzip2 is bzip2 compression algorithm.
    95  	Bzip2
    96  	// Gzip is gzip compression algorithm.
    97  	Gzip
    98  	// Xz is xz compression algorithm.
    99  	Xz
   100  )
   101  
   102  const (
   103  	// AUFSWhiteoutFormat is the default format for whiteouts
   104  	AUFSWhiteoutFormat WhiteoutFormat = iota
   105  	// OverlayWhiteoutFormat formats whiteout according to the overlay
   106  	// standard.
   107  	OverlayWhiteoutFormat
   108  )
   109  
   110  // IsArchive checks for the magic bytes of a tar or any supported compression
   111  // algorithm.
   112  func IsArchive(header []byte) bool {
   113  	compression := DetectCompression(header)
   114  	if compression != Uncompressed {
   115  		return true
   116  	}
   117  	r := tar.NewReader(bytes.NewBuffer(header))
   118  	_, err := r.Next()
   119  	return err == nil
   120  }
   121  
   122  // IsArchivePath checks if the (possibly compressed) file at the given path
   123  // starts with a tar file header.
   124  func IsArchivePath(path string) bool {
   125  	file, err := os.Open(path)
   126  	if err != nil {
   127  		return false
   128  	}
   129  	defer file.Close()
   130  	rdr, err := DecompressStream(file)
   131  	if err != nil {
   132  		return false
   133  	}
   134  	r := tar.NewReader(rdr)
   135  	_, err = r.Next()
   136  	return err == nil
   137  }
   138  
   139  // DetectCompression detects the compression algorithm of the source.
   140  func DetectCompression(source []byte) Compression {
   141  	for compression, m := range map[Compression][]byte{
   142  		Bzip2: {0x42, 0x5A, 0x68},
   143  		Gzip:  {0x1F, 0x8B, 0x08},
   144  		Xz:    {0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00},
   145  	} {
   146  		if len(source) < len(m) {
   147  			logrus.Debug("Len too short")
   148  			continue
   149  		}
   150  		if bytes.Compare(m, source[:len(m)]) == 0 {
   151  			return compression
   152  		}
   153  	}
   154  	return Uncompressed
   155  }
   156  
   157  func xzDecompress(archive io.Reader) (io.ReadCloser, <-chan struct{}, error) {
   158  	args := []string{"xz", "-d", "-c", "-q"}
   159  
   160  	return cmdStream(exec.Command(args[0], args[1:]...), archive)
   161  }
   162  
   163  // DecompressStream decompresses the archive and returns a ReaderCloser with the decompressed archive.
   164  func DecompressStream(archive io.Reader) (io.ReadCloser, error) {
   165  	p := pools.BufioReader32KPool
   166  	buf := p.Get(archive)
   167  	bs, err := buf.Peek(10)
   168  	if err != nil && err != io.EOF {
   169  		// Note: we'll ignore any io.EOF error because there are some odd
   170  		// cases where the layer.tar file will be empty (zero bytes) and
   171  		// that results in an io.EOF from the Peek() call. So, in those
   172  		// cases we'll just treat it as a non-compressed stream and
   173  		// that means just create an empty layer.
   174  		// See Issue 18170
   175  		return nil, err
   176  	}
   177  
   178  	compression := DetectCompression(bs)
   179  	switch compression {
   180  	case Uncompressed:
   181  		readBufWrapper := p.NewReadCloserWrapper(buf, buf)
   182  		return readBufWrapper, nil
   183  	case Gzip:
   184  		gzReader, err := gzip.NewReader(buf)
   185  		if err != nil {
   186  			return nil, err
   187  		}
   188  		readBufWrapper := p.NewReadCloserWrapper(buf, gzReader)
   189  		return readBufWrapper, nil
   190  	case Bzip2:
   191  		bz2Reader := bzip2.NewReader(buf)
   192  		readBufWrapper := p.NewReadCloserWrapper(buf, bz2Reader)
   193  		return readBufWrapper, nil
   194  	case Xz:
   195  		xzReader, chdone, err := xzDecompress(buf)
   196  		if err != nil {
   197  			return nil, err
   198  		}
   199  		readBufWrapper := p.NewReadCloserWrapper(buf, xzReader)
   200  		return ioutils.NewReadCloserWrapper(readBufWrapper, func() error {
   201  			<-chdone
   202  			return readBufWrapper.Close()
   203  		}), nil
   204  	default:
   205  		return nil, fmt.Errorf("Unsupported compression format %s", (&compression).Extension())
   206  	}
   207  }
   208  
   209  // CompressStream compresseses the dest with specified compression algorithm.
   210  func CompressStream(dest io.Writer, compression Compression) (io.WriteCloser, error) {
   211  	p := pools.BufioWriter32KPool
   212  	buf := p.Get(dest)
   213  	switch compression {
   214  	case Uncompressed:
   215  		writeBufWrapper := p.NewWriteCloserWrapper(buf, buf)
   216  		return writeBufWrapper, nil
   217  	case Gzip:
   218  		gzWriter := gzip.NewWriter(dest)
   219  		writeBufWrapper := p.NewWriteCloserWrapper(buf, gzWriter)
   220  		return writeBufWrapper, nil
   221  	case Bzip2, Xz:
   222  		// archive/bzip2 does not support writing, and there is no xz support at all
   223  		// However, this is not a problem as docker only currently generates gzipped tars
   224  		return nil, fmt.Errorf("Unsupported compression format %s", (&compression).Extension())
   225  	default:
   226  		return nil, fmt.Errorf("Unsupported compression format %s", (&compression).Extension())
   227  	}
   228  }
   229  
   230  // Extension returns the extension of a file that uses the specified compression algorithm.
   231  func (compression *Compression) Extension() string {
   232  	switch *compression {
   233  	case Uncompressed:
   234  		return "tar"
   235  	case Bzip2:
   236  		return "tar.bz2"
   237  	case Gzip:
   238  		return "tar.gz"
   239  	case Xz:
   240  		return "tar.xz"
   241  	}
   242  	return ""
   243  }
   244  
   245  type tarWhiteoutConverter interface {
   246  	ConvertWrite(*tar.Header, string, os.FileInfo) error
   247  	ConvertRead(*tar.Header, string) (bool, error)
   248  }
   249  
   250  type tarAppender struct {
   251  	TarWriter *tar.Writer
   252  	Buffer    *bufio.Writer
   253  
   254  	// for hardlink mapping
   255  	SeenFiles map[uint64]string
   256  	UIDMaps   []idtools.IDMap
   257  	GIDMaps   []idtools.IDMap
   258  
   259  	// For packing and unpacking whiteout files in the
   260  	// non standard format. The whiteout files defined
   261  	// by the AUFS standard are used as the tar whiteout
   262  	// standard.
   263  	WhiteoutConverter tarWhiteoutConverter
   264  }
   265  
   266  // canonicalTarName provides a platform-independent and consistent posix-style
   267  //path for files and directories to be archived regardless of the platform.
   268  func canonicalTarName(name string, isDir bool) (string, error) {
   269  	name, err := CanonicalTarNameForPath(name)
   270  	if err != nil {
   271  		return "", err
   272  	}
   273  
   274  	// suffix with '/' for directories
   275  	if isDir && !strings.HasSuffix(name, "/") {
   276  		name += "/"
   277  	}
   278  	return name, nil
   279  }
   280  
   281  // addTarFile adds to the tar archive a file from `path` as `name`
   282  func (ta *tarAppender) addTarFile(path, name string) error {
   283  	fi, err := os.Lstat(path)
   284  	if err != nil {
   285  		return err
   286  	}
   287  
   288  	link := ""
   289  	if fi.Mode()&os.ModeSymlink != 0 {
   290  		if link, err = os.Readlink(path); err != nil {
   291  			return err
   292  		}
   293  	}
   294  
   295  	hdr, err := tar.FileInfoHeader(fi, link)
   296  	if err != nil {
   297  		return err
   298  	}
   299  	hdr.Mode = int64(chmodTarEntry(os.FileMode(hdr.Mode)))
   300  
   301  	name, err = canonicalTarName(name, fi.IsDir())
   302  	if err != nil {
   303  		return fmt.Errorf("tar: cannot canonicalize path: %v", err)
   304  	}
   305  	hdr.Name = name
   306  
   307  	inode, err := setHeaderForSpecialDevice(hdr, ta, name, fi.Sys())
   308  	if err != nil {
   309  		return err
   310  	}
   311  
   312  	// if it's not a directory and has more than 1 link,
   313  	// it's hardlinked, so set the type flag accordingly
   314  	if !fi.IsDir() && hasHardlinks(fi) {
   315  		// a link should have a name that it links too
   316  		// and that linked name should be first in the tar archive
   317  		if oldpath, ok := ta.SeenFiles[inode]; ok {
   318  			hdr.Typeflag = tar.TypeLink
   319  			hdr.Linkname = oldpath
   320  			hdr.Size = 0 // This Must be here for the writer math to add up!
   321  		} else {
   322  			ta.SeenFiles[inode] = name
   323  		}
   324  	}
   325  
   326  	capability, _ := system.Lgetxattr(path, "security.capability")
   327  	if capability != nil {
   328  		hdr.Xattrs = make(map[string]string)
   329  		hdr.Xattrs["security.capability"] = string(capability)
   330  	}
   331  
   332  	//handle re-mapping container ID mappings back to host ID mappings before
   333  	//writing tar headers/files. We skip whiteout files because they were written
   334  	//by the kernel and already have proper ownership relative to the host
   335  	if !strings.HasPrefix(filepath.Base(hdr.Name), WhiteoutPrefix) && (ta.UIDMaps != nil || ta.GIDMaps != nil) {
   336  		uid, gid, err := getFileUIDGID(fi.Sys())
   337  		if err != nil {
   338  			return err
   339  		}
   340  		xUID, err := idtools.ToContainer(uid, ta.UIDMaps)
   341  		if err != nil {
   342  			return err
   343  		}
   344  		xGID, err := idtools.ToContainer(gid, ta.GIDMaps)
   345  		if err != nil {
   346  			return err
   347  		}
   348  		hdr.Uid = xUID
   349  		hdr.Gid = xGID
   350  	}
   351  
   352  	if ta.WhiteoutConverter != nil {
   353  		if err := ta.WhiteoutConverter.ConvertWrite(hdr, path, fi); err != nil {
   354  			return err
   355  		}
   356  	}
   357  
   358  	if err := ta.TarWriter.WriteHeader(hdr); err != nil {
   359  		return err
   360  	}
   361  
   362  	if hdr.Typeflag == tar.TypeReg && hdr.Size > 0 {
   363  		file, err := os.Open(path)
   364  		if err != nil {
   365  			return err
   366  		}
   367  
   368  		ta.Buffer.Reset(ta.TarWriter)
   369  		defer ta.Buffer.Reset(nil)
   370  		_, err = io.Copy(ta.Buffer, file)
   371  		file.Close()
   372  		if err != nil {
   373  			return err
   374  		}
   375  		err = ta.Buffer.Flush()
   376  		if err != nil {
   377  			return err
   378  		}
   379  	}
   380  
   381  	return nil
   382  }
   383  
   384  func createTarFile(path, extractDir string, hdr *tar.Header, reader io.Reader, Lchown bool, chownOpts *TarChownOptions) error {
   385  	// hdr.Mode is in linux format, which we can use for sycalls,
   386  	// but for os.Foo() calls we need the mode converted to os.FileMode,
   387  	// so use hdrInfo.Mode() (they differ for e.g. setuid bits)
   388  	hdrInfo := hdr.FileInfo()
   389  
   390  	switch hdr.Typeflag {
   391  	case tar.TypeDir:
   392  		// Create directory unless it exists as a directory already.
   393  		// In that case we just want to merge the two
   394  		if fi, err := os.Lstat(path); !(err == nil && fi.IsDir()) {
   395  			if err := os.Mkdir(path, hdrInfo.Mode()); err != nil {
   396  				return err
   397  			}
   398  		}
   399  
   400  	case tar.TypeReg, tar.TypeRegA:
   401  		// Source is regular file
   402  		file, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, hdrInfo.Mode())
   403  		if err != nil {
   404  			return err
   405  		}
   406  		if _, err := io.Copy(file, reader); err != nil {
   407  			file.Close()
   408  			return err
   409  		}
   410  		file.Close()
   411  
   412  	case tar.TypeBlock, tar.TypeChar, tar.TypeFifo:
   413  		// Handle this is an OS-specific way
   414  		if err := handleTarTypeBlockCharFifo(hdr, path); err != nil {
   415  			return err
   416  		}
   417  
   418  	case tar.TypeLink:
   419  		targetPath := filepath.Join(extractDir, hdr.Linkname)
   420  		// check for hardlink breakout
   421  		if !strings.HasPrefix(targetPath, extractDir) {
   422  			return breakoutError(fmt.Errorf("invalid hardlink %q -> %q", targetPath, hdr.Linkname))
   423  		}
   424  		if err := os.Link(targetPath, path); err != nil {
   425  			return err
   426  		}
   427  
   428  	case tar.TypeSymlink:
   429  		// 	path 				-> hdr.Linkname = targetPath
   430  		// e.g. /extractDir/path/to/symlink 	-> ../2/file	= /extractDir/path/2/file
   431  		targetPath := filepath.Join(filepath.Dir(path), hdr.Linkname)
   432  
   433  		// the reason we don't need to check symlinks in the path (with FollowSymlinkInScope) is because
   434  		// that symlink would first have to be created, which would be caught earlier, at this very check:
   435  		if !strings.HasPrefix(targetPath, extractDir) {
   436  			return breakoutError(fmt.Errorf("invalid symlink %q -> %q", path, hdr.Linkname))
   437  		}
   438  		if err := os.Symlink(hdr.Linkname, path); err != nil {
   439  			return err
   440  		}
   441  
   442  	case tar.TypeXGlobalHeader:
   443  		logrus.Debug("PAX Global Extended Headers found and ignored")
   444  		return nil
   445  
   446  	default:
   447  		return fmt.Errorf("Unhandled tar header type %d\n", hdr.Typeflag)
   448  	}
   449  
   450  	// Lchown is not supported on Windows.
   451  	if Lchown && runtime.GOOS != "windows" {
   452  		if chownOpts == nil {
   453  			chownOpts = &TarChownOptions{UID: hdr.Uid, GID: hdr.Gid}
   454  		}
   455  		if err := os.Lchown(path, chownOpts.UID, chownOpts.GID); err != nil {
   456  			return err
   457  		}
   458  	}
   459  
   460  	var errors []string
   461  	for key, value := range hdr.Xattrs {
   462  		if err := system.Lsetxattr(path, key, []byte(value), 0); err != nil {
   463  			if err == syscall.ENOTSUP {
   464  				// We ignore errors here because not all graphdrivers support
   465  				// xattrs *cough* old versions of AUFS *cough*. However only
   466  				// ENOTSUP should be emitted in that case, otherwise we still
   467  				// bail.
   468  				errors = append(errors, err.Error())
   469  				continue
   470  			}
   471  			return err
   472  		}
   473  
   474  	}
   475  
   476  	if len(errors) > 0 {
   477  		logrus.WithFields(logrus.Fields{
   478  			"errors": errors,
   479  		}).Warn("ignored xattrs in archive: underlying filesystem doesn't support them")
   480  	}
   481  
   482  	// There is no LChmod, so ignore mode for symlink. Also, this
   483  	// must happen after chown, as that can modify the file mode
   484  	if err := handleLChmod(hdr, path, hdrInfo); err != nil {
   485  		return err
   486  	}
   487  
   488  	aTime := hdr.AccessTime
   489  	if aTime.Before(hdr.ModTime) {
   490  		// Last access time should never be before last modified time.
   491  		aTime = hdr.ModTime
   492  	}
   493  
   494  	// system.Chtimes doesn't support a NOFOLLOW flag atm
   495  	if hdr.Typeflag == tar.TypeLink {
   496  		if fi, err := os.Lstat(hdr.Linkname); err == nil && (fi.Mode()&os.ModeSymlink == 0) {
   497  			if err := system.Chtimes(path, aTime, hdr.ModTime); err != nil {
   498  				return err
   499  			}
   500  		}
   501  	} else if hdr.Typeflag != tar.TypeSymlink {
   502  		if err := system.Chtimes(path, aTime, hdr.ModTime); err != nil {
   503  			return err
   504  		}
   505  	} else {
   506  		ts := []syscall.Timespec{timeToTimespec(aTime), timeToTimespec(hdr.ModTime)}
   507  		if err := system.LUtimesNano(path, ts); err != nil && err != system.ErrNotSupportedPlatform {
   508  			return err
   509  		}
   510  	}
   511  	return nil
   512  }
   513  
   514  // Tar creates an archive from the directory at `path`, and returns it as a
   515  // stream of bytes.
   516  func Tar(path string, compression Compression) (io.ReadCloser, error) {
   517  	return TarWithOptions(path, &TarOptions{Compression: compression})
   518  }
   519  
   520  // TarWithOptions creates an archive from the directory at `path`, only including files whose relative
   521  // paths are included in `options.IncludeFiles` (if non-nil) or not in `options.ExcludePatterns`.
   522  func TarWithOptions(srcPath string, options *TarOptions) (io.ReadCloser, error) {
   523  
   524  	// Fix the source path to work with long path names. This is a no-op
   525  	// on platforms other than Windows.
   526  	srcPath = fixVolumePathPrefix(srcPath)
   527  
   528  	patterns, patDirs, exceptions, err := fileutils.CleanPatterns(options.ExcludePatterns)
   529  
   530  	if err != nil {
   531  		return nil, err
   532  	}
   533  
   534  	pipeReader, pipeWriter := io.Pipe()
   535  
   536  	compressWriter, err := CompressStream(pipeWriter, options.Compression)
   537  	if err != nil {
   538  		return nil, err
   539  	}
   540  
   541  	go func() {
   542  		ta := &tarAppender{
   543  			TarWriter:         tar.NewWriter(compressWriter),
   544  			Buffer:            pools.BufioWriter32KPool.Get(nil),
   545  			SeenFiles:         make(map[uint64]string),
   546  			UIDMaps:           options.UIDMaps,
   547  			GIDMaps:           options.GIDMaps,
   548  			WhiteoutConverter: getWhiteoutConverter(options.WhiteoutFormat),
   549  		}
   550  
   551  		defer func() {
   552  			// Make sure to check the error on Close.
   553  			if err := ta.TarWriter.Close(); err != nil {
   554  				logrus.Errorf("Can't close tar writer: %s", err)
   555  			}
   556  			if err := compressWriter.Close(); err != nil {
   557  				logrus.Errorf("Can't close compress writer: %s", err)
   558  			}
   559  			if err := pipeWriter.Close(); err != nil {
   560  				logrus.Errorf("Can't close pipe writer: %s", err)
   561  			}
   562  		}()
   563  
   564  		// this buffer is needed for the duration of this piped stream
   565  		defer pools.BufioWriter32KPool.Put(ta.Buffer)
   566  
   567  		// In general we log errors here but ignore them because
   568  		// during e.g. a diff operation the container can continue
   569  		// mutating the filesystem and we can see transient errors
   570  		// from this
   571  
   572  		stat, err := os.Lstat(srcPath)
   573  		if err != nil {
   574  			return
   575  		}
   576  
   577  		if !stat.IsDir() {
   578  			// We can't later join a non-dir with any includes because the
   579  			// 'walk' will error if "file/." is stat-ed and "file" is not a
   580  			// directory. So, we must split the source path and use the
   581  			// basename as the include.
   582  			if len(options.IncludeFiles) > 0 {
   583  				logrus.Warn("Tar: Can't archive a file with includes")
   584  			}
   585  
   586  			dir, base := SplitPathDirEntry(srcPath)
   587  			srcPath = dir
   588  			options.IncludeFiles = []string{base}
   589  		}
   590  
   591  		if len(options.IncludeFiles) == 0 {
   592  			options.IncludeFiles = []string{"."}
   593  		}
   594  
   595  		seen := make(map[string]bool)
   596  
   597  		for _, include := range options.IncludeFiles {
   598  			rebaseName := options.RebaseNames[include]
   599  
   600  			walkRoot := getWalkRoot(srcPath, include)
   601  			filepath.Walk(walkRoot, func(filePath string, f os.FileInfo, err error) error {
   602  				if err != nil {
   603  					logrus.Errorf("Tar: Can't stat file %s to tar: %s", srcPath, err)
   604  					return nil
   605  				}
   606  
   607  				relFilePath, err := filepath.Rel(srcPath, filePath)
   608  				if err != nil || (!options.IncludeSourceDir && relFilePath == "." && f.IsDir()) {
   609  					// Error getting relative path OR we are looking
   610  					// at the source directory path. Skip in both situations.
   611  					return nil
   612  				}
   613  
   614  				if options.IncludeSourceDir && include == "." && relFilePath != "." {
   615  					relFilePath = strings.Join([]string{".", relFilePath}, string(filepath.Separator))
   616  				}
   617  
   618  				skip := false
   619  
   620  				// If "include" is an exact match for the current file
   621  				// then even if there's an "excludePatterns" pattern that
   622  				// matches it, don't skip it. IOW, assume an explicit 'include'
   623  				// is asking for that file no matter what - which is true
   624  				// for some files, like .dockerignore and Dockerfile (sometimes)
   625  				if include != relFilePath {
   626  					skip, err = fileutils.OptimizedMatches(relFilePath, patterns, patDirs)
   627  					if err != nil {
   628  						logrus.Errorf("Error matching %s: %v", relFilePath, err)
   629  						return err
   630  					}
   631  				}
   632  
   633  				if skip {
   634  					// If we want to skip this file and its a directory
   635  					// then we should first check to see if there's an
   636  					// excludes pattern (eg !dir/file) that starts with this
   637  					// dir. If so then we can't skip this dir.
   638  
   639  					// Its not a dir then so we can just return/skip.
   640  					if !f.IsDir() {
   641  						return nil
   642  					}
   643  
   644  					// No exceptions (!...) in patterns so just skip dir
   645  					if !exceptions {
   646  						return filepath.SkipDir
   647  					}
   648  
   649  					dirSlash := relFilePath + string(filepath.Separator)
   650  
   651  					for _, pat := range patterns {
   652  						if pat[0] != '!' {
   653  							continue
   654  						}
   655  						pat = pat[1:] + string(filepath.Separator)
   656  						if strings.HasPrefix(pat, dirSlash) {
   657  							// found a match - so can't skip this dir
   658  							return nil
   659  						}
   660  					}
   661  
   662  					// No matching exclusion dir so just skip dir
   663  					return filepath.SkipDir
   664  				}
   665  
   666  				if seen[relFilePath] {
   667  					return nil
   668  				}
   669  				seen[relFilePath] = true
   670  
   671  				// Rename the base resource.
   672  				if rebaseName != "" {
   673  					var replacement string
   674  					if rebaseName != string(filepath.Separator) {
   675  						// Special case the root directory to replace with an
   676  						// empty string instead so that we don't end up with
   677  						// double slashes in the paths.
   678  						replacement = rebaseName
   679  					}
   680  
   681  					relFilePath = strings.Replace(relFilePath, include, replacement, 1)
   682  				}
   683  
   684  				if err := ta.addTarFile(filePath, relFilePath); err != nil {
   685  					logrus.Errorf("Can't add file %s to tar: %s", filePath, err)
   686  					// if pipe is broken, stop writing tar stream to it
   687  					if err == io.ErrClosedPipe {
   688  						return err
   689  					}
   690  				}
   691  				return nil
   692  			})
   693  		}
   694  	}()
   695  
   696  	return pipeReader, nil
   697  }
   698  
   699  // Unpack unpacks the decompressedArchive to dest with options.
   700  func Unpack(decompressedArchive io.Reader, dest string, options *TarOptions) error {
   701  	tr := tar.NewReader(decompressedArchive)
   702  	trBuf := pools.BufioReader32KPool.Get(nil)
   703  	defer pools.BufioReader32KPool.Put(trBuf)
   704  
   705  	var dirs []*tar.Header
   706  	remappedRootUID, remappedRootGID, err := idtools.GetRootUIDGID(options.UIDMaps, options.GIDMaps)
   707  	if err != nil {
   708  		return err
   709  	}
   710  	whiteoutConverter := getWhiteoutConverter(options.WhiteoutFormat)
   711  
   712  	// Iterate through the files in the archive.
   713  loop:
   714  	for {
   715  		hdr, err := tr.Next()
   716  		if err == io.EOF {
   717  			// end of tar archive
   718  			break
   719  		}
   720  		if err != nil {
   721  			return err
   722  		}
   723  
   724  		// Normalize name, for safety and for a simple is-root check
   725  		// This keeps "../" as-is, but normalizes "/../" to "/". Or Windows:
   726  		// This keeps "..\" as-is, but normalizes "\..\" to "\".
   727  		hdr.Name = filepath.Clean(hdr.Name)
   728  
   729  		for _, exclude := range options.ExcludePatterns {
   730  			if strings.HasPrefix(hdr.Name, exclude) {
   731  				continue loop
   732  			}
   733  		}
   734  
   735  		// After calling filepath.Clean(hdr.Name) above, hdr.Name will now be in
   736  		// the filepath format for the OS on which the daemon is running. Hence
   737  		// the check for a slash-suffix MUST be done in an OS-agnostic way.
   738  		if !strings.HasSuffix(hdr.Name, string(os.PathSeparator)) {
   739  			// Not the root directory, ensure that the parent directory exists
   740  			parent := filepath.Dir(hdr.Name)
   741  			parentPath := filepath.Join(dest, parent)
   742  			if _, err := os.Lstat(parentPath); err != nil && os.IsNotExist(err) {
   743  				err = idtools.MkdirAllNewAs(parentPath, 0777, remappedRootUID, remappedRootGID)
   744  				if err != nil {
   745  					return err
   746  				}
   747  			}
   748  		}
   749  
   750  		path := filepath.Join(dest, hdr.Name)
   751  		rel, err := filepath.Rel(dest, path)
   752  		if err != nil {
   753  			return err
   754  		}
   755  		if strings.HasPrefix(rel, ".."+string(os.PathSeparator)) {
   756  			return breakoutError(fmt.Errorf("%q is outside of %q", hdr.Name, dest))
   757  		}
   758  
   759  		// If path exits we almost always just want to remove and replace it
   760  		// The only exception is when it is a directory *and* the file from
   761  		// the layer is also a directory. Then we want to merge them (i.e.
   762  		// just apply the metadata from the layer).
   763  		if fi, err := os.Lstat(path); err == nil {
   764  			if options.NoOverwriteDirNonDir && fi.IsDir() && hdr.Typeflag != tar.TypeDir {
   765  				// If NoOverwriteDirNonDir is true then we cannot replace
   766  				// an existing directory with a non-directory from the archive.
   767  				return fmt.Errorf("cannot overwrite directory %q with non-directory %q", path, dest)
   768  			}
   769  
   770  			if options.NoOverwriteDirNonDir && !fi.IsDir() && hdr.Typeflag == tar.TypeDir {
   771  				// If NoOverwriteDirNonDir is true then we cannot replace
   772  				// an existing non-directory with a directory from the archive.
   773  				return fmt.Errorf("cannot overwrite non-directory %q with directory %q", path, dest)
   774  			}
   775  
   776  			if fi.IsDir() && hdr.Name == "." {
   777  				continue
   778  			}
   779  
   780  			if !(fi.IsDir() && hdr.Typeflag == tar.TypeDir) {
   781  				if err := os.RemoveAll(path); err != nil {
   782  					return err
   783  				}
   784  			}
   785  		}
   786  		trBuf.Reset(tr)
   787  
   788  		// if the options contain a uid & gid maps, convert header uid/gid
   789  		// entries using the maps such that lchown sets the proper mapped
   790  		// uid/gid after writing the file. We only perform this mapping if
   791  		// the file isn't already owned by the remapped root UID or GID, as
   792  		// that specific uid/gid has no mapping from container -> host, and
   793  		// those files already have the proper ownership for inside the
   794  		// container.
   795  		if hdr.Uid != remappedRootUID {
   796  			xUID, err := idtools.ToHost(hdr.Uid, options.UIDMaps)
   797  			if err != nil {
   798  				return err
   799  			}
   800  			hdr.Uid = xUID
   801  		}
   802  		if hdr.Gid != remappedRootGID {
   803  			xGID, err := idtools.ToHost(hdr.Gid, options.GIDMaps)
   804  			if err != nil {
   805  				return err
   806  			}
   807  			hdr.Gid = xGID
   808  		}
   809  
   810  		if whiteoutConverter != nil {
   811  			writeFile, err := whiteoutConverter.ConvertRead(hdr, path)
   812  			if err != nil {
   813  				return err
   814  			}
   815  			if !writeFile {
   816  				continue
   817  			}
   818  		}
   819  
   820  		if err := createTarFile(path, dest, hdr, trBuf, !options.NoLchown, options.ChownOpts); err != nil {
   821  			return err
   822  		}
   823  
   824  		// Directory mtimes must be handled at the end to avoid further
   825  		// file creation in them to modify the directory mtime
   826  		if hdr.Typeflag == tar.TypeDir {
   827  			dirs = append(dirs, hdr)
   828  		}
   829  	}
   830  
   831  	for _, hdr := range dirs {
   832  		path := filepath.Join(dest, hdr.Name)
   833  
   834  		if err := system.Chtimes(path, hdr.AccessTime, hdr.ModTime); err != nil {
   835  			return err
   836  		}
   837  	}
   838  	return nil
   839  }
   840  
   841  // Untar reads a stream of bytes from `archive`, parses it as a tar archive,
   842  // and unpacks it into the directory at `dest`.
   843  // The archive may be compressed with one of the following algorithms:
   844  //  identity (uncompressed), gzip, bzip2, xz.
   845  // FIXME: specify behavior when target path exists vs. doesn't exist.
   846  func Untar(tarArchive io.Reader, dest string, options *TarOptions) error {
   847  	return untarHandler(tarArchive, dest, options, true)
   848  }
   849  
   850  // UntarUncompressed reads a stream of bytes from `archive`, parses it as a tar archive,
   851  // and unpacks it into the directory at `dest`.
   852  // The archive must be an uncompressed stream.
   853  func UntarUncompressed(tarArchive io.Reader, dest string, options *TarOptions) error {
   854  	return untarHandler(tarArchive, dest, options, false)
   855  }
   856  
   857  // Handler for teasing out the automatic decompression
   858  func untarHandler(tarArchive io.Reader, dest string, options *TarOptions, decompress bool) error {
   859  	if tarArchive == nil {
   860  		return fmt.Errorf("Empty archive")
   861  	}
   862  	dest = filepath.Clean(dest)
   863  	if options == nil {
   864  		options = &TarOptions{}
   865  	}
   866  	if options.ExcludePatterns == nil {
   867  		options.ExcludePatterns = []string{}
   868  	}
   869  
   870  	r := tarArchive
   871  	if decompress {
   872  		decompressedArchive, err := DecompressStream(tarArchive)
   873  		if err != nil {
   874  			return err
   875  		}
   876  		defer decompressedArchive.Close()
   877  		r = decompressedArchive
   878  	}
   879  
   880  	return Unpack(r, dest, options)
   881  }
   882  
   883  // TarUntar is a convenience function which calls Tar and Untar, with the output of one piped into the other.
   884  // If either Tar or Untar fails, TarUntar aborts and returns the error.
   885  func (archiver *Archiver) TarUntar(src, dst string) error {
   886  	logrus.Debugf("TarUntar(%s %s)", src, dst)
   887  	archive, err := TarWithOptions(src, &TarOptions{Compression: Uncompressed})
   888  	if err != nil {
   889  		return err
   890  	}
   891  	defer archive.Close()
   892  
   893  	var options *TarOptions
   894  	if archiver.UIDMaps != nil || archiver.GIDMaps != nil {
   895  		options = &TarOptions{
   896  			UIDMaps: archiver.UIDMaps,
   897  			GIDMaps: archiver.GIDMaps,
   898  		}
   899  	}
   900  	return archiver.Untar(archive, dst, options)
   901  }
   902  
   903  // TarUntar is a convenience function which calls Tar and Untar, with the output of one piped into the other.
   904  // If either Tar or Untar fails, TarUntar aborts and returns the error.
   905  func TarUntar(src, dst string) error {
   906  	return defaultArchiver.TarUntar(src, dst)
   907  }
   908  
   909  // UntarPath untar a file from path to a destination, src is the source tar file path.
   910  func (archiver *Archiver) UntarPath(src, dst string) error {
   911  	archive, err := os.Open(src)
   912  	if err != nil {
   913  		return err
   914  	}
   915  	defer archive.Close()
   916  	var options *TarOptions
   917  	if archiver.UIDMaps != nil || archiver.GIDMaps != nil {
   918  		options = &TarOptions{
   919  			UIDMaps: archiver.UIDMaps,
   920  			GIDMaps: archiver.GIDMaps,
   921  		}
   922  	}
   923  	return archiver.Untar(archive, dst, options)
   924  }
   925  
   926  // UntarPath is a convenience function which looks for an archive
   927  // at filesystem path `src`, and unpacks it at `dst`.
   928  func UntarPath(src, dst string) error {
   929  	return defaultArchiver.UntarPath(src, dst)
   930  }
   931  
   932  // CopyWithTar creates a tar archive of filesystem path `src`, and
   933  // unpacks it at filesystem path `dst`.
   934  // The archive is streamed directly with fixed buffering and no
   935  // intermediary disk IO.
   936  func (archiver *Archiver) CopyWithTar(src, dst string) error {
   937  	srcSt, err := os.Stat(src)
   938  	if err != nil {
   939  		return err
   940  	}
   941  	if !srcSt.IsDir() {
   942  		return archiver.CopyFileWithTar(src, dst)
   943  	}
   944  
   945  	// if this archiver is set up with ID mapping we need to create
   946  	// the new destination directory with the remapped root UID/GID pair
   947  	// as owner
   948  	rootUID, rootGID, err := idtools.GetRootUIDGID(archiver.UIDMaps, archiver.GIDMaps)
   949  	if err != nil {
   950  		return err
   951  	}
   952  	// Create dst, copy src's content into it
   953  	logrus.Debugf("Creating dest directory: %s", dst)
   954  	if err := idtools.MkdirAllNewAs(dst, 0755, rootUID, rootGID); err != nil {
   955  		return err
   956  	}
   957  	logrus.Debugf("Calling TarUntar(%s, %s)", src, dst)
   958  	return archiver.TarUntar(src, dst)
   959  }
   960  
   961  // CopyWithTar creates a tar archive of filesystem path `src`, and
   962  // unpacks it at filesystem path `dst`.
   963  // The archive is streamed directly with fixed buffering and no
   964  // intermediary disk IO.
   965  func CopyWithTar(src, dst string) error {
   966  	return defaultArchiver.CopyWithTar(src, dst)
   967  }
   968  
   969  // CopyFileWithTar emulates the behavior of the 'cp' command-line
   970  // for a single file. It copies a regular file from path `src` to
   971  // path `dst`, and preserves all its metadata.
   972  func (archiver *Archiver) CopyFileWithTar(src, dst string) (err error) {
   973  	logrus.Debugf("CopyFileWithTar(%s, %s)", src, dst)
   974  	srcSt, err := os.Stat(src)
   975  	if err != nil {
   976  		return err
   977  	}
   978  
   979  	if srcSt.IsDir() {
   980  		return fmt.Errorf("Can't copy a directory")
   981  	}
   982  
   983  	// Clean up the trailing slash. This must be done in an operating
   984  	// system specific manner.
   985  	if dst[len(dst)-1] == os.PathSeparator {
   986  		dst = filepath.Join(dst, filepath.Base(src))
   987  	}
   988  	// Create the holding directory if necessary
   989  	if err := system.MkdirAll(filepath.Dir(dst), 0700); err != nil {
   990  		return err
   991  	}
   992  
   993  	r, w := io.Pipe()
   994  	errC := promise.Go(func() error {
   995  		defer w.Close()
   996  
   997  		srcF, err := os.Open(src)
   998  		if err != nil {
   999  			return err
  1000  		}
  1001  		defer srcF.Close()
  1002  
  1003  		hdr, err := tar.FileInfoHeader(srcSt, "")
  1004  		if err != nil {
  1005  			return err
  1006  		}
  1007  		hdr.Name = filepath.Base(dst)
  1008  		hdr.Mode = int64(chmodTarEntry(os.FileMode(hdr.Mode)))
  1009  
  1010  		remappedRootUID, remappedRootGID, err := idtools.GetRootUIDGID(archiver.UIDMaps, archiver.GIDMaps)
  1011  		if err != nil {
  1012  			return err
  1013  		}
  1014  
  1015  		// only perform mapping if the file being copied isn't already owned by the
  1016  		// uid or gid of the remapped root in the container
  1017  		if remappedRootUID != hdr.Uid {
  1018  			xUID, err := idtools.ToHost(hdr.Uid, archiver.UIDMaps)
  1019  			if err != nil {
  1020  				return err
  1021  			}
  1022  			hdr.Uid = xUID
  1023  		}
  1024  		if remappedRootGID != hdr.Gid {
  1025  			xGID, err := idtools.ToHost(hdr.Gid, archiver.GIDMaps)
  1026  			if err != nil {
  1027  				return err
  1028  			}
  1029  			hdr.Gid = xGID
  1030  		}
  1031  
  1032  		tw := tar.NewWriter(w)
  1033  		defer tw.Close()
  1034  		if err := tw.WriteHeader(hdr); err != nil {
  1035  			return err
  1036  		}
  1037  		if _, err := io.Copy(tw, srcF); err != nil {
  1038  			return err
  1039  		}
  1040  		return nil
  1041  	})
  1042  	defer func() {
  1043  		if er := <-errC; err != nil {
  1044  			err = er
  1045  		}
  1046  	}()
  1047  
  1048  	err = archiver.Untar(r, filepath.Dir(dst), nil)
  1049  	if err != nil {
  1050  		r.CloseWithError(err)
  1051  	}
  1052  	return err
  1053  }
  1054  
  1055  // CopyFileWithTar emulates the behavior of the 'cp' command-line
  1056  // for a single file. It copies a regular file from path `src` to
  1057  // path `dst`, and preserves all its metadata.
  1058  //
  1059  // Destination handling is in an operating specific manner depending
  1060  // where the daemon is running. If `dst` ends with a trailing slash
  1061  // the final destination path will be `dst/base(src)`  (Linux) or
  1062  // `dst\base(src)` (Windows).
  1063  func CopyFileWithTar(src, dst string) (err error) {
  1064  	return defaultArchiver.CopyFileWithTar(src, dst)
  1065  }
  1066  
  1067  // cmdStream executes a command, and returns its stdout as a stream.
  1068  // If the command fails to run or doesn't complete successfully, an error
  1069  // will be returned, including anything written on stderr.
  1070  func cmdStream(cmd *exec.Cmd, input io.Reader) (io.ReadCloser, <-chan struct{}, error) {
  1071  	chdone := make(chan struct{})
  1072  	cmd.Stdin = input
  1073  	pipeR, pipeW := io.Pipe()
  1074  	cmd.Stdout = pipeW
  1075  	var errBuf bytes.Buffer
  1076  	cmd.Stderr = &errBuf
  1077  
  1078  	// Run the command and return the pipe
  1079  	if err := cmd.Start(); err != nil {
  1080  		return nil, nil, err
  1081  	}
  1082  
  1083  	// Copy stdout to the returned pipe
  1084  	go func() {
  1085  		if err := cmd.Wait(); err != nil {
  1086  			pipeW.CloseWithError(fmt.Errorf("%s: %s", err, errBuf.String()))
  1087  		} else {
  1088  			pipeW.Close()
  1089  		}
  1090  		close(chdone)
  1091  	}()
  1092  
  1093  	return pipeR, chdone, nil
  1094  }
  1095  
  1096  // NewTempArchive reads the content of src into a temporary file, and returns the contents
  1097  // of that file as an archive. The archive can only be read once - as soon as reading completes,
  1098  // the file will be deleted.
  1099  func NewTempArchive(src Archive, dir string) (*TempArchive, error) {
  1100  	f, err := ioutil.TempFile(dir, "")
  1101  	if err != nil {
  1102  		return nil, err
  1103  	}
  1104  	if _, err := io.Copy(f, src); err != nil {
  1105  		return nil, err
  1106  	}
  1107  	if _, err := f.Seek(0, 0); err != nil {
  1108  		return nil, err
  1109  	}
  1110  	st, err := f.Stat()
  1111  	if err != nil {
  1112  		return nil, err
  1113  	}
  1114  	size := st.Size()
  1115  	return &TempArchive{File: f, Size: size}, nil
  1116  }
  1117  
  1118  // TempArchive is a temporary archive. The archive can only be read once - as soon as reading completes,
  1119  // the file will be deleted.
  1120  type TempArchive struct {
  1121  	*os.File
  1122  	Size   int64 // Pre-computed from Stat().Size() as a convenience
  1123  	read   int64
  1124  	closed bool
  1125  }
  1126  
  1127  // Close closes the underlying file if it's still open, or does a no-op
  1128  // to allow callers to try to close the TempArchive multiple times safely.
  1129  func (archive *TempArchive) Close() error {
  1130  	if archive.closed {
  1131  		return nil
  1132  	}
  1133  
  1134  	archive.closed = true
  1135  
  1136  	return archive.File.Close()
  1137  }
  1138  
  1139  func (archive *TempArchive) Read(data []byte) (int, error) {
  1140  	n, err := archive.File.Read(data)
  1141  	archive.read += int64(n)
  1142  	if err != nil || archive.read == archive.Size {
  1143  		archive.Close()
  1144  		os.Remove(archive.File.Name())
  1145  	}
  1146  	return n, err
  1147  }