github.com/hustcat/docker@v1.3.3-0.20160314103604-901c67a8eeab/pkg/archive/archive.go (about)

     1  package archive
     2  
     3  import (
     4  	"archive/tar"
     5  	"bufio"
     6  	"bytes"
     7  	"compress/bzip2"
     8  	"compress/gzip"
     9  	"errors"
    10  	"fmt"
    11  	"io"
    12  	"io/ioutil"
    13  	"os"
    14  	"os/exec"
    15  	"path/filepath"
    16  	"runtime"
    17  	"strings"
    18  	"syscall"
    19  
    20  	"github.com/Sirupsen/logrus"
    21  	"github.com/docker/docker/pkg/fileutils"
    22  	"github.com/docker/docker/pkg/idtools"
    23  	"github.com/docker/docker/pkg/ioutils"
    24  	"github.com/docker/docker/pkg/pools"
    25  	"github.com/docker/docker/pkg/promise"
    26  	"github.com/docker/docker/pkg/system"
    27  )
    28  
    29  type (
    30  	// Archive is a type of io.ReadCloser which has two interfaces Read and Closer.
    31  	Archive io.ReadCloser
    32  	// Reader is a type of io.Reader.
    33  	Reader io.Reader
    34  	// Compression is the state represents if compressed or not.
    35  	Compression int
    36  	// TarChownOptions wraps the chown options UID and GID.
    37  	TarChownOptions struct {
    38  		UID, GID int
    39  	}
    40  	// TarOptions wraps the tar options.
    41  	TarOptions struct {
    42  		IncludeFiles     []string
    43  		ExcludePatterns  []string
    44  		Compression      Compression
    45  		NoLchown         bool
    46  		UIDMaps          []idtools.IDMap
    47  		GIDMaps          []idtools.IDMap
    48  		ChownOpts        *TarChownOptions
    49  		IncludeSourceDir bool
    50  		// When unpacking, specifies whether overwriting a directory with a
    51  		// non-directory is allowed and vice versa.
    52  		NoOverwriteDirNonDir bool
    53  		// For each include when creating an archive, the included name will be
    54  		// replaced with the matching name from this map.
    55  		RebaseNames map[string]string
    56  	}
    57  
    58  	// Archiver allows the reuse of most utility functions of this package
    59  	// with a pluggable Untar function. Also, to facilitate the passing of
    60  	// specific id mappings for untar, an archiver can be created with maps
    61  	// which will then be passed to Untar operations
    62  	Archiver struct {
    63  		Untar   func(io.Reader, string, *TarOptions) error
    64  		UIDMaps []idtools.IDMap
    65  		GIDMaps []idtools.IDMap
    66  	}
    67  
    68  	// breakoutError is used to differentiate errors related to breaking out
    69  	// When testing archive breakout in the unit tests, this error is expected
    70  	// in order for the test to pass.
    71  	breakoutError error
    72  )
    73  
    74  var (
    75  	// ErrNotImplemented is the error message of function not implemented.
    76  	ErrNotImplemented = errors.New("Function not implemented")
    77  	defaultArchiver   = &Archiver{Untar: Untar, UIDMaps: nil, GIDMaps: nil}
    78  )
    79  
    80  const (
    81  	// HeaderSize is the size in bytes of a tar header
    82  	HeaderSize = 512
    83  )
    84  
    85  const (
    86  	// Uncompressed represents the uncompressed.
    87  	Uncompressed Compression = iota
    88  	// Bzip2 is bzip2 compression algorithm.
    89  	Bzip2
    90  	// Gzip is gzip compression algorithm.
    91  	Gzip
    92  	// Xz is xz compression algorithm.
    93  	Xz
    94  )
    95  
    96  // IsArchive checks for the magic bytes of a tar or any supported compression
    97  // algorithm.
    98  func IsArchive(header []byte) bool {
    99  	compression := DetectCompression(header)
   100  	if compression != Uncompressed {
   101  		return true
   102  	}
   103  	r := tar.NewReader(bytes.NewBuffer(header))
   104  	_, err := r.Next()
   105  	return err == nil
   106  }
   107  
   108  // IsArchivePath checks if the (possibly compressed) file at the given path
   109  // starts with a tar file header.
   110  func IsArchivePath(path string) bool {
   111  	file, err := os.Open(path)
   112  	if err != nil {
   113  		return false
   114  	}
   115  	defer file.Close()
   116  	rdr, err := DecompressStream(file)
   117  	if err != nil {
   118  		return false
   119  	}
   120  	r := tar.NewReader(rdr)
   121  	_, err = r.Next()
   122  	return err == nil
   123  }
   124  
   125  // DetectCompression detects the compression algorithm of the source.
   126  func DetectCompression(source []byte) Compression {
   127  	for compression, m := range map[Compression][]byte{
   128  		Bzip2: {0x42, 0x5A, 0x68},
   129  		Gzip:  {0x1F, 0x8B, 0x08},
   130  		Xz:    {0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00},
   131  	} {
   132  		if len(source) < len(m) {
   133  			logrus.Debugf("Len too short")
   134  			continue
   135  		}
   136  		if bytes.Compare(m, source[:len(m)]) == 0 {
   137  			return compression
   138  		}
   139  	}
   140  	return Uncompressed
   141  }
   142  
   143  func xzDecompress(archive io.Reader) (io.ReadCloser, <-chan struct{}, error) {
   144  	args := []string{"xz", "-d", "-c", "-q"}
   145  
   146  	return cmdStream(exec.Command(args[0], args[1:]...), archive)
   147  }
   148  
   149  // DecompressStream decompress the archive and returns a ReaderCloser with the decompressed archive.
   150  func DecompressStream(archive io.Reader) (io.ReadCloser, error) {
   151  	p := pools.BufioReader32KPool
   152  	buf := p.Get(archive)
   153  	bs, err := buf.Peek(10)
   154  	if err != nil && err != io.EOF {
   155  		// Note: we'll ignore any io.EOF error because there are some odd
   156  		// cases where the layer.tar file will be empty (zero bytes) and
   157  		// that results in an io.EOF from the Peek() call. So, in those
   158  		// cases we'll just treat it as a non-compressed stream and
   159  		// that means just create an empty layer.
   160  		// See Issue 18170
   161  		return nil, err
   162  	}
   163  
   164  	compression := DetectCompression(bs)
   165  	switch compression {
   166  	case Uncompressed:
   167  		readBufWrapper := p.NewReadCloserWrapper(buf, buf)
   168  		return readBufWrapper, nil
   169  	case Gzip:
   170  		gzReader, err := gzip.NewReader(buf)
   171  		if err != nil {
   172  			return nil, err
   173  		}
   174  		readBufWrapper := p.NewReadCloserWrapper(buf, gzReader)
   175  		return readBufWrapper, nil
   176  	case Bzip2:
   177  		bz2Reader := bzip2.NewReader(buf)
   178  		readBufWrapper := p.NewReadCloserWrapper(buf, bz2Reader)
   179  		return readBufWrapper, nil
   180  	case Xz:
   181  		xzReader, chdone, err := xzDecompress(buf)
   182  		if err != nil {
   183  			return nil, err
   184  		}
   185  		readBufWrapper := p.NewReadCloserWrapper(buf, xzReader)
   186  		return ioutils.NewReadCloserWrapper(readBufWrapper, func() error {
   187  			<-chdone
   188  			return readBufWrapper.Close()
   189  		}), nil
   190  	default:
   191  		return nil, fmt.Errorf("Unsupported compression format %s", (&compression).Extension())
   192  	}
   193  }
   194  
   195  // CompressStream compresses the dest with specified compression algorithm.
   196  func CompressStream(dest io.WriteCloser, compression Compression) (io.WriteCloser, error) {
   197  	p := pools.BufioWriter32KPool
   198  	buf := p.Get(dest)
   199  	switch compression {
   200  	case Uncompressed:
   201  		writeBufWrapper := p.NewWriteCloserWrapper(buf, buf)
   202  		return writeBufWrapper, nil
   203  	case Gzip:
   204  		gzWriter := gzip.NewWriter(dest)
   205  		writeBufWrapper := p.NewWriteCloserWrapper(buf, gzWriter)
   206  		return writeBufWrapper, nil
   207  	case Bzip2, Xz:
   208  		// archive/bzip2 does not support writing, and there is no xz support at all
   209  		// However, this is not a problem as docker only currently generates gzipped tars
   210  		return nil, fmt.Errorf("Unsupported compression format %s", (&compression).Extension())
   211  	default:
   212  		return nil, fmt.Errorf("Unsupported compression format %s", (&compression).Extension())
   213  	}
   214  }
   215  
   216  // Extension returns the extension of a file that uses the specified compression algorithm.
   217  func (compression *Compression) Extension() string {
   218  	switch *compression {
   219  	case Uncompressed:
   220  		return "tar"
   221  	case Bzip2:
   222  		return "tar.bz2"
   223  	case Gzip:
   224  		return "tar.gz"
   225  	case Xz:
   226  		return "tar.xz"
   227  	}
   228  	return ""
   229  }
   230  
   231  type tarAppender struct {
   232  	TarWriter *tar.Writer
   233  	Buffer    *bufio.Writer
   234  
   235  	// for hardlink mapping
   236  	SeenFiles map[uint64]string
   237  	UIDMaps   []idtools.IDMap
   238  	GIDMaps   []idtools.IDMap
   239  }
   240  
   241  // canonicalTarName provides a platform-independent and consistent posix-style
   242  //path for files and directories to be archived regardless of the platform.
   243  func canonicalTarName(name string, isDir bool) (string, error) {
   244  	name, err := CanonicalTarNameForPath(name)
   245  	if err != nil {
   246  		return "", err
   247  	}
   248  
   249  	// suffix with '/' for directories
   250  	if isDir && !strings.HasSuffix(name, "/") {
   251  		name += "/"
   252  	}
   253  	return name, nil
   254  }
   255  
   256  func (ta *tarAppender) addTarFile(path, name string) error {
   257  	fi, err := os.Lstat(path)
   258  	if err != nil {
   259  		return err
   260  	}
   261  
   262  	link := ""
   263  	if fi.Mode()&os.ModeSymlink != 0 {
   264  		if link, err = os.Readlink(path); err != nil {
   265  			return err
   266  		}
   267  	}
   268  
   269  	hdr, err := tar.FileInfoHeader(fi, link)
   270  	if err != nil {
   271  		return err
   272  	}
   273  	hdr.Mode = int64(chmodTarEntry(os.FileMode(hdr.Mode)))
   274  
   275  	name, err = canonicalTarName(name, fi.IsDir())
   276  	if err != nil {
   277  		return fmt.Errorf("tar: cannot canonicalize path: %v", err)
   278  	}
   279  	hdr.Name = name
   280  
   281  	inode, err := setHeaderForSpecialDevice(hdr, ta, name, fi.Sys())
   282  	if err != nil {
   283  		return err
   284  	}
   285  
   286  	// if it's not a directory and has more than 1 link,
   287  	// it's hardlinked, so set the type flag accordingly
   288  	if !fi.IsDir() && hasHardlinks(fi) {
   289  		// a link should have a name that it links too
   290  		// and that linked name should be first in the tar archive
   291  		if oldpath, ok := ta.SeenFiles[inode]; ok {
   292  			hdr.Typeflag = tar.TypeLink
   293  			hdr.Linkname = oldpath
   294  			hdr.Size = 0 // This Must be here for the writer math to add up!
   295  		} else {
   296  			ta.SeenFiles[inode] = name
   297  		}
   298  	}
   299  
   300  	capability, _ := system.Lgetxattr(path, "security.capability")
   301  	if capability != nil {
   302  		hdr.Xattrs = make(map[string]string)
   303  		hdr.Xattrs["security.capability"] = string(capability)
   304  	}
   305  
   306  	//handle re-mapping container ID mappings back to host ID mappings before
   307  	//writing tar headers/files. We skip whiteout files because they were written
   308  	//by the kernel and already have proper ownership relative to the host
   309  	if !strings.HasPrefix(filepath.Base(hdr.Name), WhiteoutPrefix) && (ta.UIDMaps != nil || ta.GIDMaps != nil) {
   310  		uid, gid, err := getFileUIDGID(fi.Sys())
   311  		if err != nil {
   312  			return err
   313  		}
   314  		xUID, err := idtools.ToContainer(uid, ta.UIDMaps)
   315  		if err != nil {
   316  			return err
   317  		}
   318  		xGID, err := idtools.ToContainer(gid, ta.GIDMaps)
   319  		if err != nil {
   320  			return err
   321  		}
   322  		hdr.Uid = xUID
   323  		hdr.Gid = xGID
   324  	}
   325  
   326  	if err := ta.TarWriter.WriteHeader(hdr); err != nil {
   327  		return err
   328  	}
   329  
   330  	if hdr.Typeflag == tar.TypeReg {
   331  		file, err := os.Open(path)
   332  		if err != nil {
   333  			return err
   334  		}
   335  
   336  		ta.Buffer.Reset(ta.TarWriter)
   337  		defer ta.Buffer.Reset(nil)
   338  		_, err = io.Copy(ta.Buffer, file)
   339  		file.Close()
   340  		if err != nil {
   341  			return err
   342  		}
   343  		err = ta.Buffer.Flush()
   344  		if err != nil {
   345  			return err
   346  		}
   347  	}
   348  
   349  	return nil
   350  }
   351  
   352  func createTarFile(path, extractDir string, hdr *tar.Header, reader io.Reader, Lchown bool, chownOpts *TarChownOptions) error {
   353  	// hdr.Mode is in linux format, which we can use for sycalls,
   354  	// but for os.Foo() calls we need the mode converted to os.FileMode,
   355  	// so use hdrInfo.Mode() (they differ for e.g. setuid bits)
   356  	hdrInfo := hdr.FileInfo()
   357  
   358  	switch hdr.Typeflag {
   359  	case tar.TypeDir:
   360  		// Create directory unless it exists as a directory already.
   361  		// In that case we just want to merge the two
   362  		if fi, err := os.Lstat(path); !(err == nil && fi.IsDir()) {
   363  			if err := os.Mkdir(path, hdrInfo.Mode()); err != nil {
   364  				return err
   365  			}
   366  		}
   367  
   368  	case tar.TypeReg, tar.TypeRegA:
   369  		// Source is regular file
   370  		file, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, hdrInfo.Mode())
   371  		if err != nil {
   372  			return err
   373  		}
   374  		if _, err := io.Copy(file, reader); err != nil {
   375  			file.Close()
   376  			return err
   377  		}
   378  		file.Close()
   379  
   380  	case tar.TypeBlock, tar.TypeChar, tar.TypeFifo:
   381  		// Handle this is an OS-specific way
   382  		if err := handleTarTypeBlockCharFifo(hdr, path); err != nil {
   383  			return err
   384  		}
   385  
   386  	case tar.TypeLink:
   387  		targetPath := filepath.Join(extractDir, hdr.Linkname)
   388  		// check for hardlink breakout
   389  		if !strings.HasPrefix(targetPath, extractDir) {
   390  			return breakoutError(fmt.Errorf("invalid hardlink %q -> %q", targetPath, hdr.Linkname))
   391  		}
   392  		if err := os.Link(targetPath, path); err != nil {
   393  			return err
   394  		}
   395  
   396  	case tar.TypeSymlink:
   397  		// 	path 				-> hdr.Linkname = targetPath
   398  		// e.g. /extractDir/path/to/symlink 	-> ../2/file	= /extractDir/path/2/file
   399  		targetPath := filepath.Join(filepath.Dir(path), hdr.Linkname)
   400  
   401  		// the reason we don't need to check symlinks in the path (with FollowSymlinkInScope) is because
   402  		// that symlink would first have to be created, which would be caught earlier, at this very check:
   403  		if !strings.HasPrefix(targetPath, extractDir) {
   404  			return breakoutError(fmt.Errorf("invalid symlink %q -> %q", path, hdr.Linkname))
   405  		}
   406  		if err := os.Symlink(hdr.Linkname, path); err != nil {
   407  			return err
   408  		}
   409  
   410  	case tar.TypeXGlobalHeader:
   411  		logrus.Debugf("PAX Global Extended Headers found and ignored")
   412  		return nil
   413  
   414  	default:
   415  		return fmt.Errorf("Unhandled tar header type %d\n", hdr.Typeflag)
   416  	}
   417  
   418  	// Lchown is not supported on Windows.
   419  	if Lchown && runtime.GOOS != "windows" {
   420  		if chownOpts == nil {
   421  			chownOpts = &TarChownOptions{UID: hdr.Uid, GID: hdr.Gid}
   422  		}
   423  		if err := os.Lchown(path, chownOpts.UID, chownOpts.GID); err != nil {
   424  			return err
   425  		}
   426  	}
   427  
   428  	for key, value := range hdr.Xattrs {
   429  		if err := system.Lsetxattr(path, key, []byte(value), 0); err != nil {
   430  			return err
   431  		}
   432  	}
   433  
   434  	// There is no LChmod, so ignore mode for symlink. Also, this
   435  	// must happen after chown, as that can modify the file mode
   436  	if err := handleLChmod(hdr, path, hdrInfo); err != nil {
   437  		return err
   438  	}
   439  
   440  	aTime := hdr.AccessTime
   441  	if aTime.Before(hdr.ModTime) {
   442  		// Last access time should never be before last modified time.
   443  		aTime = hdr.ModTime
   444  	}
   445  
   446  	// system.Chtimes doesn't support a NOFOLLOW flag atm
   447  	if hdr.Typeflag == tar.TypeLink {
   448  		if fi, err := os.Lstat(hdr.Linkname); err == nil && (fi.Mode()&os.ModeSymlink == 0) {
   449  			if err := system.Chtimes(path, aTime, hdr.ModTime); err != nil {
   450  				return err
   451  			}
   452  		}
   453  	} else if hdr.Typeflag != tar.TypeSymlink {
   454  		if err := system.Chtimes(path, aTime, hdr.ModTime); err != nil {
   455  			return err
   456  		}
   457  	} else {
   458  		ts := []syscall.Timespec{timeToTimespec(aTime), timeToTimespec(hdr.ModTime)}
   459  		if err := system.LUtimesNano(path, ts); err != nil && err != system.ErrNotSupportedPlatform {
   460  			return err
   461  		}
   462  	}
   463  	return nil
   464  }
   465  
   466  // Tar creates an archive from the directory at `path`, and returns it as a
   467  // stream of bytes.
   468  func Tar(path string, compression Compression) (io.ReadCloser, error) {
   469  	return TarWithOptions(path, &TarOptions{Compression: compression})
   470  }
   471  
   472  // TarWithOptions creates an archive from the directory at `path`, only including files whose relative
   473  // paths are included in `options.IncludeFiles` (if non-nil) or not in `options.ExcludePatterns`.
   474  func TarWithOptions(srcPath string, options *TarOptions) (io.ReadCloser, error) {
   475  
   476  	// Fix the source path to work with long path names. This is a no-op
   477  	// on platforms other than Windows.
   478  	srcPath = fixVolumePathPrefix(srcPath)
   479  
   480  	patterns, patDirs, exceptions, err := fileutils.CleanPatterns(options.ExcludePatterns)
   481  
   482  	if err != nil {
   483  		return nil, err
   484  	}
   485  
   486  	pipeReader, pipeWriter := io.Pipe()
   487  
   488  	compressWriter, err := CompressStream(pipeWriter, options.Compression)
   489  	if err != nil {
   490  		return nil, err
   491  	}
   492  
   493  	go func() {
   494  		ta := &tarAppender{
   495  			TarWriter: tar.NewWriter(compressWriter),
   496  			Buffer:    pools.BufioWriter32KPool.Get(nil),
   497  			SeenFiles: make(map[uint64]string),
   498  			UIDMaps:   options.UIDMaps,
   499  			GIDMaps:   options.GIDMaps,
   500  		}
   501  
   502  		defer func() {
   503  			// Make sure to check the error on Close.
   504  			if err := ta.TarWriter.Close(); err != nil {
   505  				logrus.Errorf("Can't close tar writer: %s", err)
   506  			}
   507  			if err := compressWriter.Close(); err != nil {
   508  				logrus.Errorf("Can't close compress writer: %s", err)
   509  			}
   510  			if err := pipeWriter.Close(); err != nil {
   511  				logrus.Errorf("Can't close pipe writer: %s", err)
   512  			}
   513  		}()
   514  
   515  		// this buffer is needed for the duration of this piped stream
   516  		defer pools.BufioWriter32KPool.Put(ta.Buffer)
   517  
   518  		// In general we log errors here but ignore them because
   519  		// during e.g. a diff operation the container can continue
   520  		// mutating the filesystem and we can see transient errors
   521  		// from this
   522  
   523  		stat, err := os.Lstat(srcPath)
   524  		if err != nil {
   525  			return
   526  		}
   527  
   528  		if !stat.IsDir() {
   529  			// We can't later join a non-dir with any includes because the
   530  			// 'walk' will error if "file/." is stat-ed and "file" is not a
   531  			// directory. So, we must split the source path and use the
   532  			// basename as the include.
   533  			if len(options.IncludeFiles) > 0 {
   534  				logrus.Warn("Tar: Can't archive a file with includes")
   535  			}
   536  
   537  			dir, base := SplitPathDirEntry(srcPath)
   538  			srcPath = dir
   539  			options.IncludeFiles = []string{base}
   540  		}
   541  
   542  		if len(options.IncludeFiles) == 0 {
   543  			options.IncludeFiles = []string{"."}
   544  		}
   545  
   546  		seen := make(map[string]bool)
   547  
   548  		for _, include := range options.IncludeFiles {
   549  			rebaseName := options.RebaseNames[include]
   550  
   551  			walkRoot := getWalkRoot(srcPath, include)
   552  			filepath.Walk(walkRoot, func(filePath string, f os.FileInfo, err error) error {
   553  				if err != nil {
   554  					logrus.Errorf("Tar: Can't stat file %s to tar: %s", srcPath, err)
   555  					return nil
   556  				}
   557  
   558  				relFilePath, err := filepath.Rel(srcPath, filePath)
   559  				if err != nil || (!options.IncludeSourceDir && relFilePath == "." && f.IsDir()) {
   560  					// Error getting relative path OR we are looking
   561  					// at the source directory path. Skip in both situations.
   562  					return nil
   563  				}
   564  
   565  				if options.IncludeSourceDir && include == "." && relFilePath != "." {
   566  					relFilePath = strings.Join([]string{".", relFilePath}, string(filepath.Separator))
   567  				}
   568  
   569  				skip := false
   570  
   571  				// If "include" is an exact match for the current file
   572  				// then even if there's an "excludePatterns" pattern that
   573  				// matches it, don't skip it. IOW, assume an explicit 'include'
   574  				// is asking for that file no matter what - which is true
   575  				// for some files, like .dockerignore and Dockerfile (sometimes)
   576  				if include != relFilePath {
   577  					skip, err = fileutils.OptimizedMatches(relFilePath, patterns, patDirs)
   578  					if err != nil {
   579  						logrus.Errorf("Error matching %s: %v", relFilePath, err)
   580  						return err
   581  					}
   582  				}
   583  
   584  				if skip {
   585  					// If we want to skip this file and its a directory
   586  					// then we should first check to see if there's an
   587  					// excludes pattern (eg !dir/file) that starts with this
   588  					// dir. If so then we can't skip this dir.
   589  
   590  					// Its not a dir then so we can just return/skip.
   591  					if !f.IsDir() {
   592  						return nil
   593  					}
   594  
   595  					// No exceptions (!...) in patterns so just skip dir
   596  					if !exceptions {
   597  						return filepath.SkipDir
   598  					}
   599  
   600  					dirSlash := relFilePath + string(filepath.Separator)
   601  
   602  					for _, pat := range patterns {
   603  						if pat[0] != '!' {
   604  							continue
   605  						}
   606  						pat = pat[1:] + string(filepath.Separator)
   607  						if strings.HasPrefix(pat, dirSlash) {
   608  							// found a match - so can't skip this dir
   609  							return nil
   610  						}
   611  					}
   612  
   613  					// No matching exclusion dir so just skip dir
   614  					return filepath.SkipDir
   615  				}
   616  
   617  				if seen[relFilePath] {
   618  					return nil
   619  				}
   620  				seen[relFilePath] = true
   621  
   622  				// Rename the base resource.
   623  				if rebaseName != "" {
   624  					var replacement string
   625  					if rebaseName != string(filepath.Separator) {
   626  						// Special case the root directory to replace with an
   627  						// empty string instead so that we don't end up with
   628  						// double slashes in the paths.
   629  						replacement = rebaseName
   630  					}
   631  
   632  					relFilePath = strings.Replace(relFilePath, include, replacement, 1)
   633  				}
   634  
   635  				if err := ta.addTarFile(filePath, relFilePath); err != nil {
   636  					logrus.Errorf("Can't add file %s to tar: %s", filePath, err)
   637  					// if pipe is broken, stop writting tar stream to it
   638  					if err == io.ErrClosedPipe {
   639  						return err
   640  					}
   641  				}
   642  				return nil
   643  			})
   644  		}
   645  	}()
   646  
   647  	return pipeReader, nil
   648  }
   649  
   650  // Unpack unpacks the decompressedArchive to dest with options.
   651  func Unpack(decompressedArchive io.Reader, dest string, options *TarOptions) error {
   652  	tr := tar.NewReader(decompressedArchive)
   653  	trBuf := pools.BufioReader32KPool.Get(nil)
   654  	defer pools.BufioReader32KPool.Put(trBuf)
   655  
   656  	var dirs []*tar.Header
   657  	remappedRootUID, remappedRootGID, err := idtools.GetRootUIDGID(options.UIDMaps, options.GIDMaps)
   658  	if err != nil {
   659  		return err
   660  	}
   661  
   662  	// Iterate through the files in the archive.
   663  loop:
   664  	for {
   665  		hdr, err := tr.Next()
   666  		if err == io.EOF {
   667  			// end of tar archive
   668  			break
   669  		}
   670  		if err != nil {
   671  			return err
   672  		}
   673  
   674  		// Normalize name, for safety and for a simple is-root check
   675  		// This keeps "../" as-is, but normalizes "/../" to "/". Or Windows:
   676  		// This keeps "..\" as-is, but normalizes "\..\" to "\".
   677  		hdr.Name = filepath.Clean(hdr.Name)
   678  
   679  		for _, exclude := range options.ExcludePatterns {
   680  			if strings.HasPrefix(hdr.Name, exclude) {
   681  				continue loop
   682  			}
   683  		}
   684  
   685  		// After calling filepath.Clean(hdr.Name) above, hdr.Name will now be in
   686  		// the filepath format for the OS on which the daemon is running. Hence
   687  		// the check for a slash-suffix MUST be done in an OS-agnostic way.
   688  		if !strings.HasSuffix(hdr.Name, string(os.PathSeparator)) {
   689  			// Not the root directory, ensure that the parent directory exists
   690  			parent := filepath.Dir(hdr.Name)
   691  			parentPath := filepath.Join(dest, parent)
   692  			if _, err := os.Lstat(parentPath); err != nil && os.IsNotExist(err) {
   693  				err = idtools.MkdirAllNewAs(parentPath, 0777, remappedRootUID, remappedRootGID)
   694  				if err != nil {
   695  					return err
   696  				}
   697  			}
   698  		}
   699  
   700  		path := filepath.Join(dest, hdr.Name)
   701  		rel, err := filepath.Rel(dest, path)
   702  		if err != nil {
   703  			return err
   704  		}
   705  		if strings.HasPrefix(rel, ".."+string(os.PathSeparator)) {
   706  			return breakoutError(fmt.Errorf("%q is outside of %q", hdr.Name, dest))
   707  		}
   708  
   709  		// If path exits we almost always just want to remove and replace it
   710  		// The only exception is when it is a directory *and* the file from
   711  		// the layer is also a directory. Then we want to merge them (i.e.
   712  		// just apply the metadata from the layer).
   713  		if fi, err := os.Lstat(path); err == nil {
   714  			if options.NoOverwriteDirNonDir && fi.IsDir() && hdr.Typeflag != tar.TypeDir {
   715  				// If NoOverwriteDirNonDir is true then we cannot replace
   716  				// an existing directory with a non-directory from the archive.
   717  				return fmt.Errorf("cannot overwrite directory %q with non-directory %q", path, dest)
   718  			}
   719  
   720  			if options.NoOverwriteDirNonDir && !fi.IsDir() && hdr.Typeflag == tar.TypeDir {
   721  				// If NoOverwriteDirNonDir is true then we cannot replace
   722  				// an existing non-directory with a directory from the archive.
   723  				return fmt.Errorf("cannot overwrite non-directory %q with directory %q", path, dest)
   724  			}
   725  
   726  			if fi.IsDir() && hdr.Name == "." {
   727  				continue
   728  			}
   729  
   730  			if !(fi.IsDir() && hdr.Typeflag == tar.TypeDir) {
   731  				if err := os.RemoveAll(path); err != nil {
   732  					return err
   733  				}
   734  			}
   735  		}
   736  		trBuf.Reset(tr)
   737  
   738  		// if the options contain a uid & gid maps, convert header uid/gid
   739  		// entries using the maps such that lchown sets the proper mapped
   740  		// uid/gid after writing the file. We only perform this mapping if
   741  		// the file isn't already owned by the remapped root UID or GID, as
   742  		// that specific uid/gid has no mapping from container -> host, and
   743  		// those files already have the proper ownership for inside the
   744  		// container.
   745  		if hdr.Uid != remappedRootUID {
   746  			xUID, err := idtools.ToHost(hdr.Uid, options.UIDMaps)
   747  			if err != nil {
   748  				return err
   749  			}
   750  			hdr.Uid = xUID
   751  		}
   752  		if hdr.Gid != remappedRootGID {
   753  			xGID, err := idtools.ToHost(hdr.Gid, options.GIDMaps)
   754  			if err != nil {
   755  				return err
   756  			}
   757  			hdr.Gid = xGID
   758  		}
   759  
   760  		if err := createTarFile(path, dest, hdr, trBuf, !options.NoLchown, options.ChownOpts); err != nil {
   761  			return err
   762  		}
   763  
   764  		// Directory mtimes must be handled at the end to avoid further
   765  		// file creation in them to modify the directory mtime
   766  		if hdr.Typeflag == tar.TypeDir {
   767  			dirs = append(dirs, hdr)
   768  		}
   769  	}
   770  
   771  	for _, hdr := range dirs {
   772  		path := filepath.Join(dest, hdr.Name)
   773  
   774  		if err := system.Chtimes(path, hdr.AccessTime, hdr.ModTime); err != nil {
   775  			return err
   776  		}
   777  	}
   778  	return nil
   779  }
   780  
   781  // Untar reads a stream of bytes from `archive`, parses it as a tar archive,
   782  // and unpacks it into the directory at `dest`.
   783  // The archive may be compressed with one of the following algorithms:
   784  //  identity (uncompressed), gzip, bzip2, xz.
   785  // FIXME: specify behavior when target path exists vs. doesn't exist.
   786  func Untar(tarArchive io.Reader, dest string, options *TarOptions) error {
   787  	return untarHandler(tarArchive, dest, options, true)
   788  }
   789  
   790  // UntarUncompressed reads a stream of bytes from `archive`, parses it as a tar archive,
   791  // and unpacks it into the directory at `dest`.
   792  // The archive must be an uncompressed stream.
   793  func UntarUncompressed(tarArchive io.Reader, dest string, options *TarOptions) error {
   794  	return untarHandler(tarArchive, dest, options, false)
   795  }
   796  
   797  // Handler for teasing out the automatic decompression
   798  func untarHandler(tarArchive io.Reader, dest string, options *TarOptions, decompress bool) error {
   799  	if tarArchive == nil {
   800  		return fmt.Errorf("Empty archive")
   801  	}
   802  	dest = filepath.Clean(dest)
   803  	if options == nil {
   804  		options = &TarOptions{}
   805  	}
   806  	if options.ExcludePatterns == nil {
   807  		options.ExcludePatterns = []string{}
   808  	}
   809  
   810  	r := tarArchive
   811  	if decompress {
   812  		decompressedArchive, err := DecompressStream(tarArchive)
   813  		if err != nil {
   814  			return err
   815  		}
   816  		defer decompressedArchive.Close()
   817  		r = decompressedArchive
   818  	}
   819  
   820  	return Unpack(r, dest, options)
   821  }
   822  
   823  // TarUntar is a convenience function which calls Tar and Untar, with the output of one piped into the other.
   824  // If either Tar or Untar fails, TarUntar aborts and returns the error.
   825  func (archiver *Archiver) TarUntar(src, dst string) error {
   826  	logrus.Debugf("TarUntar(%s %s)", src, dst)
   827  	archive, err := TarWithOptions(src, &TarOptions{Compression: Uncompressed})
   828  	if err != nil {
   829  		return err
   830  	}
   831  	defer archive.Close()
   832  
   833  	var options *TarOptions
   834  	if archiver.UIDMaps != nil || archiver.GIDMaps != nil {
   835  		options = &TarOptions{
   836  			UIDMaps: archiver.UIDMaps,
   837  			GIDMaps: archiver.GIDMaps,
   838  		}
   839  	}
   840  	return archiver.Untar(archive, dst, options)
   841  }
   842  
   843  // TarUntar is a convenience function which calls Tar and Untar, with the output of one piped into the other.
   844  // If either Tar or Untar fails, TarUntar aborts and returns the error.
   845  func TarUntar(src, dst string) error {
   846  	return defaultArchiver.TarUntar(src, dst)
   847  }
   848  
   849  // UntarPath untar a file from path to a destination, src is the source tar file path.
   850  func (archiver *Archiver) UntarPath(src, dst string) error {
   851  	archive, err := os.Open(src)
   852  	if err != nil {
   853  		return err
   854  	}
   855  	defer archive.Close()
   856  	var options *TarOptions
   857  	if archiver.UIDMaps != nil || archiver.GIDMaps != nil {
   858  		options = &TarOptions{
   859  			UIDMaps: archiver.UIDMaps,
   860  			GIDMaps: archiver.GIDMaps,
   861  		}
   862  	}
   863  	return archiver.Untar(archive, dst, options)
   864  }
   865  
   866  // UntarPath is a convenience function which looks for an archive
   867  // at filesystem path `src`, and unpacks it at `dst`.
   868  func UntarPath(src, dst string) error {
   869  	return defaultArchiver.UntarPath(src, dst)
   870  }
   871  
   872  // CopyWithTar creates a tar archive of filesystem path `src`, and
   873  // unpacks it at filesystem path `dst`.
   874  // The archive is streamed directly with fixed buffering and no
   875  // intermediary disk IO.
   876  func (archiver *Archiver) CopyWithTar(src, dst string) error {
   877  	srcSt, err := os.Stat(src)
   878  	if err != nil {
   879  		return err
   880  	}
   881  	if !srcSt.IsDir() {
   882  		return archiver.CopyFileWithTar(src, dst)
   883  	}
   884  	// Create dst, copy src's content into it
   885  	logrus.Debugf("Creating dest directory: %s", dst)
   886  	if err := system.MkdirAll(dst, 0755); err != nil {
   887  		return err
   888  	}
   889  	logrus.Debugf("Calling TarUntar(%s, %s)", src, dst)
   890  	return archiver.TarUntar(src, dst)
   891  }
   892  
   893  // CopyWithTar creates a tar archive of filesystem path `src`, and
   894  // unpacks it at filesystem path `dst`.
   895  // The archive is streamed directly with fixed buffering and no
   896  // intermediary disk IO.
   897  func CopyWithTar(src, dst string) error {
   898  	return defaultArchiver.CopyWithTar(src, dst)
   899  }
   900  
   901  // CopyFileWithTar emulates the behavior of the 'cp' command-line
   902  // for a single file. It copies a regular file from path `src` to
   903  // path `dst`, and preserves all its metadata.
   904  func (archiver *Archiver) CopyFileWithTar(src, dst string) (err error) {
   905  	logrus.Debugf("CopyFileWithTar(%s, %s)", src, dst)
   906  	srcSt, err := os.Stat(src)
   907  	if err != nil {
   908  		return err
   909  	}
   910  
   911  	if srcSt.IsDir() {
   912  		return fmt.Errorf("Can't copy a directory")
   913  	}
   914  
   915  	// Clean up the trailing slash. This must be done in an operating
   916  	// system specific manner.
   917  	if dst[len(dst)-1] == os.PathSeparator {
   918  		dst = filepath.Join(dst, filepath.Base(src))
   919  	}
   920  	// Create the holding directory if necessary
   921  	if err := system.MkdirAll(filepath.Dir(dst), 0700); err != nil {
   922  		return err
   923  	}
   924  
   925  	r, w := io.Pipe()
   926  	errC := promise.Go(func() error {
   927  		defer w.Close()
   928  
   929  		srcF, err := os.Open(src)
   930  		if err != nil {
   931  			return err
   932  		}
   933  		defer srcF.Close()
   934  
   935  		hdr, err := tar.FileInfoHeader(srcSt, "")
   936  		if err != nil {
   937  			return err
   938  		}
   939  		hdr.Name = filepath.Base(dst)
   940  		hdr.Mode = int64(chmodTarEntry(os.FileMode(hdr.Mode)))
   941  
   942  		remappedRootUID, remappedRootGID, err := idtools.GetRootUIDGID(archiver.UIDMaps, archiver.GIDMaps)
   943  		if err != nil {
   944  			return err
   945  		}
   946  
   947  		// only perform mapping if the file being copied isn't already owned by the
   948  		// uid or gid of the remapped root in the container
   949  		if remappedRootUID != hdr.Uid {
   950  			xUID, err := idtools.ToHost(hdr.Uid, archiver.UIDMaps)
   951  			if err != nil {
   952  				return err
   953  			}
   954  			hdr.Uid = xUID
   955  		}
   956  		if remappedRootGID != hdr.Gid {
   957  			xGID, err := idtools.ToHost(hdr.Gid, archiver.GIDMaps)
   958  			if err != nil {
   959  				return err
   960  			}
   961  			hdr.Gid = xGID
   962  		}
   963  
   964  		tw := tar.NewWriter(w)
   965  		defer tw.Close()
   966  		if err := tw.WriteHeader(hdr); err != nil {
   967  			return err
   968  		}
   969  		if _, err := io.Copy(tw, srcF); err != nil {
   970  			return err
   971  		}
   972  		return nil
   973  	})
   974  	defer func() {
   975  		if er := <-errC; err != nil {
   976  			err = er
   977  		}
   978  	}()
   979  
   980  	err = archiver.Untar(r, filepath.Dir(dst), nil)
   981  	if err != nil {
   982  		r.CloseWithError(err)
   983  	}
   984  	return err
   985  }
   986  
   987  // CopyFileWithTar emulates the behavior of the 'cp' command-line
   988  // for a single file. It copies a regular file from path `src` to
   989  // path `dst`, and preserves all its metadata.
   990  //
   991  // Destination handling is in an operating specific manner depending
   992  // where the daemon is running. If `dst` ends with a trailing slash
   993  // the final destination path will be `dst/base(src)`  (Linux) or
   994  // `dst\base(src)` (Windows).
   995  func CopyFileWithTar(src, dst string) (err error) {
   996  	return defaultArchiver.CopyFileWithTar(src, dst)
   997  }
   998  
   999  // cmdStream executes a command, and returns its stdout as a stream.
  1000  // If the command fails to run or doesn't complete successfully, an error
  1001  // will be returned, including anything written on stderr.
  1002  func cmdStream(cmd *exec.Cmd, input io.Reader) (io.ReadCloser, <-chan struct{}, error) {
  1003  	chdone := make(chan struct{})
  1004  	cmd.Stdin = input
  1005  	pipeR, pipeW := io.Pipe()
  1006  	cmd.Stdout = pipeW
  1007  	var errBuf bytes.Buffer
  1008  	cmd.Stderr = &errBuf
  1009  
  1010  	// Run the command and return the pipe
  1011  	if err := cmd.Start(); err != nil {
  1012  		return nil, nil, err
  1013  	}
  1014  
  1015  	// Copy stdout to the returned pipe
  1016  	go func() {
  1017  		if err := cmd.Wait(); err != nil {
  1018  			pipeW.CloseWithError(fmt.Errorf("%s: %s", err, errBuf.String()))
  1019  		} else {
  1020  			pipeW.Close()
  1021  		}
  1022  		close(chdone)
  1023  	}()
  1024  
  1025  	return pipeR, chdone, nil
  1026  }
  1027  
  1028  // NewTempArchive reads the content of src into a temporary file, and returns the contents
  1029  // of that file as an archive. The archive can only be read once - as soon as reading completes,
  1030  // the file will be deleted.
  1031  func NewTempArchive(src Archive, dir string) (*TempArchive, error) {
  1032  	f, err := ioutil.TempFile(dir, "")
  1033  	if err != nil {
  1034  		return nil, err
  1035  	}
  1036  	if _, err := io.Copy(f, src); err != nil {
  1037  		return nil, err
  1038  	}
  1039  	if _, err := f.Seek(0, 0); err != nil {
  1040  		return nil, err
  1041  	}
  1042  	st, err := f.Stat()
  1043  	if err != nil {
  1044  		return nil, err
  1045  	}
  1046  	size := st.Size()
  1047  	return &TempArchive{File: f, Size: size}, nil
  1048  }
  1049  
  1050  // TempArchive is a temporary archive. The archive can only be read once - as soon as reading completes,
  1051  // the file will be deleted.
  1052  type TempArchive struct {
  1053  	*os.File
  1054  	Size   int64 // Pre-computed from Stat().Size() as a convenience
  1055  	read   int64
  1056  	closed bool
  1057  }
  1058  
  1059  // Close closes the underlying file if it's still open, or does a no-op
  1060  // to allow callers to try to close the TempArchive multiple times safely.
  1061  func (archive *TempArchive) Close() error {
  1062  	if archive.closed {
  1063  		return nil
  1064  	}
  1065  
  1066  	archive.closed = true
  1067  
  1068  	return archive.File.Close()
  1069  }
  1070  
  1071  func (archive *TempArchive) Read(data []byte) (int, error) {
  1072  	n, err := archive.File.Read(data)
  1073  	archive.read += int64(n)
  1074  	if err != nil || archive.read == archive.Size {
  1075  		archive.Close()
  1076  		os.Remove(archive.File.Name())
  1077  	}
  1078  	return n, err
  1079  }