github.com/shishir-a412ed/docker@v1.3.2-0.20180103180333-fda904911d87/pkg/archive/archive.go (about)

     1  package archive
     2  
     3  import (
     4  	"archive/tar"
     5  	"bufio"
     6  	"bytes"
     7  	"compress/bzip2"
     8  	"compress/gzip"
     9  	"fmt"
    10  	"io"
    11  	"io/ioutil"
    12  	"os"
    13  	"os/exec"
    14  	"path/filepath"
    15  	"runtime"
    16  	"strings"
    17  	"syscall"
    18  
    19  	"github.com/docker/docker/pkg/fileutils"
    20  	"github.com/docker/docker/pkg/idtools"
    21  	"github.com/docker/docker/pkg/ioutils"
    22  	"github.com/docker/docker/pkg/pools"
    23  	"github.com/docker/docker/pkg/system"
    24  	"github.com/sirupsen/logrus"
    25  )
    26  
    27  type (
    28  	// Compression is the state represents if compressed or not.
    29  	Compression int
    30  	// WhiteoutFormat is the format of whiteouts unpacked
    31  	WhiteoutFormat int
    32  
    33  	// TarOptions wraps the tar options.
    34  	TarOptions struct {
    35  		IncludeFiles     []string
    36  		ExcludePatterns  []string
    37  		Compression      Compression
    38  		NoLchown         bool
    39  		UIDMaps          []idtools.IDMap
    40  		GIDMaps          []idtools.IDMap
    41  		ChownOpts        *idtools.IDPair
    42  		IncludeSourceDir bool
    43  		// WhiteoutFormat is the expected on disk format for whiteout files.
    44  		// This format will be converted to the standard format on pack
    45  		// and from the standard format on unpack.
    46  		WhiteoutFormat WhiteoutFormat
    47  		// When unpacking, specifies whether overwriting a directory with a
    48  		// non-directory is allowed and vice versa.
    49  		NoOverwriteDirNonDir bool
    50  		// For each include when creating an archive, the included name will be
    51  		// replaced with the matching name from this map.
    52  		RebaseNames map[string]string
    53  		InUserNS    bool
    54  	}
    55  )
    56  
    57  // Archiver implements the Archiver interface and allows the reuse of most utility functions of
    58  // this package with a pluggable Untar function. Also, to facilitate the passing of specific id
    59  // mappings for untar, an Archiver can be created with maps which will then be passed to Untar operations.
    60  type Archiver struct {
    61  	Untar         func(io.Reader, string, *TarOptions) error
    62  	IDMappingsVar *idtools.IDMappings
    63  }
    64  
    65  // NewDefaultArchiver returns a new Archiver without any IDMappings
    66  func NewDefaultArchiver() *Archiver {
    67  	return &Archiver{Untar: Untar, IDMappingsVar: &idtools.IDMappings{}}
    68  }
    69  
    70  // breakoutError is used to differentiate errors related to breaking out
    71  // When testing archive breakout in the unit tests, this error is expected
    72  // in order for the test to pass.
    73  type breakoutError error
    74  
    75  const (
    76  	// Uncompressed represents the uncompressed.
    77  	Uncompressed Compression = iota
    78  	// Bzip2 is bzip2 compression algorithm.
    79  	Bzip2
    80  	// Gzip is gzip compression algorithm.
    81  	Gzip
    82  	// Xz is xz compression algorithm.
    83  	Xz
    84  )
    85  
    86  const (
    87  	// AUFSWhiteoutFormat is the default format for whiteouts
    88  	AUFSWhiteoutFormat WhiteoutFormat = iota
    89  	// OverlayWhiteoutFormat formats whiteout according to the overlay
    90  	// standard.
    91  	OverlayWhiteoutFormat
    92  )
    93  
    94  const (
    95  	modeISDIR  = 040000  // Directory
    96  	modeISFIFO = 010000  // FIFO
    97  	modeISREG  = 0100000 // Regular file
    98  	modeISLNK  = 0120000 // Symbolic link
    99  	modeISBLK  = 060000  // Block special file
   100  	modeISCHR  = 020000  // Character special file
   101  	modeISSOCK = 0140000 // Socket
   102  )
   103  
   104  // IsArchivePath checks if the (possibly compressed) file at the given path
   105  // starts with a tar file header.
   106  func IsArchivePath(path string) bool {
   107  	file, err := os.Open(path)
   108  	if err != nil {
   109  		return false
   110  	}
   111  	defer file.Close()
   112  	rdr, err := DecompressStream(file)
   113  	if err != nil {
   114  		return false
   115  	}
   116  	r := tar.NewReader(rdr)
   117  	_, err = r.Next()
   118  	return err == nil
   119  }
   120  
   121  // DetectCompression detects the compression algorithm of the source.
   122  func DetectCompression(source []byte) Compression {
   123  	for compression, m := range map[Compression][]byte{
   124  		Bzip2: {0x42, 0x5A, 0x68},
   125  		Gzip:  {0x1F, 0x8B, 0x08},
   126  		Xz:    {0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00},
   127  	} {
   128  		if len(source) < len(m) {
   129  			logrus.Debug("Len too short")
   130  			continue
   131  		}
   132  		if bytes.Equal(m, source[:len(m)]) {
   133  			return compression
   134  		}
   135  	}
   136  	return Uncompressed
   137  }
   138  
   139  func xzDecompress(archive io.Reader) (io.ReadCloser, <-chan struct{}, error) {
   140  	args := []string{"xz", "-d", "-c", "-q"}
   141  
   142  	return cmdStream(exec.Command(args[0], args[1:]...), archive)
   143  }
   144  
   145  // DecompressStream decompresses the archive and returns a ReaderCloser with the decompressed archive.
   146  func DecompressStream(archive io.Reader) (io.ReadCloser, error) {
   147  	p := pools.BufioReader32KPool
   148  	buf := p.Get(archive)
   149  	bs, err := buf.Peek(10)
   150  	if err != nil && err != io.EOF {
   151  		// Note: we'll ignore any io.EOF error because there are some odd
   152  		// cases where the layer.tar file will be empty (zero bytes) and
   153  		// that results in an io.EOF from the Peek() call. So, in those
   154  		// cases we'll just treat it as a non-compressed stream and
   155  		// that means just create an empty layer.
   156  		// See Issue 18170
   157  		return nil, err
   158  	}
   159  
   160  	compression := DetectCompression(bs)
   161  	switch compression {
   162  	case Uncompressed:
   163  		readBufWrapper := p.NewReadCloserWrapper(buf, buf)
   164  		return readBufWrapper, nil
   165  	case Gzip:
   166  		gzReader, err := gzip.NewReader(buf)
   167  		if err != nil {
   168  			return nil, err
   169  		}
   170  		readBufWrapper := p.NewReadCloserWrapper(buf, gzReader)
   171  		return readBufWrapper, nil
   172  	case Bzip2:
   173  		bz2Reader := bzip2.NewReader(buf)
   174  		readBufWrapper := p.NewReadCloserWrapper(buf, bz2Reader)
   175  		return readBufWrapper, nil
   176  	case Xz:
   177  		xzReader, chdone, err := xzDecompress(buf)
   178  		if err != nil {
   179  			return nil, err
   180  		}
   181  		readBufWrapper := p.NewReadCloserWrapper(buf, xzReader)
   182  		return ioutils.NewReadCloserWrapper(readBufWrapper, func() error {
   183  			<-chdone
   184  			return readBufWrapper.Close()
   185  		}), nil
   186  	default:
   187  		return nil, fmt.Errorf("Unsupported compression format %s", (&compression).Extension())
   188  	}
   189  }
   190  
   191  // CompressStream compresses the dest with specified compression algorithm.
   192  func CompressStream(dest io.Writer, compression Compression) (io.WriteCloser, error) {
   193  	p := pools.BufioWriter32KPool
   194  	buf := p.Get(dest)
   195  	switch compression {
   196  	case Uncompressed:
   197  		writeBufWrapper := p.NewWriteCloserWrapper(buf, buf)
   198  		return writeBufWrapper, nil
   199  	case Gzip:
   200  		gzWriter := gzip.NewWriter(dest)
   201  		writeBufWrapper := p.NewWriteCloserWrapper(buf, gzWriter)
   202  		return writeBufWrapper, nil
   203  	case Bzip2, Xz:
   204  		// archive/bzip2 does not support writing, and there is no xz support at all
   205  		// However, this is not a problem as docker only currently generates gzipped tars
   206  		return nil, fmt.Errorf("Unsupported compression format %s", (&compression).Extension())
   207  	default:
   208  		return nil, fmt.Errorf("Unsupported compression format %s", (&compression).Extension())
   209  	}
   210  }
   211  
   212  // TarModifierFunc is a function that can be passed to ReplaceFileTarWrapper to
   213  // modify the contents or header of an entry in the archive. If the file already
   214  // exists in the archive the TarModifierFunc will be called with the Header and
   215  // a reader which will return the files content. If the file does not exist both
   216  // header and content will be nil.
   217  type TarModifierFunc func(path string, header *tar.Header, content io.Reader) (*tar.Header, []byte, error)
   218  
   219  // ReplaceFileTarWrapper converts inputTarStream to a new tar stream. Files in the
   220  // tar stream are modified if they match any of the keys in mods.
   221  func ReplaceFileTarWrapper(inputTarStream io.ReadCloser, mods map[string]TarModifierFunc) io.ReadCloser {
   222  	pipeReader, pipeWriter := io.Pipe()
   223  
   224  	go func() {
   225  		tarReader := tar.NewReader(inputTarStream)
   226  		tarWriter := tar.NewWriter(pipeWriter)
   227  		defer inputTarStream.Close()
   228  		defer tarWriter.Close()
   229  
   230  		modify := func(name string, original *tar.Header, modifier TarModifierFunc, tarReader io.Reader) error {
   231  			header, data, err := modifier(name, original, tarReader)
   232  			switch {
   233  			case err != nil:
   234  				return err
   235  			case header == nil:
   236  				return nil
   237  			}
   238  
   239  			header.Name = name
   240  			header.Size = int64(len(data))
   241  			if err := tarWriter.WriteHeader(header); err != nil {
   242  				return err
   243  			}
   244  			if len(data) != 0 {
   245  				if _, err := tarWriter.Write(data); err != nil {
   246  					return err
   247  				}
   248  			}
   249  			return nil
   250  		}
   251  
   252  		var err error
   253  		var originalHeader *tar.Header
   254  		for {
   255  			originalHeader, err = tarReader.Next()
   256  			if err == io.EOF {
   257  				break
   258  			}
   259  			if err != nil {
   260  				pipeWriter.CloseWithError(err)
   261  				return
   262  			}
   263  
   264  			modifier, ok := mods[originalHeader.Name]
   265  			if !ok {
   266  				// No modifiers for this file, copy the header and data
   267  				if err := tarWriter.WriteHeader(originalHeader); err != nil {
   268  					pipeWriter.CloseWithError(err)
   269  					return
   270  				}
   271  				if _, err := pools.Copy(tarWriter, tarReader); err != nil {
   272  					pipeWriter.CloseWithError(err)
   273  					return
   274  				}
   275  				continue
   276  			}
   277  			delete(mods, originalHeader.Name)
   278  
   279  			if err := modify(originalHeader.Name, originalHeader, modifier, tarReader); err != nil {
   280  				pipeWriter.CloseWithError(err)
   281  				return
   282  			}
   283  		}
   284  
   285  		// Apply the modifiers that haven't matched any files in the archive
   286  		for name, modifier := range mods {
   287  			if err := modify(name, nil, modifier, nil); err != nil {
   288  				pipeWriter.CloseWithError(err)
   289  				return
   290  			}
   291  		}
   292  
   293  		pipeWriter.Close()
   294  
   295  	}()
   296  	return pipeReader
   297  }
   298  
   299  // Extension returns the extension of a file that uses the specified compression algorithm.
   300  func (compression *Compression) Extension() string {
   301  	switch *compression {
   302  	case Uncompressed:
   303  		return "tar"
   304  	case Bzip2:
   305  		return "tar.bz2"
   306  	case Gzip:
   307  		return "tar.gz"
   308  	case Xz:
   309  		return "tar.xz"
   310  	}
   311  	return ""
   312  }
   313  
   314  // FileInfoHeader creates a populated Header from fi.
   315  // Compared to archive pkg this function fills in more information.
   316  // Also, regardless of Go version, this function fills file type bits (e.g. hdr.Mode |= modeISDIR),
   317  // which have been deleted since Go 1.9 archive/tar.
   318  func FileInfoHeader(name string, fi os.FileInfo, link string) (*tar.Header, error) {
   319  	hdr, err := tar.FileInfoHeader(fi, link)
   320  	if err != nil {
   321  		return nil, err
   322  	}
   323  	hdr.Mode = fillGo18FileTypeBits(int64(chmodTarEntry(os.FileMode(hdr.Mode))), fi)
   324  	name, err = canonicalTarName(name, fi.IsDir())
   325  	if err != nil {
   326  		return nil, fmt.Errorf("tar: cannot canonicalize path: %v", err)
   327  	}
   328  	hdr.Name = name
   329  	if err := setHeaderForSpecialDevice(hdr, name, fi.Sys()); err != nil {
   330  		return nil, err
   331  	}
   332  	return hdr, nil
   333  }
   334  
   335  // fillGo18FileTypeBits fills type bits which have been removed on Go 1.9 archive/tar
   336  // https://github.com/golang/go/commit/66b5a2f
   337  func fillGo18FileTypeBits(mode int64, fi os.FileInfo) int64 {
   338  	fm := fi.Mode()
   339  	switch {
   340  	case fm.IsRegular():
   341  		mode |= modeISREG
   342  	case fi.IsDir():
   343  		mode |= modeISDIR
   344  	case fm&os.ModeSymlink != 0:
   345  		mode |= modeISLNK
   346  	case fm&os.ModeDevice != 0:
   347  		if fm&os.ModeCharDevice != 0 {
   348  			mode |= modeISCHR
   349  		} else {
   350  			mode |= modeISBLK
   351  		}
   352  	case fm&os.ModeNamedPipe != 0:
   353  		mode |= modeISFIFO
   354  	case fm&os.ModeSocket != 0:
   355  		mode |= modeISSOCK
   356  	}
   357  	return mode
   358  }
   359  
   360  // ReadSecurityXattrToTarHeader reads security.capability xattr from filesystem
   361  // to a tar header
   362  func ReadSecurityXattrToTarHeader(path string, hdr *tar.Header) error {
   363  	capability, _ := system.Lgetxattr(path, "security.capability")
   364  	if capability != nil {
   365  		hdr.Xattrs = make(map[string]string)
   366  		hdr.Xattrs["security.capability"] = string(capability)
   367  	}
   368  	return nil
   369  }
   370  
   371  type tarWhiteoutConverter interface {
   372  	ConvertWrite(*tar.Header, string, os.FileInfo) (*tar.Header, error)
   373  	ConvertRead(*tar.Header, string) (bool, error)
   374  }
   375  
   376  type tarAppender struct {
   377  	TarWriter *tar.Writer
   378  	Buffer    *bufio.Writer
   379  
   380  	// for hardlink mapping
   381  	SeenFiles  map[uint64]string
   382  	IDMappings *idtools.IDMappings
   383  	ChownOpts  *idtools.IDPair
   384  
   385  	// For packing and unpacking whiteout files in the
   386  	// non standard format. The whiteout files defined
   387  	// by the AUFS standard are used as the tar whiteout
   388  	// standard.
   389  	WhiteoutConverter tarWhiteoutConverter
   390  }
   391  
   392  func newTarAppender(idMapping *idtools.IDMappings, writer io.Writer, chownOpts *idtools.IDPair) *tarAppender {
   393  	return &tarAppender{
   394  		SeenFiles:  make(map[uint64]string),
   395  		TarWriter:  tar.NewWriter(writer),
   396  		Buffer:     pools.BufioWriter32KPool.Get(nil),
   397  		IDMappings: idMapping,
   398  		ChownOpts:  chownOpts,
   399  	}
   400  }
   401  
   402  // canonicalTarName provides a platform-independent and consistent posix-style
   403  //path for files and directories to be archived regardless of the platform.
   404  func canonicalTarName(name string, isDir bool) (string, error) {
   405  	name, err := CanonicalTarNameForPath(name)
   406  	if err != nil {
   407  		return "", err
   408  	}
   409  
   410  	// suffix with '/' for directories
   411  	if isDir && !strings.HasSuffix(name, "/") {
   412  		name += "/"
   413  	}
   414  	return name, nil
   415  }
   416  
   417  // addTarFile adds to the tar archive a file from `path` as `name`
   418  func (ta *tarAppender) addTarFile(path, name string) error {
   419  	fi, err := os.Lstat(path)
   420  	if err != nil {
   421  		return err
   422  	}
   423  
   424  	var link string
   425  	if fi.Mode()&os.ModeSymlink != 0 {
   426  		var err error
   427  		link, err = os.Readlink(path)
   428  		if err != nil {
   429  			return err
   430  		}
   431  	}
   432  
   433  	hdr, err := FileInfoHeader(name, fi, link)
   434  	if err != nil {
   435  		return err
   436  	}
   437  	if err := ReadSecurityXattrToTarHeader(path, hdr); err != nil {
   438  		return err
   439  	}
   440  
   441  	// if it's not a directory and has more than 1 link,
   442  	// it's hard linked, so set the type flag accordingly
   443  	if !fi.IsDir() && hasHardlinks(fi) {
   444  		inode, err := getInodeFromStat(fi.Sys())
   445  		if err != nil {
   446  			return err
   447  		}
   448  		// a link should have a name that it links too
   449  		// and that linked name should be first in the tar archive
   450  		if oldpath, ok := ta.SeenFiles[inode]; ok {
   451  			hdr.Typeflag = tar.TypeLink
   452  			hdr.Linkname = oldpath
   453  			hdr.Size = 0 // This Must be here for the writer math to add up!
   454  		} else {
   455  			ta.SeenFiles[inode] = name
   456  		}
   457  	}
   458  
   459  	//check whether the file is overlayfs whiteout
   460  	//if yes, skip re-mapping container ID mappings.
   461  	isOverlayWhiteout := fi.Mode()&os.ModeCharDevice != 0 && hdr.Devmajor == 0 && hdr.Devminor == 0
   462  
   463  	//handle re-mapping container ID mappings back to host ID mappings before
   464  	//writing tar headers/files. We skip whiteout files because they were written
   465  	//by the kernel and already have proper ownership relative to the host
   466  	if !isOverlayWhiteout &&
   467  		!strings.HasPrefix(filepath.Base(hdr.Name), WhiteoutPrefix) &&
   468  		!ta.IDMappings.Empty() {
   469  		fileIDPair, err := getFileUIDGID(fi.Sys())
   470  		if err != nil {
   471  			return err
   472  		}
   473  		hdr.Uid, hdr.Gid, err = ta.IDMappings.ToContainer(fileIDPair)
   474  		if err != nil {
   475  			return err
   476  		}
   477  	}
   478  
   479  	// explicitly override with ChownOpts
   480  	if ta.ChownOpts != nil {
   481  		hdr.Uid = ta.ChownOpts.UID
   482  		hdr.Gid = ta.ChownOpts.GID
   483  	}
   484  
   485  	if ta.WhiteoutConverter != nil {
   486  		wo, err := ta.WhiteoutConverter.ConvertWrite(hdr, path, fi)
   487  		if err != nil {
   488  			return err
   489  		}
   490  
   491  		// If a new whiteout file exists, write original hdr, then
   492  		// replace hdr with wo to be written after. Whiteouts should
   493  		// always be written after the original. Note the original
   494  		// hdr may have been updated to be a whiteout with returning
   495  		// a whiteout header
   496  		if wo != nil {
   497  			if err := ta.TarWriter.WriteHeader(hdr); err != nil {
   498  				return err
   499  			}
   500  			if hdr.Typeflag == tar.TypeReg && hdr.Size > 0 {
   501  				return fmt.Errorf("tar: cannot use whiteout for non-empty file")
   502  			}
   503  			hdr = wo
   504  		}
   505  	}
   506  
   507  	if err := ta.TarWriter.WriteHeader(hdr); err != nil {
   508  		return err
   509  	}
   510  
   511  	if hdr.Typeflag == tar.TypeReg && hdr.Size > 0 {
   512  		// We use system.OpenSequential to ensure we use sequential file
   513  		// access on Windows to avoid depleting the standby list.
   514  		// On Linux, this equates to a regular os.Open.
   515  		file, err := system.OpenSequential(path)
   516  		if err != nil {
   517  			return err
   518  		}
   519  
   520  		ta.Buffer.Reset(ta.TarWriter)
   521  		defer ta.Buffer.Reset(nil)
   522  		_, err = io.Copy(ta.Buffer, file)
   523  		file.Close()
   524  		if err != nil {
   525  			return err
   526  		}
   527  		err = ta.Buffer.Flush()
   528  		if err != nil {
   529  			return err
   530  		}
   531  	}
   532  
   533  	return nil
   534  }
   535  
   536  func createTarFile(path, extractDir string, hdr *tar.Header, reader io.Reader, Lchown bool, chownOpts *idtools.IDPair, inUserns bool) error {
   537  	// hdr.Mode is in linux format, which we can use for sycalls,
   538  	// but for os.Foo() calls we need the mode converted to os.FileMode,
   539  	// so use hdrInfo.Mode() (they differ for e.g. setuid bits)
   540  	hdrInfo := hdr.FileInfo()
   541  
   542  	switch hdr.Typeflag {
   543  	case tar.TypeDir:
   544  		// Create directory unless it exists as a directory already.
   545  		// In that case we just want to merge the two
   546  		if fi, err := os.Lstat(path); !(err == nil && fi.IsDir()) {
   547  			if err := os.Mkdir(path, hdrInfo.Mode()); err != nil {
   548  				return err
   549  			}
   550  		}
   551  
   552  	case tar.TypeReg, tar.TypeRegA:
   553  		// Source is regular file. We use system.OpenFileSequential to use sequential
   554  		// file access to avoid depleting the standby list on Windows.
   555  		// On Linux, this equates to a regular os.OpenFile
   556  		file, err := system.OpenFileSequential(path, os.O_CREATE|os.O_WRONLY, hdrInfo.Mode())
   557  		if err != nil {
   558  			return err
   559  		}
   560  		if _, err := io.Copy(file, reader); err != nil {
   561  			file.Close()
   562  			return err
   563  		}
   564  		file.Close()
   565  
   566  	case tar.TypeBlock, tar.TypeChar:
   567  		if inUserns { // cannot create devices in a userns
   568  			return nil
   569  		}
   570  		// Handle this is an OS-specific way
   571  		if err := handleTarTypeBlockCharFifo(hdr, path); err != nil {
   572  			return err
   573  		}
   574  
   575  	case tar.TypeFifo:
   576  		// Handle this is an OS-specific way
   577  		if err := handleTarTypeBlockCharFifo(hdr, path); err != nil {
   578  			return err
   579  		}
   580  
   581  	case tar.TypeLink:
   582  		targetPath := filepath.Join(extractDir, hdr.Linkname)
   583  		// check for hardlink breakout
   584  		if !strings.HasPrefix(targetPath, extractDir) {
   585  			return breakoutError(fmt.Errorf("invalid hardlink %q -> %q", targetPath, hdr.Linkname))
   586  		}
   587  		if err := os.Link(targetPath, path); err != nil {
   588  			return err
   589  		}
   590  
   591  	case tar.TypeSymlink:
   592  		// 	path 				-> hdr.Linkname = targetPath
   593  		// e.g. /extractDir/path/to/symlink 	-> ../2/file	= /extractDir/path/2/file
   594  		targetPath := filepath.Join(filepath.Dir(path), hdr.Linkname)
   595  
   596  		// the reason we don't need to check symlinks in the path (with FollowSymlinkInScope) is because
   597  		// that symlink would first have to be created, which would be caught earlier, at this very check:
   598  		if !strings.HasPrefix(targetPath, extractDir) {
   599  			return breakoutError(fmt.Errorf("invalid symlink %q -> %q", path, hdr.Linkname))
   600  		}
   601  		if err := os.Symlink(hdr.Linkname, path); err != nil {
   602  			return err
   603  		}
   604  
   605  	case tar.TypeXGlobalHeader:
   606  		logrus.Debug("PAX Global Extended Headers found and ignored")
   607  		return nil
   608  
   609  	default:
   610  		return fmt.Errorf("unhandled tar header type %d", hdr.Typeflag)
   611  	}
   612  
   613  	// Lchown is not supported on Windows.
   614  	if Lchown && runtime.GOOS != "windows" {
   615  		if chownOpts == nil {
   616  			chownOpts = &idtools.IDPair{UID: hdr.Uid, GID: hdr.Gid}
   617  		}
   618  		if err := os.Lchown(path, chownOpts.UID, chownOpts.GID); err != nil {
   619  			return err
   620  		}
   621  	}
   622  
   623  	var errors []string
   624  	for key, value := range hdr.Xattrs {
   625  		if err := system.Lsetxattr(path, key, []byte(value), 0); err != nil {
   626  			if err == syscall.ENOTSUP {
   627  				// We ignore errors here because not all graphdrivers support
   628  				// xattrs *cough* old versions of AUFS *cough*. However only
   629  				// ENOTSUP should be emitted in that case, otherwise we still
   630  				// bail.
   631  				errors = append(errors, err.Error())
   632  				continue
   633  			}
   634  			return err
   635  		}
   636  
   637  	}
   638  
   639  	if len(errors) > 0 {
   640  		logrus.WithFields(logrus.Fields{
   641  			"errors": errors,
   642  		}).Warn("ignored xattrs in archive: underlying filesystem doesn't support them")
   643  	}
   644  
   645  	// There is no LChmod, so ignore mode for symlink. Also, this
   646  	// must happen after chown, as that can modify the file mode
   647  	if err := handleLChmod(hdr, path, hdrInfo); err != nil {
   648  		return err
   649  	}
   650  
   651  	aTime := hdr.AccessTime
   652  	if aTime.Before(hdr.ModTime) {
   653  		// Last access time should never be before last modified time.
   654  		aTime = hdr.ModTime
   655  	}
   656  
   657  	// system.Chtimes doesn't support a NOFOLLOW flag atm
   658  	if hdr.Typeflag == tar.TypeLink {
   659  		if fi, err := os.Lstat(hdr.Linkname); err == nil && (fi.Mode()&os.ModeSymlink == 0) {
   660  			if err := system.Chtimes(path, aTime, hdr.ModTime); err != nil {
   661  				return err
   662  			}
   663  		}
   664  	} else if hdr.Typeflag != tar.TypeSymlink {
   665  		if err := system.Chtimes(path, aTime, hdr.ModTime); err != nil {
   666  			return err
   667  		}
   668  	} else {
   669  		ts := []syscall.Timespec{timeToTimespec(aTime), timeToTimespec(hdr.ModTime)}
   670  		if err := system.LUtimesNano(path, ts); err != nil && err != system.ErrNotSupportedPlatform {
   671  			return err
   672  		}
   673  	}
   674  	return nil
   675  }
   676  
   677  // Tar creates an archive from the directory at `path`, and returns it as a
   678  // stream of bytes.
   679  func Tar(path string, compression Compression) (io.ReadCloser, error) {
   680  	return TarWithOptions(path, &TarOptions{Compression: compression})
   681  }
   682  
   683  // TarWithOptions creates an archive from the directory at `path`, only including files whose relative
   684  // paths are included in `options.IncludeFiles` (if non-nil) or not in `options.ExcludePatterns`.
   685  func TarWithOptions(srcPath string, options *TarOptions) (io.ReadCloser, error) {
   686  
   687  	// Fix the source path to work with long path names. This is a no-op
   688  	// on platforms other than Windows.
   689  	srcPath = fixVolumePathPrefix(srcPath)
   690  
   691  	pm, err := fileutils.NewPatternMatcher(options.ExcludePatterns)
   692  	if err != nil {
   693  		return nil, err
   694  	}
   695  
   696  	pipeReader, pipeWriter := io.Pipe()
   697  
   698  	compressWriter, err := CompressStream(pipeWriter, options.Compression)
   699  	if err != nil {
   700  		return nil, err
   701  	}
   702  
   703  	go func() {
   704  		ta := newTarAppender(
   705  			idtools.NewIDMappingsFromMaps(options.UIDMaps, options.GIDMaps),
   706  			compressWriter,
   707  			options.ChownOpts,
   708  		)
   709  		ta.WhiteoutConverter = getWhiteoutConverter(options.WhiteoutFormat)
   710  
   711  		defer func() {
   712  			// Make sure to check the error on Close.
   713  			if err := ta.TarWriter.Close(); err != nil {
   714  				logrus.Errorf("Can't close tar writer: %s", err)
   715  			}
   716  			if err := compressWriter.Close(); err != nil {
   717  				logrus.Errorf("Can't close compress writer: %s", err)
   718  			}
   719  			if err := pipeWriter.Close(); err != nil {
   720  				logrus.Errorf("Can't close pipe writer: %s", err)
   721  			}
   722  		}()
   723  
   724  		// this buffer is needed for the duration of this piped stream
   725  		defer pools.BufioWriter32KPool.Put(ta.Buffer)
   726  
   727  		// In general we log errors here but ignore them because
   728  		// during e.g. a diff operation the container can continue
   729  		// mutating the filesystem and we can see transient errors
   730  		// from this
   731  
   732  		stat, err := os.Lstat(srcPath)
   733  		if err != nil {
   734  			return
   735  		}
   736  
   737  		if !stat.IsDir() {
   738  			// We can't later join a non-dir with any includes because the
   739  			// 'walk' will error if "file/." is stat-ed and "file" is not a
   740  			// directory. So, we must split the source path and use the
   741  			// basename as the include.
   742  			if len(options.IncludeFiles) > 0 {
   743  				logrus.Warn("Tar: Can't archive a file with includes")
   744  			}
   745  
   746  			dir, base := SplitPathDirEntry(srcPath)
   747  			srcPath = dir
   748  			options.IncludeFiles = []string{base}
   749  		}
   750  
   751  		if len(options.IncludeFiles) == 0 {
   752  			options.IncludeFiles = []string{"."}
   753  		}
   754  
   755  		seen := make(map[string]bool)
   756  
   757  		for _, include := range options.IncludeFiles {
   758  			rebaseName := options.RebaseNames[include]
   759  
   760  			walkRoot := getWalkRoot(srcPath, include)
   761  			filepath.Walk(walkRoot, func(filePath string, f os.FileInfo, err error) error {
   762  				if err != nil {
   763  					logrus.Errorf("Tar: Can't stat file %s to tar: %s", srcPath, err)
   764  					return nil
   765  				}
   766  
   767  				relFilePath, err := filepath.Rel(srcPath, filePath)
   768  				if err != nil || (!options.IncludeSourceDir && relFilePath == "." && f.IsDir()) {
   769  					// Error getting relative path OR we are looking
   770  					// at the source directory path. Skip in both situations.
   771  					return nil
   772  				}
   773  
   774  				if options.IncludeSourceDir && include == "." && relFilePath != "." {
   775  					relFilePath = strings.Join([]string{".", relFilePath}, string(filepath.Separator))
   776  				}
   777  
   778  				skip := false
   779  
   780  				// If "include" is an exact match for the current file
   781  				// then even if there's an "excludePatterns" pattern that
   782  				// matches it, don't skip it. IOW, assume an explicit 'include'
   783  				// is asking for that file no matter what - which is true
   784  				// for some files, like .dockerignore and Dockerfile (sometimes)
   785  				if include != relFilePath {
   786  					skip, err = pm.Matches(relFilePath)
   787  					if err != nil {
   788  						logrus.Errorf("Error matching %s: %v", relFilePath, err)
   789  						return err
   790  					}
   791  				}
   792  
   793  				if skip {
   794  					// If we want to skip this file and its a directory
   795  					// then we should first check to see if there's an
   796  					// excludes pattern (e.g. !dir/file) that starts with this
   797  					// dir. If so then we can't skip this dir.
   798  
   799  					// Its not a dir then so we can just return/skip.
   800  					if !f.IsDir() {
   801  						return nil
   802  					}
   803  
   804  					// No exceptions (!...) in patterns so just skip dir
   805  					if !pm.Exclusions() {
   806  						return filepath.SkipDir
   807  					}
   808  
   809  					dirSlash := relFilePath + string(filepath.Separator)
   810  
   811  					for _, pat := range pm.Patterns() {
   812  						if !pat.Exclusion() {
   813  							continue
   814  						}
   815  						if strings.HasPrefix(pat.String()+string(filepath.Separator), dirSlash) {
   816  							// found a match - so can't skip this dir
   817  							return nil
   818  						}
   819  					}
   820  
   821  					// No matching exclusion dir so just skip dir
   822  					return filepath.SkipDir
   823  				}
   824  
   825  				if seen[relFilePath] {
   826  					return nil
   827  				}
   828  				seen[relFilePath] = true
   829  
   830  				// Rename the base resource.
   831  				if rebaseName != "" {
   832  					var replacement string
   833  					if rebaseName != string(filepath.Separator) {
   834  						// Special case the root directory to replace with an
   835  						// empty string instead so that we don't end up with
   836  						// double slashes in the paths.
   837  						replacement = rebaseName
   838  					}
   839  
   840  					relFilePath = strings.Replace(relFilePath, include, replacement, 1)
   841  				}
   842  
   843  				if err := ta.addTarFile(filePath, relFilePath); err != nil {
   844  					logrus.Errorf("Can't add file %s to tar: %s", filePath, err)
   845  					// if pipe is broken, stop writing tar stream to it
   846  					if err == io.ErrClosedPipe {
   847  						return err
   848  					}
   849  				}
   850  				return nil
   851  			})
   852  		}
   853  	}()
   854  
   855  	return pipeReader, nil
   856  }
   857  
   858  // Unpack unpacks the decompressedArchive to dest with options.
   859  func Unpack(decompressedArchive io.Reader, dest string, options *TarOptions) error {
   860  	tr := tar.NewReader(decompressedArchive)
   861  	trBuf := pools.BufioReader32KPool.Get(nil)
   862  	defer pools.BufioReader32KPool.Put(trBuf)
   863  
   864  	var dirs []*tar.Header
   865  	idMappings := idtools.NewIDMappingsFromMaps(options.UIDMaps, options.GIDMaps)
   866  	rootIDs := idMappings.RootPair()
   867  	whiteoutConverter := getWhiteoutConverter(options.WhiteoutFormat)
   868  
   869  	// Iterate through the files in the archive.
   870  loop:
   871  	for {
   872  		hdr, err := tr.Next()
   873  		if err == io.EOF {
   874  			// end of tar archive
   875  			break
   876  		}
   877  		if err != nil {
   878  			return err
   879  		}
   880  
   881  		// Normalize name, for safety and for a simple is-root check
   882  		// This keeps "../" as-is, but normalizes "/../" to "/". Or Windows:
   883  		// This keeps "..\" as-is, but normalizes "\..\" to "\".
   884  		hdr.Name = filepath.Clean(hdr.Name)
   885  
   886  		for _, exclude := range options.ExcludePatterns {
   887  			if strings.HasPrefix(hdr.Name, exclude) {
   888  				continue loop
   889  			}
   890  		}
   891  
   892  		// After calling filepath.Clean(hdr.Name) above, hdr.Name will now be in
   893  		// the filepath format for the OS on which the daemon is running. Hence
   894  		// the check for a slash-suffix MUST be done in an OS-agnostic way.
   895  		if !strings.HasSuffix(hdr.Name, string(os.PathSeparator)) {
   896  			// Not the root directory, ensure that the parent directory exists
   897  			parent := filepath.Dir(hdr.Name)
   898  			parentPath := filepath.Join(dest, parent)
   899  			if _, err := os.Lstat(parentPath); err != nil && os.IsNotExist(err) {
   900  				err = idtools.MkdirAllAndChownNew(parentPath, 0777, rootIDs)
   901  				if err != nil {
   902  					return err
   903  				}
   904  			}
   905  		}
   906  
   907  		path := filepath.Join(dest, hdr.Name)
   908  		rel, err := filepath.Rel(dest, path)
   909  		if err != nil {
   910  			return err
   911  		}
   912  		if strings.HasPrefix(rel, ".."+string(os.PathSeparator)) {
   913  			return breakoutError(fmt.Errorf("%q is outside of %q", hdr.Name, dest))
   914  		}
   915  
   916  		// If path exits we almost always just want to remove and replace it
   917  		// The only exception is when it is a directory *and* the file from
   918  		// the layer is also a directory. Then we want to merge them (i.e.
   919  		// just apply the metadata from the layer).
   920  		if fi, err := os.Lstat(path); err == nil {
   921  			if options.NoOverwriteDirNonDir && fi.IsDir() && hdr.Typeflag != tar.TypeDir {
   922  				// If NoOverwriteDirNonDir is true then we cannot replace
   923  				// an existing directory with a non-directory from the archive.
   924  				return fmt.Errorf("cannot overwrite directory %q with non-directory %q", path, dest)
   925  			}
   926  
   927  			if options.NoOverwriteDirNonDir && !fi.IsDir() && hdr.Typeflag == tar.TypeDir {
   928  				// If NoOverwriteDirNonDir is true then we cannot replace
   929  				// an existing non-directory with a directory from the archive.
   930  				return fmt.Errorf("cannot overwrite non-directory %q with directory %q", path, dest)
   931  			}
   932  
   933  			if fi.IsDir() && hdr.Name == "." {
   934  				continue
   935  			}
   936  
   937  			if !(fi.IsDir() && hdr.Typeflag == tar.TypeDir) {
   938  				if err := os.RemoveAll(path); err != nil {
   939  					return err
   940  				}
   941  			}
   942  		}
   943  		trBuf.Reset(tr)
   944  
   945  		if err := remapIDs(idMappings, hdr); err != nil {
   946  			return err
   947  		}
   948  
   949  		if whiteoutConverter != nil {
   950  			writeFile, err := whiteoutConverter.ConvertRead(hdr, path)
   951  			if err != nil {
   952  				return err
   953  			}
   954  			if !writeFile {
   955  				continue
   956  			}
   957  		}
   958  
   959  		if err := createTarFile(path, dest, hdr, trBuf, !options.NoLchown, options.ChownOpts, options.InUserNS); err != nil {
   960  			return err
   961  		}
   962  
   963  		// Directory mtimes must be handled at the end to avoid further
   964  		// file creation in them to modify the directory mtime
   965  		if hdr.Typeflag == tar.TypeDir {
   966  			dirs = append(dirs, hdr)
   967  		}
   968  	}
   969  
   970  	for _, hdr := range dirs {
   971  		path := filepath.Join(dest, hdr.Name)
   972  
   973  		if err := system.Chtimes(path, hdr.AccessTime, hdr.ModTime); err != nil {
   974  			return err
   975  		}
   976  	}
   977  	return nil
   978  }
   979  
   980  // Untar reads a stream of bytes from `archive`, parses it as a tar archive,
   981  // and unpacks it into the directory at `dest`.
   982  // The archive may be compressed with one of the following algorithms:
   983  //  identity (uncompressed), gzip, bzip2, xz.
   984  // FIXME: specify behavior when target path exists vs. doesn't exist.
   985  func Untar(tarArchive io.Reader, dest string, options *TarOptions) error {
   986  	return untarHandler(tarArchive, dest, options, true)
   987  }
   988  
   989  // UntarUncompressed reads a stream of bytes from `archive`, parses it as a tar archive,
   990  // and unpacks it into the directory at `dest`.
   991  // The archive must be an uncompressed stream.
   992  func UntarUncompressed(tarArchive io.Reader, dest string, options *TarOptions) error {
   993  	return untarHandler(tarArchive, dest, options, false)
   994  }
   995  
   996  // Handler for teasing out the automatic decompression
   997  func untarHandler(tarArchive io.Reader, dest string, options *TarOptions, decompress bool) error {
   998  	if tarArchive == nil {
   999  		return fmt.Errorf("Empty archive")
  1000  	}
  1001  	dest = filepath.Clean(dest)
  1002  	if options == nil {
  1003  		options = &TarOptions{}
  1004  	}
  1005  	if options.ExcludePatterns == nil {
  1006  		options.ExcludePatterns = []string{}
  1007  	}
  1008  
  1009  	r := tarArchive
  1010  	if decompress {
  1011  		decompressedArchive, err := DecompressStream(tarArchive)
  1012  		if err != nil {
  1013  			return err
  1014  		}
  1015  		defer decompressedArchive.Close()
  1016  		r = decompressedArchive
  1017  	}
  1018  
  1019  	return Unpack(r, dest, options)
  1020  }
  1021  
  1022  // TarUntar is a convenience function which calls Tar and Untar, with the output of one piped into the other.
  1023  // If either Tar or Untar fails, TarUntar aborts and returns the error.
  1024  func (archiver *Archiver) TarUntar(src, dst string) error {
  1025  	logrus.Debugf("TarUntar(%s %s)", src, dst)
  1026  	archive, err := TarWithOptions(src, &TarOptions{Compression: Uncompressed})
  1027  	if err != nil {
  1028  		return err
  1029  	}
  1030  	defer archive.Close()
  1031  	options := &TarOptions{
  1032  		UIDMaps: archiver.IDMappingsVar.UIDs(),
  1033  		GIDMaps: archiver.IDMappingsVar.GIDs(),
  1034  	}
  1035  	return archiver.Untar(archive, dst, options)
  1036  }
  1037  
  1038  // UntarPath untar a file from path to a destination, src is the source tar file path.
  1039  func (archiver *Archiver) UntarPath(src, dst string) error {
  1040  	archive, err := os.Open(src)
  1041  	if err != nil {
  1042  		return err
  1043  	}
  1044  	defer archive.Close()
  1045  	options := &TarOptions{
  1046  		UIDMaps: archiver.IDMappingsVar.UIDs(),
  1047  		GIDMaps: archiver.IDMappingsVar.GIDs(),
  1048  	}
  1049  	return archiver.Untar(archive, dst, options)
  1050  }
  1051  
  1052  // CopyWithTar creates a tar archive of filesystem path `src`, and
  1053  // unpacks it at filesystem path `dst`.
  1054  // The archive is streamed directly with fixed buffering and no
  1055  // intermediary disk IO.
  1056  func (archiver *Archiver) CopyWithTar(src, dst string) error {
  1057  	srcSt, err := os.Stat(src)
  1058  	if err != nil {
  1059  		return err
  1060  	}
  1061  	if !srcSt.IsDir() {
  1062  		return archiver.CopyFileWithTar(src, dst)
  1063  	}
  1064  
  1065  	// if this Archiver is set up with ID mapping we need to create
  1066  	// the new destination directory with the remapped root UID/GID pair
  1067  	// as owner
  1068  	rootIDs := archiver.IDMappingsVar.RootPair()
  1069  	// Create dst, copy src's content into it
  1070  	logrus.Debugf("Creating dest directory: %s", dst)
  1071  	if err := idtools.MkdirAllAndChownNew(dst, 0755, rootIDs); err != nil {
  1072  		return err
  1073  	}
  1074  	logrus.Debugf("Calling TarUntar(%s, %s)", src, dst)
  1075  	return archiver.TarUntar(src, dst)
  1076  }
  1077  
  1078  // CopyFileWithTar emulates the behavior of the 'cp' command-line
  1079  // for a single file. It copies a regular file from path `src` to
  1080  // path `dst`, and preserves all its metadata.
  1081  func (archiver *Archiver) CopyFileWithTar(src, dst string) (err error) {
  1082  	logrus.Debugf("CopyFileWithTar(%s, %s)", src, dst)
  1083  	srcSt, err := os.Stat(src)
  1084  	if err != nil {
  1085  		return err
  1086  	}
  1087  
  1088  	if srcSt.IsDir() {
  1089  		return fmt.Errorf("Can't copy a directory")
  1090  	}
  1091  
  1092  	// Clean up the trailing slash. This must be done in an operating
  1093  	// system specific manner.
  1094  	if dst[len(dst)-1] == os.PathSeparator {
  1095  		dst = filepath.Join(dst, filepath.Base(src))
  1096  	}
  1097  	// Create the holding directory if necessary
  1098  	if err := system.MkdirAll(filepath.Dir(dst), 0700, ""); err != nil {
  1099  		return err
  1100  	}
  1101  
  1102  	r, w := io.Pipe()
  1103  	errC := make(chan error, 1)
  1104  
  1105  	go func() {
  1106  		defer close(errC)
  1107  
  1108  		errC <- func() error {
  1109  			defer w.Close()
  1110  
  1111  			srcF, err := os.Open(src)
  1112  			if err != nil {
  1113  				return err
  1114  			}
  1115  			defer srcF.Close()
  1116  
  1117  			hdr, err := tar.FileInfoHeader(srcSt, "")
  1118  			if err != nil {
  1119  				return err
  1120  			}
  1121  			hdr.Name = filepath.Base(dst)
  1122  			hdr.Mode = int64(chmodTarEntry(os.FileMode(hdr.Mode)))
  1123  
  1124  			if err := remapIDs(archiver.IDMappingsVar, hdr); err != nil {
  1125  				return err
  1126  			}
  1127  
  1128  			tw := tar.NewWriter(w)
  1129  			defer tw.Close()
  1130  			if err := tw.WriteHeader(hdr); err != nil {
  1131  				return err
  1132  			}
  1133  			if _, err := io.Copy(tw, srcF); err != nil {
  1134  				return err
  1135  			}
  1136  			return nil
  1137  		}()
  1138  	}()
  1139  	defer func() {
  1140  		if er := <-errC; err == nil && er != nil {
  1141  			err = er
  1142  		}
  1143  	}()
  1144  
  1145  	err = archiver.Untar(r, filepath.Dir(dst), nil)
  1146  	if err != nil {
  1147  		r.CloseWithError(err)
  1148  	}
  1149  	return err
  1150  }
  1151  
  1152  // IDMappings returns the IDMappings of the archiver.
  1153  func (archiver *Archiver) IDMappings() *idtools.IDMappings {
  1154  	return archiver.IDMappingsVar
  1155  }
  1156  
  1157  func remapIDs(idMappings *idtools.IDMappings, hdr *tar.Header) error {
  1158  	ids, err := idMappings.ToHost(idtools.IDPair{UID: hdr.Uid, GID: hdr.Gid})
  1159  	hdr.Uid, hdr.Gid = ids.UID, ids.GID
  1160  	return err
  1161  }
  1162  
  1163  // cmdStream executes a command, and returns its stdout as a stream.
  1164  // If the command fails to run or doesn't complete successfully, an error
  1165  // will be returned, including anything written on stderr.
  1166  func cmdStream(cmd *exec.Cmd, input io.Reader) (io.ReadCloser, <-chan struct{}, error) {
  1167  	chdone := make(chan struct{})
  1168  	cmd.Stdin = input
  1169  	pipeR, pipeW := io.Pipe()
  1170  	cmd.Stdout = pipeW
  1171  	var errBuf bytes.Buffer
  1172  	cmd.Stderr = &errBuf
  1173  
  1174  	// Run the command and return the pipe
  1175  	if err := cmd.Start(); err != nil {
  1176  		return nil, nil, err
  1177  	}
  1178  
  1179  	// Copy stdout to the returned pipe
  1180  	go func() {
  1181  		if err := cmd.Wait(); err != nil {
  1182  			pipeW.CloseWithError(fmt.Errorf("%s: %s", err, errBuf.String()))
  1183  		} else {
  1184  			pipeW.Close()
  1185  		}
  1186  		close(chdone)
  1187  	}()
  1188  
  1189  	return pipeR, chdone, nil
  1190  }
  1191  
  1192  // NewTempArchive reads the content of src into a temporary file, and returns the contents
  1193  // of that file as an archive. The archive can only be read once - as soon as reading completes,
  1194  // the file will be deleted.
  1195  func NewTempArchive(src io.Reader, dir string) (*TempArchive, error) {
  1196  	f, err := ioutil.TempFile(dir, "")
  1197  	if err != nil {
  1198  		return nil, err
  1199  	}
  1200  	if _, err := io.Copy(f, src); err != nil {
  1201  		return nil, err
  1202  	}
  1203  	if _, err := f.Seek(0, 0); err != nil {
  1204  		return nil, err
  1205  	}
  1206  	st, err := f.Stat()
  1207  	if err != nil {
  1208  		return nil, err
  1209  	}
  1210  	size := st.Size()
  1211  	return &TempArchive{File: f, Size: size}, nil
  1212  }
  1213  
  1214  // TempArchive is a temporary archive. The archive can only be read once - as soon as reading completes,
  1215  // the file will be deleted.
  1216  type TempArchive struct {
  1217  	*os.File
  1218  	Size   int64 // Pre-computed from Stat().Size() as a convenience
  1219  	read   int64
  1220  	closed bool
  1221  }
  1222  
  1223  // Close closes the underlying file if it's still open, or does a no-op
  1224  // to allow callers to try to close the TempArchive multiple times safely.
  1225  func (archive *TempArchive) Close() error {
  1226  	if archive.closed {
  1227  		return nil
  1228  	}
  1229  
  1230  	archive.closed = true
  1231  
  1232  	return archive.File.Close()
  1233  }
  1234  
  1235  func (archive *TempArchive) Read(data []byte) (int, error) {
  1236  	n, err := archive.File.Read(data)
  1237  	archive.read += int64(n)
  1238  	if err != nil || archive.read == archive.Size {
  1239  		archive.Close()
  1240  		os.Remove(archive.File.Name())
  1241  	}
  1242  	return n, err
  1243  }