github.com/shishir-a412ed/docker@v1.3.2-0.20180103180333-fda904911d87/pkg/archive/archive.go (about) 1 package archive 2 3 import ( 4 "archive/tar" 5 "bufio" 6 "bytes" 7 "compress/bzip2" 8 "compress/gzip" 9 "fmt" 10 "io" 11 "io/ioutil" 12 "os" 13 "os/exec" 14 "path/filepath" 15 "runtime" 16 "strings" 17 "syscall" 18 19 "github.com/docker/docker/pkg/fileutils" 20 "github.com/docker/docker/pkg/idtools" 21 "github.com/docker/docker/pkg/ioutils" 22 "github.com/docker/docker/pkg/pools" 23 "github.com/docker/docker/pkg/system" 24 "github.com/sirupsen/logrus" 25 ) 26 27 type ( 28 // Compression is the state represents if compressed or not. 29 Compression int 30 // WhiteoutFormat is the format of whiteouts unpacked 31 WhiteoutFormat int 32 33 // TarOptions wraps the tar options. 34 TarOptions struct { 35 IncludeFiles []string 36 ExcludePatterns []string 37 Compression Compression 38 NoLchown bool 39 UIDMaps []idtools.IDMap 40 GIDMaps []idtools.IDMap 41 ChownOpts *idtools.IDPair 42 IncludeSourceDir bool 43 // WhiteoutFormat is the expected on disk format for whiteout files. 44 // This format will be converted to the standard format on pack 45 // and from the standard format on unpack. 46 WhiteoutFormat WhiteoutFormat 47 // When unpacking, specifies whether overwriting a directory with a 48 // non-directory is allowed and vice versa. 49 NoOverwriteDirNonDir bool 50 // For each include when creating an archive, the included name will be 51 // replaced with the matching name from this map. 52 RebaseNames map[string]string 53 InUserNS bool 54 } 55 ) 56 57 // Archiver implements the Archiver interface and allows the reuse of most utility functions of 58 // this package with a pluggable Untar function. Also, to facilitate the passing of specific id 59 // mappings for untar, an Archiver can be created with maps which will then be passed to Untar operations. 60 type Archiver struct { 61 Untar func(io.Reader, string, *TarOptions) error 62 IDMappingsVar *idtools.IDMappings 63 } 64 65 // NewDefaultArchiver returns a new Archiver without any IDMappings 66 func NewDefaultArchiver() *Archiver { 67 return &Archiver{Untar: Untar, IDMappingsVar: &idtools.IDMappings{}} 68 } 69 70 // breakoutError is used to differentiate errors related to breaking out 71 // When testing archive breakout in the unit tests, this error is expected 72 // in order for the test to pass. 73 type breakoutError error 74 75 const ( 76 // Uncompressed represents the uncompressed. 77 Uncompressed Compression = iota 78 // Bzip2 is bzip2 compression algorithm. 79 Bzip2 80 // Gzip is gzip compression algorithm. 81 Gzip 82 // Xz is xz compression algorithm. 83 Xz 84 ) 85 86 const ( 87 // AUFSWhiteoutFormat is the default format for whiteouts 88 AUFSWhiteoutFormat WhiteoutFormat = iota 89 // OverlayWhiteoutFormat formats whiteout according to the overlay 90 // standard. 91 OverlayWhiteoutFormat 92 ) 93 94 const ( 95 modeISDIR = 040000 // Directory 96 modeISFIFO = 010000 // FIFO 97 modeISREG = 0100000 // Regular file 98 modeISLNK = 0120000 // Symbolic link 99 modeISBLK = 060000 // Block special file 100 modeISCHR = 020000 // Character special file 101 modeISSOCK = 0140000 // Socket 102 ) 103 104 // IsArchivePath checks if the (possibly compressed) file at the given path 105 // starts with a tar file header. 106 func IsArchivePath(path string) bool { 107 file, err := os.Open(path) 108 if err != nil { 109 return false 110 } 111 defer file.Close() 112 rdr, err := DecompressStream(file) 113 if err != nil { 114 return false 115 } 116 r := tar.NewReader(rdr) 117 _, err = r.Next() 118 return err == nil 119 } 120 121 // DetectCompression detects the compression algorithm of the source. 122 func DetectCompression(source []byte) Compression { 123 for compression, m := range map[Compression][]byte{ 124 Bzip2: {0x42, 0x5A, 0x68}, 125 Gzip: {0x1F, 0x8B, 0x08}, 126 Xz: {0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00}, 127 } { 128 if len(source) < len(m) { 129 logrus.Debug("Len too short") 130 continue 131 } 132 if bytes.Equal(m, source[:len(m)]) { 133 return compression 134 } 135 } 136 return Uncompressed 137 } 138 139 func xzDecompress(archive io.Reader) (io.ReadCloser, <-chan struct{}, error) { 140 args := []string{"xz", "-d", "-c", "-q"} 141 142 return cmdStream(exec.Command(args[0], args[1:]...), archive) 143 } 144 145 // DecompressStream decompresses the archive and returns a ReaderCloser with the decompressed archive. 146 func DecompressStream(archive io.Reader) (io.ReadCloser, error) { 147 p := pools.BufioReader32KPool 148 buf := p.Get(archive) 149 bs, err := buf.Peek(10) 150 if err != nil && err != io.EOF { 151 // Note: we'll ignore any io.EOF error because there are some odd 152 // cases where the layer.tar file will be empty (zero bytes) and 153 // that results in an io.EOF from the Peek() call. So, in those 154 // cases we'll just treat it as a non-compressed stream and 155 // that means just create an empty layer. 156 // See Issue 18170 157 return nil, err 158 } 159 160 compression := DetectCompression(bs) 161 switch compression { 162 case Uncompressed: 163 readBufWrapper := p.NewReadCloserWrapper(buf, buf) 164 return readBufWrapper, nil 165 case Gzip: 166 gzReader, err := gzip.NewReader(buf) 167 if err != nil { 168 return nil, err 169 } 170 readBufWrapper := p.NewReadCloserWrapper(buf, gzReader) 171 return readBufWrapper, nil 172 case Bzip2: 173 bz2Reader := bzip2.NewReader(buf) 174 readBufWrapper := p.NewReadCloserWrapper(buf, bz2Reader) 175 return readBufWrapper, nil 176 case Xz: 177 xzReader, chdone, err := xzDecompress(buf) 178 if err != nil { 179 return nil, err 180 } 181 readBufWrapper := p.NewReadCloserWrapper(buf, xzReader) 182 return ioutils.NewReadCloserWrapper(readBufWrapper, func() error { 183 <-chdone 184 return readBufWrapper.Close() 185 }), nil 186 default: 187 return nil, fmt.Errorf("Unsupported compression format %s", (&compression).Extension()) 188 } 189 } 190 191 // CompressStream compresses the dest with specified compression algorithm. 192 func CompressStream(dest io.Writer, compression Compression) (io.WriteCloser, error) { 193 p := pools.BufioWriter32KPool 194 buf := p.Get(dest) 195 switch compression { 196 case Uncompressed: 197 writeBufWrapper := p.NewWriteCloserWrapper(buf, buf) 198 return writeBufWrapper, nil 199 case Gzip: 200 gzWriter := gzip.NewWriter(dest) 201 writeBufWrapper := p.NewWriteCloserWrapper(buf, gzWriter) 202 return writeBufWrapper, nil 203 case Bzip2, Xz: 204 // archive/bzip2 does not support writing, and there is no xz support at all 205 // However, this is not a problem as docker only currently generates gzipped tars 206 return nil, fmt.Errorf("Unsupported compression format %s", (&compression).Extension()) 207 default: 208 return nil, fmt.Errorf("Unsupported compression format %s", (&compression).Extension()) 209 } 210 } 211 212 // TarModifierFunc is a function that can be passed to ReplaceFileTarWrapper to 213 // modify the contents or header of an entry in the archive. If the file already 214 // exists in the archive the TarModifierFunc will be called with the Header and 215 // a reader which will return the files content. If the file does not exist both 216 // header and content will be nil. 217 type TarModifierFunc func(path string, header *tar.Header, content io.Reader) (*tar.Header, []byte, error) 218 219 // ReplaceFileTarWrapper converts inputTarStream to a new tar stream. Files in the 220 // tar stream are modified if they match any of the keys in mods. 221 func ReplaceFileTarWrapper(inputTarStream io.ReadCloser, mods map[string]TarModifierFunc) io.ReadCloser { 222 pipeReader, pipeWriter := io.Pipe() 223 224 go func() { 225 tarReader := tar.NewReader(inputTarStream) 226 tarWriter := tar.NewWriter(pipeWriter) 227 defer inputTarStream.Close() 228 defer tarWriter.Close() 229 230 modify := func(name string, original *tar.Header, modifier TarModifierFunc, tarReader io.Reader) error { 231 header, data, err := modifier(name, original, tarReader) 232 switch { 233 case err != nil: 234 return err 235 case header == nil: 236 return nil 237 } 238 239 header.Name = name 240 header.Size = int64(len(data)) 241 if err := tarWriter.WriteHeader(header); err != nil { 242 return err 243 } 244 if len(data) != 0 { 245 if _, err := tarWriter.Write(data); err != nil { 246 return err 247 } 248 } 249 return nil 250 } 251 252 var err error 253 var originalHeader *tar.Header 254 for { 255 originalHeader, err = tarReader.Next() 256 if err == io.EOF { 257 break 258 } 259 if err != nil { 260 pipeWriter.CloseWithError(err) 261 return 262 } 263 264 modifier, ok := mods[originalHeader.Name] 265 if !ok { 266 // No modifiers for this file, copy the header and data 267 if err := tarWriter.WriteHeader(originalHeader); err != nil { 268 pipeWriter.CloseWithError(err) 269 return 270 } 271 if _, err := pools.Copy(tarWriter, tarReader); err != nil { 272 pipeWriter.CloseWithError(err) 273 return 274 } 275 continue 276 } 277 delete(mods, originalHeader.Name) 278 279 if err := modify(originalHeader.Name, originalHeader, modifier, tarReader); err != nil { 280 pipeWriter.CloseWithError(err) 281 return 282 } 283 } 284 285 // Apply the modifiers that haven't matched any files in the archive 286 for name, modifier := range mods { 287 if err := modify(name, nil, modifier, nil); err != nil { 288 pipeWriter.CloseWithError(err) 289 return 290 } 291 } 292 293 pipeWriter.Close() 294 295 }() 296 return pipeReader 297 } 298 299 // Extension returns the extension of a file that uses the specified compression algorithm. 300 func (compression *Compression) Extension() string { 301 switch *compression { 302 case Uncompressed: 303 return "tar" 304 case Bzip2: 305 return "tar.bz2" 306 case Gzip: 307 return "tar.gz" 308 case Xz: 309 return "tar.xz" 310 } 311 return "" 312 } 313 314 // FileInfoHeader creates a populated Header from fi. 315 // Compared to archive pkg this function fills in more information. 316 // Also, regardless of Go version, this function fills file type bits (e.g. hdr.Mode |= modeISDIR), 317 // which have been deleted since Go 1.9 archive/tar. 318 func FileInfoHeader(name string, fi os.FileInfo, link string) (*tar.Header, error) { 319 hdr, err := tar.FileInfoHeader(fi, link) 320 if err != nil { 321 return nil, err 322 } 323 hdr.Mode = fillGo18FileTypeBits(int64(chmodTarEntry(os.FileMode(hdr.Mode))), fi) 324 name, err = canonicalTarName(name, fi.IsDir()) 325 if err != nil { 326 return nil, fmt.Errorf("tar: cannot canonicalize path: %v", err) 327 } 328 hdr.Name = name 329 if err := setHeaderForSpecialDevice(hdr, name, fi.Sys()); err != nil { 330 return nil, err 331 } 332 return hdr, nil 333 } 334 335 // fillGo18FileTypeBits fills type bits which have been removed on Go 1.9 archive/tar 336 // https://github.com/golang/go/commit/66b5a2f 337 func fillGo18FileTypeBits(mode int64, fi os.FileInfo) int64 { 338 fm := fi.Mode() 339 switch { 340 case fm.IsRegular(): 341 mode |= modeISREG 342 case fi.IsDir(): 343 mode |= modeISDIR 344 case fm&os.ModeSymlink != 0: 345 mode |= modeISLNK 346 case fm&os.ModeDevice != 0: 347 if fm&os.ModeCharDevice != 0 { 348 mode |= modeISCHR 349 } else { 350 mode |= modeISBLK 351 } 352 case fm&os.ModeNamedPipe != 0: 353 mode |= modeISFIFO 354 case fm&os.ModeSocket != 0: 355 mode |= modeISSOCK 356 } 357 return mode 358 } 359 360 // ReadSecurityXattrToTarHeader reads security.capability xattr from filesystem 361 // to a tar header 362 func ReadSecurityXattrToTarHeader(path string, hdr *tar.Header) error { 363 capability, _ := system.Lgetxattr(path, "security.capability") 364 if capability != nil { 365 hdr.Xattrs = make(map[string]string) 366 hdr.Xattrs["security.capability"] = string(capability) 367 } 368 return nil 369 } 370 371 type tarWhiteoutConverter interface { 372 ConvertWrite(*tar.Header, string, os.FileInfo) (*tar.Header, error) 373 ConvertRead(*tar.Header, string) (bool, error) 374 } 375 376 type tarAppender struct { 377 TarWriter *tar.Writer 378 Buffer *bufio.Writer 379 380 // for hardlink mapping 381 SeenFiles map[uint64]string 382 IDMappings *idtools.IDMappings 383 ChownOpts *idtools.IDPair 384 385 // For packing and unpacking whiteout files in the 386 // non standard format. The whiteout files defined 387 // by the AUFS standard are used as the tar whiteout 388 // standard. 389 WhiteoutConverter tarWhiteoutConverter 390 } 391 392 func newTarAppender(idMapping *idtools.IDMappings, writer io.Writer, chownOpts *idtools.IDPair) *tarAppender { 393 return &tarAppender{ 394 SeenFiles: make(map[uint64]string), 395 TarWriter: tar.NewWriter(writer), 396 Buffer: pools.BufioWriter32KPool.Get(nil), 397 IDMappings: idMapping, 398 ChownOpts: chownOpts, 399 } 400 } 401 402 // canonicalTarName provides a platform-independent and consistent posix-style 403 //path for files and directories to be archived regardless of the platform. 404 func canonicalTarName(name string, isDir bool) (string, error) { 405 name, err := CanonicalTarNameForPath(name) 406 if err != nil { 407 return "", err 408 } 409 410 // suffix with '/' for directories 411 if isDir && !strings.HasSuffix(name, "/") { 412 name += "/" 413 } 414 return name, nil 415 } 416 417 // addTarFile adds to the tar archive a file from `path` as `name` 418 func (ta *tarAppender) addTarFile(path, name string) error { 419 fi, err := os.Lstat(path) 420 if err != nil { 421 return err 422 } 423 424 var link string 425 if fi.Mode()&os.ModeSymlink != 0 { 426 var err error 427 link, err = os.Readlink(path) 428 if err != nil { 429 return err 430 } 431 } 432 433 hdr, err := FileInfoHeader(name, fi, link) 434 if err != nil { 435 return err 436 } 437 if err := ReadSecurityXattrToTarHeader(path, hdr); err != nil { 438 return err 439 } 440 441 // if it's not a directory and has more than 1 link, 442 // it's hard linked, so set the type flag accordingly 443 if !fi.IsDir() && hasHardlinks(fi) { 444 inode, err := getInodeFromStat(fi.Sys()) 445 if err != nil { 446 return err 447 } 448 // a link should have a name that it links too 449 // and that linked name should be first in the tar archive 450 if oldpath, ok := ta.SeenFiles[inode]; ok { 451 hdr.Typeflag = tar.TypeLink 452 hdr.Linkname = oldpath 453 hdr.Size = 0 // This Must be here for the writer math to add up! 454 } else { 455 ta.SeenFiles[inode] = name 456 } 457 } 458 459 //check whether the file is overlayfs whiteout 460 //if yes, skip re-mapping container ID mappings. 461 isOverlayWhiteout := fi.Mode()&os.ModeCharDevice != 0 && hdr.Devmajor == 0 && hdr.Devminor == 0 462 463 //handle re-mapping container ID mappings back to host ID mappings before 464 //writing tar headers/files. We skip whiteout files because they were written 465 //by the kernel and already have proper ownership relative to the host 466 if !isOverlayWhiteout && 467 !strings.HasPrefix(filepath.Base(hdr.Name), WhiteoutPrefix) && 468 !ta.IDMappings.Empty() { 469 fileIDPair, err := getFileUIDGID(fi.Sys()) 470 if err != nil { 471 return err 472 } 473 hdr.Uid, hdr.Gid, err = ta.IDMappings.ToContainer(fileIDPair) 474 if err != nil { 475 return err 476 } 477 } 478 479 // explicitly override with ChownOpts 480 if ta.ChownOpts != nil { 481 hdr.Uid = ta.ChownOpts.UID 482 hdr.Gid = ta.ChownOpts.GID 483 } 484 485 if ta.WhiteoutConverter != nil { 486 wo, err := ta.WhiteoutConverter.ConvertWrite(hdr, path, fi) 487 if err != nil { 488 return err 489 } 490 491 // If a new whiteout file exists, write original hdr, then 492 // replace hdr with wo to be written after. Whiteouts should 493 // always be written after the original. Note the original 494 // hdr may have been updated to be a whiteout with returning 495 // a whiteout header 496 if wo != nil { 497 if err := ta.TarWriter.WriteHeader(hdr); err != nil { 498 return err 499 } 500 if hdr.Typeflag == tar.TypeReg && hdr.Size > 0 { 501 return fmt.Errorf("tar: cannot use whiteout for non-empty file") 502 } 503 hdr = wo 504 } 505 } 506 507 if err := ta.TarWriter.WriteHeader(hdr); err != nil { 508 return err 509 } 510 511 if hdr.Typeflag == tar.TypeReg && hdr.Size > 0 { 512 // We use system.OpenSequential to ensure we use sequential file 513 // access on Windows to avoid depleting the standby list. 514 // On Linux, this equates to a regular os.Open. 515 file, err := system.OpenSequential(path) 516 if err != nil { 517 return err 518 } 519 520 ta.Buffer.Reset(ta.TarWriter) 521 defer ta.Buffer.Reset(nil) 522 _, err = io.Copy(ta.Buffer, file) 523 file.Close() 524 if err != nil { 525 return err 526 } 527 err = ta.Buffer.Flush() 528 if err != nil { 529 return err 530 } 531 } 532 533 return nil 534 } 535 536 func createTarFile(path, extractDir string, hdr *tar.Header, reader io.Reader, Lchown bool, chownOpts *idtools.IDPair, inUserns bool) error { 537 // hdr.Mode is in linux format, which we can use for sycalls, 538 // but for os.Foo() calls we need the mode converted to os.FileMode, 539 // so use hdrInfo.Mode() (they differ for e.g. setuid bits) 540 hdrInfo := hdr.FileInfo() 541 542 switch hdr.Typeflag { 543 case tar.TypeDir: 544 // Create directory unless it exists as a directory already. 545 // In that case we just want to merge the two 546 if fi, err := os.Lstat(path); !(err == nil && fi.IsDir()) { 547 if err := os.Mkdir(path, hdrInfo.Mode()); err != nil { 548 return err 549 } 550 } 551 552 case tar.TypeReg, tar.TypeRegA: 553 // Source is regular file. We use system.OpenFileSequential to use sequential 554 // file access to avoid depleting the standby list on Windows. 555 // On Linux, this equates to a regular os.OpenFile 556 file, err := system.OpenFileSequential(path, os.O_CREATE|os.O_WRONLY, hdrInfo.Mode()) 557 if err != nil { 558 return err 559 } 560 if _, err := io.Copy(file, reader); err != nil { 561 file.Close() 562 return err 563 } 564 file.Close() 565 566 case tar.TypeBlock, tar.TypeChar: 567 if inUserns { // cannot create devices in a userns 568 return nil 569 } 570 // Handle this is an OS-specific way 571 if err := handleTarTypeBlockCharFifo(hdr, path); err != nil { 572 return err 573 } 574 575 case tar.TypeFifo: 576 // Handle this is an OS-specific way 577 if err := handleTarTypeBlockCharFifo(hdr, path); err != nil { 578 return err 579 } 580 581 case tar.TypeLink: 582 targetPath := filepath.Join(extractDir, hdr.Linkname) 583 // check for hardlink breakout 584 if !strings.HasPrefix(targetPath, extractDir) { 585 return breakoutError(fmt.Errorf("invalid hardlink %q -> %q", targetPath, hdr.Linkname)) 586 } 587 if err := os.Link(targetPath, path); err != nil { 588 return err 589 } 590 591 case tar.TypeSymlink: 592 // path -> hdr.Linkname = targetPath 593 // e.g. /extractDir/path/to/symlink -> ../2/file = /extractDir/path/2/file 594 targetPath := filepath.Join(filepath.Dir(path), hdr.Linkname) 595 596 // the reason we don't need to check symlinks in the path (with FollowSymlinkInScope) is because 597 // that symlink would first have to be created, which would be caught earlier, at this very check: 598 if !strings.HasPrefix(targetPath, extractDir) { 599 return breakoutError(fmt.Errorf("invalid symlink %q -> %q", path, hdr.Linkname)) 600 } 601 if err := os.Symlink(hdr.Linkname, path); err != nil { 602 return err 603 } 604 605 case tar.TypeXGlobalHeader: 606 logrus.Debug("PAX Global Extended Headers found and ignored") 607 return nil 608 609 default: 610 return fmt.Errorf("unhandled tar header type %d", hdr.Typeflag) 611 } 612 613 // Lchown is not supported on Windows. 614 if Lchown && runtime.GOOS != "windows" { 615 if chownOpts == nil { 616 chownOpts = &idtools.IDPair{UID: hdr.Uid, GID: hdr.Gid} 617 } 618 if err := os.Lchown(path, chownOpts.UID, chownOpts.GID); err != nil { 619 return err 620 } 621 } 622 623 var errors []string 624 for key, value := range hdr.Xattrs { 625 if err := system.Lsetxattr(path, key, []byte(value), 0); err != nil { 626 if err == syscall.ENOTSUP { 627 // We ignore errors here because not all graphdrivers support 628 // xattrs *cough* old versions of AUFS *cough*. However only 629 // ENOTSUP should be emitted in that case, otherwise we still 630 // bail. 631 errors = append(errors, err.Error()) 632 continue 633 } 634 return err 635 } 636 637 } 638 639 if len(errors) > 0 { 640 logrus.WithFields(logrus.Fields{ 641 "errors": errors, 642 }).Warn("ignored xattrs in archive: underlying filesystem doesn't support them") 643 } 644 645 // There is no LChmod, so ignore mode for symlink. Also, this 646 // must happen after chown, as that can modify the file mode 647 if err := handleLChmod(hdr, path, hdrInfo); err != nil { 648 return err 649 } 650 651 aTime := hdr.AccessTime 652 if aTime.Before(hdr.ModTime) { 653 // Last access time should never be before last modified time. 654 aTime = hdr.ModTime 655 } 656 657 // system.Chtimes doesn't support a NOFOLLOW flag atm 658 if hdr.Typeflag == tar.TypeLink { 659 if fi, err := os.Lstat(hdr.Linkname); err == nil && (fi.Mode()&os.ModeSymlink == 0) { 660 if err := system.Chtimes(path, aTime, hdr.ModTime); err != nil { 661 return err 662 } 663 } 664 } else if hdr.Typeflag != tar.TypeSymlink { 665 if err := system.Chtimes(path, aTime, hdr.ModTime); err != nil { 666 return err 667 } 668 } else { 669 ts := []syscall.Timespec{timeToTimespec(aTime), timeToTimespec(hdr.ModTime)} 670 if err := system.LUtimesNano(path, ts); err != nil && err != system.ErrNotSupportedPlatform { 671 return err 672 } 673 } 674 return nil 675 } 676 677 // Tar creates an archive from the directory at `path`, and returns it as a 678 // stream of bytes. 679 func Tar(path string, compression Compression) (io.ReadCloser, error) { 680 return TarWithOptions(path, &TarOptions{Compression: compression}) 681 } 682 683 // TarWithOptions creates an archive from the directory at `path`, only including files whose relative 684 // paths are included in `options.IncludeFiles` (if non-nil) or not in `options.ExcludePatterns`. 685 func TarWithOptions(srcPath string, options *TarOptions) (io.ReadCloser, error) { 686 687 // Fix the source path to work with long path names. This is a no-op 688 // on platforms other than Windows. 689 srcPath = fixVolumePathPrefix(srcPath) 690 691 pm, err := fileutils.NewPatternMatcher(options.ExcludePatterns) 692 if err != nil { 693 return nil, err 694 } 695 696 pipeReader, pipeWriter := io.Pipe() 697 698 compressWriter, err := CompressStream(pipeWriter, options.Compression) 699 if err != nil { 700 return nil, err 701 } 702 703 go func() { 704 ta := newTarAppender( 705 idtools.NewIDMappingsFromMaps(options.UIDMaps, options.GIDMaps), 706 compressWriter, 707 options.ChownOpts, 708 ) 709 ta.WhiteoutConverter = getWhiteoutConverter(options.WhiteoutFormat) 710 711 defer func() { 712 // Make sure to check the error on Close. 713 if err := ta.TarWriter.Close(); err != nil { 714 logrus.Errorf("Can't close tar writer: %s", err) 715 } 716 if err := compressWriter.Close(); err != nil { 717 logrus.Errorf("Can't close compress writer: %s", err) 718 } 719 if err := pipeWriter.Close(); err != nil { 720 logrus.Errorf("Can't close pipe writer: %s", err) 721 } 722 }() 723 724 // this buffer is needed for the duration of this piped stream 725 defer pools.BufioWriter32KPool.Put(ta.Buffer) 726 727 // In general we log errors here but ignore them because 728 // during e.g. a diff operation the container can continue 729 // mutating the filesystem and we can see transient errors 730 // from this 731 732 stat, err := os.Lstat(srcPath) 733 if err != nil { 734 return 735 } 736 737 if !stat.IsDir() { 738 // We can't later join a non-dir with any includes because the 739 // 'walk' will error if "file/." is stat-ed and "file" is not a 740 // directory. So, we must split the source path and use the 741 // basename as the include. 742 if len(options.IncludeFiles) > 0 { 743 logrus.Warn("Tar: Can't archive a file with includes") 744 } 745 746 dir, base := SplitPathDirEntry(srcPath) 747 srcPath = dir 748 options.IncludeFiles = []string{base} 749 } 750 751 if len(options.IncludeFiles) == 0 { 752 options.IncludeFiles = []string{"."} 753 } 754 755 seen := make(map[string]bool) 756 757 for _, include := range options.IncludeFiles { 758 rebaseName := options.RebaseNames[include] 759 760 walkRoot := getWalkRoot(srcPath, include) 761 filepath.Walk(walkRoot, func(filePath string, f os.FileInfo, err error) error { 762 if err != nil { 763 logrus.Errorf("Tar: Can't stat file %s to tar: %s", srcPath, err) 764 return nil 765 } 766 767 relFilePath, err := filepath.Rel(srcPath, filePath) 768 if err != nil || (!options.IncludeSourceDir && relFilePath == "." && f.IsDir()) { 769 // Error getting relative path OR we are looking 770 // at the source directory path. Skip in both situations. 771 return nil 772 } 773 774 if options.IncludeSourceDir && include == "." && relFilePath != "." { 775 relFilePath = strings.Join([]string{".", relFilePath}, string(filepath.Separator)) 776 } 777 778 skip := false 779 780 // If "include" is an exact match for the current file 781 // then even if there's an "excludePatterns" pattern that 782 // matches it, don't skip it. IOW, assume an explicit 'include' 783 // is asking for that file no matter what - which is true 784 // for some files, like .dockerignore and Dockerfile (sometimes) 785 if include != relFilePath { 786 skip, err = pm.Matches(relFilePath) 787 if err != nil { 788 logrus.Errorf("Error matching %s: %v", relFilePath, err) 789 return err 790 } 791 } 792 793 if skip { 794 // If we want to skip this file and its a directory 795 // then we should first check to see if there's an 796 // excludes pattern (e.g. !dir/file) that starts with this 797 // dir. If so then we can't skip this dir. 798 799 // Its not a dir then so we can just return/skip. 800 if !f.IsDir() { 801 return nil 802 } 803 804 // No exceptions (!...) in patterns so just skip dir 805 if !pm.Exclusions() { 806 return filepath.SkipDir 807 } 808 809 dirSlash := relFilePath + string(filepath.Separator) 810 811 for _, pat := range pm.Patterns() { 812 if !pat.Exclusion() { 813 continue 814 } 815 if strings.HasPrefix(pat.String()+string(filepath.Separator), dirSlash) { 816 // found a match - so can't skip this dir 817 return nil 818 } 819 } 820 821 // No matching exclusion dir so just skip dir 822 return filepath.SkipDir 823 } 824 825 if seen[relFilePath] { 826 return nil 827 } 828 seen[relFilePath] = true 829 830 // Rename the base resource. 831 if rebaseName != "" { 832 var replacement string 833 if rebaseName != string(filepath.Separator) { 834 // Special case the root directory to replace with an 835 // empty string instead so that we don't end up with 836 // double slashes in the paths. 837 replacement = rebaseName 838 } 839 840 relFilePath = strings.Replace(relFilePath, include, replacement, 1) 841 } 842 843 if err := ta.addTarFile(filePath, relFilePath); err != nil { 844 logrus.Errorf("Can't add file %s to tar: %s", filePath, err) 845 // if pipe is broken, stop writing tar stream to it 846 if err == io.ErrClosedPipe { 847 return err 848 } 849 } 850 return nil 851 }) 852 } 853 }() 854 855 return pipeReader, nil 856 } 857 858 // Unpack unpacks the decompressedArchive to dest with options. 859 func Unpack(decompressedArchive io.Reader, dest string, options *TarOptions) error { 860 tr := tar.NewReader(decompressedArchive) 861 trBuf := pools.BufioReader32KPool.Get(nil) 862 defer pools.BufioReader32KPool.Put(trBuf) 863 864 var dirs []*tar.Header 865 idMappings := idtools.NewIDMappingsFromMaps(options.UIDMaps, options.GIDMaps) 866 rootIDs := idMappings.RootPair() 867 whiteoutConverter := getWhiteoutConverter(options.WhiteoutFormat) 868 869 // Iterate through the files in the archive. 870 loop: 871 for { 872 hdr, err := tr.Next() 873 if err == io.EOF { 874 // end of tar archive 875 break 876 } 877 if err != nil { 878 return err 879 } 880 881 // Normalize name, for safety and for a simple is-root check 882 // This keeps "../" as-is, but normalizes "/../" to "/". Or Windows: 883 // This keeps "..\" as-is, but normalizes "\..\" to "\". 884 hdr.Name = filepath.Clean(hdr.Name) 885 886 for _, exclude := range options.ExcludePatterns { 887 if strings.HasPrefix(hdr.Name, exclude) { 888 continue loop 889 } 890 } 891 892 // After calling filepath.Clean(hdr.Name) above, hdr.Name will now be in 893 // the filepath format for the OS on which the daemon is running. Hence 894 // the check for a slash-suffix MUST be done in an OS-agnostic way. 895 if !strings.HasSuffix(hdr.Name, string(os.PathSeparator)) { 896 // Not the root directory, ensure that the parent directory exists 897 parent := filepath.Dir(hdr.Name) 898 parentPath := filepath.Join(dest, parent) 899 if _, err := os.Lstat(parentPath); err != nil && os.IsNotExist(err) { 900 err = idtools.MkdirAllAndChownNew(parentPath, 0777, rootIDs) 901 if err != nil { 902 return err 903 } 904 } 905 } 906 907 path := filepath.Join(dest, hdr.Name) 908 rel, err := filepath.Rel(dest, path) 909 if err != nil { 910 return err 911 } 912 if strings.HasPrefix(rel, ".."+string(os.PathSeparator)) { 913 return breakoutError(fmt.Errorf("%q is outside of %q", hdr.Name, dest)) 914 } 915 916 // If path exits we almost always just want to remove and replace it 917 // The only exception is when it is a directory *and* the file from 918 // the layer is also a directory. Then we want to merge them (i.e. 919 // just apply the metadata from the layer). 920 if fi, err := os.Lstat(path); err == nil { 921 if options.NoOverwriteDirNonDir && fi.IsDir() && hdr.Typeflag != tar.TypeDir { 922 // If NoOverwriteDirNonDir is true then we cannot replace 923 // an existing directory with a non-directory from the archive. 924 return fmt.Errorf("cannot overwrite directory %q with non-directory %q", path, dest) 925 } 926 927 if options.NoOverwriteDirNonDir && !fi.IsDir() && hdr.Typeflag == tar.TypeDir { 928 // If NoOverwriteDirNonDir is true then we cannot replace 929 // an existing non-directory with a directory from the archive. 930 return fmt.Errorf("cannot overwrite non-directory %q with directory %q", path, dest) 931 } 932 933 if fi.IsDir() && hdr.Name == "." { 934 continue 935 } 936 937 if !(fi.IsDir() && hdr.Typeflag == tar.TypeDir) { 938 if err := os.RemoveAll(path); err != nil { 939 return err 940 } 941 } 942 } 943 trBuf.Reset(tr) 944 945 if err := remapIDs(idMappings, hdr); err != nil { 946 return err 947 } 948 949 if whiteoutConverter != nil { 950 writeFile, err := whiteoutConverter.ConvertRead(hdr, path) 951 if err != nil { 952 return err 953 } 954 if !writeFile { 955 continue 956 } 957 } 958 959 if err := createTarFile(path, dest, hdr, trBuf, !options.NoLchown, options.ChownOpts, options.InUserNS); err != nil { 960 return err 961 } 962 963 // Directory mtimes must be handled at the end to avoid further 964 // file creation in them to modify the directory mtime 965 if hdr.Typeflag == tar.TypeDir { 966 dirs = append(dirs, hdr) 967 } 968 } 969 970 for _, hdr := range dirs { 971 path := filepath.Join(dest, hdr.Name) 972 973 if err := system.Chtimes(path, hdr.AccessTime, hdr.ModTime); err != nil { 974 return err 975 } 976 } 977 return nil 978 } 979 980 // Untar reads a stream of bytes from `archive`, parses it as a tar archive, 981 // and unpacks it into the directory at `dest`. 982 // The archive may be compressed with one of the following algorithms: 983 // identity (uncompressed), gzip, bzip2, xz. 984 // FIXME: specify behavior when target path exists vs. doesn't exist. 985 func Untar(tarArchive io.Reader, dest string, options *TarOptions) error { 986 return untarHandler(tarArchive, dest, options, true) 987 } 988 989 // UntarUncompressed reads a stream of bytes from `archive`, parses it as a tar archive, 990 // and unpacks it into the directory at `dest`. 991 // The archive must be an uncompressed stream. 992 func UntarUncompressed(tarArchive io.Reader, dest string, options *TarOptions) error { 993 return untarHandler(tarArchive, dest, options, false) 994 } 995 996 // Handler for teasing out the automatic decompression 997 func untarHandler(tarArchive io.Reader, dest string, options *TarOptions, decompress bool) error { 998 if tarArchive == nil { 999 return fmt.Errorf("Empty archive") 1000 } 1001 dest = filepath.Clean(dest) 1002 if options == nil { 1003 options = &TarOptions{} 1004 } 1005 if options.ExcludePatterns == nil { 1006 options.ExcludePatterns = []string{} 1007 } 1008 1009 r := tarArchive 1010 if decompress { 1011 decompressedArchive, err := DecompressStream(tarArchive) 1012 if err != nil { 1013 return err 1014 } 1015 defer decompressedArchive.Close() 1016 r = decompressedArchive 1017 } 1018 1019 return Unpack(r, dest, options) 1020 } 1021 1022 // TarUntar is a convenience function which calls Tar and Untar, with the output of one piped into the other. 1023 // If either Tar or Untar fails, TarUntar aborts and returns the error. 1024 func (archiver *Archiver) TarUntar(src, dst string) error { 1025 logrus.Debugf("TarUntar(%s %s)", src, dst) 1026 archive, err := TarWithOptions(src, &TarOptions{Compression: Uncompressed}) 1027 if err != nil { 1028 return err 1029 } 1030 defer archive.Close() 1031 options := &TarOptions{ 1032 UIDMaps: archiver.IDMappingsVar.UIDs(), 1033 GIDMaps: archiver.IDMappingsVar.GIDs(), 1034 } 1035 return archiver.Untar(archive, dst, options) 1036 } 1037 1038 // UntarPath untar a file from path to a destination, src is the source tar file path. 1039 func (archiver *Archiver) UntarPath(src, dst string) error { 1040 archive, err := os.Open(src) 1041 if err != nil { 1042 return err 1043 } 1044 defer archive.Close() 1045 options := &TarOptions{ 1046 UIDMaps: archiver.IDMappingsVar.UIDs(), 1047 GIDMaps: archiver.IDMappingsVar.GIDs(), 1048 } 1049 return archiver.Untar(archive, dst, options) 1050 } 1051 1052 // CopyWithTar creates a tar archive of filesystem path `src`, and 1053 // unpacks it at filesystem path `dst`. 1054 // The archive is streamed directly with fixed buffering and no 1055 // intermediary disk IO. 1056 func (archiver *Archiver) CopyWithTar(src, dst string) error { 1057 srcSt, err := os.Stat(src) 1058 if err != nil { 1059 return err 1060 } 1061 if !srcSt.IsDir() { 1062 return archiver.CopyFileWithTar(src, dst) 1063 } 1064 1065 // if this Archiver is set up with ID mapping we need to create 1066 // the new destination directory with the remapped root UID/GID pair 1067 // as owner 1068 rootIDs := archiver.IDMappingsVar.RootPair() 1069 // Create dst, copy src's content into it 1070 logrus.Debugf("Creating dest directory: %s", dst) 1071 if err := idtools.MkdirAllAndChownNew(dst, 0755, rootIDs); err != nil { 1072 return err 1073 } 1074 logrus.Debugf("Calling TarUntar(%s, %s)", src, dst) 1075 return archiver.TarUntar(src, dst) 1076 } 1077 1078 // CopyFileWithTar emulates the behavior of the 'cp' command-line 1079 // for a single file. It copies a regular file from path `src` to 1080 // path `dst`, and preserves all its metadata. 1081 func (archiver *Archiver) CopyFileWithTar(src, dst string) (err error) { 1082 logrus.Debugf("CopyFileWithTar(%s, %s)", src, dst) 1083 srcSt, err := os.Stat(src) 1084 if err != nil { 1085 return err 1086 } 1087 1088 if srcSt.IsDir() { 1089 return fmt.Errorf("Can't copy a directory") 1090 } 1091 1092 // Clean up the trailing slash. This must be done in an operating 1093 // system specific manner. 1094 if dst[len(dst)-1] == os.PathSeparator { 1095 dst = filepath.Join(dst, filepath.Base(src)) 1096 } 1097 // Create the holding directory if necessary 1098 if err := system.MkdirAll(filepath.Dir(dst), 0700, ""); err != nil { 1099 return err 1100 } 1101 1102 r, w := io.Pipe() 1103 errC := make(chan error, 1) 1104 1105 go func() { 1106 defer close(errC) 1107 1108 errC <- func() error { 1109 defer w.Close() 1110 1111 srcF, err := os.Open(src) 1112 if err != nil { 1113 return err 1114 } 1115 defer srcF.Close() 1116 1117 hdr, err := tar.FileInfoHeader(srcSt, "") 1118 if err != nil { 1119 return err 1120 } 1121 hdr.Name = filepath.Base(dst) 1122 hdr.Mode = int64(chmodTarEntry(os.FileMode(hdr.Mode))) 1123 1124 if err := remapIDs(archiver.IDMappingsVar, hdr); err != nil { 1125 return err 1126 } 1127 1128 tw := tar.NewWriter(w) 1129 defer tw.Close() 1130 if err := tw.WriteHeader(hdr); err != nil { 1131 return err 1132 } 1133 if _, err := io.Copy(tw, srcF); err != nil { 1134 return err 1135 } 1136 return nil 1137 }() 1138 }() 1139 defer func() { 1140 if er := <-errC; err == nil && er != nil { 1141 err = er 1142 } 1143 }() 1144 1145 err = archiver.Untar(r, filepath.Dir(dst), nil) 1146 if err != nil { 1147 r.CloseWithError(err) 1148 } 1149 return err 1150 } 1151 1152 // IDMappings returns the IDMappings of the archiver. 1153 func (archiver *Archiver) IDMappings() *idtools.IDMappings { 1154 return archiver.IDMappingsVar 1155 } 1156 1157 func remapIDs(idMappings *idtools.IDMappings, hdr *tar.Header) error { 1158 ids, err := idMappings.ToHost(idtools.IDPair{UID: hdr.Uid, GID: hdr.Gid}) 1159 hdr.Uid, hdr.Gid = ids.UID, ids.GID 1160 return err 1161 } 1162 1163 // cmdStream executes a command, and returns its stdout as a stream. 1164 // If the command fails to run or doesn't complete successfully, an error 1165 // will be returned, including anything written on stderr. 1166 func cmdStream(cmd *exec.Cmd, input io.Reader) (io.ReadCloser, <-chan struct{}, error) { 1167 chdone := make(chan struct{}) 1168 cmd.Stdin = input 1169 pipeR, pipeW := io.Pipe() 1170 cmd.Stdout = pipeW 1171 var errBuf bytes.Buffer 1172 cmd.Stderr = &errBuf 1173 1174 // Run the command and return the pipe 1175 if err := cmd.Start(); err != nil { 1176 return nil, nil, err 1177 } 1178 1179 // Copy stdout to the returned pipe 1180 go func() { 1181 if err := cmd.Wait(); err != nil { 1182 pipeW.CloseWithError(fmt.Errorf("%s: %s", err, errBuf.String())) 1183 } else { 1184 pipeW.Close() 1185 } 1186 close(chdone) 1187 }() 1188 1189 return pipeR, chdone, nil 1190 } 1191 1192 // NewTempArchive reads the content of src into a temporary file, and returns the contents 1193 // of that file as an archive. The archive can only be read once - as soon as reading completes, 1194 // the file will be deleted. 1195 func NewTempArchive(src io.Reader, dir string) (*TempArchive, error) { 1196 f, err := ioutil.TempFile(dir, "") 1197 if err != nil { 1198 return nil, err 1199 } 1200 if _, err := io.Copy(f, src); err != nil { 1201 return nil, err 1202 } 1203 if _, err := f.Seek(0, 0); err != nil { 1204 return nil, err 1205 } 1206 st, err := f.Stat() 1207 if err != nil { 1208 return nil, err 1209 } 1210 size := st.Size() 1211 return &TempArchive{File: f, Size: size}, nil 1212 } 1213 1214 // TempArchive is a temporary archive. The archive can only be read once - as soon as reading completes, 1215 // the file will be deleted. 1216 type TempArchive struct { 1217 *os.File 1218 Size int64 // Pre-computed from Stat().Size() as a convenience 1219 read int64 1220 closed bool 1221 } 1222 1223 // Close closes the underlying file if it's still open, or does a no-op 1224 // to allow callers to try to close the TempArchive multiple times safely. 1225 func (archive *TempArchive) Close() error { 1226 if archive.closed { 1227 return nil 1228 } 1229 1230 archive.closed = true 1231 1232 return archive.File.Close() 1233 } 1234 1235 func (archive *TempArchive) Read(data []byte) (int, error) { 1236 n, err := archive.File.Read(data) 1237 archive.read += int64(n) 1238 if err != nil || archive.read == archive.Size { 1239 archive.Close() 1240 os.Remove(archive.File.Name()) 1241 } 1242 return n, err 1243 }