github.com/noxiouz/docker@v0.7.3-0.20160629055221-3d231c78e8c5/pkg/archive/archive.go (about) 1 package archive 2 3 import ( 4 "archive/tar" 5 "bufio" 6 "bytes" 7 "compress/bzip2" 8 "compress/gzip" 9 "errors" 10 "fmt" 11 "io" 12 "io/ioutil" 13 "os" 14 "os/exec" 15 "path/filepath" 16 "runtime" 17 "strings" 18 "syscall" 19 20 "github.com/Sirupsen/logrus" 21 "github.com/docker/docker/pkg/fileutils" 22 "github.com/docker/docker/pkg/idtools" 23 "github.com/docker/docker/pkg/ioutils" 24 "github.com/docker/docker/pkg/pools" 25 "github.com/docker/docker/pkg/promise" 26 "github.com/docker/docker/pkg/system" 27 ) 28 29 type ( 30 // Archive is a type of io.ReadCloser which has two interfaces Read and Closer. 31 Archive io.ReadCloser 32 // Reader is a type of io.Reader. 33 Reader io.Reader 34 // Compression is the state represents if compressed or not. 35 Compression int 36 // WhiteoutFormat is the format of whiteouts unpacked 37 WhiteoutFormat int 38 // TarChownOptions wraps the chown options UID and GID. 39 TarChownOptions struct { 40 UID, GID int 41 } 42 // TarOptions wraps the tar options. 43 TarOptions struct { 44 IncludeFiles []string 45 ExcludePatterns []string 46 Compression Compression 47 NoLchown bool 48 UIDMaps []idtools.IDMap 49 GIDMaps []idtools.IDMap 50 ChownOpts *TarChownOptions 51 IncludeSourceDir bool 52 // WhiteoutFormat is the expected on disk format for whiteout files. 53 // This format will be converted to the standard format on pack 54 // and from the standard format on unpack. 55 WhiteoutFormat WhiteoutFormat 56 // When unpacking, specifies whether overwriting a directory with a 57 // non-directory is allowed and vice versa. 58 NoOverwriteDirNonDir bool 59 // For each include when creating an archive, the included name will be 60 // replaced with the matching name from this map. 61 RebaseNames map[string]string 62 } 63 64 // Archiver allows the reuse of most utility functions of this package 65 // with a pluggable Untar function. Also, to facilitate the passing of 66 // specific id mappings for untar, an archiver can be created with maps 67 // which will then be passed to Untar operations 68 Archiver struct { 69 Untar func(io.Reader, string, *TarOptions) error 70 UIDMaps []idtools.IDMap 71 GIDMaps []idtools.IDMap 72 } 73 74 // breakoutError is used to differentiate errors related to breaking out 75 // When testing archive breakout in the unit tests, this error is expected 76 // in order for the test to pass. 77 breakoutError error 78 ) 79 80 var ( 81 // ErrNotImplemented is the error message of function not implemented. 82 ErrNotImplemented = errors.New("Function not implemented") 83 defaultArchiver = &Archiver{Untar: Untar, UIDMaps: nil, GIDMaps: nil} 84 ) 85 86 const ( 87 // HeaderSize is the size in bytes of a tar header 88 HeaderSize = 512 89 ) 90 91 const ( 92 // Uncompressed represents the uncompressed. 93 Uncompressed Compression = iota 94 // Bzip2 is bzip2 compression algorithm. 95 Bzip2 96 // Gzip is gzip compression algorithm. 97 Gzip 98 // Xz is xz compression algorithm. 99 Xz 100 ) 101 102 const ( 103 // AUFSWhiteoutFormat is the default format for whiteouts 104 AUFSWhiteoutFormat WhiteoutFormat = iota 105 // OverlayWhiteoutFormat formats whiteout according to the overlay 106 // standard. 107 OverlayWhiteoutFormat 108 ) 109 110 // IsArchive checks for the magic bytes of a tar or any supported compression 111 // algorithm. 112 func IsArchive(header []byte) bool { 113 compression := DetectCompression(header) 114 if compression != Uncompressed { 115 return true 116 } 117 r := tar.NewReader(bytes.NewBuffer(header)) 118 _, err := r.Next() 119 return err == nil 120 } 121 122 // IsArchivePath checks if the (possibly compressed) file at the given path 123 // starts with a tar file header. 124 func IsArchivePath(path string) bool { 125 file, err := os.Open(path) 126 if err != nil { 127 return false 128 } 129 defer file.Close() 130 rdr, err := DecompressStream(file) 131 if err != nil { 132 return false 133 } 134 r := tar.NewReader(rdr) 135 _, err = r.Next() 136 return err == nil 137 } 138 139 // DetectCompression detects the compression algorithm of the source. 140 func DetectCompression(source []byte) Compression { 141 for compression, m := range map[Compression][]byte{ 142 Bzip2: {0x42, 0x5A, 0x68}, 143 Gzip: {0x1F, 0x8B, 0x08}, 144 Xz: {0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00}, 145 } { 146 if len(source) < len(m) { 147 logrus.Debug("Len too short") 148 continue 149 } 150 if bytes.Compare(m, source[:len(m)]) == 0 { 151 return compression 152 } 153 } 154 return Uncompressed 155 } 156 157 func xzDecompress(archive io.Reader) (io.ReadCloser, <-chan struct{}, error) { 158 args := []string{"xz", "-d", "-c", "-q"} 159 160 return cmdStream(exec.Command(args[0], args[1:]...), archive) 161 } 162 163 // DecompressStream decompresses the archive and returns a ReaderCloser with the decompressed archive. 164 func DecompressStream(archive io.Reader) (io.ReadCloser, error) { 165 p := pools.BufioReader32KPool 166 buf := p.Get(archive) 167 bs, err := buf.Peek(10) 168 if err != nil && err != io.EOF { 169 // Note: we'll ignore any io.EOF error because there are some odd 170 // cases where the layer.tar file will be empty (zero bytes) and 171 // that results in an io.EOF from the Peek() call. So, in those 172 // cases we'll just treat it as a non-compressed stream and 173 // that means just create an empty layer. 174 // See Issue 18170 175 return nil, err 176 } 177 178 compression := DetectCompression(bs) 179 switch compression { 180 case Uncompressed: 181 readBufWrapper := p.NewReadCloserWrapper(buf, buf) 182 return readBufWrapper, nil 183 case Gzip: 184 gzReader, err := gzip.NewReader(buf) 185 if err != nil { 186 return nil, err 187 } 188 readBufWrapper := p.NewReadCloserWrapper(buf, gzReader) 189 return readBufWrapper, nil 190 case Bzip2: 191 bz2Reader := bzip2.NewReader(buf) 192 readBufWrapper := p.NewReadCloserWrapper(buf, bz2Reader) 193 return readBufWrapper, nil 194 case Xz: 195 xzReader, chdone, err := xzDecompress(buf) 196 if err != nil { 197 return nil, err 198 } 199 readBufWrapper := p.NewReadCloserWrapper(buf, xzReader) 200 return ioutils.NewReadCloserWrapper(readBufWrapper, func() error { 201 <-chdone 202 return readBufWrapper.Close() 203 }), nil 204 default: 205 return nil, fmt.Errorf("Unsupported compression format %s", (&compression).Extension()) 206 } 207 } 208 209 // CompressStream compresseses the dest with specified compression algorithm. 210 func CompressStream(dest io.Writer, compression Compression) (io.WriteCloser, error) { 211 p := pools.BufioWriter32KPool 212 buf := p.Get(dest) 213 switch compression { 214 case Uncompressed: 215 writeBufWrapper := p.NewWriteCloserWrapper(buf, buf) 216 return writeBufWrapper, nil 217 case Gzip: 218 gzWriter := gzip.NewWriter(dest) 219 writeBufWrapper := p.NewWriteCloserWrapper(buf, gzWriter) 220 return writeBufWrapper, nil 221 case Bzip2, Xz: 222 // archive/bzip2 does not support writing, and there is no xz support at all 223 // However, this is not a problem as docker only currently generates gzipped tars 224 return nil, fmt.Errorf("Unsupported compression format %s", (&compression).Extension()) 225 default: 226 return nil, fmt.Errorf("Unsupported compression format %s", (&compression).Extension()) 227 } 228 } 229 230 // Extension returns the extension of a file that uses the specified compression algorithm. 231 func (compression *Compression) Extension() string { 232 switch *compression { 233 case Uncompressed: 234 return "tar" 235 case Bzip2: 236 return "tar.bz2" 237 case Gzip: 238 return "tar.gz" 239 case Xz: 240 return "tar.xz" 241 } 242 return "" 243 } 244 245 type tarWhiteoutConverter interface { 246 ConvertWrite(*tar.Header, string, os.FileInfo) error 247 ConvertRead(*tar.Header, string) (bool, error) 248 } 249 250 type tarAppender struct { 251 TarWriter *tar.Writer 252 Buffer *bufio.Writer 253 254 // for hardlink mapping 255 SeenFiles map[uint64]string 256 UIDMaps []idtools.IDMap 257 GIDMaps []idtools.IDMap 258 259 // For packing and unpacking whiteout files in the 260 // non standard format. The whiteout files defined 261 // by the AUFS standard are used as the tar whiteout 262 // standard. 263 WhiteoutConverter tarWhiteoutConverter 264 } 265 266 // canonicalTarName provides a platform-independent and consistent posix-style 267 //path for files and directories to be archived regardless of the platform. 268 func canonicalTarName(name string, isDir bool) (string, error) { 269 name, err := CanonicalTarNameForPath(name) 270 if err != nil { 271 return "", err 272 } 273 274 // suffix with '/' for directories 275 if isDir && !strings.HasSuffix(name, "/") { 276 name += "/" 277 } 278 return name, nil 279 } 280 281 // addTarFile adds to the tar archive a file from `path` as `name` 282 func (ta *tarAppender) addTarFile(path, name string) error { 283 fi, err := os.Lstat(path) 284 if err != nil { 285 return err 286 } 287 288 link := "" 289 if fi.Mode()&os.ModeSymlink != 0 { 290 if link, err = os.Readlink(path); err != nil { 291 return err 292 } 293 } 294 295 hdr, err := tar.FileInfoHeader(fi, link) 296 if err != nil { 297 return err 298 } 299 hdr.Mode = int64(chmodTarEntry(os.FileMode(hdr.Mode))) 300 301 name, err = canonicalTarName(name, fi.IsDir()) 302 if err != nil { 303 return fmt.Errorf("tar: cannot canonicalize path: %v", err) 304 } 305 hdr.Name = name 306 307 inode, err := setHeaderForSpecialDevice(hdr, ta, name, fi.Sys()) 308 if err != nil { 309 return err 310 } 311 312 // if it's not a directory and has more than 1 link, 313 // it's hardlinked, so set the type flag accordingly 314 if !fi.IsDir() && hasHardlinks(fi) { 315 // a link should have a name that it links too 316 // and that linked name should be first in the tar archive 317 if oldpath, ok := ta.SeenFiles[inode]; ok { 318 hdr.Typeflag = tar.TypeLink 319 hdr.Linkname = oldpath 320 hdr.Size = 0 // This Must be here for the writer math to add up! 321 } else { 322 ta.SeenFiles[inode] = name 323 } 324 } 325 326 capability, _ := system.Lgetxattr(path, "security.capability") 327 if capability != nil { 328 hdr.Xattrs = make(map[string]string) 329 hdr.Xattrs["security.capability"] = string(capability) 330 } 331 332 //handle re-mapping container ID mappings back to host ID mappings before 333 //writing tar headers/files. We skip whiteout files because they were written 334 //by the kernel and already have proper ownership relative to the host 335 if !strings.HasPrefix(filepath.Base(hdr.Name), WhiteoutPrefix) && (ta.UIDMaps != nil || ta.GIDMaps != nil) { 336 uid, gid, err := getFileUIDGID(fi.Sys()) 337 if err != nil { 338 return err 339 } 340 xUID, err := idtools.ToContainer(uid, ta.UIDMaps) 341 if err != nil { 342 return err 343 } 344 xGID, err := idtools.ToContainer(gid, ta.GIDMaps) 345 if err != nil { 346 return err 347 } 348 hdr.Uid = xUID 349 hdr.Gid = xGID 350 } 351 352 if ta.WhiteoutConverter != nil { 353 if err := ta.WhiteoutConverter.ConvertWrite(hdr, path, fi); err != nil { 354 return err 355 } 356 } 357 358 if err := ta.TarWriter.WriteHeader(hdr); err != nil { 359 return err 360 } 361 362 if hdr.Typeflag == tar.TypeReg && hdr.Size > 0 { 363 file, err := os.Open(path) 364 if err != nil { 365 return err 366 } 367 368 ta.Buffer.Reset(ta.TarWriter) 369 defer ta.Buffer.Reset(nil) 370 _, err = io.Copy(ta.Buffer, file) 371 file.Close() 372 if err != nil { 373 return err 374 } 375 err = ta.Buffer.Flush() 376 if err != nil { 377 return err 378 } 379 } 380 381 return nil 382 } 383 384 func createTarFile(path, extractDir string, hdr *tar.Header, reader io.Reader, Lchown bool, chownOpts *TarChownOptions) error { 385 // hdr.Mode is in linux format, which we can use for sycalls, 386 // but for os.Foo() calls we need the mode converted to os.FileMode, 387 // so use hdrInfo.Mode() (they differ for e.g. setuid bits) 388 hdrInfo := hdr.FileInfo() 389 390 switch hdr.Typeflag { 391 case tar.TypeDir: 392 // Create directory unless it exists as a directory already. 393 // In that case we just want to merge the two 394 if fi, err := os.Lstat(path); !(err == nil && fi.IsDir()) { 395 if err := os.Mkdir(path, hdrInfo.Mode()); err != nil { 396 return err 397 } 398 } 399 400 case tar.TypeReg, tar.TypeRegA: 401 // Source is regular file 402 file, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, hdrInfo.Mode()) 403 if err != nil { 404 return err 405 } 406 if _, err := io.Copy(file, reader); err != nil { 407 file.Close() 408 return err 409 } 410 file.Close() 411 412 case tar.TypeBlock, tar.TypeChar, tar.TypeFifo: 413 // Handle this is an OS-specific way 414 if err := handleTarTypeBlockCharFifo(hdr, path); err != nil { 415 return err 416 } 417 418 case tar.TypeLink: 419 targetPath := filepath.Join(extractDir, hdr.Linkname) 420 // check for hardlink breakout 421 if !strings.HasPrefix(targetPath, extractDir) { 422 return breakoutError(fmt.Errorf("invalid hardlink %q -> %q", targetPath, hdr.Linkname)) 423 } 424 if err := os.Link(targetPath, path); err != nil { 425 return err 426 } 427 428 case tar.TypeSymlink: 429 // path -> hdr.Linkname = targetPath 430 // e.g. /extractDir/path/to/symlink -> ../2/file = /extractDir/path/2/file 431 targetPath := filepath.Join(filepath.Dir(path), hdr.Linkname) 432 433 // the reason we don't need to check symlinks in the path (with FollowSymlinkInScope) is because 434 // that symlink would first have to be created, which would be caught earlier, at this very check: 435 if !strings.HasPrefix(targetPath, extractDir) { 436 return breakoutError(fmt.Errorf("invalid symlink %q -> %q", path, hdr.Linkname)) 437 } 438 if err := os.Symlink(hdr.Linkname, path); err != nil { 439 return err 440 } 441 442 case tar.TypeXGlobalHeader: 443 logrus.Debug("PAX Global Extended Headers found and ignored") 444 return nil 445 446 default: 447 return fmt.Errorf("Unhandled tar header type %d\n", hdr.Typeflag) 448 } 449 450 // Lchown is not supported on Windows. 451 if Lchown && runtime.GOOS != "windows" { 452 if chownOpts == nil { 453 chownOpts = &TarChownOptions{UID: hdr.Uid, GID: hdr.Gid} 454 } 455 if err := os.Lchown(path, chownOpts.UID, chownOpts.GID); err != nil { 456 return err 457 } 458 } 459 460 var errors []string 461 for key, value := range hdr.Xattrs { 462 if err := system.Lsetxattr(path, key, []byte(value), 0); err != nil { 463 if err == syscall.ENOTSUP { 464 // We ignore errors here because not all graphdrivers support 465 // xattrs *cough* old versions of AUFS *cough*. However only 466 // ENOTSUP should be emitted in that case, otherwise we still 467 // bail. 468 errors = append(errors, err.Error()) 469 continue 470 } 471 return err 472 } 473 474 } 475 476 if len(errors) > 0 { 477 logrus.WithFields(logrus.Fields{ 478 "errors": errors, 479 }).Warn("ignored xattrs in archive: underlying filesystem doesn't support them") 480 } 481 482 // There is no LChmod, so ignore mode for symlink. Also, this 483 // must happen after chown, as that can modify the file mode 484 if err := handleLChmod(hdr, path, hdrInfo); err != nil { 485 return err 486 } 487 488 aTime := hdr.AccessTime 489 if aTime.Before(hdr.ModTime) { 490 // Last access time should never be before last modified time. 491 aTime = hdr.ModTime 492 } 493 494 // system.Chtimes doesn't support a NOFOLLOW flag atm 495 if hdr.Typeflag == tar.TypeLink { 496 if fi, err := os.Lstat(hdr.Linkname); err == nil && (fi.Mode()&os.ModeSymlink == 0) { 497 if err := system.Chtimes(path, aTime, hdr.ModTime); err != nil { 498 return err 499 } 500 } 501 } else if hdr.Typeflag != tar.TypeSymlink { 502 if err := system.Chtimes(path, aTime, hdr.ModTime); err != nil { 503 return err 504 } 505 } else { 506 ts := []syscall.Timespec{timeToTimespec(aTime), timeToTimespec(hdr.ModTime)} 507 if err := system.LUtimesNano(path, ts); err != nil && err != system.ErrNotSupportedPlatform { 508 return err 509 } 510 } 511 return nil 512 } 513 514 // Tar creates an archive from the directory at `path`, and returns it as a 515 // stream of bytes. 516 func Tar(path string, compression Compression) (io.ReadCloser, error) { 517 return TarWithOptions(path, &TarOptions{Compression: compression}) 518 } 519 520 // TarWithOptions creates an archive from the directory at `path`, only including files whose relative 521 // paths are included in `options.IncludeFiles` (if non-nil) or not in `options.ExcludePatterns`. 522 func TarWithOptions(srcPath string, options *TarOptions) (io.ReadCloser, error) { 523 524 // Fix the source path to work with long path names. This is a no-op 525 // on platforms other than Windows. 526 srcPath = fixVolumePathPrefix(srcPath) 527 528 patterns, patDirs, exceptions, err := fileutils.CleanPatterns(options.ExcludePatterns) 529 530 if err != nil { 531 return nil, err 532 } 533 534 pipeReader, pipeWriter := io.Pipe() 535 536 compressWriter, err := CompressStream(pipeWriter, options.Compression) 537 if err != nil { 538 return nil, err 539 } 540 541 go func() { 542 ta := &tarAppender{ 543 TarWriter: tar.NewWriter(compressWriter), 544 Buffer: pools.BufioWriter32KPool.Get(nil), 545 SeenFiles: make(map[uint64]string), 546 UIDMaps: options.UIDMaps, 547 GIDMaps: options.GIDMaps, 548 WhiteoutConverter: getWhiteoutConverter(options.WhiteoutFormat), 549 } 550 551 defer func() { 552 // Make sure to check the error on Close. 553 if err := ta.TarWriter.Close(); err != nil { 554 logrus.Errorf("Can't close tar writer: %s", err) 555 } 556 if err := compressWriter.Close(); err != nil { 557 logrus.Errorf("Can't close compress writer: %s", err) 558 } 559 if err := pipeWriter.Close(); err != nil { 560 logrus.Errorf("Can't close pipe writer: %s", err) 561 } 562 }() 563 564 // this buffer is needed for the duration of this piped stream 565 defer pools.BufioWriter32KPool.Put(ta.Buffer) 566 567 // In general we log errors here but ignore them because 568 // during e.g. a diff operation the container can continue 569 // mutating the filesystem and we can see transient errors 570 // from this 571 572 stat, err := os.Lstat(srcPath) 573 if err != nil { 574 return 575 } 576 577 if !stat.IsDir() { 578 // We can't later join a non-dir with any includes because the 579 // 'walk' will error if "file/." is stat-ed and "file" is not a 580 // directory. So, we must split the source path and use the 581 // basename as the include. 582 if len(options.IncludeFiles) > 0 { 583 logrus.Warn("Tar: Can't archive a file with includes") 584 } 585 586 dir, base := SplitPathDirEntry(srcPath) 587 srcPath = dir 588 options.IncludeFiles = []string{base} 589 } 590 591 if len(options.IncludeFiles) == 0 { 592 options.IncludeFiles = []string{"."} 593 } 594 595 seen := make(map[string]bool) 596 597 for _, include := range options.IncludeFiles { 598 rebaseName := options.RebaseNames[include] 599 600 walkRoot := getWalkRoot(srcPath, include) 601 filepath.Walk(walkRoot, func(filePath string, f os.FileInfo, err error) error { 602 if err != nil { 603 logrus.Errorf("Tar: Can't stat file %s to tar: %s", srcPath, err) 604 return nil 605 } 606 607 relFilePath, err := filepath.Rel(srcPath, filePath) 608 if err != nil || (!options.IncludeSourceDir && relFilePath == "." && f.IsDir()) { 609 // Error getting relative path OR we are looking 610 // at the source directory path. Skip in both situations. 611 return nil 612 } 613 614 if options.IncludeSourceDir && include == "." && relFilePath != "." { 615 relFilePath = strings.Join([]string{".", relFilePath}, string(filepath.Separator)) 616 } 617 618 skip := false 619 620 // If "include" is an exact match for the current file 621 // then even if there's an "excludePatterns" pattern that 622 // matches it, don't skip it. IOW, assume an explicit 'include' 623 // is asking for that file no matter what - which is true 624 // for some files, like .dockerignore and Dockerfile (sometimes) 625 if include != relFilePath { 626 skip, err = fileutils.OptimizedMatches(relFilePath, patterns, patDirs) 627 if err != nil { 628 logrus.Errorf("Error matching %s: %v", relFilePath, err) 629 return err 630 } 631 } 632 633 if skip { 634 // If we want to skip this file and its a directory 635 // then we should first check to see if there's an 636 // excludes pattern (eg !dir/file) that starts with this 637 // dir. If so then we can't skip this dir. 638 639 // Its not a dir then so we can just return/skip. 640 if !f.IsDir() { 641 return nil 642 } 643 644 // No exceptions (!...) in patterns so just skip dir 645 if !exceptions { 646 return filepath.SkipDir 647 } 648 649 dirSlash := relFilePath + string(filepath.Separator) 650 651 for _, pat := range patterns { 652 if pat[0] != '!' { 653 continue 654 } 655 pat = pat[1:] + string(filepath.Separator) 656 if strings.HasPrefix(pat, dirSlash) { 657 // found a match - so can't skip this dir 658 return nil 659 } 660 } 661 662 // No matching exclusion dir so just skip dir 663 return filepath.SkipDir 664 } 665 666 if seen[relFilePath] { 667 return nil 668 } 669 seen[relFilePath] = true 670 671 // Rename the base resource. 672 if rebaseName != "" { 673 var replacement string 674 if rebaseName != string(filepath.Separator) { 675 // Special case the root directory to replace with an 676 // empty string instead so that we don't end up with 677 // double slashes in the paths. 678 replacement = rebaseName 679 } 680 681 relFilePath = strings.Replace(relFilePath, include, replacement, 1) 682 } 683 684 if err := ta.addTarFile(filePath, relFilePath); err != nil { 685 logrus.Errorf("Can't add file %s to tar: %s", filePath, err) 686 // if pipe is broken, stop writing tar stream to it 687 if err == io.ErrClosedPipe { 688 return err 689 } 690 } 691 return nil 692 }) 693 } 694 }() 695 696 return pipeReader, nil 697 } 698 699 // Unpack unpacks the decompressedArchive to dest with options. 700 func Unpack(decompressedArchive io.Reader, dest string, options *TarOptions) error { 701 tr := tar.NewReader(decompressedArchive) 702 trBuf := pools.BufioReader32KPool.Get(nil) 703 defer pools.BufioReader32KPool.Put(trBuf) 704 705 var dirs []*tar.Header 706 remappedRootUID, remappedRootGID, err := idtools.GetRootUIDGID(options.UIDMaps, options.GIDMaps) 707 if err != nil { 708 return err 709 } 710 whiteoutConverter := getWhiteoutConverter(options.WhiteoutFormat) 711 712 // Iterate through the files in the archive. 713 loop: 714 for { 715 hdr, err := tr.Next() 716 if err == io.EOF { 717 // end of tar archive 718 break 719 } 720 if err != nil { 721 return err 722 } 723 724 // Normalize name, for safety and for a simple is-root check 725 // This keeps "../" as-is, but normalizes "/../" to "/". Or Windows: 726 // This keeps "..\" as-is, but normalizes "\..\" to "\". 727 hdr.Name = filepath.Clean(hdr.Name) 728 729 for _, exclude := range options.ExcludePatterns { 730 if strings.HasPrefix(hdr.Name, exclude) { 731 continue loop 732 } 733 } 734 735 // After calling filepath.Clean(hdr.Name) above, hdr.Name will now be in 736 // the filepath format for the OS on which the daemon is running. Hence 737 // the check for a slash-suffix MUST be done in an OS-agnostic way. 738 if !strings.HasSuffix(hdr.Name, string(os.PathSeparator)) { 739 // Not the root directory, ensure that the parent directory exists 740 parent := filepath.Dir(hdr.Name) 741 parentPath := filepath.Join(dest, parent) 742 if _, err := os.Lstat(parentPath); err != nil && os.IsNotExist(err) { 743 err = idtools.MkdirAllNewAs(parentPath, 0777, remappedRootUID, remappedRootGID) 744 if err != nil { 745 return err 746 } 747 } 748 } 749 750 path := filepath.Join(dest, hdr.Name) 751 rel, err := filepath.Rel(dest, path) 752 if err != nil { 753 return err 754 } 755 if strings.HasPrefix(rel, ".."+string(os.PathSeparator)) { 756 return breakoutError(fmt.Errorf("%q is outside of %q", hdr.Name, dest)) 757 } 758 759 // If path exits we almost always just want to remove and replace it 760 // The only exception is when it is a directory *and* the file from 761 // the layer is also a directory. Then we want to merge them (i.e. 762 // just apply the metadata from the layer). 763 if fi, err := os.Lstat(path); err == nil { 764 if options.NoOverwriteDirNonDir && fi.IsDir() && hdr.Typeflag != tar.TypeDir { 765 // If NoOverwriteDirNonDir is true then we cannot replace 766 // an existing directory with a non-directory from the archive. 767 return fmt.Errorf("cannot overwrite directory %q with non-directory %q", path, dest) 768 } 769 770 if options.NoOverwriteDirNonDir && !fi.IsDir() && hdr.Typeflag == tar.TypeDir { 771 // If NoOverwriteDirNonDir is true then we cannot replace 772 // an existing non-directory with a directory from the archive. 773 return fmt.Errorf("cannot overwrite non-directory %q with directory %q", path, dest) 774 } 775 776 if fi.IsDir() && hdr.Name == "." { 777 continue 778 } 779 780 if !(fi.IsDir() && hdr.Typeflag == tar.TypeDir) { 781 if err := os.RemoveAll(path); err != nil { 782 return err 783 } 784 } 785 } 786 trBuf.Reset(tr) 787 788 // if the options contain a uid & gid maps, convert header uid/gid 789 // entries using the maps such that lchown sets the proper mapped 790 // uid/gid after writing the file. We only perform this mapping if 791 // the file isn't already owned by the remapped root UID or GID, as 792 // that specific uid/gid has no mapping from container -> host, and 793 // those files already have the proper ownership for inside the 794 // container. 795 if hdr.Uid != remappedRootUID { 796 xUID, err := idtools.ToHost(hdr.Uid, options.UIDMaps) 797 if err != nil { 798 return err 799 } 800 hdr.Uid = xUID 801 } 802 if hdr.Gid != remappedRootGID { 803 xGID, err := idtools.ToHost(hdr.Gid, options.GIDMaps) 804 if err != nil { 805 return err 806 } 807 hdr.Gid = xGID 808 } 809 810 if whiteoutConverter != nil { 811 writeFile, err := whiteoutConverter.ConvertRead(hdr, path) 812 if err != nil { 813 return err 814 } 815 if !writeFile { 816 continue 817 } 818 } 819 820 if err := createTarFile(path, dest, hdr, trBuf, !options.NoLchown, options.ChownOpts); err != nil { 821 return err 822 } 823 824 // Directory mtimes must be handled at the end to avoid further 825 // file creation in them to modify the directory mtime 826 if hdr.Typeflag == tar.TypeDir { 827 dirs = append(dirs, hdr) 828 } 829 } 830 831 for _, hdr := range dirs { 832 path := filepath.Join(dest, hdr.Name) 833 834 if err := system.Chtimes(path, hdr.AccessTime, hdr.ModTime); err != nil { 835 return err 836 } 837 } 838 return nil 839 } 840 841 // Untar reads a stream of bytes from `archive`, parses it as a tar archive, 842 // and unpacks it into the directory at `dest`. 843 // The archive may be compressed with one of the following algorithms: 844 // identity (uncompressed), gzip, bzip2, xz. 845 // FIXME: specify behavior when target path exists vs. doesn't exist. 846 func Untar(tarArchive io.Reader, dest string, options *TarOptions) error { 847 return untarHandler(tarArchive, dest, options, true) 848 } 849 850 // UntarUncompressed reads a stream of bytes from `archive`, parses it as a tar archive, 851 // and unpacks it into the directory at `dest`. 852 // The archive must be an uncompressed stream. 853 func UntarUncompressed(tarArchive io.Reader, dest string, options *TarOptions) error { 854 return untarHandler(tarArchive, dest, options, false) 855 } 856 857 // Handler for teasing out the automatic decompression 858 func untarHandler(tarArchive io.Reader, dest string, options *TarOptions, decompress bool) error { 859 if tarArchive == nil { 860 return fmt.Errorf("Empty archive") 861 } 862 dest = filepath.Clean(dest) 863 if options == nil { 864 options = &TarOptions{} 865 } 866 if options.ExcludePatterns == nil { 867 options.ExcludePatterns = []string{} 868 } 869 870 r := tarArchive 871 if decompress { 872 decompressedArchive, err := DecompressStream(tarArchive) 873 if err != nil { 874 return err 875 } 876 defer decompressedArchive.Close() 877 r = decompressedArchive 878 } 879 880 return Unpack(r, dest, options) 881 } 882 883 // TarUntar is a convenience function which calls Tar and Untar, with the output of one piped into the other. 884 // If either Tar or Untar fails, TarUntar aborts and returns the error. 885 func (archiver *Archiver) TarUntar(src, dst string) error { 886 logrus.Debugf("TarUntar(%s %s)", src, dst) 887 archive, err := TarWithOptions(src, &TarOptions{Compression: Uncompressed}) 888 if err != nil { 889 return err 890 } 891 defer archive.Close() 892 893 var options *TarOptions 894 if archiver.UIDMaps != nil || archiver.GIDMaps != nil { 895 options = &TarOptions{ 896 UIDMaps: archiver.UIDMaps, 897 GIDMaps: archiver.GIDMaps, 898 } 899 } 900 return archiver.Untar(archive, dst, options) 901 } 902 903 // TarUntar is a convenience function which calls Tar and Untar, with the output of one piped into the other. 904 // If either Tar or Untar fails, TarUntar aborts and returns the error. 905 func TarUntar(src, dst string) error { 906 return defaultArchiver.TarUntar(src, dst) 907 } 908 909 // UntarPath untar a file from path to a destination, src is the source tar file path. 910 func (archiver *Archiver) UntarPath(src, dst string) error { 911 archive, err := os.Open(src) 912 if err != nil { 913 return err 914 } 915 defer archive.Close() 916 var options *TarOptions 917 if archiver.UIDMaps != nil || archiver.GIDMaps != nil { 918 options = &TarOptions{ 919 UIDMaps: archiver.UIDMaps, 920 GIDMaps: archiver.GIDMaps, 921 } 922 } 923 return archiver.Untar(archive, dst, options) 924 } 925 926 // UntarPath is a convenience function which looks for an archive 927 // at filesystem path `src`, and unpacks it at `dst`. 928 func UntarPath(src, dst string) error { 929 return defaultArchiver.UntarPath(src, dst) 930 } 931 932 // CopyWithTar creates a tar archive of filesystem path `src`, and 933 // unpacks it at filesystem path `dst`. 934 // The archive is streamed directly with fixed buffering and no 935 // intermediary disk IO. 936 func (archiver *Archiver) CopyWithTar(src, dst string) error { 937 srcSt, err := os.Stat(src) 938 if err != nil { 939 return err 940 } 941 if !srcSt.IsDir() { 942 return archiver.CopyFileWithTar(src, dst) 943 } 944 945 // if this archiver is set up with ID mapping we need to create 946 // the new destination directory with the remapped root UID/GID pair 947 // as owner 948 rootUID, rootGID, err := idtools.GetRootUIDGID(archiver.UIDMaps, archiver.GIDMaps) 949 if err != nil { 950 return err 951 } 952 // Create dst, copy src's content into it 953 logrus.Debugf("Creating dest directory: %s", dst) 954 if err := idtools.MkdirAllNewAs(dst, 0755, rootUID, rootGID); err != nil { 955 return err 956 } 957 logrus.Debugf("Calling TarUntar(%s, %s)", src, dst) 958 return archiver.TarUntar(src, dst) 959 } 960 961 // CopyWithTar creates a tar archive of filesystem path `src`, and 962 // unpacks it at filesystem path `dst`. 963 // The archive is streamed directly with fixed buffering and no 964 // intermediary disk IO. 965 func CopyWithTar(src, dst string) error { 966 return defaultArchiver.CopyWithTar(src, dst) 967 } 968 969 // CopyFileWithTar emulates the behavior of the 'cp' command-line 970 // for a single file. It copies a regular file from path `src` to 971 // path `dst`, and preserves all its metadata. 972 func (archiver *Archiver) CopyFileWithTar(src, dst string) (err error) { 973 logrus.Debugf("CopyFileWithTar(%s, %s)", src, dst) 974 srcSt, err := os.Stat(src) 975 if err != nil { 976 return err 977 } 978 979 if srcSt.IsDir() { 980 return fmt.Errorf("Can't copy a directory") 981 } 982 983 // Clean up the trailing slash. This must be done in an operating 984 // system specific manner. 985 if dst[len(dst)-1] == os.PathSeparator { 986 dst = filepath.Join(dst, filepath.Base(src)) 987 } 988 // Create the holding directory if necessary 989 if err := system.MkdirAll(filepath.Dir(dst), 0700); err != nil { 990 return err 991 } 992 993 r, w := io.Pipe() 994 errC := promise.Go(func() error { 995 defer w.Close() 996 997 srcF, err := os.Open(src) 998 if err != nil { 999 return err 1000 } 1001 defer srcF.Close() 1002 1003 hdr, err := tar.FileInfoHeader(srcSt, "") 1004 if err != nil { 1005 return err 1006 } 1007 hdr.Name = filepath.Base(dst) 1008 hdr.Mode = int64(chmodTarEntry(os.FileMode(hdr.Mode))) 1009 1010 remappedRootUID, remappedRootGID, err := idtools.GetRootUIDGID(archiver.UIDMaps, archiver.GIDMaps) 1011 if err != nil { 1012 return err 1013 } 1014 1015 // only perform mapping if the file being copied isn't already owned by the 1016 // uid or gid of the remapped root in the container 1017 if remappedRootUID != hdr.Uid { 1018 xUID, err := idtools.ToHost(hdr.Uid, archiver.UIDMaps) 1019 if err != nil { 1020 return err 1021 } 1022 hdr.Uid = xUID 1023 } 1024 if remappedRootGID != hdr.Gid { 1025 xGID, err := idtools.ToHost(hdr.Gid, archiver.GIDMaps) 1026 if err != nil { 1027 return err 1028 } 1029 hdr.Gid = xGID 1030 } 1031 1032 tw := tar.NewWriter(w) 1033 defer tw.Close() 1034 if err := tw.WriteHeader(hdr); err != nil { 1035 return err 1036 } 1037 if _, err := io.Copy(tw, srcF); err != nil { 1038 return err 1039 } 1040 return nil 1041 }) 1042 defer func() { 1043 if er := <-errC; err != nil { 1044 err = er 1045 } 1046 }() 1047 1048 err = archiver.Untar(r, filepath.Dir(dst), nil) 1049 if err != nil { 1050 r.CloseWithError(err) 1051 } 1052 return err 1053 } 1054 1055 // CopyFileWithTar emulates the behavior of the 'cp' command-line 1056 // for a single file. It copies a regular file from path `src` to 1057 // path `dst`, and preserves all its metadata. 1058 // 1059 // Destination handling is in an operating specific manner depending 1060 // where the daemon is running. If `dst` ends with a trailing slash 1061 // the final destination path will be `dst/base(src)` (Linux) or 1062 // `dst\base(src)` (Windows). 1063 func CopyFileWithTar(src, dst string) (err error) { 1064 return defaultArchiver.CopyFileWithTar(src, dst) 1065 } 1066 1067 // cmdStream executes a command, and returns its stdout as a stream. 1068 // If the command fails to run or doesn't complete successfully, an error 1069 // will be returned, including anything written on stderr. 1070 func cmdStream(cmd *exec.Cmd, input io.Reader) (io.ReadCloser, <-chan struct{}, error) { 1071 chdone := make(chan struct{}) 1072 cmd.Stdin = input 1073 pipeR, pipeW := io.Pipe() 1074 cmd.Stdout = pipeW 1075 var errBuf bytes.Buffer 1076 cmd.Stderr = &errBuf 1077 1078 // Run the command and return the pipe 1079 if err := cmd.Start(); err != nil { 1080 return nil, nil, err 1081 } 1082 1083 // Copy stdout to the returned pipe 1084 go func() { 1085 if err := cmd.Wait(); err != nil { 1086 pipeW.CloseWithError(fmt.Errorf("%s: %s", err, errBuf.String())) 1087 } else { 1088 pipeW.Close() 1089 } 1090 close(chdone) 1091 }() 1092 1093 return pipeR, chdone, nil 1094 } 1095 1096 // NewTempArchive reads the content of src into a temporary file, and returns the contents 1097 // of that file as an archive. The archive can only be read once - as soon as reading completes, 1098 // the file will be deleted. 1099 func NewTempArchive(src Archive, dir string) (*TempArchive, error) { 1100 f, err := ioutil.TempFile(dir, "") 1101 if err != nil { 1102 return nil, err 1103 } 1104 if _, err := io.Copy(f, src); err != nil { 1105 return nil, err 1106 } 1107 if _, err := f.Seek(0, 0); err != nil { 1108 return nil, err 1109 } 1110 st, err := f.Stat() 1111 if err != nil { 1112 return nil, err 1113 } 1114 size := st.Size() 1115 return &TempArchive{File: f, Size: size}, nil 1116 } 1117 1118 // TempArchive is a temporary archive. The archive can only be read once - as soon as reading completes, 1119 // the file will be deleted. 1120 type TempArchive struct { 1121 *os.File 1122 Size int64 // Pre-computed from Stat().Size() as a convenience 1123 read int64 1124 closed bool 1125 } 1126 1127 // Close closes the underlying file if it's still open, or does a no-op 1128 // to allow callers to try to close the TempArchive multiple times safely. 1129 func (archive *TempArchive) Close() error { 1130 if archive.closed { 1131 return nil 1132 } 1133 1134 archive.closed = true 1135 1136 return archive.File.Close() 1137 } 1138 1139 func (archive *TempArchive) Read(data []byte) (int, error) { 1140 n, err := archive.File.Read(data) 1141 archive.read += int64(n) 1142 if err != nil || archive.read == archive.Size { 1143 archive.Close() 1144 os.Remove(archive.File.Name()) 1145 } 1146 return n, err 1147 }