github.com/lacework-dev/go-moby@v20.10.12+incompatible/pkg/archive/archive.go (about) 1 package archive // import "github.com/docker/docker/pkg/archive" 2 3 import ( 4 "archive/tar" 5 "bufio" 6 "bytes" 7 "compress/bzip2" 8 "compress/gzip" 9 "context" 10 "fmt" 11 "io" 12 "io/ioutil" 13 "os" 14 "path/filepath" 15 "runtime" 16 "strconv" 17 "strings" 18 "syscall" 19 "time" 20 21 "github.com/docker/docker/pkg/fileutils" 22 "github.com/docker/docker/pkg/idtools" 23 "github.com/docker/docker/pkg/ioutils" 24 "github.com/docker/docker/pkg/pools" 25 "github.com/docker/docker/pkg/system" 26 "github.com/sirupsen/logrus" 27 exec "golang.org/x/sys/execabs" 28 ) 29 30 type ( 31 // Compression is the state represents if compressed or not. 32 Compression int 33 // WhiteoutFormat is the format of whiteouts unpacked 34 WhiteoutFormat int 35 36 // TarOptions wraps the tar options. 37 TarOptions struct { 38 IncludeFiles []string 39 ExcludePatterns []string 40 Compression Compression 41 NoLchown bool 42 UIDMaps []idtools.IDMap 43 GIDMaps []idtools.IDMap 44 ChownOpts *idtools.Identity 45 IncludeSourceDir bool 46 // WhiteoutFormat is the expected on disk format for whiteout files. 47 // This format will be converted to the standard format on pack 48 // and from the standard format on unpack. 49 WhiteoutFormat WhiteoutFormat 50 // When unpacking, specifies whether overwriting a directory with a 51 // non-directory is allowed and vice versa. 52 NoOverwriteDirNonDir bool 53 // For each include when creating an archive, the included name will be 54 // replaced with the matching name from this map. 55 RebaseNames map[string]string 56 InUserNS bool 57 } 58 ) 59 60 // Archiver implements the Archiver interface and allows the reuse of most utility functions of 61 // this package with a pluggable Untar function. Also, to facilitate the passing of specific id 62 // mappings for untar, an Archiver can be created with maps which will then be passed to Untar operations. 63 type Archiver struct { 64 Untar func(io.Reader, string, *TarOptions) error 65 IDMapping *idtools.IdentityMapping 66 } 67 68 // NewDefaultArchiver returns a new Archiver without any IdentityMapping 69 func NewDefaultArchiver() *Archiver { 70 return &Archiver{Untar: Untar, IDMapping: &idtools.IdentityMapping{}} 71 } 72 73 // breakoutError is used to differentiate errors related to breaking out 74 // When testing archive breakout in the unit tests, this error is expected 75 // in order for the test to pass. 76 type breakoutError error 77 78 const ( 79 // Uncompressed represents the uncompressed. 80 Uncompressed Compression = iota 81 // Bzip2 is bzip2 compression algorithm. 82 Bzip2 83 // Gzip is gzip compression algorithm. 84 Gzip 85 // Xz is xz compression algorithm. 86 Xz 87 ) 88 89 const ( 90 // AUFSWhiteoutFormat is the default format for whiteouts 91 AUFSWhiteoutFormat WhiteoutFormat = iota 92 // OverlayWhiteoutFormat formats whiteout according to the overlay 93 // standard. 94 OverlayWhiteoutFormat 95 ) 96 97 const ( 98 modeISDIR = 040000 // Directory 99 modeISFIFO = 010000 // FIFO 100 modeISREG = 0100000 // Regular file 101 modeISLNK = 0120000 // Symbolic link 102 modeISBLK = 060000 // Block special file 103 modeISCHR = 020000 // Character special file 104 modeISSOCK = 0140000 // Socket 105 ) 106 107 // IsArchivePath checks if the (possibly compressed) file at the given path 108 // starts with a tar file header. 109 func IsArchivePath(path string) bool { 110 file, err := os.Open(path) 111 if err != nil { 112 return false 113 } 114 defer file.Close() 115 rdr, err := DecompressStream(file) 116 if err != nil { 117 return false 118 } 119 defer rdr.Close() 120 r := tar.NewReader(rdr) 121 _, err = r.Next() 122 return err == nil 123 } 124 125 // DetectCompression detects the compression algorithm of the source. 126 func DetectCompression(source []byte) Compression { 127 for compression, m := range map[Compression][]byte{ 128 Bzip2: {0x42, 0x5A, 0x68}, 129 Gzip: {0x1F, 0x8B, 0x08}, 130 Xz: {0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00}, 131 } { 132 if len(source) < len(m) { 133 logrus.Debug("Len too short") 134 continue 135 } 136 if bytes.Equal(m, source[:len(m)]) { 137 return compression 138 } 139 } 140 return Uncompressed 141 } 142 143 func xzDecompress(ctx context.Context, archive io.Reader) (io.ReadCloser, error) { 144 args := []string{"xz", "-d", "-c", "-q"} 145 146 return cmdStream(exec.CommandContext(ctx, args[0], args[1:]...), archive) 147 } 148 149 func gzDecompress(ctx context.Context, buf io.Reader) (io.ReadCloser, error) { 150 noPigzEnv := os.Getenv("MOBY_DISABLE_PIGZ") 151 var noPigz bool 152 153 if noPigzEnv != "" { 154 var err error 155 noPigz, err = strconv.ParseBool(noPigzEnv) 156 if err != nil { 157 logrus.WithError(err).Warn("invalid value in MOBY_DISABLE_PIGZ env var") 158 } 159 } 160 161 if noPigz { 162 logrus.Debugf("Use of pigz is disabled due to MOBY_DISABLE_PIGZ=%s", noPigzEnv) 163 return gzip.NewReader(buf) 164 } 165 166 unpigzPath, err := exec.LookPath("unpigz") 167 if err != nil { 168 logrus.Debugf("unpigz binary not found, falling back to go gzip library") 169 return gzip.NewReader(buf) 170 } 171 172 logrus.Debugf("Using %s to decompress", unpigzPath) 173 174 return cmdStream(exec.CommandContext(ctx, unpigzPath, "-d", "-c"), buf) 175 } 176 177 func wrapReadCloser(readBuf io.ReadCloser, cancel context.CancelFunc) io.ReadCloser { 178 return ioutils.NewReadCloserWrapper(readBuf, func() error { 179 cancel() 180 return readBuf.Close() 181 }) 182 } 183 184 // DecompressStream decompresses the archive and returns a ReaderCloser with the decompressed archive. 185 func DecompressStream(archive io.Reader) (io.ReadCloser, error) { 186 p := pools.BufioReader32KPool 187 buf := p.Get(archive) 188 bs, err := buf.Peek(10) 189 if err != nil && err != io.EOF { 190 // Note: we'll ignore any io.EOF error because there are some odd 191 // cases where the layer.tar file will be empty (zero bytes) and 192 // that results in an io.EOF from the Peek() call. So, in those 193 // cases we'll just treat it as a non-compressed stream and 194 // that means just create an empty layer. 195 // See Issue 18170 196 return nil, err 197 } 198 199 compression := DetectCompression(bs) 200 switch compression { 201 case Uncompressed: 202 readBufWrapper := p.NewReadCloserWrapper(buf, buf) 203 return readBufWrapper, nil 204 case Gzip: 205 ctx, cancel := context.WithCancel(context.Background()) 206 207 gzReader, err := gzDecompress(ctx, buf) 208 if err != nil { 209 cancel() 210 return nil, err 211 } 212 readBufWrapper := p.NewReadCloserWrapper(buf, gzReader) 213 return wrapReadCloser(readBufWrapper, cancel), nil 214 case Bzip2: 215 bz2Reader := bzip2.NewReader(buf) 216 readBufWrapper := p.NewReadCloserWrapper(buf, bz2Reader) 217 return readBufWrapper, nil 218 case Xz: 219 ctx, cancel := context.WithCancel(context.Background()) 220 221 xzReader, err := xzDecompress(ctx, buf) 222 if err != nil { 223 cancel() 224 return nil, err 225 } 226 readBufWrapper := p.NewReadCloserWrapper(buf, xzReader) 227 return wrapReadCloser(readBufWrapper, cancel), nil 228 default: 229 return nil, fmt.Errorf("Unsupported compression format %s", (&compression).Extension()) 230 } 231 } 232 233 // CompressStream compresses the dest with specified compression algorithm. 234 func CompressStream(dest io.Writer, compression Compression) (io.WriteCloser, error) { 235 p := pools.BufioWriter32KPool 236 buf := p.Get(dest) 237 switch compression { 238 case Uncompressed: 239 writeBufWrapper := p.NewWriteCloserWrapper(buf, buf) 240 return writeBufWrapper, nil 241 case Gzip: 242 gzWriter := gzip.NewWriter(dest) 243 writeBufWrapper := p.NewWriteCloserWrapper(buf, gzWriter) 244 return writeBufWrapper, nil 245 case Bzip2, Xz: 246 // archive/bzip2 does not support writing, and there is no xz support at all 247 // However, this is not a problem as docker only currently generates gzipped tars 248 return nil, fmt.Errorf("Unsupported compression format %s", (&compression).Extension()) 249 default: 250 return nil, fmt.Errorf("Unsupported compression format %s", (&compression).Extension()) 251 } 252 } 253 254 // TarModifierFunc is a function that can be passed to ReplaceFileTarWrapper to 255 // modify the contents or header of an entry in the archive. If the file already 256 // exists in the archive the TarModifierFunc will be called with the Header and 257 // a reader which will return the files content. If the file does not exist both 258 // header and content will be nil. 259 type TarModifierFunc func(path string, header *tar.Header, content io.Reader) (*tar.Header, []byte, error) 260 261 // ReplaceFileTarWrapper converts inputTarStream to a new tar stream. Files in the 262 // tar stream are modified if they match any of the keys in mods. 263 func ReplaceFileTarWrapper(inputTarStream io.ReadCloser, mods map[string]TarModifierFunc) io.ReadCloser { 264 pipeReader, pipeWriter := io.Pipe() 265 266 go func() { 267 tarReader := tar.NewReader(inputTarStream) 268 tarWriter := tar.NewWriter(pipeWriter) 269 defer inputTarStream.Close() 270 defer tarWriter.Close() 271 272 modify := func(name string, original *tar.Header, modifier TarModifierFunc, tarReader io.Reader) error { 273 header, data, err := modifier(name, original, tarReader) 274 switch { 275 case err != nil: 276 return err 277 case header == nil: 278 return nil 279 } 280 281 header.Name = name 282 header.Size = int64(len(data)) 283 if err := tarWriter.WriteHeader(header); err != nil { 284 return err 285 } 286 if len(data) != 0 { 287 if _, err := tarWriter.Write(data); err != nil { 288 return err 289 } 290 } 291 return nil 292 } 293 294 var err error 295 var originalHeader *tar.Header 296 for { 297 originalHeader, err = tarReader.Next() 298 if err == io.EOF { 299 break 300 } 301 if err != nil { 302 pipeWriter.CloseWithError(err) 303 return 304 } 305 306 modifier, ok := mods[originalHeader.Name] 307 if !ok { 308 // No modifiers for this file, copy the header and data 309 if err := tarWriter.WriteHeader(originalHeader); err != nil { 310 pipeWriter.CloseWithError(err) 311 return 312 } 313 if _, err := pools.Copy(tarWriter, tarReader); err != nil { 314 pipeWriter.CloseWithError(err) 315 return 316 } 317 continue 318 } 319 delete(mods, originalHeader.Name) 320 321 if err := modify(originalHeader.Name, originalHeader, modifier, tarReader); err != nil { 322 pipeWriter.CloseWithError(err) 323 return 324 } 325 } 326 327 // Apply the modifiers that haven't matched any files in the archive 328 for name, modifier := range mods { 329 if err := modify(name, nil, modifier, nil); err != nil { 330 pipeWriter.CloseWithError(err) 331 return 332 } 333 } 334 335 pipeWriter.Close() 336 337 }() 338 return pipeReader 339 } 340 341 // Extension returns the extension of a file that uses the specified compression algorithm. 342 func (compression *Compression) Extension() string { 343 switch *compression { 344 case Uncompressed: 345 return "tar" 346 case Bzip2: 347 return "tar.bz2" 348 case Gzip: 349 return "tar.gz" 350 case Xz: 351 return "tar.xz" 352 } 353 return "" 354 } 355 356 // FileInfoHeader creates a populated Header from fi. 357 // Compared to archive pkg this function fills in more information. 358 // Also, regardless of Go version, this function fills file type bits (e.g. hdr.Mode |= modeISDIR), 359 // which have been deleted since Go 1.9 archive/tar. 360 func FileInfoHeader(name string, fi os.FileInfo, link string) (*tar.Header, error) { 361 hdr, err := tar.FileInfoHeader(fi, link) 362 if err != nil { 363 return nil, err 364 } 365 hdr.Format = tar.FormatPAX 366 hdr.ModTime = hdr.ModTime.Truncate(time.Second) 367 hdr.AccessTime = time.Time{} 368 hdr.ChangeTime = time.Time{} 369 hdr.Mode = fillGo18FileTypeBits(int64(chmodTarEntry(os.FileMode(hdr.Mode))), fi) 370 hdr.Name = canonicalTarName(name, fi.IsDir()) 371 if err := setHeaderForSpecialDevice(hdr, name, fi.Sys()); err != nil { 372 return nil, err 373 } 374 return hdr, nil 375 } 376 377 // fillGo18FileTypeBits fills type bits which have been removed on Go 1.9 archive/tar 378 // https://github.com/golang/go/commit/66b5a2f 379 func fillGo18FileTypeBits(mode int64, fi os.FileInfo) int64 { 380 fm := fi.Mode() 381 switch { 382 case fm.IsRegular(): 383 mode |= modeISREG 384 case fi.IsDir(): 385 mode |= modeISDIR 386 case fm&os.ModeSymlink != 0: 387 mode |= modeISLNK 388 case fm&os.ModeDevice != 0: 389 if fm&os.ModeCharDevice != 0 { 390 mode |= modeISCHR 391 } else { 392 mode |= modeISBLK 393 } 394 case fm&os.ModeNamedPipe != 0: 395 mode |= modeISFIFO 396 case fm&os.ModeSocket != 0: 397 mode |= modeISSOCK 398 } 399 return mode 400 } 401 402 // ReadSecurityXattrToTarHeader reads security.capability xattr from filesystem 403 // to a tar header 404 func ReadSecurityXattrToTarHeader(path string, hdr *tar.Header) error { 405 const ( 406 // Values based on linux/include/uapi/linux/capability.h 407 xattrCapsSz2 = 20 408 versionOffset = 3 409 vfsCapRevision2 = 2 410 vfsCapRevision3 = 3 411 ) 412 capability, _ := system.Lgetxattr(path, "security.capability") 413 if capability != nil { 414 length := len(capability) 415 if capability[versionOffset] == vfsCapRevision3 { 416 // Convert VFS_CAP_REVISION_3 to VFS_CAP_REVISION_2 as root UID makes no 417 // sense outside the user namespace the archive is built in. 418 capability[versionOffset] = vfsCapRevision2 419 length = xattrCapsSz2 420 } 421 hdr.Xattrs = make(map[string]string) 422 hdr.Xattrs["security.capability"] = string(capability[:length]) 423 } 424 return nil 425 } 426 427 type tarWhiteoutConverter interface { 428 ConvertWrite(*tar.Header, string, os.FileInfo) (*tar.Header, error) 429 ConvertRead(*tar.Header, string) (bool, error) 430 } 431 432 type tarAppender struct { 433 TarWriter *tar.Writer 434 Buffer *bufio.Writer 435 436 // for hardlink mapping 437 SeenFiles map[uint64]string 438 IdentityMapping *idtools.IdentityMapping 439 ChownOpts *idtools.Identity 440 441 // For packing and unpacking whiteout files in the 442 // non standard format. The whiteout files defined 443 // by the AUFS standard are used as the tar whiteout 444 // standard. 445 WhiteoutConverter tarWhiteoutConverter 446 } 447 448 func newTarAppender(idMapping *idtools.IdentityMapping, writer io.Writer, chownOpts *idtools.Identity) *tarAppender { 449 return &tarAppender{ 450 SeenFiles: make(map[uint64]string), 451 TarWriter: tar.NewWriter(writer), 452 Buffer: pools.BufioWriter32KPool.Get(nil), 453 IdentityMapping: idMapping, 454 ChownOpts: chownOpts, 455 } 456 } 457 458 // canonicalTarName provides a platform-independent and consistent posix-style 459 // path for files and directories to be archived regardless of the platform. 460 func canonicalTarName(name string, isDir bool) string { 461 name = CanonicalTarNameForPath(name) 462 463 // suffix with '/' for directories 464 if isDir && !strings.HasSuffix(name, "/") { 465 name += "/" 466 } 467 return name 468 } 469 470 // addTarFile adds to the tar archive a file from `path` as `name` 471 func (ta *tarAppender) addTarFile(path, name string) error { 472 fi, err := os.Lstat(path) 473 if err != nil { 474 return err 475 } 476 477 var link string 478 if fi.Mode()&os.ModeSymlink != 0 { 479 var err error 480 link, err = os.Readlink(path) 481 if err != nil { 482 return err 483 } 484 } 485 486 hdr, err := FileInfoHeader(name, fi, link) 487 if err != nil { 488 return err 489 } 490 if err := ReadSecurityXattrToTarHeader(path, hdr); err != nil { 491 return err 492 } 493 494 // if it's not a directory and has more than 1 link, 495 // it's hard linked, so set the type flag accordingly 496 if !fi.IsDir() && hasHardlinks(fi) { 497 inode, err := getInodeFromStat(fi.Sys()) 498 if err != nil { 499 return err 500 } 501 // a link should have a name that it links too 502 // and that linked name should be first in the tar archive 503 if oldpath, ok := ta.SeenFiles[inode]; ok { 504 hdr.Typeflag = tar.TypeLink 505 hdr.Linkname = oldpath 506 hdr.Size = 0 // This Must be here for the writer math to add up! 507 } else { 508 ta.SeenFiles[inode] = name 509 } 510 } 511 512 // check whether the file is overlayfs whiteout 513 // if yes, skip re-mapping container ID mappings. 514 isOverlayWhiteout := fi.Mode()&os.ModeCharDevice != 0 && hdr.Devmajor == 0 && hdr.Devminor == 0 515 516 // handle re-mapping container ID mappings back to host ID mappings before 517 // writing tar headers/files. We skip whiteout files because they were written 518 // by the kernel and already have proper ownership relative to the host 519 if !isOverlayWhiteout && !strings.HasPrefix(filepath.Base(hdr.Name), WhiteoutPrefix) && !ta.IdentityMapping.Empty() { 520 fileIDPair, err := getFileUIDGID(fi.Sys()) 521 if err != nil { 522 return err 523 } 524 hdr.Uid, hdr.Gid, err = ta.IdentityMapping.ToContainer(fileIDPair) 525 if err != nil { 526 return err 527 } 528 } 529 530 // explicitly override with ChownOpts 531 if ta.ChownOpts != nil { 532 hdr.Uid = ta.ChownOpts.UID 533 hdr.Gid = ta.ChownOpts.GID 534 } 535 536 if ta.WhiteoutConverter != nil { 537 wo, err := ta.WhiteoutConverter.ConvertWrite(hdr, path, fi) 538 if err != nil { 539 return err 540 } 541 542 // If a new whiteout file exists, write original hdr, then 543 // replace hdr with wo to be written after. Whiteouts should 544 // always be written after the original. Note the original 545 // hdr may have been updated to be a whiteout with returning 546 // a whiteout header 547 if wo != nil { 548 if err := ta.TarWriter.WriteHeader(hdr); err != nil { 549 return err 550 } 551 if hdr.Typeflag == tar.TypeReg && hdr.Size > 0 { 552 return fmt.Errorf("tar: cannot use whiteout for non-empty file") 553 } 554 hdr = wo 555 } 556 } 557 558 if err := ta.TarWriter.WriteHeader(hdr); err != nil { 559 return err 560 } 561 562 if hdr.Typeflag == tar.TypeReg && hdr.Size > 0 { 563 // We use system.OpenSequential to ensure we use sequential file 564 // access on Windows to avoid depleting the standby list. 565 // On Linux, this equates to a regular os.Open. 566 file, err := system.OpenSequential(path) 567 if err != nil { 568 return err 569 } 570 571 ta.Buffer.Reset(ta.TarWriter) 572 defer ta.Buffer.Reset(nil) 573 _, err = io.Copy(ta.Buffer, file) 574 file.Close() 575 if err != nil { 576 return err 577 } 578 err = ta.Buffer.Flush() 579 if err != nil { 580 return err 581 } 582 } 583 584 return nil 585 } 586 587 func createTarFile(path, extractDir string, hdr *tar.Header, reader io.Reader, Lchown bool, chownOpts *idtools.Identity, inUserns bool) error { 588 // hdr.Mode is in linux format, which we can use for sycalls, 589 // but for os.Foo() calls we need the mode converted to os.FileMode, 590 // so use hdrInfo.Mode() (they differ for e.g. setuid bits) 591 hdrInfo := hdr.FileInfo() 592 593 switch hdr.Typeflag { 594 case tar.TypeDir: 595 // Create directory unless it exists as a directory already. 596 // In that case we just want to merge the two 597 if fi, err := os.Lstat(path); !(err == nil && fi.IsDir()) { 598 if err := os.Mkdir(path, hdrInfo.Mode()); err != nil { 599 return err 600 } 601 } 602 603 case tar.TypeReg, tar.TypeRegA: 604 // Source is regular file. We use system.OpenFileSequential to use sequential 605 // file access to avoid depleting the standby list on Windows. 606 // On Linux, this equates to a regular os.OpenFile 607 file, err := system.OpenFileSequential(path, os.O_CREATE|os.O_WRONLY, hdrInfo.Mode()) 608 if err != nil { 609 return err 610 } 611 if _, err := io.Copy(file, reader); err != nil { 612 file.Close() 613 return err 614 } 615 file.Close() 616 617 case tar.TypeBlock, tar.TypeChar: 618 if inUserns { // cannot create devices in a userns 619 return nil 620 } 621 // Handle this is an OS-specific way 622 if err := handleTarTypeBlockCharFifo(hdr, path); err != nil { 623 return err 624 } 625 626 case tar.TypeFifo: 627 // Handle this is an OS-specific way 628 if err := handleTarTypeBlockCharFifo(hdr, path); err != nil { 629 return err 630 } 631 632 case tar.TypeLink: 633 targetPath := filepath.Join(extractDir, hdr.Linkname) 634 // check for hardlink breakout 635 if !strings.HasPrefix(targetPath, extractDir) { 636 return breakoutError(fmt.Errorf("invalid hardlink %q -> %q", targetPath, hdr.Linkname)) 637 } 638 if err := os.Link(targetPath, path); err != nil { 639 return err 640 } 641 642 case tar.TypeSymlink: 643 // path -> hdr.Linkname = targetPath 644 // e.g. /extractDir/path/to/symlink -> ../2/file = /extractDir/path/2/file 645 targetPath := filepath.Join(filepath.Dir(path), hdr.Linkname) 646 647 // the reason we don't need to check symlinks in the path (with FollowSymlinkInScope) is because 648 // that symlink would first have to be created, which would be caught earlier, at this very check: 649 if !strings.HasPrefix(targetPath, extractDir) { 650 return breakoutError(fmt.Errorf("invalid symlink %q -> %q", path, hdr.Linkname)) 651 } 652 if err := os.Symlink(hdr.Linkname, path); err != nil { 653 return err 654 } 655 656 case tar.TypeXGlobalHeader: 657 logrus.Debug("PAX Global Extended Headers found and ignored") 658 return nil 659 660 default: 661 return fmt.Errorf("unhandled tar header type %d", hdr.Typeflag) 662 } 663 664 // Lchown is not supported on Windows. 665 if Lchown && runtime.GOOS != "windows" { 666 if chownOpts == nil { 667 chownOpts = &idtools.Identity{UID: hdr.Uid, GID: hdr.Gid} 668 } 669 if err := os.Lchown(path, chownOpts.UID, chownOpts.GID); err != nil { 670 return err 671 } 672 } 673 674 var errors []string 675 for key, value := range hdr.Xattrs { 676 if err := system.Lsetxattr(path, key, []byte(value), 0); err != nil { 677 if err == syscall.ENOTSUP || err == syscall.EPERM { 678 // We ignore errors here because not all graphdrivers support 679 // xattrs *cough* old versions of AUFS *cough*. However only 680 // ENOTSUP should be emitted in that case, otherwise we still 681 // bail. 682 // EPERM occurs if modifying xattrs is not allowed. This can 683 // happen when running in userns with restrictions (ChromeOS). 684 errors = append(errors, err.Error()) 685 continue 686 } 687 return err 688 } 689 690 } 691 692 if len(errors) > 0 { 693 logrus.WithFields(logrus.Fields{ 694 "errors": errors, 695 }).Warn("ignored xattrs in archive: underlying filesystem doesn't support them") 696 } 697 698 // There is no LChmod, so ignore mode for symlink. Also, this 699 // must happen after chown, as that can modify the file mode 700 if err := handleLChmod(hdr, path, hdrInfo); err != nil { 701 return err 702 } 703 704 aTime := hdr.AccessTime 705 if aTime.Before(hdr.ModTime) { 706 // Last access time should never be before last modified time. 707 aTime = hdr.ModTime 708 } 709 710 // system.Chtimes doesn't support a NOFOLLOW flag atm 711 if hdr.Typeflag == tar.TypeLink { 712 if fi, err := os.Lstat(hdr.Linkname); err == nil && (fi.Mode()&os.ModeSymlink == 0) { 713 if err := system.Chtimes(path, aTime, hdr.ModTime); err != nil { 714 return err 715 } 716 } 717 } else if hdr.Typeflag != tar.TypeSymlink { 718 if err := system.Chtimes(path, aTime, hdr.ModTime); err != nil { 719 return err 720 } 721 } else { 722 ts := []syscall.Timespec{timeToTimespec(aTime), timeToTimespec(hdr.ModTime)} 723 if err := system.LUtimesNano(path, ts); err != nil && err != system.ErrNotSupportedPlatform { 724 return err 725 } 726 } 727 return nil 728 } 729 730 // Tar creates an archive from the directory at `path`, and returns it as a 731 // stream of bytes. 732 func Tar(path string, compression Compression) (io.ReadCloser, error) { 733 return TarWithOptions(path, &TarOptions{Compression: compression}) 734 } 735 736 // TarWithOptions creates an archive from the directory at `path`, only including files whose relative 737 // paths are included in `options.IncludeFiles` (if non-nil) or not in `options.ExcludePatterns`. 738 func TarWithOptions(srcPath string, options *TarOptions) (io.ReadCloser, error) { 739 740 // Fix the source path to work with long path names. This is a no-op 741 // on platforms other than Windows. 742 srcPath = fixVolumePathPrefix(srcPath) 743 744 pm, err := fileutils.NewPatternMatcher(options.ExcludePatterns) 745 if err != nil { 746 return nil, err 747 } 748 749 pipeReader, pipeWriter := io.Pipe() 750 751 compressWriter, err := CompressStream(pipeWriter, options.Compression) 752 if err != nil { 753 return nil, err 754 } 755 756 whiteoutConverter, err := getWhiteoutConverter(options.WhiteoutFormat, options.InUserNS) 757 if err != nil { 758 return nil, err 759 } 760 761 go func() { 762 ta := newTarAppender( 763 idtools.NewIDMappingsFromMaps(options.UIDMaps, options.GIDMaps), 764 compressWriter, 765 options.ChownOpts, 766 ) 767 ta.WhiteoutConverter = whiteoutConverter 768 769 defer func() { 770 // Make sure to check the error on Close. 771 if err := ta.TarWriter.Close(); err != nil { 772 logrus.Errorf("Can't close tar writer: %s", err) 773 } 774 if err := compressWriter.Close(); err != nil { 775 logrus.Errorf("Can't close compress writer: %s", err) 776 } 777 if err := pipeWriter.Close(); err != nil { 778 logrus.Errorf("Can't close pipe writer: %s", err) 779 } 780 }() 781 782 // this buffer is needed for the duration of this piped stream 783 defer pools.BufioWriter32KPool.Put(ta.Buffer) 784 785 // In general we log errors here but ignore them because 786 // during e.g. a diff operation the container can continue 787 // mutating the filesystem and we can see transient errors 788 // from this 789 790 stat, err := os.Lstat(srcPath) 791 if err != nil { 792 return 793 } 794 795 if !stat.IsDir() { 796 // We can't later join a non-dir with any includes because the 797 // 'walk' will error if "file/." is stat-ed and "file" is not a 798 // directory. So, we must split the source path and use the 799 // basename as the include. 800 if len(options.IncludeFiles) > 0 { 801 logrus.Warn("Tar: Can't archive a file with includes") 802 } 803 804 dir, base := SplitPathDirEntry(srcPath) 805 srcPath = dir 806 options.IncludeFiles = []string{base} 807 } 808 809 if len(options.IncludeFiles) == 0 { 810 options.IncludeFiles = []string{"."} 811 } 812 813 seen := make(map[string]bool) 814 815 for _, include := range options.IncludeFiles { 816 rebaseName := options.RebaseNames[include] 817 818 walkRoot := getWalkRoot(srcPath, include) 819 filepath.Walk(walkRoot, func(filePath string, f os.FileInfo, err error) error { 820 if err != nil { 821 logrus.Errorf("Tar: Can't stat file %s to tar: %s", srcPath, err) 822 return nil 823 } 824 825 relFilePath, err := filepath.Rel(srcPath, filePath) 826 if err != nil || (!options.IncludeSourceDir && relFilePath == "." && f.IsDir()) { 827 // Error getting relative path OR we are looking 828 // at the source directory path. Skip in both situations. 829 return nil 830 } 831 832 if options.IncludeSourceDir && include == "." && relFilePath != "." { 833 relFilePath = strings.Join([]string{".", relFilePath}, string(filepath.Separator)) 834 } 835 836 skip := false 837 838 // If "include" is an exact match for the current file 839 // then even if there's an "excludePatterns" pattern that 840 // matches it, don't skip it. IOW, assume an explicit 'include' 841 // is asking for that file no matter what - which is true 842 // for some files, like .dockerignore and Dockerfile (sometimes) 843 if include != relFilePath { 844 skip, err = pm.Matches(relFilePath) 845 if err != nil { 846 logrus.Errorf("Error matching %s: %v", relFilePath, err) 847 return err 848 } 849 } 850 851 if skip { 852 // If we want to skip this file and its a directory 853 // then we should first check to see if there's an 854 // excludes pattern (e.g. !dir/file) that starts with this 855 // dir. If so then we can't skip this dir. 856 857 // Its not a dir then so we can just return/skip. 858 if !f.IsDir() { 859 return nil 860 } 861 862 // No exceptions (!...) in patterns so just skip dir 863 if !pm.Exclusions() { 864 return filepath.SkipDir 865 } 866 867 dirSlash := relFilePath + string(filepath.Separator) 868 869 for _, pat := range pm.Patterns() { 870 if !pat.Exclusion() { 871 continue 872 } 873 if strings.HasPrefix(pat.String()+string(filepath.Separator), dirSlash) { 874 // found a match - so can't skip this dir 875 return nil 876 } 877 } 878 879 // No matching exclusion dir so just skip dir 880 return filepath.SkipDir 881 } 882 883 if seen[relFilePath] { 884 return nil 885 } 886 seen[relFilePath] = true 887 888 // Rename the base resource. 889 if rebaseName != "" { 890 var replacement string 891 if rebaseName != string(filepath.Separator) { 892 // Special case the root directory to replace with an 893 // empty string instead so that we don't end up with 894 // double slashes in the paths. 895 replacement = rebaseName 896 } 897 898 relFilePath = strings.Replace(relFilePath, include, replacement, 1) 899 } 900 901 if err := ta.addTarFile(filePath, relFilePath); err != nil { 902 logrus.Errorf("Can't add file %s to tar: %s", filePath, err) 903 // if pipe is broken, stop writing tar stream to it 904 if err == io.ErrClosedPipe { 905 return err 906 } 907 } 908 return nil 909 }) 910 } 911 }() 912 913 return pipeReader, nil 914 } 915 916 // Unpack unpacks the decompressedArchive to dest with options. 917 func Unpack(decompressedArchive io.Reader, dest string, options *TarOptions) error { 918 tr := tar.NewReader(decompressedArchive) 919 trBuf := pools.BufioReader32KPool.Get(nil) 920 defer pools.BufioReader32KPool.Put(trBuf) 921 922 var dirs []*tar.Header 923 idMapping := idtools.NewIDMappingsFromMaps(options.UIDMaps, options.GIDMaps) 924 rootIDs := idMapping.RootPair() 925 whiteoutConverter, err := getWhiteoutConverter(options.WhiteoutFormat, options.InUserNS) 926 if err != nil { 927 return err 928 } 929 930 // Iterate through the files in the archive. 931 loop: 932 for { 933 hdr, err := tr.Next() 934 if err == io.EOF { 935 // end of tar archive 936 break 937 } 938 if err != nil { 939 return err 940 } 941 942 // ignore XGlobalHeader early to avoid creating parent directories for them 943 if hdr.Typeflag == tar.TypeXGlobalHeader { 944 logrus.Debugf("PAX Global Extended Headers found for %s and ignored", hdr.Name) 945 continue 946 } 947 948 // Normalize name, for safety and for a simple is-root check 949 // This keeps "../" as-is, but normalizes "/../" to "/". Or Windows: 950 // This keeps "..\" as-is, but normalizes "\..\" to "\". 951 hdr.Name = filepath.Clean(hdr.Name) 952 953 for _, exclude := range options.ExcludePatterns { 954 if strings.HasPrefix(hdr.Name, exclude) { 955 continue loop 956 } 957 } 958 959 // After calling filepath.Clean(hdr.Name) above, hdr.Name will now be in 960 // the filepath format for the OS on which the daemon is running. Hence 961 // the check for a slash-suffix MUST be done in an OS-agnostic way. 962 if !strings.HasSuffix(hdr.Name, string(os.PathSeparator)) { 963 // Not the root directory, ensure that the parent directory exists 964 parent := filepath.Dir(hdr.Name) 965 parentPath := filepath.Join(dest, parent) 966 if _, err := os.Lstat(parentPath); err != nil && os.IsNotExist(err) { 967 err = idtools.MkdirAllAndChownNew(parentPath, 0755, rootIDs) 968 if err != nil { 969 return err 970 } 971 } 972 } 973 974 path := filepath.Join(dest, hdr.Name) 975 rel, err := filepath.Rel(dest, path) 976 if err != nil { 977 return err 978 } 979 if strings.HasPrefix(rel, ".."+string(os.PathSeparator)) { 980 return breakoutError(fmt.Errorf("%q is outside of %q", hdr.Name, dest)) 981 } 982 983 // If path exits we almost always just want to remove and replace it 984 // The only exception is when it is a directory *and* the file from 985 // the layer is also a directory. Then we want to merge them (i.e. 986 // just apply the metadata from the layer). 987 if fi, err := os.Lstat(path); err == nil { 988 if options.NoOverwriteDirNonDir && fi.IsDir() && hdr.Typeflag != tar.TypeDir { 989 // If NoOverwriteDirNonDir is true then we cannot replace 990 // an existing directory with a non-directory from the archive. 991 return fmt.Errorf("cannot overwrite directory %q with non-directory %q", path, dest) 992 } 993 994 if options.NoOverwriteDirNonDir && !fi.IsDir() && hdr.Typeflag == tar.TypeDir { 995 // If NoOverwriteDirNonDir is true then we cannot replace 996 // an existing non-directory with a directory from the archive. 997 return fmt.Errorf("cannot overwrite non-directory %q with directory %q", path, dest) 998 } 999 1000 if fi.IsDir() && hdr.Name == "." { 1001 continue 1002 } 1003 1004 if !(fi.IsDir() && hdr.Typeflag == tar.TypeDir) { 1005 if err := os.RemoveAll(path); err != nil { 1006 return err 1007 } 1008 } 1009 } 1010 trBuf.Reset(tr) 1011 1012 if err := remapIDs(idMapping, hdr); err != nil { 1013 return err 1014 } 1015 1016 if whiteoutConverter != nil { 1017 writeFile, err := whiteoutConverter.ConvertRead(hdr, path) 1018 if err != nil { 1019 return err 1020 } 1021 if !writeFile { 1022 continue 1023 } 1024 } 1025 1026 if err := createTarFile(path, dest, hdr, trBuf, !options.NoLchown, options.ChownOpts, options.InUserNS); err != nil { 1027 return err 1028 } 1029 1030 // Directory mtimes must be handled at the end to avoid further 1031 // file creation in them to modify the directory mtime 1032 if hdr.Typeflag == tar.TypeDir { 1033 dirs = append(dirs, hdr) 1034 } 1035 } 1036 1037 for _, hdr := range dirs { 1038 path := filepath.Join(dest, hdr.Name) 1039 1040 if err := system.Chtimes(path, hdr.AccessTime, hdr.ModTime); err != nil { 1041 return err 1042 } 1043 } 1044 return nil 1045 } 1046 1047 // Untar reads a stream of bytes from `archive`, parses it as a tar archive, 1048 // and unpacks it into the directory at `dest`. 1049 // The archive may be compressed with one of the following algorithms: 1050 // identity (uncompressed), gzip, bzip2, xz. 1051 // FIXME: specify behavior when target path exists vs. doesn't exist. 1052 func Untar(tarArchive io.Reader, dest string, options *TarOptions) error { 1053 return untarHandler(tarArchive, dest, options, true) 1054 } 1055 1056 // UntarUncompressed reads a stream of bytes from `archive`, parses it as a tar archive, 1057 // and unpacks it into the directory at `dest`. 1058 // The archive must be an uncompressed stream. 1059 func UntarUncompressed(tarArchive io.Reader, dest string, options *TarOptions) error { 1060 return untarHandler(tarArchive, dest, options, false) 1061 } 1062 1063 // Handler for teasing out the automatic decompression 1064 func untarHandler(tarArchive io.Reader, dest string, options *TarOptions, decompress bool) error { 1065 if tarArchive == nil { 1066 return fmt.Errorf("Empty archive") 1067 } 1068 dest = filepath.Clean(dest) 1069 if options == nil { 1070 options = &TarOptions{} 1071 } 1072 if options.ExcludePatterns == nil { 1073 options.ExcludePatterns = []string{} 1074 } 1075 1076 r := tarArchive 1077 if decompress { 1078 decompressedArchive, err := DecompressStream(tarArchive) 1079 if err != nil { 1080 return err 1081 } 1082 defer decompressedArchive.Close() 1083 r = decompressedArchive 1084 } 1085 1086 return Unpack(r, dest, options) 1087 } 1088 1089 // TarUntar is a convenience function which calls Tar and Untar, with the output of one piped into the other. 1090 // If either Tar or Untar fails, TarUntar aborts and returns the error. 1091 func (archiver *Archiver) TarUntar(src, dst string) error { 1092 logrus.Debugf("TarUntar(%s %s)", src, dst) 1093 archive, err := TarWithOptions(src, &TarOptions{Compression: Uncompressed}) 1094 if err != nil { 1095 return err 1096 } 1097 defer archive.Close() 1098 options := &TarOptions{ 1099 UIDMaps: archiver.IDMapping.UIDs(), 1100 GIDMaps: archiver.IDMapping.GIDs(), 1101 } 1102 return archiver.Untar(archive, dst, options) 1103 } 1104 1105 // UntarPath untar a file from path to a destination, src is the source tar file path. 1106 func (archiver *Archiver) UntarPath(src, dst string) error { 1107 archive, err := os.Open(src) 1108 if err != nil { 1109 return err 1110 } 1111 defer archive.Close() 1112 options := &TarOptions{ 1113 UIDMaps: archiver.IDMapping.UIDs(), 1114 GIDMaps: archiver.IDMapping.GIDs(), 1115 } 1116 return archiver.Untar(archive, dst, options) 1117 } 1118 1119 // CopyWithTar creates a tar archive of filesystem path `src`, and 1120 // unpacks it at filesystem path `dst`. 1121 // The archive is streamed directly with fixed buffering and no 1122 // intermediary disk IO. 1123 func (archiver *Archiver) CopyWithTar(src, dst string) error { 1124 srcSt, err := os.Stat(src) 1125 if err != nil { 1126 return err 1127 } 1128 if !srcSt.IsDir() { 1129 return archiver.CopyFileWithTar(src, dst) 1130 } 1131 1132 // if this Archiver is set up with ID mapping we need to create 1133 // the new destination directory with the remapped root UID/GID pair 1134 // as owner 1135 rootIDs := archiver.IDMapping.RootPair() 1136 // Create dst, copy src's content into it 1137 logrus.Debugf("Creating dest directory: %s", dst) 1138 if err := idtools.MkdirAllAndChownNew(dst, 0755, rootIDs); err != nil { 1139 return err 1140 } 1141 logrus.Debugf("Calling TarUntar(%s, %s)", src, dst) 1142 return archiver.TarUntar(src, dst) 1143 } 1144 1145 // CopyFileWithTar emulates the behavior of the 'cp' command-line 1146 // for a single file. It copies a regular file from path `src` to 1147 // path `dst`, and preserves all its metadata. 1148 func (archiver *Archiver) CopyFileWithTar(src, dst string) (err error) { 1149 logrus.Debugf("CopyFileWithTar(%s, %s)", src, dst) 1150 srcSt, err := os.Stat(src) 1151 if err != nil { 1152 return err 1153 } 1154 1155 if srcSt.IsDir() { 1156 return fmt.Errorf("Can't copy a directory") 1157 } 1158 1159 // Clean up the trailing slash. This must be done in an operating 1160 // system specific manner. 1161 if dst[len(dst)-1] == os.PathSeparator { 1162 dst = filepath.Join(dst, filepath.Base(src)) 1163 } 1164 // Create the holding directory if necessary 1165 if err := system.MkdirAll(filepath.Dir(dst), 0700); err != nil { 1166 return err 1167 } 1168 1169 r, w := io.Pipe() 1170 errC := make(chan error, 1) 1171 1172 go func() { 1173 defer close(errC) 1174 1175 errC <- func() error { 1176 defer w.Close() 1177 1178 srcF, err := os.Open(src) 1179 if err != nil { 1180 return err 1181 } 1182 defer srcF.Close() 1183 1184 hdr, err := tar.FileInfoHeader(srcSt, "") 1185 if err != nil { 1186 return err 1187 } 1188 hdr.Format = tar.FormatPAX 1189 hdr.ModTime = hdr.ModTime.Truncate(time.Second) 1190 hdr.AccessTime = time.Time{} 1191 hdr.ChangeTime = time.Time{} 1192 hdr.Name = filepath.Base(dst) 1193 hdr.Mode = int64(chmodTarEntry(os.FileMode(hdr.Mode))) 1194 1195 if err := remapIDs(archiver.IDMapping, hdr); err != nil { 1196 return err 1197 } 1198 1199 tw := tar.NewWriter(w) 1200 defer tw.Close() 1201 if err := tw.WriteHeader(hdr); err != nil { 1202 return err 1203 } 1204 if _, err := io.Copy(tw, srcF); err != nil { 1205 return err 1206 } 1207 return nil 1208 }() 1209 }() 1210 defer func() { 1211 if er := <-errC; err == nil && er != nil { 1212 err = er 1213 } 1214 }() 1215 1216 err = archiver.Untar(r, filepath.Dir(dst), nil) 1217 if err != nil { 1218 r.CloseWithError(err) 1219 } 1220 return err 1221 } 1222 1223 // IdentityMapping returns the IdentityMapping of the archiver. 1224 func (archiver *Archiver) IdentityMapping() *idtools.IdentityMapping { 1225 return archiver.IDMapping 1226 } 1227 1228 func remapIDs(idMapping *idtools.IdentityMapping, hdr *tar.Header) error { 1229 ids, err := idMapping.ToHost(idtools.Identity{UID: hdr.Uid, GID: hdr.Gid}) 1230 hdr.Uid, hdr.Gid = ids.UID, ids.GID 1231 return err 1232 } 1233 1234 // cmdStream executes a command, and returns its stdout as a stream. 1235 // If the command fails to run or doesn't complete successfully, an error 1236 // will be returned, including anything written on stderr. 1237 func cmdStream(cmd *exec.Cmd, input io.Reader) (io.ReadCloser, error) { 1238 cmd.Stdin = input 1239 pipeR, pipeW := io.Pipe() 1240 cmd.Stdout = pipeW 1241 var errBuf bytes.Buffer 1242 cmd.Stderr = &errBuf 1243 1244 // Run the command and return the pipe 1245 if err := cmd.Start(); err != nil { 1246 return nil, err 1247 } 1248 1249 // Ensure the command has exited before we clean anything up 1250 done := make(chan struct{}) 1251 1252 // Copy stdout to the returned pipe 1253 go func() { 1254 if err := cmd.Wait(); err != nil { 1255 pipeW.CloseWithError(fmt.Errorf("%s: %s", err, errBuf.String())) 1256 } else { 1257 pipeW.Close() 1258 } 1259 close(done) 1260 }() 1261 1262 return ioutils.NewReadCloserWrapper(pipeR, func() error { 1263 // Close pipeR, and then wait for the command to complete before returning. We have to close pipeR first, as 1264 // cmd.Wait waits for any non-file stdout/stderr/stdin to close. 1265 err := pipeR.Close() 1266 <-done 1267 return err 1268 }), nil 1269 } 1270 1271 // NewTempArchive reads the content of src into a temporary file, and returns the contents 1272 // of that file as an archive. The archive can only be read once - as soon as reading completes, 1273 // the file will be deleted. 1274 func NewTempArchive(src io.Reader, dir string) (*TempArchive, error) { 1275 f, err := ioutil.TempFile(dir, "") 1276 if err != nil { 1277 return nil, err 1278 } 1279 if _, err := io.Copy(f, src); err != nil { 1280 return nil, err 1281 } 1282 if _, err := f.Seek(0, 0); err != nil { 1283 return nil, err 1284 } 1285 st, err := f.Stat() 1286 if err != nil { 1287 return nil, err 1288 } 1289 size := st.Size() 1290 return &TempArchive{File: f, Size: size}, nil 1291 } 1292 1293 // TempArchive is a temporary archive. The archive can only be read once - as soon as reading completes, 1294 // the file will be deleted. 1295 type TempArchive struct { 1296 *os.File 1297 Size int64 // Pre-computed from Stat().Size() as a convenience 1298 read int64 1299 closed bool 1300 } 1301 1302 // Close closes the underlying file if it's still open, or does a no-op 1303 // to allow callers to try to close the TempArchive multiple times safely. 1304 func (archive *TempArchive) Close() error { 1305 if archive.closed { 1306 return nil 1307 } 1308 1309 archive.closed = true 1310 1311 return archive.File.Close() 1312 } 1313 1314 func (archive *TempArchive) Read(data []byte) (int, error) { 1315 n, err := archive.File.Read(data) 1316 archive.read += int64(n) 1317 if err != nil || archive.read == archive.Size { 1318 archive.Close() 1319 os.Remove(archive.File.Name()) 1320 } 1321 return n, err 1322 }