github.com/codemac/docker@v1.2.1-0.20150518222241-6a18412d5b9c/pkg/archive/archive.go (about) 1 package archive 2 3 import ( 4 "archive/tar" 5 "bufio" 6 "bytes" 7 "compress/bzip2" 8 "compress/gzip" 9 "errors" 10 "fmt" 11 "io" 12 "io/ioutil" 13 "os" 14 "os/exec" 15 "path" 16 "path/filepath" 17 "strings" 18 "syscall" 19 20 "github.com/Sirupsen/logrus" 21 "github.com/docker/docker/pkg/fileutils" 22 "github.com/docker/docker/pkg/pools" 23 "github.com/docker/docker/pkg/promise" 24 "github.com/docker/docker/pkg/system" 25 ) 26 27 type ( 28 Archive io.ReadCloser 29 ArchiveReader io.Reader 30 Compression int 31 TarOptions struct { 32 IncludeFiles []string 33 ExcludePatterns []string 34 Compression Compression 35 NoLchown bool 36 Name string 37 } 38 39 // Archiver allows the reuse of most utility functions of this package 40 // with a pluggable Untar function. 41 Archiver struct { 42 Untar func(io.Reader, string, *TarOptions) error 43 } 44 45 // breakoutError is used to differentiate errors related to breaking out 46 // When testing archive breakout in the unit tests, this error is expected 47 // in order for the test to pass. 48 breakoutError error 49 ) 50 51 var ( 52 ErrNotImplemented = errors.New("Function not implemented") 53 defaultArchiver = &Archiver{Untar} 54 ) 55 56 const ( 57 Uncompressed Compression = iota 58 Bzip2 59 Gzip 60 Xz 61 ) 62 63 func IsArchive(header []byte) bool { 64 compression := DetectCompression(header) 65 if compression != Uncompressed { 66 return true 67 } 68 r := tar.NewReader(bytes.NewBuffer(header)) 69 _, err := r.Next() 70 return err == nil 71 } 72 73 func DetectCompression(source []byte) Compression { 74 for compression, m := range map[Compression][]byte{ 75 Bzip2: {0x42, 0x5A, 0x68}, 76 Gzip: {0x1F, 0x8B, 0x08}, 77 Xz: {0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00}, 78 } { 79 if len(source) < len(m) { 80 logrus.Debugf("Len too short") 81 continue 82 } 83 if bytes.Compare(m, source[:len(m)]) == 0 { 84 return compression 85 } 86 } 87 return Uncompressed 88 } 89 90 func xzDecompress(archive io.Reader) (io.ReadCloser, error) { 91 args := []string{"xz", "-d", "-c", "-q"} 92 93 return CmdStream(exec.Command(args[0], args[1:]...), archive) 94 } 95 96 func DecompressStream(archive io.Reader) (io.ReadCloser, error) { 97 p := pools.BufioReader32KPool 98 buf := p.Get(archive) 99 bs, err := buf.Peek(10) 100 if err != nil { 101 return nil, err 102 } 103 104 compression := DetectCompression(bs) 105 switch compression { 106 case Uncompressed: 107 readBufWrapper := p.NewReadCloserWrapper(buf, buf) 108 return readBufWrapper, nil 109 case Gzip: 110 gzReader, err := gzip.NewReader(buf) 111 if err != nil { 112 return nil, err 113 } 114 readBufWrapper := p.NewReadCloserWrapper(buf, gzReader) 115 return readBufWrapper, nil 116 case Bzip2: 117 bz2Reader := bzip2.NewReader(buf) 118 readBufWrapper := p.NewReadCloserWrapper(buf, bz2Reader) 119 return readBufWrapper, nil 120 case Xz: 121 xzReader, err := xzDecompress(buf) 122 if err != nil { 123 return nil, err 124 } 125 readBufWrapper := p.NewReadCloserWrapper(buf, xzReader) 126 return readBufWrapper, nil 127 default: 128 return nil, fmt.Errorf("Unsupported compression format %s", (&compression).Extension()) 129 } 130 } 131 132 func CompressStream(dest io.WriteCloser, compression Compression) (io.WriteCloser, error) { 133 p := pools.BufioWriter32KPool 134 buf := p.Get(dest) 135 switch compression { 136 case Uncompressed: 137 writeBufWrapper := p.NewWriteCloserWrapper(buf, buf) 138 return writeBufWrapper, nil 139 case Gzip: 140 gzWriter := gzip.NewWriter(dest) 141 writeBufWrapper := p.NewWriteCloserWrapper(buf, gzWriter) 142 return writeBufWrapper, nil 143 case Bzip2, Xz: 144 // archive/bzip2 does not support writing, and there is no xz support at all 145 // However, this is not a problem as docker only currently generates gzipped tars 146 return nil, fmt.Errorf("Unsupported compression format %s", (&compression).Extension()) 147 default: 148 return nil, fmt.Errorf("Unsupported compression format %s", (&compression).Extension()) 149 } 150 } 151 152 func (compression *Compression) Extension() string { 153 switch *compression { 154 case Uncompressed: 155 return "tar" 156 case Bzip2: 157 return "tar.bz2" 158 case Gzip: 159 return "tar.gz" 160 case Xz: 161 return "tar.xz" 162 } 163 return "" 164 } 165 166 type tarAppender struct { 167 TarWriter *tar.Writer 168 Buffer *bufio.Writer 169 170 // for hardlink mapping 171 SeenFiles map[uint64]string 172 } 173 174 // canonicalTarName provides a platform-independent and consistent posix-style 175 //path for files and directories to be archived regardless of the platform. 176 func canonicalTarName(name string, isDir bool) (string, error) { 177 name, err := CanonicalTarNameForPath(name) 178 if err != nil { 179 return "", err 180 } 181 182 // suffix with '/' for directories 183 if isDir && !strings.HasSuffix(name, "/") { 184 name += "/" 185 } 186 return name, nil 187 } 188 189 func (ta *tarAppender) addTarFile(path, name string) error { 190 fi, err := os.Lstat(path) 191 if err != nil { 192 return err 193 } 194 195 link := "" 196 if fi.Mode()&os.ModeSymlink != 0 { 197 if link, err = os.Readlink(path); err != nil { 198 return err 199 } 200 } 201 202 hdr, err := tar.FileInfoHeader(fi, link) 203 if err != nil { 204 return err 205 } 206 hdr.Mode = int64(chmodTarEntry(os.FileMode(hdr.Mode))) 207 208 name, err = canonicalTarName(name, fi.IsDir()) 209 if err != nil { 210 return fmt.Errorf("tar: cannot canonicalize path: %v", err) 211 } 212 hdr.Name = name 213 214 nlink, inode, err := setHeaderForSpecialDevice(hdr, ta, name, fi.Sys()) 215 if err != nil { 216 return err 217 } 218 219 // if it's a regular file and has more than 1 link, 220 // it's hardlinked, so set the type flag accordingly 221 if fi.Mode().IsRegular() && nlink > 1 { 222 // a link should have a name that it links too 223 // and that linked name should be first in the tar archive 224 if oldpath, ok := ta.SeenFiles[inode]; ok { 225 hdr.Typeflag = tar.TypeLink 226 hdr.Linkname = oldpath 227 hdr.Size = 0 // This Must be here for the writer math to add up! 228 } else { 229 ta.SeenFiles[inode] = name 230 } 231 } 232 233 capability, _ := system.Lgetxattr(path, "security.capability") 234 if capability != nil { 235 hdr.Xattrs = make(map[string]string) 236 hdr.Xattrs["security.capability"] = string(capability) 237 } 238 239 if err := ta.TarWriter.WriteHeader(hdr); err != nil { 240 return err 241 } 242 243 if hdr.Typeflag == tar.TypeReg { 244 file, err := os.Open(path) 245 if err != nil { 246 return err 247 } 248 249 ta.Buffer.Reset(ta.TarWriter) 250 defer ta.Buffer.Reset(nil) 251 _, err = io.Copy(ta.Buffer, file) 252 file.Close() 253 if err != nil { 254 return err 255 } 256 err = ta.Buffer.Flush() 257 if err != nil { 258 return err 259 } 260 } 261 262 return nil 263 } 264 265 func createTarFile(path, extractDir string, hdr *tar.Header, reader io.Reader, Lchown bool) error { 266 // hdr.Mode is in linux format, which we can use for sycalls, 267 // but for os.Foo() calls we need the mode converted to os.FileMode, 268 // so use hdrInfo.Mode() (they differ for e.g. setuid bits) 269 hdrInfo := hdr.FileInfo() 270 271 switch hdr.Typeflag { 272 case tar.TypeDir: 273 // Create directory unless it exists as a directory already. 274 // In that case we just want to merge the two 275 if fi, err := os.Lstat(path); !(err == nil && fi.IsDir()) { 276 if err := os.Mkdir(path, hdrInfo.Mode()); err != nil { 277 return err 278 } 279 } 280 281 case tar.TypeReg, tar.TypeRegA: 282 // Source is regular file 283 file, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, hdrInfo.Mode()) 284 if err != nil { 285 return err 286 } 287 if _, err := io.Copy(file, reader); err != nil { 288 file.Close() 289 return err 290 } 291 file.Close() 292 293 case tar.TypeBlock, tar.TypeChar, tar.TypeFifo: 294 mode := uint32(hdr.Mode & 07777) 295 switch hdr.Typeflag { 296 case tar.TypeBlock: 297 mode |= syscall.S_IFBLK 298 case tar.TypeChar: 299 mode |= syscall.S_IFCHR 300 case tar.TypeFifo: 301 mode |= syscall.S_IFIFO 302 } 303 304 if err := system.Mknod(path, mode, int(system.Mkdev(hdr.Devmajor, hdr.Devminor))); err != nil { 305 return err 306 } 307 308 case tar.TypeLink: 309 targetPath := filepath.Join(extractDir, hdr.Linkname) 310 // check for hardlink breakout 311 if !strings.HasPrefix(targetPath, extractDir) { 312 return breakoutError(fmt.Errorf("invalid hardlink %q -> %q", targetPath, hdr.Linkname)) 313 } 314 if err := os.Link(targetPath, path); err != nil { 315 return err 316 } 317 318 case tar.TypeSymlink: 319 // path -> hdr.Linkname = targetPath 320 // e.g. /extractDir/path/to/symlink -> ../2/file = /extractDir/path/2/file 321 targetPath := filepath.Join(filepath.Dir(path), hdr.Linkname) 322 323 // the reason we don't need to check symlinks in the path (with FollowSymlinkInScope) is because 324 // that symlink would first have to be created, which would be caught earlier, at this very check: 325 if !strings.HasPrefix(targetPath, extractDir) { 326 return breakoutError(fmt.Errorf("invalid symlink %q -> %q", path, hdr.Linkname)) 327 } 328 if err := os.Symlink(hdr.Linkname, path); err != nil { 329 return err 330 } 331 332 case tar.TypeXGlobalHeader: 333 logrus.Debugf("PAX Global Extended Headers found and ignored") 334 return nil 335 336 default: 337 return fmt.Errorf("Unhandled tar header type %d\n", hdr.Typeflag) 338 } 339 340 if err := os.Lchown(path, hdr.Uid, hdr.Gid); err != nil && Lchown { 341 return err 342 } 343 344 for key, value := range hdr.Xattrs { 345 if err := system.Lsetxattr(path, key, []byte(value), 0); err != nil { 346 return err 347 } 348 } 349 350 // There is no LChmod, so ignore mode for symlink. Also, this 351 // must happen after chown, as that can modify the file mode 352 if hdr.Typeflag == tar.TypeLink { 353 if fi, err := os.Lstat(hdr.Linkname); err == nil && (fi.Mode()&os.ModeSymlink == 0) { 354 if err := os.Chmod(path, hdrInfo.Mode()); err != nil { 355 return err 356 } 357 } 358 } else if hdr.Typeflag != tar.TypeSymlink { 359 if err := os.Chmod(path, hdrInfo.Mode()); err != nil { 360 return err 361 } 362 } 363 364 ts := []syscall.Timespec{timeToTimespec(hdr.AccessTime), timeToTimespec(hdr.ModTime)} 365 // syscall.UtimesNano doesn't support a NOFOLLOW flag atm, and 366 if hdr.Typeflag == tar.TypeLink { 367 if fi, err := os.Lstat(hdr.Linkname); err == nil && (fi.Mode()&os.ModeSymlink == 0) { 368 if err := system.UtimesNano(path, ts); err != nil && err != system.ErrNotSupportedPlatform { 369 return err 370 } 371 } 372 } else if hdr.Typeflag != tar.TypeSymlink { 373 if err := system.UtimesNano(path, ts); err != nil && err != system.ErrNotSupportedPlatform { 374 return err 375 } 376 } else { 377 if err := system.LUtimesNano(path, ts); err != nil && err != system.ErrNotSupportedPlatform { 378 return err 379 } 380 } 381 return nil 382 } 383 384 // Tar creates an archive from the directory at `path`, and returns it as a 385 // stream of bytes. 386 func Tar(path string, compression Compression) (io.ReadCloser, error) { 387 return TarWithOptions(path, &TarOptions{Compression: compression}) 388 } 389 390 // TarWithOptions creates an archive from the directory at `path`, only including files whose relative 391 // paths are included in `options.IncludeFiles` (if non-nil) or not in `options.ExcludePatterns`. 392 func TarWithOptions(srcPath string, options *TarOptions) (io.ReadCloser, error) { 393 394 patterns, patDirs, exceptions, err := fileutils.CleanPatterns(options.ExcludePatterns) 395 396 if err != nil { 397 return nil, err 398 } 399 400 pipeReader, pipeWriter := io.Pipe() 401 402 compressWriter, err := CompressStream(pipeWriter, options.Compression) 403 if err != nil { 404 return nil, err 405 } 406 407 go func() { 408 ta := &tarAppender{ 409 TarWriter: tar.NewWriter(compressWriter), 410 Buffer: pools.BufioWriter32KPool.Get(nil), 411 SeenFiles: make(map[uint64]string), 412 } 413 // this buffer is needed for the duration of this piped stream 414 defer pools.BufioWriter32KPool.Put(ta.Buffer) 415 416 // In general we log errors here but ignore them because 417 // during e.g. a diff operation the container can continue 418 // mutating the filesystem and we can see transient errors 419 // from this 420 421 if options.IncludeFiles == nil { 422 options.IncludeFiles = []string{"."} 423 } 424 425 seen := make(map[string]bool) 426 427 var renamedRelFilePath string // For when tar.Options.Name is set 428 for _, include := range options.IncludeFiles { 429 filepath.Walk(filepath.Join(srcPath, include), func(filePath string, f os.FileInfo, err error) error { 430 if err != nil { 431 logrus.Debugf("Tar: Can't stat file %s to tar: %s", srcPath, err) 432 return nil 433 } 434 435 relFilePath, err := filepath.Rel(srcPath, filePath) 436 if err != nil || (relFilePath == "." && f.IsDir()) { 437 // Error getting relative path OR we are looking 438 // at the root path. Skip in both situations. 439 return nil 440 } 441 442 skip := false 443 444 // If "include" is an exact match for the current file 445 // then even if there's an "excludePatterns" pattern that 446 // matches it, don't skip it. IOW, assume an explicit 'include' 447 // is asking for that file no matter what - which is true 448 // for some files, like .dockerignore and Dockerfile (sometimes) 449 if include != relFilePath { 450 skip, err = fileutils.OptimizedMatches(relFilePath, patterns, patDirs) 451 if err != nil { 452 logrus.Debugf("Error matching %s", relFilePath, err) 453 return err 454 } 455 } 456 457 if skip { 458 if !exceptions && f.IsDir() { 459 return filepath.SkipDir 460 } 461 return nil 462 } 463 464 if seen[relFilePath] { 465 return nil 466 } 467 seen[relFilePath] = true 468 469 // Rename the base resource 470 if options.Name != "" && filePath == srcPath+"/"+filepath.Base(relFilePath) { 471 renamedRelFilePath = relFilePath 472 } 473 // Set this to make sure the items underneath also get renamed 474 if options.Name != "" { 475 relFilePath = strings.Replace(relFilePath, renamedRelFilePath, options.Name, 1) 476 } 477 478 if err := ta.addTarFile(filePath, relFilePath); err != nil { 479 logrus.Debugf("Can't add file %s to tar: %s", filePath, err) 480 } 481 return nil 482 }) 483 } 484 485 // Make sure to check the error on Close. 486 if err := ta.TarWriter.Close(); err != nil { 487 logrus.Debugf("Can't close tar writer: %s", err) 488 } 489 if err := compressWriter.Close(); err != nil { 490 logrus.Debugf("Can't close compress writer: %s", err) 491 } 492 if err := pipeWriter.Close(); err != nil { 493 logrus.Debugf("Can't close pipe writer: %s", err) 494 } 495 }() 496 497 return pipeReader, nil 498 } 499 500 func Unpack(decompressedArchive io.Reader, dest string, options *TarOptions) error { 501 tr := tar.NewReader(decompressedArchive) 502 trBuf := pools.BufioReader32KPool.Get(nil) 503 defer pools.BufioReader32KPool.Put(trBuf) 504 505 var dirs []*tar.Header 506 507 // Iterate through the files in the archive. 508 loop: 509 for { 510 hdr, err := tr.Next() 511 if err == io.EOF { 512 // end of tar archive 513 break 514 } 515 if err != nil { 516 return err 517 } 518 519 // Normalize name, for safety and for a simple is-root check 520 // This keeps "../" as-is, but normalizes "/../" to "/" 521 hdr.Name = filepath.Clean(hdr.Name) 522 523 for _, exclude := range options.ExcludePatterns { 524 if strings.HasPrefix(hdr.Name, exclude) { 525 continue loop 526 } 527 } 528 529 if !strings.HasSuffix(hdr.Name, "/") { 530 // Not the root directory, ensure that the parent directory exists 531 parent := filepath.Dir(hdr.Name) 532 parentPath := filepath.Join(dest, parent) 533 if _, err := os.Lstat(parentPath); err != nil && os.IsNotExist(err) { 534 err = os.MkdirAll(parentPath, 0777) 535 if err != nil { 536 return err 537 } 538 } 539 } 540 541 path := filepath.Join(dest, hdr.Name) 542 rel, err := filepath.Rel(dest, path) 543 if err != nil { 544 return err 545 } 546 if strings.HasPrefix(rel, "../") { 547 return breakoutError(fmt.Errorf("%q is outside of %q", hdr.Name, dest)) 548 } 549 550 // If path exits we almost always just want to remove and replace it 551 // The only exception is when it is a directory *and* the file from 552 // the layer is also a directory. Then we want to merge them (i.e. 553 // just apply the metadata from the layer). 554 if fi, err := os.Lstat(path); err == nil { 555 if fi.IsDir() && hdr.Name == "." { 556 continue 557 } 558 if !(fi.IsDir() && hdr.Typeflag == tar.TypeDir) { 559 if err := os.RemoveAll(path); err != nil { 560 return err 561 } 562 } 563 } 564 trBuf.Reset(tr) 565 if err := createTarFile(path, dest, hdr, trBuf, !options.NoLchown); err != nil { 566 return err 567 } 568 569 // Directory mtimes must be handled at the end to avoid further 570 // file creation in them to modify the directory mtime 571 if hdr.Typeflag == tar.TypeDir { 572 dirs = append(dirs, hdr) 573 } 574 } 575 576 for _, hdr := range dirs { 577 path := filepath.Join(dest, hdr.Name) 578 ts := []syscall.Timespec{timeToTimespec(hdr.AccessTime), timeToTimespec(hdr.ModTime)} 579 if err := syscall.UtimesNano(path, ts); err != nil { 580 return err 581 } 582 } 583 return nil 584 } 585 586 // Untar reads a stream of bytes from `archive`, parses it as a tar archive, 587 // and unpacks it into the directory at `dest`. 588 // The archive may be compressed with one of the following algorithms: 589 // identity (uncompressed), gzip, bzip2, xz. 590 // FIXME: specify behavior when target path exists vs. doesn't exist. 591 func Untar(archive io.Reader, dest string, options *TarOptions) error { 592 if archive == nil { 593 return fmt.Errorf("Empty archive") 594 } 595 dest = filepath.Clean(dest) 596 if options == nil { 597 options = &TarOptions{} 598 } 599 if options.ExcludePatterns == nil { 600 options.ExcludePatterns = []string{} 601 } 602 decompressedArchive, err := DecompressStream(archive) 603 if err != nil { 604 return err 605 } 606 defer decompressedArchive.Close() 607 return Unpack(decompressedArchive, dest, options) 608 } 609 610 func (archiver *Archiver) TarUntar(src, dst string) error { 611 logrus.Debugf("TarUntar(%s %s)", src, dst) 612 archive, err := TarWithOptions(src, &TarOptions{Compression: Uncompressed}) 613 if err != nil { 614 return err 615 } 616 defer archive.Close() 617 return archiver.Untar(archive, dst, nil) 618 } 619 620 // TarUntar is a convenience function which calls Tar and Untar, with the output of one piped into the other. 621 // If either Tar or Untar fails, TarUntar aborts and returns the error. 622 func TarUntar(src, dst string) error { 623 return defaultArchiver.TarUntar(src, dst) 624 } 625 626 func (archiver *Archiver) UntarPath(src, dst string) error { 627 archive, err := os.Open(src) 628 if err != nil { 629 return err 630 } 631 defer archive.Close() 632 if err := archiver.Untar(archive, dst, nil); err != nil { 633 return err 634 } 635 return nil 636 } 637 638 // UntarPath is a convenience function which looks for an archive 639 // at filesystem path `src`, and unpacks it at `dst`. 640 func UntarPath(src, dst string) error { 641 return defaultArchiver.UntarPath(src, dst) 642 } 643 644 func (archiver *Archiver) CopyWithTar(src, dst string) error { 645 srcSt, err := os.Stat(src) 646 if err != nil { 647 return err 648 } 649 if !srcSt.IsDir() { 650 return archiver.CopyFileWithTar(src, dst) 651 } 652 // Create dst, copy src's content into it 653 logrus.Debugf("Creating dest directory: %s", dst) 654 if err := os.MkdirAll(dst, 0755); err != nil && !os.IsExist(err) { 655 return err 656 } 657 logrus.Debugf("Calling TarUntar(%s, %s)", src, dst) 658 return archiver.TarUntar(src, dst) 659 } 660 661 // CopyWithTar creates a tar archive of filesystem path `src`, and 662 // unpacks it at filesystem path `dst`. 663 // The archive is streamed directly with fixed buffering and no 664 // intermediary disk IO. 665 func CopyWithTar(src, dst string) error { 666 return defaultArchiver.CopyWithTar(src, dst) 667 } 668 669 func (archiver *Archiver) CopyFileWithTar(src, dst string) (err error) { 670 logrus.Debugf("CopyFileWithTar(%s, %s)", src, dst) 671 srcSt, err := os.Stat(src) 672 if err != nil { 673 return err 674 } 675 if srcSt.IsDir() { 676 return fmt.Errorf("Can't copy a directory") 677 } 678 // Clean up the trailing / 679 if dst[len(dst)-1] == '/' { 680 dst = path.Join(dst, filepath.Base(src)) 681 } 682 // Create the holding directory if necessary 683 if err := os.MkdirAll(filepath.Dir(dst), 0700); err != nil && !os.IsExist(err) { 684 return err 685 } 686 687 r, w := io.Pipe() 688 errC := promise.Go(func() error { 689 defer w.Close() 690 691 srcF, err := os.Open(src) 692 if err != nil { 693 return err 694 } 695 defer srcF.Close() 696 697 hdr, err := tar.FileInfoHeader(srcSt, "") 698 if err != nil { 699 return err 700 } 701 hdr.Name = filepath.Base(dst) 702 hdr.Mode = int64(chmodTarEntry(os.FileMode(hdr.Mode))) 703 704 tw := tar.NewWriter(w) 705 defer tw.Close() 706 if err := tw.WriteHeader(hdr); err != nil { 707 return err 708 } 709 if _, err := io.Copy(tw, srcF); err != nil { 710 return err 711 } 712 return nil 713 }) 714 defer func() { 715 if er := <-errC; err != nil { 716 err = er 717 } 718 }() 719 return archiver.Untar(r, filepath.Dir(dst), nil) 720 } 721 722 // CopyFileWithTar emulates the behavior of the 'cp' command-line 723 // for a single file. It copies a regular file from path `src` to 724 // path `dst`, and preserves all its metadata. 725 // 726 // If `dst` ends with a trailing slash '/', the final destination path 727 // will be `dst/base(src)`. 728 func CopyFileWithTar(src, dst string) (err error) { 729 return defaultArchiver.CopyFileWithTar(src, dst) 730 } 731 732 // CmdStream executes a command, and returns its stdout as a stream. 733 // If the command fails to run or doesn't complete successfully, an error 734 // will be returned, including anything written on stderr. 735 func CmdStream(cmd *exec.Cmd, input io.Reader) (io.ReadCloser, error) { 736 if input != nil { 737 stdin, err := cmd.StdinPipe() 738 if err != nil { 739 return nil, err 740 } 741 // Write stdin if any 742 go func() { 743 io.Copy(stdin, input) 744 stdin.Close() 745 }() 746 } 747 stdout, err := cmd.StdoutPipe() 748 if err != nil { 749 return nil, err 750 } 751 stderr, err := cmd.StderrPipe() 752 if err != nil { 753 return nil, err 754 } 755 pipeR, pipeW := io.Pipe() 756 errChan := make(chan []byte) 757 // Collect stderr, we will use it in case of an error 758 go func() { 759 errText, e := ioutil.ReadAll(stderr) 760 if e != nil { 761 errText = []byte("(...couldn't fetch stderr: " + e.Error() + ")") 762 } 763 errChan <- errText 764 }() 765 // Copy stdout to the returned pipe 766 go func() { 767 _, err := io.Copy(pipeW, stdout) 768 if err != nil { 769 pipeW.CloseWithError(err) 770 } 771 errText := <-errChan 772 if err := cmd.Wait(); err != nil { 773 pipeW.CloseWithError(fmt.Errorf("%s: %s", err, errText)) 774 } else { 775 pipeW.Close() 776 } 777 }() 778 // Run the command and return the pipe 779 if err := cmd.Start(); err != nil { 780 return nil, err 781 } 782 return pipeR, nil 783 } 784 785 // NewTempArchive reads the content of src into a temporary file, and returns the contents 786 // of that file as an archive. The archive can only be read once - as soon as reading completes, 787 // the file will be deleted. 788 func NewTempArchive(src Archive, dir string) (*TempArchive, error) { 789 f, err := ioutil.TempFile(dir, "") 790 if err != nil { 791 return nil, err 792 } 793 if _, err := io.Copy(f, src); err != nil { 794 return nil, err 795 } 796 if _, err := f.Seek(0, 0); err != nil { 797 return nil, err 798 } 799 st, err := f.Stat() 800 if err != nil { 801 return nil, err 802 } 803 size := st.Size() 804 return &TempArchive{File: f, Size: size}, nil 805 } 806 807 type TempArchive struct { 808 *os.File 809 Size int64 // Pre-computed from Stat().Size() as a convenience 810 read int64 811 closed bool 812 } 813 814 // Close closes the underlying file if it's still open, or does a no-op 815 // to allow callers to try to close the TempArchive multiple times safely. 816 func (archive *TempArchive) Close() error { 817 if archive.closed { 818 return nil 819 } 820 821 archive.closed = true 822 823 return archive.File.Close() 824 } 825 826 func (archive *TempArchive) Read(data []byte) (int, error) { 827 n, err := archive.File.Read(data) 828 archive.read += int64(n) 829 if err != nil || archive.read == archive.Size { 830 archive.Close() 831 os.Remove(archive.File.Name()) 832 } 833 return n, err 834 }