github.com/chenchun/docker@v1.3.2-0.20150629222414-20467faf132b/pkg/archive/archive.go (about) 1 package archive 2 3 import ( 4 "archive/tar" 5 "bufio" 6 "bytes" 7 "compress/bzip2" 8 "compress/gzip" 9 "errors" 10 "fmt" 11 "io" 12 "io/ioutil" 13 "os" 14 "os/exec" 15 "path/filepath" 16 "runtime" 17 "strings" 18 "syscall" 19 20 "github.com/Sirupsen/logrus" 21 "github.com/docker/docker/pkg/fileutils" 22 "github.com/docker/docker/pkg/pools" 23 "github.com/docker/docker/pkg/promise" 24 "github.com/docker/docker/pkg/system" 25 ) 26 27 type ( 28 Archive io.ReadCloser 29 ArchiveReader io.Reader 30 Compression int 31 TarOptions struct { 32 IncludeFiles []string 33 ExcludePatterns []string 34 Compression Compression 35 NoLchown bool 36 Name string 37 } 38 39 // Archiver allows the reuse of most utility functions of this package 40 // with a pluggable Untar function. 41 Archiver struct { 42 Untar func(io.Reader, string, *TarOptions) error 43 } 44 45 // breakoutError is used to differentiate errors related to breaking out 46 // When testing archive breakout in the unit tests, this error is expected 47 // in order for the test to pass. 48 breakoutError error 49 ) 50 51 var ( 52 ErrNotImplemented = errors.New("Function not implemented") 53 defaultArchiver = &Archiver{Untar} 54 ) 55 56 const ( 57 Uncompressed Compression = iota 58 Bzip2 59 Gzip 60 Xz 61 ) 62 63 func IsArchive(header []byte) bool { 64 compression := DetectCompression(header) 65 if compression != Uncompressed { 66 return true 67 } 68 r := tar.NewReader(bytes.NewBuffer(header)) 69 _, err := r.Next() 70 return err == nil 71 } 72 73 func DetectCompression(source []byte) Compression { 74 for compression, m := range map[Compression][]byte{ 75 Bzip2: {0x42, 0x5A, 0x68}, 76 Gzip: {0x1F, 0x8B, 0x08}, 77 Xz: {0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00}, 78 } { 79 if len(source) < len(m) { 80 logrus.Debugf("Len too short") 81 continue 82 } 83 if bytes.Compare(m, source[:len(m)]) == 0 { 84 return compression 85 } 86 } 87 return Uncompressed 88 } 89 90 func xzDecompress(archive io.Reader) (io.ReadCloser, error) { 91 args := []string{"xz", "-d", "-c", "-q"} 92 93 return CmdStream(exec.Command(args[0], args[1:]...), archive) 94 } 95 96 func DecompressStream(archive io.Reader) (io.ReadCloser, error) { 97 p := pools.BufioReader32KPool 98 buf := p.Get(archive) 99 bs, err := buf.Peek(10) 100 if err != nil { 101 return nil, err 102 } 103 104 compression := DetectCompression(bs) 105 switch compression { 106 case Uncompressed: 107 readBufWrapper := p.NewReadCloserWrapper(buf, buf) 108 return readBufWrapper, nil 109 case Gzip: 110 gzReader, err := gzip.NewReader(buf) 111 if err != nil { 112 return nil, err 113 } 114 readBufWrapper := p.NewReadCloserWrapper(buf, gzReader) 115 return readBufWrapper, nil 116 case Bzip2: 117 bz2Reader := bzip2.NewReader(buf) 118 readBufWrapper := p.NewReadCloserWrapper(buf, bz2Reader) 119 return readBufWrapper, nil 120 case Xz: 121 xzReader, err := xzDecompress(buf) 122 if err != nil { 123 return nil, err 124 } 125 readBufWrapper := p.NewReadCloserWrapper(buf, xzReader) 126 return readBufWrapper, nil 127 default: 128 return nil, fmt.Errorf("Unsupported compression format %s", (&compression).Extension()) 129 } 130 } 131 132 func CompressStream(dest io.WriteCloser, compression Compression) (io.WriteCloser, error) { 133 p := pools.BufioWriter32KPool 134 buf := p.Get(dest) 135 switch compression { 136 case Uncompressed: 137 writeBufWrapper := p.NewWriteCloserWrapper(buf, buf) 138 return writeBufWrapper, nil 139 case Gzip: 140 gzWriter := gzip.NewWriter(dest) 141 writeBufWrapper := p.NewWriteCloserWrapper(buf, gzWriter) 142 return writeBufWrapper, nil 143 case Bzip2, Xz: 144 // archive/bzip2 does not support writing, and there is no xz support at all 145 // However, this is not a problem as docker only currently generates gzipped tars 146 return nil, fmt.Errorf("Unsupported compression format %s", (&compression).Extension()) 147 default: 148 return nil, fmt.Errorf("Unsupported compression format %s", (&compression).Extension()) 149 } 150 } 151 152 func (compression *Compression) Extension() string { 153 switch *compression { 154 case Uncompressed: 155 return "tar" 156 case Bzip2: 157 return "tar.bz2" 158 case Gzip: 159 return "tar.gz" 160 case Xz: 161 return "tar.xz" 162 } 163 return "" 164 } 165 166 type tarAppender struct { 167 TarWriter *tar.Writer 168 Buffer *bufio.Writer 169 170 // for hardlink mapping 171 SeenFiles map[uint64]string 172 } 173 174 // canonicalTarName provides a platform-independent and consistent posix-style 175 //path for files and directories to be archived regardless of the platform. 176 func canonicalTarName(name string, isDir bool) (string, error) { 177 name, err := CanonicalTarNameForPath(name) 178 if err != nil { 179 return "", err 180 } 181 182 // suffix with '/' for directories 183 if isDir && !strings.HasSuffix(name, "/") { 184 name += "/" 185 } 186 return name, nil 187 } 188 189 func (ta *tarAppender) addTarFile(path, name string) error { 190 fi, err := os.Lstat(path) 191 if err != nil { 192 return err 193 } 194 195 link := "" 196 if fi.Mode()&os.ModeSymlink != 0 { 197 if link, err = os.Readlink(path); err != nil { 198 return err 199 } 200 } 201 202 hdr, err := tar.FileInfoHeader(fi, link) 203 if err != nil { 204 return err 205 } 206 hdr.Mode = int64(chmodTarEntry(os.FileMode(hdr.Mode))) 207 208 name, err = canonicalTarName(name, fi.IsDir()) 209 if err != nil { 210 return fmt.Errorf("tar: cannot canonicalize path: %v", err) 211 } 212 hdr.Name = name 213 214 nlink, inode, err := setHeaderForSpecialDevice(hdr, ta, name, fi.Sys()) 215 if err != nil { 216 return err 217 } 218 219 // if it's a regular file and has more than 1 link, 220 // it's hardlinked, so set the type flag accordingly 221 if fi.Mode().IsRegular() && nlink > 1 { 222 // a link should have a name that it links too 223 // and that linked name should be first in the tar archive 224 if oldpath, ok := ta.SeenFiles[inode]; ok { 225 hdr.Typeflag = tar.TypeLink 226 hdr.Linkname = oldpath 227 hdr.Size = 0 // This Must be here for the writer math to add up! 228 } else { 229 ta.SeenFiles[inode] = name 230 } 231 } 232 233 capability, _ := system.Lgetxattr(path, "security.capability") 234 if capability != nil { 235 hdr.Xattrs = make(map[string]string) 236 hdr.Xattrs["security.capability"] = string(capability) 237 } 238 239 if err := ta.TarWriter.WriteHeader(hdr); err != nil { 240 return err 241 } 242 243 if hdr.Typeflag == tar.TypeReg { 244 file, err := os.Open(path) 245 if err != nil { 246 return err 247 } 248 249 ta.Buffer.Reset(ta.TarWriter) 250 defer ta.Buffer.Reset(nil) 251 _, err = io.Copy(ta.Buffer, file) 252 file.Close() 253 if err != nil { 254 return err 255 } 256 err = ta.Buffer.Flush() 257 if err != nil { 258 return err 259 } 260 } 261 262 return nil 263 } 264 265 func createTarFile(path, extractDir string, hdr *tar.Header, reader io.Reader, Lchown bool) error { 266 // hdr.Mode is in linux format, which we can use for sycalls, 267 // but for os.Foo() calls we need the mode converted to os.FileMode, 268 // so use hdrInfo.Mode() (they differ for e.g. setuid bits) 269 hdrInfo := hdr.FileInfo() 270 271 switch hdr.Typeflag { 272 case tar.TypeDir: 273 // Create directory unless it exists as a directory already. 274 // In that case we just want to merge the two 275 if fi, err := os.Lstat(path); !(err == nil && fi.IsDir()) { 276 if err := os.Mkdir(path, hdrInfo.Mode()); err != nil { 277 return err 278 } 279 } 280 281 case tar.TypeReg, tar.TypeRegA: 282 // Source is regular file 283 file, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, hdrInfo.Mode()) 284 if err != nil { 285 return err 286 } 287 if _, err := io.Copy(file, reader); err != nil { 288 file.Close() 289 return err 290 } 291 file.Close() 292 293 case tar.TypeBlock, tar.TypeChar, tar.TypeFifo: 294 // Handle this is an OS-specific way 295 if err := handleTarTypeBlockCharFifo(hdr, path); err != nil { 296 return err 297 } 298 299 case tar.TypeLink: 300 targetPath := filepath.Join(extractDir, hdr.Linkname) 301 // check for hardlink breakout 302 if !strings.HasPrefix(targetPath, extractDir) { 303 return breakoutError(fmt.Errorf("invalid hardlink %q -> %q", targetPath, hdr.Linkname)) 304 } 305 if err := os.Link(targetPath, path); err != nil { 306 return err 307 } 308 309 case tar.TypeSymlink: 310 // path -> hdr.Linkname = targetPath 311 // e.g. /extractDir/path/to/symlink -> ../2/file = /extractDir/path/2/file 312 targetPath := filepath.Join(filepath.Dir(path), hdr.Linkname) 313 314 // the reason we don't need to check symlinks in the path (with FollowSymlinkInScope) is because 315 // that symlink would first have to be created, which would be caught earlier, at this very check: 316 if !strings.HasPrefix(targetPath, extractDir) { 317 return breakoutError(fmt.Errorf("invalid symlink %q -> %q", path, hdr.Linkname)) 318 } 319 if err := os.Symlink(hdr.Linkname, path); err != nil { 320 return err 321 } 322 323 case tar.TypeXGlobalHeader: 324 logrus.Debugf("PAX Global Extended Headers found and ignored") 325 return nil 326 327 default: 328 return fmt.Errorf("Unhandled tar header type %d\n", hdr.Typeflag) 329 } 330 331 // Lchown is not supported on Windows 332 if runtime.GOOS != "windows" { 333 if err := os.Lchown(path, hdr.Uid, hdr.Gid); err != nil && Lchown { 334 return err 335 } 336 } 337 338 for key, value := range hdr.Xattrs { 339 if err := system.Lsetxattr(path, key, []byte(value), 0); err != nil { 340 return err 341 } 342 } 343 344 // There is no LChmod, so ignore mode for symlink. Also, this 345 // must happen after chown, as that can modify the file mode 346 if err := handleLChmod(hdr, path, hdrInfo); err != nil { 347 return err 348 } 349 350 ts := []syscall.Timespec{timeToTimespec(hdr.AccessTime), timeToTimespec(hdr.ModTime)} 351 // syscall.UtimesNano doesn't support a NOFOLLOW flag atm 352 if hdr.Typeflag == tar.TypeLink { 353 if fi, err := os.Lstat(hdr.Linkname); err == nil && (fi.Mode()&os.ModeSymlink == 0) { 354 if err := system.UtimesNano(path, ts); err != nil && err != system.ErrNotSupportedPlatform { 355 return err 356 } 357 } 358 } else if hdr.Typeflag != tar.TypeSymlink { 359 if err := system.UtimesNano(path, ts); err != nil && err != system.ErrNotSupportedPlatform { 360 return err 361 } 362 } else { 363 if err := system.LUtimesNano(path, ts); err != nil && err != system.ErrNotSupportedPlatform { 364 return err 365 } 366 } 367 return nil 368 } 369 370 // Tar creates an archive from the directory at `path`, and returns it as a 371 // stream of bytes. 372 func Tar(path string, compression Compression) (io.ReadCloser, error) { 373 return TarWithOptions(path, &TarOptions{Compression: compression}) 374 } 375 376 // TarWithOptions creates an archive from the directory at `path`, only including files whose relative 377 // paths are included in `options.IncludeFiles` (if non-nil) or not in `options.ExcludePatterns`. 378 func TarWithOptions(srcPath string, options *TarOptions) (io.ReadCloser, error) { 379 380 patterns, patDirs, exceptions, err := fileutils.CleanPatterns(options.ExcludePatterns) 381 382 if err != nil { 383 return nil, err 384 } 385 386 pipeReader, pipeWriter := io.Pipe() 387 388 compressWriter, err := CompressStream(pipeWriter, options.Compression) 389 if err != nil { 390 return nil, err 391 } 392 393 go func() { 394 ta := &tarAppender{ 395 TarWriter: tar.NewWriter(compressWriter), 396 Buffer: pools.BufioWriter32KPool.Get(nil), 397 SeenFiles: make(map[uint64]string), 398 } 399 // this buffer is needed for the duration of this piped stream 400 defer pools.BufioWriter32KPool.Put(ta.Buffer) 401 402 // In general we log errors here but ignore them because 403 // during e.g. a diff operation the container can continue 404 // mutating the filesystem and we can see transient errors 405 // from this 406 407 if options.IncludeFiles == nil { 408 options.IncludeFiles = []string{"."} 409 } 410 411 seen := make(map[string]bool) 412 413 var renamedRelFilePath string // For when tar.Options.Name is set 414 for _, include := range options.IncludeFiles { 415 filepath.Walk(filepath.Join(srcPath, include), func(filePath string, f os.FileInfo, err error) error { 416 if err != nil { 417 logrus.Debugf("Tar: Can't stat file %s to tar: %s", srcPath, err) 418 return nil 419 } 420 421 relFilePath, err := filepath.Rel(srcPath, filePath) 422 if err != nil || (relFilePath == "." && f.IsDir()) { 423 // Error getting relative path OR we are looking 424 // at the root path. Skip in both situations. 425 return nil 426 } 427 428 skip := false 429 430 // If "include" is an exact match for the current file 431 // then even if there's an "excludePatterns" pattern that 432 // matches it, don't skip it. IOW, assume an explicit 'include' 433 // is asking for that file no matter what - which is true 434 // for some files, like .dockerignore and Dockerfile (sometimes) 435 if include != relFilePath { 436 skip, err = fileutils.OptimizedMatches(relFilePath, patterns, patDirs) 437 if err != nil { 438 logrus.Debugf("Error matching %s", relFilePath, err) 439 return err 440 } 441 } 442 443 if skip { 444 if !exceptions && f.IsDir() { 445 return filepath.SkipDir 446 } 447 return nil 448 } 449 450 if seen[relFilePath] { 451 return nil 452 } 453 seen[relFilePath] = true 454 455 // TODO Windows: Verify if this needs to be os.Pathseparator 456 // Rename the base resource 457 if options.Name != "" && filePath == srcPath+"/"+filepath.Base(relFilePath) { 458 renamedRelFilePath = relFilePath 459 } 460 // Set this to make sure the items underneath also get renamed 461 if options.Name != "" { 462 relFilePath = strings.Replace(relFilePath, renamedRelFilePath, options.Name, 1) 463 } 464 465 if err := ta.addTarFile(filePath, relFilePath); err != nil { 466 logrus.Debugf("Can't add file %s to tar: %s", filePath, err) 467 } 468 return nil 469 }) 470 } 471 472 // Make sure to check the error on Close. 473 if err := ta.TarWriter.Close(); err != nil { 474 logrus.Debugf("Can't close tar writer: %s", err) 475 } 476 if err := compressWriter.Close(); err != nil { 477 logrus.Debugf("Can't close compress writer: %s", err) 478 } 479 if err := pipeWriter.Close(); err != nil { 480 logrus.Debugf("Can't close pipe writer: %s", err) 481 } 482 }() 483 484 return pipeReader, nil 485 } 486 487 func Unpack(decompressedArchive io.Reader, dest string, options *TarOptions) error { 488 tr := tar.NewReader(decompressedArchive) 489 trBuf := pools.BufioReader32KPool.Get(nil) 490 defer pools.BufioReader32KPool.Put(trBuf) 491 492 var dirs []*tar.Header 493 494 // Iterate through the files in the archive. 495 loop: 496 for { 497 hdr, err := tr.Next() 498 if err == io.EOF { 499 // end of tar archive 500 break 501 } 502 if err != nil { 503 return err 504 } 505 506 // Normalize name, for safety and for a simple is-root check 507 // This keeps "../" as-is, but normalizes "/../" to "/". Or Windows: 508 // This keeps "..\" as-is, but normalizes "\..\" to "\". 509 hdr.Name = filepath.Clean(hdr.Name) 510 511 for _, exclude := range options.ExcludePatterns { 512 if strings.HasPrefix(hdr.Name, exclude) { 513 continue loop 514 } 515 } 516 517 // After calling filepath.Clean(hdr.Name) above, hdr.Name will now be in 518 // the filepath format for the OS on which the daemon is running. Hence 519 // the check for a slash-suffix MUST be done in an OS-agnostic way. 520 if !strings.HasSuffix(hdr.Name, string(os.PathSeparator)) { 521 // Not the root directory, ensure that the parent directory exists 522 parent := filepath.Dir(hdr.Name) 523 parentPath := filepath.Join(dest, parent) 524 if _, err := os.Lstat(parentPath); err != nil && os.IsNotExist(err) { 525 err = system.MkdirAll(parentPath, 0777) 526 if err != nil { 527 return err 528 } 529 } 530 } 531 532 path := filepath.Join(dest, hdr.Name) 533 rel, err := filepath.Rel(dest, path) 534 if err != nil { 535 return err 536 } 537 if strings.HasPrefix(rel, ".."+string(os.PathSeparator)) { 538 return breakoutError(fmt.Errorf("%q is outside of %q", hdr.Name, dest)) 539 } 540 541 // If path exits we almost always just want to remove and replace it 542 // The only exception is when it is a directory *and* the file from 543 // the layer is also a directory. Then we want to merge them (i.e. 544 // just apply the metadata from the layer). 545 if fi, err := os.Lstat(path); err == nil { 546 if fi.IsDir() && hdr.Name == "." { 547 continue 548 } 549 if !(fi.IsDir() && hdr.Typeflag == tar.TypeDir) { 550 if err := os.RemoveAll(path); err != nil { 551 return err 552 } 553 } 554 } 555 trBuf.Reset(tr) 556 if err := createTarFile(path, dest, hdr, trBuf, !options.NoLchown); err != nil { 557 return err 558 } 559 560 // Directory mtimes must be handled at the end to avoid further 561 // file creation in them to modify the directory mtime 562 if hdr.Typeflag == tar.TypeDir { 563 dirs = append(dirs, hdr) 564 } 565 } 566 567 for _, hdr := range dirs { 568 path := filepath.Join(dest, hdr.Name) 569 ts := []syscall.Timespec{timeToTimespec(hdr.AccessTime), timeToTimespec(hdr.ModTime)} 570 if err := syscall.UtimesNano(path, ts); err != nil { 571 return err 572 } 573 } 574 return nil 575 } 576 577 // Untar reads a stream of bytes from `archive`, parses it as a tar archive, 578 // and unpacks it into the directory at `dest`. 579 // The archive may be compressed with one of the following algorithms: 580 // identity (uncompressed), gzip, bzip2, xz. 581 // FIXME: specify behavior when target path exists vs. doesn't exist. 582 func Untar(archive io.Reader, dest string, options *TarOptions) error { 583 if archive == nil { 584 return fmt.Errorf("Empty archive") 585 } 586 dest = filepath.Clean(dest) 587 if options == nil { 588 options = &TarOptions{} 589 } 590 if options.ExcludePatterns == nil { 591 options.ExcludePatterns = []string{} 592 } 593 decompressedArchive, err := DecompressStream(archive) 594 if err != nil { 595 return err 596 } 597 defer decompressedArchive.Close() 598 return Unpack(decompressedArchive, dest, options) 599 } 600 601 func (archiver *Archiver) TarUntar(src, dst string) error { 602 logrus.Debugf("TarUntar(%s %s)", src, dst) 603 archive, err := TarWithOptions(src, &TarOptions{Compression: Uncompressed}) 604 if err != nil { 605 return err 606 } 607 defer archive.Close() 608 return archiver.Untar(archive, dst, nil) 609 } 610 611 // TarUntar is a convenience function which calls Tar and Untar, with the output of one piped into the other. 612 // If either Tar or Untar fails, TarUntar aborts and returns the error. 613 func TarUntar(src, dst string) error { 614 return defaultArchiver.TarUntar(src, dst) 615 } 616 617 func (archiver *Archiver) UntarPath(src, dst string) error { 618 archive, err := os.Open(src) 619 if err != nil { 620 return err 621 } 622 defer archive.Close() 623 if err := archiver.Untar(archive, dst, nil); err != nil { 624 return err 625 } 626 return nil 627 } 628 629 // UntarPath is a convenience function which looks for an archive 630 // at filesystem path `src`, and unpacks it at `dst`. 631 func UntarPath(src, dst string) error { 632 return defaultArchiver.UntarPath(src, dst) 633 } 634 635 func (archiver *Archiver) CopyWithTar(src, dst string) error { 636 srcSt, err := os.Stat(src) 637 if err != nil { 638 return err 639 } 640 if !srcSt.IsDir() { 641 return archiver.CopyFileWithTar(src, dst) 642 } 643 // Create dst, copy src's content into it 644 logrus.Debugf("Creating dest directory: %s", dst) 645 if err := system.MkdirAll(dst, 0755); err != nil && !os.IsExist(err) { 646 return err 647 } 648 logrus.Debugf("Calling TarUntar(%s, %s)", src, dst) 649 return archiver.TarUntar(src, dst) 650 } 651 652 // CopyWithTar creates a tar archive of filesystem path `src`, and 653 // unpacks it at filesystem path `dst`. 654 // The archive is streamed directly with fixed buffering and no 655 // intermediary disk IO. 656 func CopyWithTar(src, dst string) error { 657 return defaultArchiver.CopyWithTar(src, dst) 658 } 659 660 func (archiver *Archiver) CopyFileWithTar(src, dst string) (err error) { 661 logrus.Debugf("CopyFileWithTar(%s, %s)", src, dst) 662 srcSt, err := os.Stat(src) 663 if err != nil { 664 return err 665 } 666 667 if srcSt.IsDir() { 668 return fmt.Errorf("Can't copy a directory") 669 } 670 671 // Clean up the trailing slash. This must be done in an operating 672 // system specific manner. 673 if dst[len(dst)-1] == os.PathSeparator { 674 dst = filepath.Join(dst, filepath.Base(src)) 675 } 676 // Create the holding directory if necessary 677 if err := system.MkdirAll(filepath.Dir(dst), 0700); err != nil && !os.IsExist(err) { 678 return err 679 } 680 681 r, w := io.Pipe() 682 errC := promise.Go(func() error { 683 defer w.Close() 684 685 srcF, err := os.Open(src) 686 if err != nil { 687 return err 688 } 689 defer srcF.Close() 690 691 hdr, err := tar.FileInfoHeader(srcSt, "") 692 if err != nil { 693 return err 694 } 695 hdr.Name = filepath.Base(dst) 696 hdr.Mode = int64(chmodTarEntry(os.FileMode(hdr.Mode))) 697 698 tw := tar.NewWriter(w) 699 defer tw.Close() 700 if err := tw.WriteHeader(hdr); err != nil { 701 return err 702 } 703 if _, err := io.Copy(tw, srcF); err != nil { 704 return err 705 } 706 return nil 707 }) 708 defer func() { 709 if er := <-errC; err != nil { 710 err = er 711 } 712 }() 713 return archiver.Untar(r, filepath.Dir(dst), nil) 714 } 715 716 // CopyFileWithTar emulates the behavior of the 'cp' command-line 717 // for a single file. It copies a regular file from path `src` to 718 // path `dst`, and preserves all its metadata. 719 // 720 // Destination handling is in an operating specific manner depending 721 // where the daemon is running. If `dst` ends with a trailing slash 722 // the final destination path will be `dst/base(src)` (Linux) or 723 // `dst\base(src)` (Windows). 724 func CopyFileWithTar(src, dst string) (err error) { 725 return defaultArchiver.CopyFileWithTar(src, dst) 726 } 727 728 // CmdStream executes a command, and returns its stdout as a stream. 729 // If the command fails to run or doesn't complete successfully, an error 730 // will be returned, including anything written on stderr. 731 func CmdStream(cmd *exec.Cmd, input io.Reader) (io.ReadCloser, error) { 732 if input != nil { 733 stdin, err := cmd.StdinPipe() 734 if err != nil { 735 return nil, err 736 } 737 // Write stdin if any 738 go func() { 739 io.Copy(stdin, input) 740 stdin.Close() 741 }() 742 } 743 stdout, err := cmd.StdoutPipe() 744 if err != nil { 745 return nil, err 746 } 747 stderr, err := cmd.StderrPipe() 748 if err != nil { 749 return nil, err 750 } 751 pipeR, pipeW := io.Pipe() 752 errChan := make(chan []byte) 753 // Collect stderr, we will use it in case of an error 754 go func() { 755 errText, e := ioutil.ReadAll(stderr) 756 if e != nil { 757 errText = []byte("(...couldn't fetch stderr: " + e.Error() + ")") 758 } 759 errChan <- errText 760 }() 761 // Copy stdout to the returned pipe 762 go func() { 763 _, err := io.Copy(pipeW, stdout) 764 if err != nil { 765 pipeW.CloseWithError(err) 766 } 767 errText := <-errChan 768 if err := cmd.Wait(); err != nil { 769 pipeW.CloseWithError(fmt.Errorf("%s: %s", err, errText)) 770 } else { 771 pipeW.Close() 772 } 773 }() 774 // Run the command and return the pipe 775 if err := cmd.Start(); err != nil { 776 return nil, err 777 } 778 return pipeR, nil 779 } 780 781 // NewTempArchive reads the content of src into a temporary file, and returns the contents 782 // of that file as an archive. The archive can only be read once - as soon as reading completes, 783 // the file will be deleted. 784 func NewTempArchive(src Archive, dir string) (*TempArchive, error) { 785 f, err := ioutil.TempFile(dir, "") 786 if err != nil { 787 return nil, err 788 } 789 if _, err := io.Copy(f, src); err != nil { 790 return nil, err 791 } 792 if _, err := f.Seek(0, 0); err != nil { 793 return nil, err 794 } 795 st, err := f.Stat() 796 if err != nil { 797 return nil, err 798 } 799 size := st.Size() 800 return &TempArchive{File: f, Size: size}, nil 801 } 802 803 type TempArchive struct { 804 *os.File 805 Size int64 // Pre-computed from Stat().Size() as a convenience 806 read int64 807 closed bool 808 } 809 810 // Close closes the underlying file if it's still open, or does a no-op 811 // to allow callers to try to close the TempArchive multiple times safely. 812 func (archive *TempArchive) Close() error { 813 if archive.closed { 814 return nil 815 } 816 817 archive.closed = true 818 819 return archive.File.Close() 820 } 821 822 func (archive *TempArchive) Read(data []byte) (int, error) { 823 n, err := archive.File.Read(data) 824 archive.read += int64(n) 825 if err != nil || archive.read == archive.Size { 826 archive.Close() 827 os.Remove(archive.File.Name()) 828 } 829 return n, err 830 }