github.com/jiasir/docker@v1.3.3-0.20170609024000-252e610103e7/builder/dockerfile/copy.go (about)

     1  package dockerfile
     2  
     3  import (
     4  	"fmt"
     5  	"io"
     6  	"net/http"
     7  	"net/url"
     8  	"os"
     9  	"path/filepath"
    10  	"sort"
    11  	"strings"
    12  	"time"
    13  
    14  	"github.com/docker/docker/builder"
    15  	"github.com/docker/docker/builder/remotecontext"
    16  	"github.com/docker/docker/pkg/ioutils"
    17  	"github.com/docker/docker/pkg/progress"
    18  	"github.com/docker/docker/pkg/streamformatter"
    19  	"github.com/docker/docker/pkg/system"
    20  	"github.com/docker/docker/pkg/urlutil"
    21  	"github.com/pkg/errors"
    22  )
    23  
    24  type pathCache interface {
    25  	Load(key interface{}) (value interface{}, ok bool)
    26  	Store(key, value interface{})
    27  }
    28  
    29  // copyInfo is a data object which stores the metadata about each source file in
    30  // a copyInstruction
    31  type copyInfo struct {
    32  	root string
    33  	path string
    34  	hash string
    35  }
    36  
    37  func newCopyInfoFromSource(source builder.Source, path string, hash string) copyInfo {
    38  	return copyInfo{root: source.Root(), path: path, hash: hash}
    39  }
    40  
    41  func newCopyInfos(copyInfos ...copyInfo) []copyInfo {
    42  	return copyInfos
    43  }
    44  
    45  // copyInstruction is a fully parsed COPY or ADD command that is passed to
    46  // Builder.performCopy to copy files into the image filesystem
    47  type copyInstruction struct {
    48  	cmdName                 string
    49  	infos                   []copyInfo
    50  	dest                    string
    51  	allowLocalDecompression bool
    52  }
    53  
    54  // copier reads a raw COPY or ADD command, fetches remote sources using a downloader,
    55  // and creates a copyInstruction
    56  type copier struct {
    57  	imageSource *imageMount
    58  	source      builder.Source
    59  	pathCache   pathCache
    60  	download    sourceDownloader
    61  	tmpPaths    []string
    62  }
    63  
    64  func copierFromDispatchRequest(req dispatchRequest, download sourceDownloader, imageSource *imageMount) copier {
    65  	return copier{
    66  		source:      req.source,
    67  		pathCache:   req.builder.pathCache,
    68  		download:    download,
    69  		imageSource: imageSource,
    70  	}
    71  }
    72  
    73  func (o *copier) createCopyInstruction(args []string, cmdName string) (copyInstruction, error) {
    74  	inst := copyInstruction{cmdName: cmdName}
    75  	last := len(args) - 1
    76  
    77  	// Work in daemon-specific filepath semantics
    78  	inst.dest = filepath.FromSlash(args[last])
    79  
    80  	infos, err := o.getCopyInfosForSourcePaths(args[0:last])
    81  	if err != nil {
    82  		return inst, errors.Wrapf(err, "%s failed", cmdName)
    83  	}
    84  	if len(infos) > 1 && !strings.HasSuffix(inst.dest, string(os.PathSeparator)) {
    85  		return inst, errors.Errorf("When using %s with more than one source file, the destination must be a directory and end with a /", cmdName)
    86  	}
    87  	inst.infos = infos
    88  	return inst, nil
    89  }
    90  
    91  // getCopyInfosForSourcePaths iterates over the source files and calculate the info
    92  // needed to copy (e.g. hash value if cached)
    93  func (o *copier) getCopyInfosForSourcePaths(sources []string) ([]copyInfo, error) {
    94  	var infos []copyInfo
    95  	for _, orig := range sources {
    96  		subinfos, err := o.getCopyInfoForSourcePath(orig)
    97  		if err != nil {
    98  			return nil, err
    99  		}
   100  		infos = append(infos, subinfos...)
   101  	}
   102  
   103  	if len(infos) == 0 {
   104  		return nil, errors.New("no source files were specified")
   105  	}
   106  	return infos, nil
   107  }
   108  
   109  func (o *copier) getCopyInfoForSourcePath(orig string) ([]copyInfo, error) {
   110  	if !urlutil.IsURL(orig) {
   111  		return o.calcCopyInfo(orig, true)
   112  	}
   113  	remote, path, err := o.download(orig)
   114  	if err != nil {
   115  		return nil, err
   116  	}
   117  	o.tmpPaths = append(o.tmpPaths, remote.Root())
   118  
   119  	hash, err := remote.Hash(path)
   120  	return newCopyInfos(newCopyInfoFromSource(remote, path, hash)), err
   121  }
   122  
   123  // Cleanup removes any temporary directories created as part of downloading
   124  // remote files.
   125  func (o *copier) Cleanup() {
   126  	for _, path := range o.tmpPaths {
   127  		os.RemoveAll(path)
   128  	}
   129  	o.tmpPaths = []string{}
   130  }
   131  
   132  // TODO: allowWildcards can probably be removed by refactoring this function further.
   133  func (o *copier) calcCopyInfo(origPath string, allowWildcards bool) ([]copyInfo, error) {
   134  	imageSource := o.imageSource
   135  	if err := validateCopySourcePath(imageSource, origPath); err != nil {
   136  		return nil, err
   137  	}
   138  
   139  	// Work in daemon-specific OS filepath semantics
   140  	origPath = filepath.FromSlash(origPath)
   141  	origPath = strings.TrimPrefix(origPath, string(os.PathSeparator))
   142  	origPath = strings.TrimPrefix(origPath, "."+string(os.PathSeparator))
   143  
   144  	// TODO: do this when creating copier. Requires validateCopySourcePath
   145  	// (and other below) to be aware of the difference sources. Why is it only
   146  	// done on image Source?
   147  	if imageSource != nil {
   148  		var err error
   149  		o.source, err = imageSource.Source()
   150  		if err != nil {
   151  			return nil, errors.Wrapf(err, "failed to copy")
   152  		}
   153  	}
   154  
   155  	if o.source == nil {
   156  		return nil, errors.Errorf("missing build context")
   157  	}
   158  
   159  	// Deal with wildcards
   160  	if allowWildcards && containsWildcards(origPath) {
   161  		return o.copyWithWildcards(origPath)
   162  	}
   163  
   164  	if imageSource != nil && imageSource.ImageID() != "" {
   165  		// return a cached copy if one exists
   166  		if h, ok := o.pathCache.Load(imageSource.ImageID() + origPath); ok {
   167  			return newCopyInfos(newCopyInfoFromSource(o.source, origPath, h.(string))), nil
   168  		}
   169  	}
   170  
   171  	// Deal with the single file case
   172  	copyInfo, err := copyInfoForFile(o.source, origPath)
   173  	switch {
   174  	case err != nil:
   175  		return nil, err
   176  	case copyInfo.hash != "":
   177  		o.storeInPathCache(imageSource, origPath, copyInfo.hash)
   178  		return newCopyInfos(copyInfo), err
   179  	}
   180  
   181  	// TODO: remove, handle dirs in Hash()
   182  	subfiles, err := walkSource(o.source, origPath)
   183  	if err != nil {
   184  		return nil, err
   185  	}
   186  
   187  	hash := hashStringSlice("dir", subfiles)
   188  	o.storeInPathCache(imageSource, origPath, hash)
   189  	return newCopyInfos(newCopyInfoFromSource(o.source, origPath, hash)), nil
   190  }
   191  
   192  func (o *copier) storeInPathCache(im *imageMount, path string, hash string) {
   193  	if im != nil {
   194  		o.pathCache.Store(im.ImageID()+path, hash)
   195  	}
   196  }
   197  
   198  func (o *copier) copyWithWildcards(origPath string) ([]copyInfo, error) {
   199  	var copyInfos []copyInfo
   200  	if err := filepath.Walk(o.source.Root(), func(path string, info os.FileInfo, err error) error {
   201  		if err != nil {
   202  			return err
   203  		}
   204  		rel, err := remotecontext.Rel(o.source.Root(), path)
   205  		if err != nil {
   206  			return err
   207  		}
   208  
   209  		if rel == "." {
   210  			return nil
   211  		}
   212  		if match, _ := filepath.Match(origPath, rel); !match {
   213  			return nil
   214  		}
   215  
   216  		// Note we set allowWildcards to false in case the name has
   217  		// a * in it
   218  		subInfos, err := o.calcCopyInfo(rel, false)
   219  		if err != nil {
   220  			return err
   221  		}
   222  		copyInfos = append(copyInfos, subInfos...)
   223  		return nil
   224  	}); err != nil {
   225  		return nil, err
   226  	}
   227  	return copyInfos, nil
   228  }
   229  
   230  func copyInfoForFile(source builder.Source, path string) (copyInfo, error) {
   231  	fi, err := remotecontext.StatAt(source, path)
   232  	if err != nil {
   233  		return copyInfo{}, err
   234  	}
   235  
   236  	if fi.IsDir() {
   237  		return copyInfo{}, nil
   238  	}
   239  	hash, err := source.Hash(path)
   240  	if err != nil {
   241  		return copyInfo{}, err
   242  	}
   243  	return newCopyInfoFromSource(source, path, "file:"+hash), nil
   244  }
   245  
   246  // TODO: dedupe with copyWithWildcards()
   247  func walkSource(source builder.Source, origPath string) ([]string, error) {
   248  	fp, err := remotecontext.FullPath(source, origPath)
   249  	if err != nil {
   250  		return nil, err
   251  	}
   252  	// Must be a dir
   253  	var subfiles []string
   254  	err = filepath.Walk(fp, func(path string, info os.FileInfo, err error) error {
   255  		if err != nil {
   256  			return err
   257  		}
   258  		rel, err := remotecontext.Rel(source.Root(), path)
   259  		if err != nil {
   260  			return err
   261  		}
   262  		if rel == "." {
   263  			return nil
   264  		}
   265  		hash, err := source.Hash(rel)
   266  		if err != nil {
   267  			return nil
   268  		}
   269  		// we already checked handleHash above
   270  		subfiles = append(subfiles, hash)
   271  		return nil
   272  	})
   273  	if err != nil {
   274  		return nil, err
   275  	}
   276  
   277  	sort.Strings(subfiles)
   278  	return subfiles, nil
   279  }
   280  
   281  type sourceDownloader func(string) (builder.Source, string, error)
   282  
   283  func newRemoteSourceDownloader(output, stdout io.Writer) sourceDownloader {
   284  	return func(url string) (builder.Source, string, error) {
   285  		return downloadSource(output, stdout, url)
   286  	}
   287  }
   288  
   289  func errOnSourceDownload(_ string) (builder.Source, string, error) {
   290  	return nil, "", errors.New("source can't be a URL for COPY")
   291  }
   292  
   293  func downloadSource(output io.Writer, stdout io.Writer, srcURL string) (remote builder.Source, p string, err error) {
   294  	u, err := url.Parse(srcURL)
   295  	if err != nil {
   296  		return
   297  	}
   298  	filename := filepath.Base(filepath.FromSlash(u.Path)) // Ensure in platform semantics
   299  	if filename == "" {
   300  		err = errors.Errorf("cannot determine filename from url: %s", u)
   301  		return
   302  	}
   303  
   304  	resp, err := remotecontext.GetWithStatusError(srcURL)
   305  	if err != nil {
   306  		return
   307  	}
   308  
   309  	// Prepare file in a tmp dir
   310  	tmpDir, err := ioutils.TempDir("", "docker-remote")
   311  	if err != nil {
   312  		return
   313  	}
   314  	defer func() {
   315  		if err != nil {
   316  			os.RemoveAll(tmpDir)
   317  		}
   318  	}()
   319  	tmpFileName := filepath.Join(tmpDir, filename)
   320  	tmpFile, err := os.OpenFile(tmpFileName, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0600)
   321  	if err != nil {
   322  		return
   323  	}
   324  
   325  	progressOutput := streamformatter.NewJSONProgressOutput(output, true)
   326  	progressReader := progress.NewProgressReader(resp.Body, progressOutput, resp.ContentLength, "", "Downloading")
   327  	// Download and dump result to tmp file
   328  	// TODO: add filehash directly
   329  	if _, err = io.Copy(tmpFile, progressReader); err != nil {
   330  		tmpFile.Close()
   331  		return
   332  	}
   333  	// TODO: how important is this random blank line to the output?
   334  	fmt.Fprintln(stdout)
   335  
   336  	// Set the mtime to the Last-Modified header value if present
   337  	// Otherwise just remove atime and mtime
   338  	mTime := time.Time{}
   339  
   340  	lastMod := resp.Header.Get("Last-Modified")
   341  	if lastMod != "" {
   342  		// If we can't parse it then just let it default to 'zero'
   343  		// otherwise use the parsed time value
   344  		if parsedMTime, err := http.ParseTime(lastMod); err == nil {
   345  			mTime = parsedMTime
   346  		}
   347  	}
   348  
   349  	tmpFile.Close()
   350  
   351  	if err = system.Chtimes(tmpFileName, mTime, mTime); err != nil {
   352  		return
   353  	}
   354  
   355  	lc, err := remotecontext.NewLazyContext(tmpDir)
   356  	return lc, filename, err
   357  }