github.com/jiasir/docker@v1.3.3-0.20170609024000-252e610103e7/builder/dockerfile/copy.go (about) 1 package dockerfile 2 3 import ( 4 "fmt" 5 "io" 6 "net/http" 7 "net/url" 8 "os" 9 "path/filepath" 10 "sort" 11 "strings" 12 "time" 13 14 "github.com/docker/docker/builder" 15 "github.com/docker/docker/builder/remotecontext" 16 "github.com/docker/docker/pkg/ioutils" 17 "github.com/docker/docker/pkg/progress" 18 "github.com/docker/docker/pkg/streamformatter" 19 "github.com/docker/docker/pkg/system" 20 "github.com/docker/docker/pkg/urlutil" 21 "github.com/pkg/errors" 22 ) 23 24 type pathCache interface { 25 Load(key interface{}) (value interface{}, ok bool) 26 Store(key, value interface{}) 27 } 28 29 // copyInfo is a data object which stores the metadata about each source file in 30 // a copyInstruction 31 type copyInfo struct { 32 root string 33 path string 34 hash string 35 } 36 37 func newCopyInfoFromSource(source builder.Source, path string, hash string) copyInfo { 38 return copyInfo{root: source.Root(), path: path, hash: hash} 39 } 40 41 func newCopyInfos(copyInfos ...copyInfo) []copyInfo { 42 return copyInfos 43 } 44 45 // copyInstruction is a fully parsed COPY or ADD command that is passed to 46 // Builder.performCopy to copy files into the image filesystem 47 type copyInstruction struct { 48 cmdName string 49 infos []copyInfo 50 dest string 51 allowLocalDecompression bool 52 } 53 54 // copier reads a raw COPY or ADD command, fetches remote sources using a downloader, 55 // and creates a copyInstruction 56 type copier struct { 57 imageSource *imageMount 58 source builder.Source 59 pathCache pathCache 60 download sourceDownloader 61 tmpPaths []string 62 } 63 64 func copierFromDispatchRequest(req dispatchRequest, download sourceDownloader, imageSource *imageMount) copier { 65 return copier{ 66 source: req.source, 67 pathCache: req.builder.pathCache, 68 download: download, 69 imageSource: imageSource, 70 } 71 } 72 73 func (o *copier) createCopyInstruction(args []string, cmdName string) (copyInstruction, error) { 74 inst := copyInstruction{cmdName: cmdName} 75 last := len(args) - 1 76 77 // Work in daemon-specific filepath semantics 78 inst.dest = filepath.FromSlash(args[last]) 79 80 infos, err := o.getCopyInfosForSourcePaths(args[0:last]) 81 if err != nil { 82 return inst, errors.Wrapf(err, "%s failed", cmdName) 83 } 84 if len(infos) > 1 && !strings.HasSuffix(inst.dest, string(os.PathSeparator)) { 85 return inst, errors.Errorf("When using %s with more than one source file, the destination must be a directory and end with a /", cmdName) 86 } 87 inst.infos = infos 88 return inst, nil 89 } 90 91 // getCopyInfosForSourcePaths iterates over the source files and calculate the info 92 // needed to copy (e.g. hash value if cached) 93 func (o *copier) getCopyInfosForSourcePaths(sources []string) ([]copyInfo, error) { 94 var infos []copyInfo 95 for _, orig := range sources { 96 subinfos, err := o.getCopyInfoForSourcePath(orig) 97 if err != nil { 98 return nil, err 99 } 100 infos = append(infos, subinfos...) 101 } 102 103 if len(infos) == 0 { 104 return nil, errors.New("no source files were specified") 105 } 106 return infos, nil 107 } 108 109 func (o *copier) getCopyInfoForSourcePath(orig string) ([]copyInfo, error) { 110 if !urlutil.IsURL(orig) { 111 return o.calcCopyInfo(orig, true) 112 } 113 remote, path, err := o.download(orig) 114 if err != nil { 115 return nil, err 116 } 117 o.tmpPaths = append(o.tmpPaths, remote.Root()) 118 119 hash, err := remote.Hash(path) 120 return newCopyInfos(newCopyInfoFromSource(remote, path, hash)), err 121 } 122 123 // Cleanup removes any temporary directories created as part of downloading 124 // remote files. 125 func (o *copier) Cleanup() { 126 for _, path := range o.tmpPaths { 127 os.RemoveAll(path) 128 } 129 o.tmpPaths = []string{} 130 } 131 132 // TODO: allowWildcards can probably be removed by refactoring this function further. 133 func (o *copier) calcCopyInfo(origPath string, allowWildcards bool) ([]copyInfo, error) { 134 imageSource := o.imageSource 135 if err := validateCopySourcePath(imageSource, origPath); err != nil { 136 return nil, err 137 } 138 139 // Work in daemon-specific OS filepath semantics 140 origPath = filepath.FromSlash(origPath) 141 origPath = strings.TrimPrefix(origPath, string(os.PathSeparator)) 142 origPath = strings.TrimPrefix(origPath, "."+string(os.PathSeparator)) 143 144 // TODO: do this when creating copier. Requires validateCopySourcePath 145 // (and other below) to be aware of the difference sources. Why is it only 146 // done on image Source? 147 if imageSource != nil { 148 var err error 149 o.source, err = imageSource.Source() 150 if err != nil { 151 return nil, errors.Wrapf(err, "failed to copy") 152 } 153 } 154 155 if o.source == nil { 156 return nil, errors.Errorf("missing build context") 157 } 158 159 // Deal with wildcards 160 if allowWildcards && containsWildcards(origPath) { 161 return o.copyWithWildcards(origPath) 162 } 163 164 if imageSource != nil && imageSource.ImageID() != "" { 165 // return a cached copy if one exists 166 if h, ok := o.pathCache.Load(imageSource.ImageID() + origPath); ok { 167 return newCopyInfos(newCopyInfoFromSource(o.source, origPath, h.(string))), nil 168 } 169 } 170 171 // Deal with the single file case 172 copyInfo, err := copyInfoForFile(o.source, origPath) 173 switch { 174 case err != nil: 175 return nil, err 176 case copyInfo.hash != "": 177 o.storeInPathCache(imageSource, origPath, copyInfo.hash) 178 return newCopyInfos(copyInfo), err 179 } 180 181 // TODO: remove, handle dirs in Hash() 182 subfiles, err := walkSource(o.source, origPath) 183 if err != nil { 184 return nil, err 185 } 186 187 hash := hashStringSlice("dir", subfiles) 188 o.storeInPathCache(imageSource, origPath, hash) 189 return newCopyInfos(newCopyInfoFromSource(o.source, origPath, hash)), nil 190 } 191 192 func (o *copier) storeInPathCache(im *imageMount, path string, hash string) { 193 if im != nil { 194 o.pathCache.Store(im.ImageID()+path, hash) 195 } 196 } 197 198 func (o *copier) copyWithWildcards(origPath string) ([]copyInfo, error) { 199 var copyInfos []copyInfo 200 if err := filepath.Walk(o.source.Root(), func(path string, info os.FileInfo, err error) error { 201 if err != nil { 202 return err 203 } 204 rel, err := remotecontext.Rel(o.source.Root(), path) 205 if err != nil { 206 return err 207 } 208 209 if rel == "." { 210 return nil 211 } 212 if match, _ := filepath.Match(origPath, rel); !match { 213 return nil 214 } 215 216 // Note we set allowWildcards to false in case the name has 217 // a * in it 218 subInfos, err := o.calcCopyInfo(rel, false) 219 if err != nil { 220 return err 221 } 222 copyInfos = append(copyInfos, subInfos...) 223 return nil 224 }); err != nil { 225 return nil, err 226 } 227 return copyInfos, nil 228 } 229 230 func copyInfoForFile(source builder.Source, path string) (copyInfo, error) { 231 fi, err := remotecontext.StatAt(source, path) 232 if err != nil { 233 return copyInfo{}, err 234 } 235 236 if fi.IsDir() { 237 return copyInfo{}, nil 238 } 239 hash, err := source.Hash(path) 240 if err != nil { 241 return copyInfo{}, err 242 } 243 return newCopyInfoFromSource(source, path, "file:"+hash), nil 244 } 245 246 // TODO: dedupe with copyWithWildcards() 247 func walkSource(source builder.Source, origPath string) ([]string, error) { 248 fp, err := remotecontext.FullPath(source, origPath) 249 if err != nil { 250 return nil, err 251 } 252 // Must be a dir 253 var subfiles []string 254 err = filepath.Walk(fp, func(path string, info os.FileInfo, err error) error { 255 if err != nil { 256 return err 257 } 258 rel, err := remotecontext.Rel(source.Root(), path) 259 if err != nil { 260 return err 261 } 262 if rel == "." { 263 return nil 264 } 265 hash, err := source.Hash(rel) 266 if err != nil { 267 return nil 268 } 269 // we already checked handleHash above 270 subfiles = append(subfiles, hash) 271 return nil 272 }) 273 if err != nil { 274 return nil, err 275 } 276 277 sort.Strings(subfiles) 278 return subfiles, nil 279 } 280 281 type sourceDownloader func(string) (builder.Source, string, error) 282 283 func newRemoteSourceDownloader(output, stdout io.Writer) sourceDownloader { 284 return func(url string) (builder.Source, string, error) { 285 return downloadSource(output, stdout, url) 286 } 287 } 288 289 func errOnSourceDownload(_ string) (builder.Source, string, error) { 290 return nil, "", errors.New("source can't be a URL for COPY") 291 } 292 293 func downloadSource(output io.Writer, stdout io.Writer, srcURL string) (remote builder.Source, p string, err error) { 294 u, err := url.Parse(srcURL) 295 if err != nil { 296 return 297 } 298 filename := filepath.Base(filepath.FromSlash(u.Path)) // Ensure in platform semantics 299 if filename == "" { 300 err = errors.Errorf("cannot determine filename from url: %s", u) 301 return 302 } 303 304 resp, err := remotecontext.GetWithStatusError(srcURL) 305 if err != nil { 306 return 307 } 308 309 // Prepare file in a tmp dir 310 tmpDir, err := ioutils.TempDir("", "docker-remote") 311 if err != nil { 312 return 313 } 314 defer func() { 315 if err != nil { 316 os.RemoveAll(tmpDir) 317 } 318 }() 319 tmpFileName := filepath.Join(tmpDir, filename) 320 tmpFile, err := os.OpenFile(tmpFileName, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0600) 321 if err != nil { 322 return 323 } 324 325 progressOutput := streamformatter.NewJSONProgressOutput(output, true) 326 progressReader := progress.NewProgressReader(resp.Body, progressOutput, resp.ContentLength, "", "Downloading") 327 // Download and dump result to tmp file 328 // TODO: add filehash directly 329 if _, err = io.Copy(tmpFile, progressReader); err != nil { 330 tmpFile.Close() 331 return 332 } 333 // TODO: how important is this random blank line to the output? 334 fmt.Fprintln(stdout) 335 336 // Set the mtime to the Last-Modified header value if present 337 // Otherwise just remove atime and mtime 338 mTime := time.Time{} 339 340 lastMod := resp.Header.Get("Last-Modified") 341 if lastMod != "" { 342 // If we can't parse it then just let it default to 'zero' 343 // otherwise use the parsed time value 344 if parsedMTime, err := http.ParseTime(lastMod); err == nil { 345 mTime = parsedMTime 346 } 347 } 348 349 tmpFile.Close() 350 351 if err = system.Chtimes(tmpFileName, mTime, mTime); err != nil { 352 return 353 } 354 355 lc, err := remotecontext.NewLazyContext(tmpDir) 356 return lc, filename, err 357 }