github.com/demonoid81/moby@v0.0.0-20200517203328-62dd8e17c460/builder/dockerfile/copy.go (about) 1 package dockerfile // import "github.com/demonoid81/moby/builder/dockerfile" 2 3 import ( 4 "archive/tar" 5 "fmt" 6 "io" 7 "mime" 8 "net/http" 9 "net/url" 10 "os" 11 "path/filepath" 12 "runtime" 13 "sort" 14 "strings" 15 "time" 16 17 "github.com/demonoid81/moby/builder" 18 "github.com/demonoid81/moby/builder/remotecontext" 19 "github.com/demonoid81/moby/pkg/archive" 20 "github.com/demonoid81/moby/pkg/containerfs" 21 "github.com/demonoid81/moby/pkg/idtools" 22 "github.com/demonoid81/moby/pkg/ioutils" 23 "github.com/demonoid81/moby/pkg/progress" 24 "github.com/demonoid81/moby/pkg/streamformatter" 25 "github.com/demonoid81/moby/pkg/system" 26 "github.com/demonoid81/moby/pkg/urlutil" 27 specs "github.com/opencontainers/image-spec/specs-go/v1" 28 "github.com/pkg/errors" 29 ) 30 31 const unnamedFilename = "__unnamed__" 32 33 type pathCache interface { 34 Load(key interface{}) (value interface{}, ok bool) 35 Store(key, value interface{}) 36 } 37 38 // copyInfo is a data object which stores the metadata about each source file in 39 // a copyInstruction 40 type copyInfo struct { 41 root containerfs.ContainerFS 42 path string 43 hash string 44 noDecompress bool 45 } 46 47 func (c copyInfo) fullPath() (string, error) { 48 return c.root.ResolveScopedPath(c.path, true) 49 } 50 51 func newCopyInfoFromSource(source builder.Source, path string, hash string) copyInfo { 52 return copyInfo{root: source.Root(), path: path, hash: hash} 53 } 54 55 func newCopyInfos(copyInfos ...copyInfo) []copyInfo { 56 return copyInfos 57 } 58 59 // copyInstruction is a fully parsed COPY or ADD command that is passed to 60 // Builder.performCopy to copy files into the image filesystem 61 type copyInstruction struct { 62 cmdName string 63 infos []copyInfo 64 dest string 65 chownStr string 66 allowLocalDecompression bool 67 preserveOwnership bool 68 } 69 70 // copier reads a raw COPY or ADD command, fetches remote sources using a downloader, 71 // and creates a copyInstruction 72 type copier struct { 73 imageSource *imageMount 74 source builder.Source 75 pathCache pathCache 76 download sourceDownloader 77 platform *specs.Platform 78 // for cleanup. TODO: having copier.cleanup() is error prone and hard to 79 // follow. Code calling performCopy should manage the lifecycle of its params. 80 // Copier should take override source as input, not imageMount. 81 activeLayer builder.RWLayer 82 tmpPaths []string 83 } 84 85 func copierFromDispatchRequest(req dispatchRequest, download sourceDownloader, imageSource *imageMount) copier { 86 platform := req.builder.platform 87 if platform == nil { 88 // May be nil if not explicitly set in API/dockerfile 89 platform = &specs.Platform{} 90 } 91 if platform.OS == "" { 92 // Default to the dispatch requests operating system if not explicit in API/dockerfile 93 platform.OS = req.state.operatingSystem 94 } 95 if platform.OS == "" { 96 // This is a failsafe just in case. Shouldn't be hit. 97 platform.OS = runtime.GOOS 98 } 99 100 return copier{ 101 source: req.source, 102 pathCache: req.builder.pathCache, 103 download: download, 104 imageSource: imageSource, 105 platform: platform, 106 } 107 108 } 109 110 func (o *copier) createCopyInstruction(args []string, cmdName string) (copyInstruction, error) { 111 inst := copyInstruction{cmdName: cmdName} 112 last := len(args) - 1 113 114 // Work in platform-specific filepath semantics 115 // TODO: This OS switch for paths is NOT correct and should not be supported. 116 // Maintained for backwards compatibility 117 pathOS := runtime.GOOS 118 if o.platform != nil { 119 pathOS = o.platform.OS 120 } 121 inst.dest = fromSlash(args[last], pathOS) 122 separator := string(separator(pathOS)) 123 infos, err := o.getCopyInfosForSourcePaths(args[0:last], inst.dest) 124 if err != nil { 125 return inst, errors.Wrapf(err, "%s failed", cmdName) 126 } 127 if len(infos) > 1 && !strings.HasSuffix(inst.dest, separator) { 128 return inst, errors.Errorf("When using %s with more than one source file, the destination must be a directory and end with a /", cmdName) 129 } 130 inst.infos = infos 131 return inst, nil 132 } 133 134 // getCopyInfosForSourcePaths iterates over the source files and calculate the info 135 // needed to copy (e.g. hash value if cached) 136 // The dest is used in case source is URL (and ends with "/") 137 func (o *copier) getCopyInfosForSourcePaths(sources []string, dest string) ([]copyInfo, error) { 138 var infos []copyInfo 139 for _, orig := range sources { 140 subinfos, err := o.getCopyInfoForSourcePath(orig, dest) 141 if err != nil { 142 return nil, err 143 } 144 infos = append(infos, subinfos...) 145 } 146 147 if len(infos) == 0 { 148 return nil, errors.New("no source files were specified") 149 } 150 return infos, nil 151 } 152 153 func (o *copier) getCopyInfoForSourcePath(orig, dest string) ([]copyInfo, error) { 154 if !urlutil.IsURL(orig) { 155 return o.calcCopyInfo(orig, true) 156 } 157 158 remote, path, err := o.download(orig) 159 if err != nil { 160 return nil, err 161 } 162 // If path == "" then we are unable to determine filename from src 163 // We have to make sure dest is available 164 if path == "" { 165 if strings.HasSuffix(dest, "/") { 166 return nil, errors.Errorf("cannot determine filename for source %s", orig) 167 } 168 path = unnamedFilename 169 } 170 o.tmpPaths = append(o.tmpPaths, remote.Root().Path()) 171 172 hash, err := remote.Hash(path) 173 ci := newCopyInfoFromSource(remote, path, hash) 174 ci.noDecompress = true // data from http shouldn't be extracted even on ADD 175 return newCopyInfos(ci), err 176 } 177 178 // Cleanup removes any temporary directories created as part of downloading 179 // remote files. 180 func (o *copier) Cleanup() { 181 for _, path := range o.tmpPaths { 182 os.RemoveAll(path) 183 } 184 o.tmpPaths = []string{} 185 if o.activeLayer != nil { 186 o.activeLayer.Release() 187 o.activeLayer = nil 188 } 189 } 190 191 // TODO: allowWildcards can probably be removed by refactoring this function further. 192 func (o *copier) calcCopyInfo(origPath string, allowWildcards bool) ([]copyInfo, error) { 193 imageSource := o.imageSource 194 195 // TODO: do this when creating copier. Requires validateCopySourcePath 196 // (and other below) to be aware of the difference sources. Why is it only 197 // done on image Source? 198 if imageSource != nil && o.activeLayer == nil { 199 // this needs to be protected against repeated calls as wildcard copy 200 // will call it multiple times for a single COPY 201 var err error 202 rwLayer, err := imageSource.NewRWLayer() 203 if err != nil { 204 return nil, err 205 } 206 o.activeLayer = rwLayer 207 208 o.source, err = remotecontext.NewLazySource(rwLayer.Root()) 209 if err != nil { 210 return nil, errors.Wrapf(err, "failed to create context for copy from %s", rwLayer.Root().Path()) 211 } 212 } 213 214 if o.source == nil { 215 return nil, errors.Errorf("missing build context") 216 } 217 218 root := o.source.Root() 219 220 if err := validateCopySourcePath(imageSource, origPath, root.OS()); err != nil { 221 return nil, err 222 } 223 224 // Work in source OS specific filepath semantics 225 // For LCOW, this is NOT the daemon OS. 226 origPath = root.FromSlash(origPath) 227 origPath = strings.TrimPrefix(origPath, string(root.Separator())) 228 origPath = strings.TrimPrefix(origPath, "."+string(root.Separator())) 229 230 // Deal with wildcards 231 if allowWildcards && containsWildcards(origPath, root.OS()) { 232 return o.copyWithWildcards(origPath) 233 } 234 235 if imageSource != nil && imageSource.ImageID() != "" { 236 // return a cached copy if one exists 237 if h, ok := o.pathCache.Load(imageSource.ImageID() + origPath); ok { 238 return newCopyInfos(newCopyInfoFromSource(o.source, origPath, h.(string))), nil 239 } 240 } 241 242 // Deal with the single file case 243 copyInfo, err := copyInfoForFile(o.source, origPath) 244 switch { 245 case err != nil: 246 return nil, err 247 case copyInfo.hash != "": 248 o.storeInPathCache(imageSource, origPath, copyInfo.hash) 249 return newCopyInfos(copyInfo), err 250 } 251 252 // TODO: remove, handle dirs in Hash() 253 subfiles, err := walkSource(o.source, origPath) 254 if err != nil { 255 return nil, err 256 } 257 258 hash := hashStringSlice("dir", subfiles) 259 o.storeInPathCache(imageSource, origPath, hash) 260 return newCopyInfos(newCopyInfoFromSource(o.source, origPath, hash)), nil 261 } 262 263 func containsWildcards(name, platform string) bool { 264 isWindows := platform == "windows" 265 for i := 0; i < len(name); i++ { 266 ch := name[i] 267 if ch == '\\' && !isWindows { 268 i++ 269 } else if ch == '*' || ch == '?' || ch == '[' { 270 return true 271 } 272 } 273 return false 274 } 275 276 func (o *copier) storeInPathCache(im *imageMount, path string, hash string) { 277 if im != nil { 278 o.pathCache.Store(im.ImageID()+path, hash) 279 } 280 } 281 282 func (o *copier) copyWithWildcards(origPath string) ([]copyInfo, error) { 283 root := o.source.Root() 284 var copyInfos []copyInfo 285 if err := root.Walk(root.Path(), func(path string, info os.FileInfo, err error) error { 286 if err != nil { 287 return err 288 } 289 rel, err := remotecontext.Rel(root, path) 290 if err != nil { 291 return err 292 } 293 294 if rel == "." { 295 return nil 296 } 297 if match, _ := root.Match(origPath, rel); !match { 298 return nil 299 } 300 301 // Note we set allowWildcards to false in case the name has 302 // a * in it 303 subInfos, err := o.calcCopyInfo(rel, false) 304 if err != nil { 305 return err 306 } 307 copyInfos = append(copyInfos, subInfos...) 308 return nil 309 }); err != nil { 310 return nil, err 311 } 312 return copyInfos, nil 313 } 314 315 func copyInfoForFile(source builder.Source, path string) (copyInfo, error) { 316 fi, err := remotecontext.StatAt(source, path) 317 if err != nil { 318 return copyInfo{}, err 319 } 320 321 if fi.IsDir() { 322 return copyInfo{}, nil 323 } 324 hash, err := source.Hash(path) 325 if err != nil { 326 return copyInfo{}, err 327 } 328 return newCopyInfoFromSource(source, path, "file:"+hash), nil 329 } 330 331 // TODO: dedupe with copyWithWildcards() 332 func walkSource(source builder.Source, origPath string) ([]string, error) { 333 fp, err := remotecontext.FullPath(source, origPath) 334 if err != nil { 335 return nil, err 336 } 337 // Must be a dir 338 var subfiles []string 339 err = source.Root().Walk(fp, func(path string, info os.FileInfo, err error) error { 340 if err != nil { 341 return err 342 } 343 rel, err := remotecontext.Rel(source.Root(), path) 344 if err != nil { 345 return err 346 } 347 if rel == "." { 348 return nil 349 } 350 hash, err := source.Hash(rel) 351 if err != nil { 352 return nil 353 } 354 // we already checked handleHash above 355 subfiles = append(subfiles, hash) 356 return nil 357 }) 358 if err != nil { 359 return nil, err 360 } 361 362 sort.Strings(subfiles) 363 return subfiles, nil 364 } 365 366 type sourceDownloader func(string) (builder.Source, string, error) 367 368 func newRemoteSourceDownloader(output, stdout io.Writer) sourceDownloader { 369 return func(url string) (builder.Source, string, error) { 370 return downloadSource(output, stdout, url) 371 } 372 } 373 374 func errOnSourceDownload(_ string) (builder.Source, string, error) { 375 return nil, "", errors.New("source can't be a URL for COPY") 376 } 377 378 func getFilenameForDownload(path string, resp *http.Response) string { 379 // Guess filename based on source 380 if path != "" && !strings.HasSuffix(path, "/") { 381 if filename := filepath.Base(filepath.FromSlash(path)); filename != "" { 382 return filename 383 } 384 } 385 386 // Guess filename based on Content-Disposition 387 if contentDisposition := resp.Header.Get("Content-Disposition"); contentDisposition != "" { 388 if _, params, err := mime.ParseMediaType(contentDisposition); err == nil { 389 if params["filename"] != "" && !strings.HasSuffix(params["filename"], "/") { 390 if filename := filepath.Base(filepath.FromSlash(params["filename"])); filename != "" { 391 return filename 392 } 393 } 394 } 395 } 396 return "" 397 } 398 399 func downloadSource(output io.Writer, stdout io.Writer, srcURL string) (remote builder.Source, p string, err error) { 400 u, err := url.Parse(srcURL) 401 if err != nil { 402 return 403 } 404 405 resp, err := remotecontext.GetWithStatusError(srcURL) 406 if err != nil { 407 return 408 } 409 410 filename := getFilenameForDownload(u.Path, resp) 411 412 // Prepare file in a tmp dir 413 tmpDir, err := ioutils.TempDir("", "docker-remote") 414 if err != nil { 415 return 416 } 417 defer func() { 418 if err != nil { 419 os.RemoveAll(tmpDir) 420 } 421 }() 422 // If filename is empty, the returned filename will be "" but 423 // the tmp filename will be created as "__unnamed__" 424 tmpFileName := filename 425 if filename == "" { 426 tmpFileName = unnamedFilename 427 } 428 tmpFileName = filepath.Join(tmpDir, tmpFileName) 429 tmpFile, err := os.OpenFile(tmpFileName, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0600) 430 if err != nil { 431 return 432 } 433 434 progressOutput := streamformatter.NewJSONProgressOutput(output, true) 435 progressReader := progress.NewProgressReader(resp.Body, progressOutput, resp.ContentLength, "", "Downloading") 436 // Download and dump result to tmp file 437 // TODO: add filehash directly 438 if _, err = io.Copy(tmpFile, progressReader); err != nil { 439 tmpFile.Close() 440 return 441 } 442 // TODO: how important is this random blank line to the output? 443 fmt.Fprintln(stdout) 444 445 // Set the mtime to the Last-Modified header value if present 446 // Otherwise just remove atime and mtime 447 mTime := time.Time{} 448 449 lastMod := resp.Header.Get("Last-Modified") 450 if lastMod != "" { 451 // If we can't parse it then just let it default to 'zero' 452 // otherwise use the parsed time value 453 if parsedMTime, err := http.ParseTime(lastMod); err == nil { 454 mTime = parsedMTime 455 } 456 } 457 458 tmpFile.Close() 459 460 if err = system.Chtimes(tmpFileName, mTime, mTime); err != nil { 461 return 462 } 463 464 lc, err := remotecontext.NewLazySource(containerfs.NewLocalContainerFS(tmpDir)) 465 return lc, filename, err 466 } 467 468 type copyFileOptions struct { 469 decompress bool 470 identity *idtools.Identity 471 archiver Archiver 472 } 473 474 type copyEndpoint struct { 475 driver containerfs.Driver 476 path string 477 } 478 479 func performCopyForInfo(dest copyInfo, source copyInfo, options copyFileOptions) error { 480 srcPath, err := source.fullPath() 481 if err != nil { 482 return err 483 } 484 485 destPath, err := dest.fullPath() 486 if err != nil { 487 return err 488 } 489 490 archiver := options.archiver 491 492 srcEndpoint := ©Endpoint{driver: source.root, path: srcPath} 493 destEndpoint := ©Endpoint{driver: dest.root, path: destPath} 494 495 src, err := source.root.Stat(srcPath) 496 if err != nil { 497 return errors.Wrapf(err, "source path not found") 498 } 499 if src.IsDir() { 500 return copyDirectory(archiver, srcEndpoint, destEndpoint, options.identity) 501 } 502 if options.decompress && isArchivePath(source.root, srcPath) && !source.noDecompress { 503 return archiver.UntarPath(srcPath, destPath) 504 } 505 506 destExistsAsDir, err := isExistingDirectory(destEndpoint) 507 if err != nil { 508 return err 509 } 510 // dest.path must be used because destPath has already been cleaned of any 511 // trailing slash 512 if endsInSlash(dest.root, dest.path) || destExistsAsDir { 513 // source.path must be used to get the correct filename when the source 514 // is a symlink 515 destPath = dest.root.Join(destPath, source.root.Base(source.path)) 516 destEndpoint = ©Endpoint{driver: dest.root, path: destPath} 517 } 518 return copyFile(archiver, srcEndpoint, destEndpoint, options.identity) 519 } 520 521 func isArchivePath(driver containerfs.ContainerFS, path string) bool { 522 file, err := driver.Open(path) 523 if err != nil { 524 return false 525 } 526 defer file.Close() 527 rdr, err := archive.DecompressStream(file) 528 if err != nil { 529 return false 530 } 531 r := tar.NewReader(rdr) 532 _, err = r.Next() 533 return err == nil 534 } 535 536 func copyDirectory(archiver Archiver, source, dest *copyEndpoint, identity *idtools.Identity) error { 537 destExists, err := isExistingDirectory(dest) 538 if err != nil { 539 return errors.Wrapf(err, "failed to query destination path") 540 } 541 542 if err := archiver.CopyWithTar(source.path, dest.path); err != nil { 543 return errors.Wrapf(err, "failed to copy directory") 544 } 545 if identity != nil { 546 // TODO: @gupta-ak. Investigate how LCOW permission mappings will work. 547 return fixPermissions(source.path, dest.path, *identity, !destExists) 548 } 549 return nil 550 } 551 552 func copyFile(archiver Archiver, source, dest *copyEndpoint, identity *idtools.Identity) error { 553 if runtime.GOOS == "windows" && dest.driver.OS() == "linux" { 554 // LCOW 555 if err := dest.driver.MkdirAll(dest.driver.Dir(dest.path), 0755); err != nil { 556 return errors.Wrapf(err, "failed to create new directory") 557 } 558 } else { 559 // Normal containers 560 if identity == nil { 561 // Use system.MkdirAll here, which is a custom version of os.MkdirAll 562 // modified for use on Windows to handle volume GUID paths. These paths 563 // are of the form \\?\Volume{<GUID>}\<path>. An example would be: 564 // \\?\Volume{dae8d3ac-b9a1-11e9-88eb-e8554b2ba1db}\bin\busybox.exe 565 566 if err := system.MkdirAll(filepath.Dir(dest.path), 0755); err != nil { 567 return err 568 } 569 } else { 570 if err := idtools.MkdirAllAndChownNew(filepath.Dir(dest.path), 0755, *identity); err != nil { 571 return errors.Wrapf(err, "failed to create new directory") 572 } 573 } 574 } 575 576 if err := archiver.CopyFileWithTar(source.path, dest.path); err != nil { 577 return errors.Wrapf(err, "failed to copy file") 578 } 579 if identity != nil { 580 // TODO: @gupta-ak. Investigate how LCOW permission mappings will work. 581 return fixPermissions(source.path, dest.path, *identity, false) 582 } 583 return nil 584 } 585 586 func endsInSlash(driver containerfs.Driver, path string) bool { 587 return strings.HasSuffix(path, string(driver.Separator())) 588 } 589 590 // isExistingDirectory returns true if the path exists and is a directory 591 func isExistingDirectory(point *copyEndpoint) (bool, error) { 592 destStat, err := point.driver.Stat(point.path) 593 switch { 594 case errors.Is(err, os.ErrNotExist): 595 return false, nil 596 case err != nil: 597 return false, err 598 } 599 return destStat.IsDir(), nil 600 }