gitee.com/bomy/docker.git@v1.13.1/builder/dockerfile/internals.go (about) 1 package dockerfile 2 3 // internals for handling commands. Covers many areas and a lot of 4 // non-contiguous functionality. Please read the comments. 5 6 import ( 7 "crypto/sha256" 8 "encoding/hex" 9 "errors" 10 "fmt" 11 "io" 12 "io/ioutil" 13 "net/http" 14 "net/url" 15 "os" 16 "path/filepath" 17 "sort" 18 "strings" 19 "time" 20 21 "github.com/Sirupsen/logrus" 22 "github.com/docker/docker/api/types" 23 "github.com/docker/docker/api/types/backend" 24 "github.com/docker/docker/api/types/container" 25 "github.com/docker/docker/api/types/strslice" 26 "github.com/docker/docker/builder" 27 "github.com/docker/docker/builder/dockerfile/parser" 28 "github.com/docker/docker/pkg/archive" 29 "github.com/docker/docker/pkg/httputils" 30 "github.com/docker/docker/pkg/ioutils" 31 "github.com/docker/docker/pkg/jsonmessage" 32 "github.com/docker/docker/pkg/progress" 33 "github.com/docker/docker/pkg/streamformatter" 34 "github.com/docker/docker/pkg/stringid" 35 "github.com/docker/docker/pkg/system" 36 "github.com/docker/docker/pkg/tarsum" 37 "github.com/docker/docker/pkg/urlutil" 38 "github.com/docker/docker/runconfig/opts" 39 ) 40 41 func (b *Builder) commit(id string, autoCmd strslice.StrSlice, comment string) error { 42 if b.disableCommit { 43 return nil 44 } 45 if b.image == "" && !b.noBaseImage { 46 return fmt.Errorf("Please provide a source image with `from` prior to commit") 47 } 48 b.runConfig.Image = b.image 49 50 if id == "" { 51 cmd := b.runConfig.Cmd 52 b.runConfig.Cmd = strslice.StrSlice(append(getShell(b.runConfig), "#(nop) ", comment)) 53 defer func(cmd strslice.StrSlice) { b.runConfig.Cmd = cmd }(cmd) 54 55 hit, err := b.probeCache() 56 if err != nil { 57 return err 58 } else if hit { 59 return nil 60 } 61 id, err = b.create() 62 if err != nil { 63 return err 64 } 65 } 66 67 // Note: Actually copy the struct 68 autoConfig := *b.runConfig 69 autoConfig.Cmd = autoCmd 70 71 commitCfg := &backend.ContainerCommitConfig{ 72 ContainerCommitConfig: types.ContainerCommitConfig{ 73 Author: b.maintainer, 74 Pause: true, 75 Config: &autoConfig, 76 }, 77 } 78 79 // Commit the container 80 imageID, err := b.docker.Commit(id, commitCfg) 81 if err != nil { 82 return err 83 } 84 85 b.image = imageID 86 return nil 87 } 88 89 type copyInfo struct { 90 builder.FileInfo 91 decompress bool 92 } 93 94 func (b *Builder) runContextCommand(args []string, allowRemote bool, allowLocalDecompression bool, cmdName string) error { 95 if b.context == nil { 96 return fmt.Errorf("No context given. Impossible to use %s", cmdName) 97 } 98 99 if len(args) < 2 { 100 return fmt.Errorf("Invalid %s format - at least two arguments required", cmdName) 101 } 102 103 // Work in daemon-specific filepath semantics 104 dest := filepath.FromSlash(args[len(args)-1]) // last one is always the dest 105 106 b.runConfig.Image = b.image 107 108 var infos []copyInfo 109 110 // Loop through each src file and calculate the info we need to 111 // do the copy (e.g. hash value if cached). Don't actually do 112 // the copy until we've looked at all src files 113 var err error 114 for _, orig := range args[0 : len(args)-1] { 115 var fi builder.FileInfo 116 decompress := allowLocalDecompression 117 if urlutil.IsURL(orig) { 118 if !allowRemote { 119 return fmt.Errorf("Source can't be a URL for %s", cmdName) 120 } 121 fi, err = b.download(orig) 122 if err != nil { 123 return err 124 } 125 defer os.RemoveAll(filepath.Dir(fi.Path())) 126 decompress = false 127 infos = append(infos, copyInfo{fi, decompress}) 128 continue 129 } 130 // not a URL 131 subInfos, err := b.calcCopyInfo(cmdName, orig, allowLocalDecompression, true) 132 if err != nil { 133 return err 134 } 135 136 infos = append(infos, subInfos...) 137 } 138 139 if len(infos) == 0 { 140 return fmt.Errorf("No source files were specified") 141 } 142 if len(infos) > 1 && !strings.HasSuffix(dest, string(os.PathSeparator)) { 143 return fmt.Errorf("When using %s with more than one source file, the destination must be a directory and end with a /", cmdName) 144 } 145 146 // For backwards compat, if there's just one info then use it as the 147 // cache look-up string, otherwise hash 'em all into one 148 var srcHash string 149 var origPaths string 150 151 if len(infos) == 1 { 152 fi := infos[0].FileInfo 153 origPaths = fi.Name() 154 if hfi, ok := fi.(builder.Hashed); ok { 155 srcHash = hfi.Hash() 156 } 157 } else { 158 var hashs []string 159 var origs []string 160 for _, info := range infos { 161 fi := info.FileInfo 162 origs = append(origs, fi.Name()) 163 if hfi, ok := fi.(builder.Hashed); ok { 164 hashs = append(hashs, hfi.Hash()) 165 } 166 } 167 hasher := sha256.New() 168 hasher.Write([]byte(strings.Join(hashs, ","))) 169 srcHash = "multi:" + hex.EncodeToString(hasher.Sum(nil)) 170 origPaths = strings.Join(origs, " ") 171 } 172 173 cmd := b.runConfig.Cmd 174 b.runConfig.Cmd = strslice.StrSlice(append(getShell(b.runConfig), fmt.Sprintf("#(nop) %s %s in %s ", cmdName, srcHash, dest))) 175 defer func(cmd strslice.StrSlice) { b.runConfig.Cmd = cmd }(cmd) 176 177 if hit, err := b.probeCache(); err != nil { 178 return err 179 } else if hit { 180 return nil 181 } 182 183 container, err := b.docker.ContainerCreate(types.ContainerCreateConfig{Config: b.runConfig}) 184 if err != nil { 185 return err 186 } 187 b.tmpContainers[container.ID] = struct{}{} 188 189 comment := fmt.Sprintf("%s %s in %s", cmdName, origPaths, dest) 190 191 // Twiddle the destination when its a relative path - meaning, make it 192 // relative to the WORKINGDIR 193 if dest, err = normaliseDest(cmdName, b.runConfig.WorkingDir, dest); err != nil { 194 return err 195 } 196 197 for _, info := range infos { 198 if err := b.docker.CopyOnBuild(container.ID, dest, info.FileInfo, info.decompress); err != nil { 199 return err 200 } 201 } 202 203 return b.commit(container.ID, cmd, comment) 204 } 205 206 func (b *Builder) download(srcURL string) (fi builder.FileInfo, err error) { 207 // get filename from URL 208 u, err := url.Parse(srcURL) 209 if err != nil { 210 return 211 } 212 path := filepath.FromSlash(u.Path) // Ensure in platform semantics 213 if strings.HasSuffix(path, string(os.PathSeparator)) { 214 path = path[:len(path)-1] 215 } 216 parts := strings.Split(path, string(os.PathSeparator)) 217 filename := parts[len(parts)-1] 218 if filename == "" { 219 err = fmt.Errorf("cannot determine filename from url: %s", u) 220 return 221 } 222 223 // Initiate the download 224 resp, err := httputils.Download(srcURL) 225 if err != nil { 226 return 227 } 228 229 // Prepare file in a tmp dir 230 tmpDir, err := ioutils.TempDir("", "docker-remote") 231 if err != nil { 232 return 233 } 234 defer func() { 235 if err != nil { 236 os.RemoveAll(tmpDir) 237 } 238 }() 239 tmpFileName := filepath.Join(tmpDir, filename) 240 tmpFile, err := os.OpenFile(tmpFileName, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0600) 241 if err != nil { 242 return 243 } 244 245 stdoutFormatter := b.Stdout.(*streamformatter.StdoutFormatter) 246 progressOutput := stdoutFormatter.StreamFormatter.NewProgressOutput(stdoutFormatter.Writer, true) 247 progressReader := progress.NewProgressReader(resp.Body, progressOutput, resp.ContentLength, "", "Downloading") 248 // Download and dump result to tmp file 249 if _, err = io.Copy(tmpFile, progressReader); err != nil { 250 tmpFile.Close() 251 return 252 } 253 fmt.Fprintln(b.Stdout) 254 // ignoring error because the file was already opened successfully 255 tmpFileSt, err := tmpFile.Stat() 256 if err != nil { 257 tmpFile.Close() 258 return 259 } 260 261 // Set the mtime to the Last-Modified header value if present 262 // Otherwise just remove atime and mtime 263 mTime := time.Time{} 264 265 lastMod := resp.Header.Get("Last-Modified") 266 if lastMod != "" { 267 // If we can't parse it then just let it default to 'zero' 268 // otherwise use the parsed time value 269 if parsedMTime, err := http.ParseTime(lastMod); err == nil { 270 mTime = parsedMTime 271 } 272 } 273 274 tmpFile.Close() 275 276 if err = system.Chtimes(tmpFileName, mTime, mTime); err != nil { 277 return 278 } 279 280 // Calc the checksum, even if we're using the cache 281 r, err := archive.Tar(tmpFileName, archive.Uncompressed) 282 if err != nil { 283 return 284 } 285 tarSum, err := tarsum.NewTarSum(r, true, tarsum.Version1) 286 if err != nil { 287 return 288 } 289 if _, err = io.Copy(ioutil.Discard, tarSum); err != nil { 290 return 291 } 292 hash := tarSum.Sum(nil) 293 r.Close() 294 return &builder.HashedFileInfo{FileInfo: builder.PathFileInfo{FileInfo: tmpFileSt, FilePath: tmpFileName}, FileHash: hash}, nil 295 } 296 297 func (b *Builder) calcCopyInfo(cmdName, origPath string, allowLocalDecompression, allowWildcards bool) ([]copyInfo, error) { 298 299 // Work in daemon-specific OS filepath semantics 300 origPath = filepath.FromSlash(origPath) 301 302 if origPath != "" && origPath[0] == os.PathSeparator && len(origPath) > 1 { 303 origPath = origPath[1:] 304 } 305 origPath = strings.TrimPrefix(origPath, "."+string(os.PathSeparator)) 306 307 // Deal with wildcards 308 if allowWildcards && containsWildcards(origPath) { 309 var copyInfos []copyInfo 310 if err := b.context.Walk("", func(path string, info builder.FileInfo, err error) error { 311 if err != nil { 312 return err 313 } 314 if info.Name() == "" { 315 // Why are we doing this check? 316 return nil 317 } 318 if match, _ := filepath.Match(origPath, path); !match { 319 return nil 320 } 321 322 // Note we set allowWildcards to false in case the name has 323 // a * in it 324 subInfos, err := b.calcCopyInfo(cmdName, path, allowLocalDecompression, false) 325 if err != nil { 326 return err 327 } 328 copyInfos = append(copyInfos, subInfos...) 329 return nil 330 }); err != nil { 331 return nil, err 332 } 333 return copyInfos, nil 334 } 335 336 // Must be a dir or a file 337 338 statPath, fi, err := b.context.Stat(origPath) 339 if err != nil { 340 return nil, err 341 } 342 343 copyInfos := []copyInfo{{FileInfo: fi, decompress: allowLocalDecompression}} 344 345 hfi, handleHash := fi.(builder.Hashed) 346 if !handleHash { 347 return copyInfos, nil 348 } 349 350 // Deal with the single file case 351 if !fi.IsDir() { 352 hfi.SetHash("file:" + hfi.Hash()) 353 return copyInfos, nil 354 } 355 // Must be a dir 356 var subfiles []string 357 err = b.context.Walk(statPath, func(path string, info builder.FileInfo, err error) error { 358 if err != nil { 359 return err 360 } 361 // we already checked handleHash above 362 subfiles = append(subfiles, info.(builder.Hashed).Hash()) 363 return nil 364 }) 365 if err != nil { 366 return nil, err 367 } 368 369 sort.Strings(subfiles) 370 hasher := sha256.New() 371 hasher.Write([]byte(strings.Join(subfiles, ","))) 372 hfi.SetHash("dir:" + hex.EncodeToString(hasher.Sum(nil))) 373 374 return copyInfos, nil 375 } 376 377 func (b *Builder) processImageFrom(img builder.Image) error { 378 if img != nil { 379 b.image = img.ImageID() 380 381 if img.RunConfig() != nil { 382 b.runConfig = img.RunConfig() 383 } 384 } 385 386 // Check to see if we have a default PATH, note that windows won't 387 // have one as its set by HCS 388 if system.DefaultPathEnv != "" { 389 // Convert the slice of strings that represent the current list 390 // of env vars into a map so we can see if PATH is already set. 391 // If its not set then go ahead and give it our default value 392 configEnv := opts.ConvertKVStringsToMap(b.runConfig.Env) 393 if _, ok := configEnv["PATH"]; !ok { 394 b.runConfig.Env = append(b.runConfig.Env, 395 "PATH="+system.DefaultPathEnv) 396 } 397 } 398 399 if img == nil { 400 // Typically this means they used "FROM scratch" 401 return nil 402 } 403 404 // Process ONBUILD triggers if they exist 405 if nTriggers := len(b.runConfig.OnBuild); nTriggers != 0 { 406 word := "trigger" 407 if nTriggers > 1 { 408 word = "triggers" 409 } 410 fmt.Fprintf(b.Stderr, "# Executing %d build %s...\n", nTriggers, word) 411 } 412 413 // Copy the ONBUILD triggers, and remove them from the config, since the config will be committed. 414 onBuildTriggers := b.runConfig.OnBuild 415 b.runConfig.OnBuild = []string{} 416 417 // parse the ONBUILD triggers by invoking the parser 418 for _, step := range onBuildTriggers { 419 ast, err := parser.Parse(strings.NewReader(step), &b.directive) 420 if err != nil { 421 return err 422 } 423 424 total := len(ast.Children) 425 for _, n := range ast.Children { 426 if err := b.checkDispatch(n, true); err != nil { 427 return err 428 } 429 } 430 for i, n := range ast.Children { 431 if err := b.dispatch(i, total, n); err != nil { 432 return err 433 } 434 } 435 } 436 437 return nil 438 } 439 440 // probeCache checks if cache match can be found for current build instruction. 441 // If an image is found, probeCache returns `(true, nil)`. 442 // If no image is found, it returns `(false, nil)`. 443 // If there is any error, it returns `(false, err)`. 444 func (b *Builder) probeCache() (bool, error) { 445 c := b.imageCache 446 if c == nil || b.options.NoCache || b.cacheBusted { 447 return false, nil 448 } 449 cache, err := c.GetCache(b.image, b.runConfig) 450 if err != nil { 451 return false, err 452 } 453 if len(cache) == 0 { 454 logrus.Debugf("[BUILDER] Cache miss: %s", b.runConfig.Cmd) 455 b.cacheBusted = true 456 return false, nil 457 } 458 459 fmt.Fprintf(b.Stdout, " ---> Using cache\n") 460 logrus.Debugf("[BUILDER] Use cached version: %s", b.runConfig.Cmd) 461 b.image = string(cache) 462 463 return true, nil 464 } 465 466 func (b *Builder) create() (string, error) { 467 if b.image == "" && !b.noBaseImage { 468 return "", fmt.Errorf("Please provide a source image with `from` prior to run") 469 } 470 b.runConfig.Image = b.image 471 472 resources := container.Resources{ 473 CgroupParent: b.options.CgroupParent, 474 CPUShares: b.options.CPUShares, 475 CPUPeriod: b.options.CPUPeriod, 476 CPUQuota: b.options.CPUQuota, 477 CpusetCpus: b.options.CPUSetCPUs, 478 CpusetMems: b.options.CPUSetMems, 479 Memory: b.options.Memory, 480 MemorySwap: b.options.MemorySwap, 481 Ulimits: b.options.Ulimits, 482 } 483 484 // TODO: why not embed a hostconfig in builder? 485 hostConfig := &container.HostConfig{ 486 SecurityOpt: b.options.SecurityOpt, 487 Isolation: b.options.Isolation, 488 ShmSize: b.options.ShmSize, 489 Resources: resources, 490 NetworkMode: container.NetworkMode(b.options.NetworkMode), 491 } 492 493 config := *b.runConfig 494 495 // Create the container 496 c, err := b.docker.ContainerCreate(types.ContainerCreateConfig{ 497 Config: b.runConfig, 498 HostConfig: hostConfig, 499 }) 500 if err != nil { 501 return "", err 502 } 503 for _, warning := range c.Warnings { 504 fmt.Fprintf(b.Stdout, " ---> [Warning] %s\n", warning) 505 } 506 507 b.tmpContainers[c.ID] = struct{}{} 508 fmt.Fprintf(b.Stdout, " ---> Running in %s\n", stringid.TruncateID(c.ID)) 509 510 // override the entry point that may have been picked up from the base image 511 if err := b.docker.ContainerUpdateCmdOnBuild(c.ID, config.Cmd); err != nil { 512 return "", err 513 } 514 515 return c.ID, nil 516 } 517 518 var errCancelled = errors.New("build cancelled") 519 520 func (b *Builder) run(cID string) (err error) { 521 errCh := make(chan error) 522 go func() { 523 errCh <- b.docker.ContainerAttachRaw(cID, nil, b.Stdout, b.Stderr, true) 524 }() 525 526 finished := make(chan struct{}) 527 cancelErrCh := make(chan error, 1) 528 go func() { 529 select { 530 case <-b.clientCtx.Done(): 531 logrus.Debugln("Build cancelled, killing and removing container:", cID) 532 b.docker.ContainerKill(cID, 0) 533 b.removeContainer(cID) 534 cancelErrCh <- errCancelled 535 case <-finished: 536 cancelErrCh <- nil 537 } 538 }() 539 540 if err := b.docker.ContainerStart(cID, nil, "", ""); err != nil { 541 close(finished) 542 if cancelErr := <-cancelErrCh; cancelErr != nil { 543 logrus.Debugf("Build cancelled (%v) and got an error from ContainerStart: %v", 544 cancelErr, err) 545 } 546 return err 547 } 548 549 // Block on reading output from container, stop on err or chan closed 550 if err := <-errCh; err != nil { 551 close(finished) 552 if cancelErr := <-cancelErrCh; cancelErr != nil { 553 logrus.Debugf("Build cancelled (%v) and got an error from errCh: %v", 554 cancelErr, err) 555 } 556 return err 557 } 558 559 if ret, _ := b.docker.ContainerWait(cID, -1); ret != 0 { 560 close(finished) 561 if cancelErr := <-cancelErrCh; cancelErr != nil { 562 logrus.Debugf("Build cancelled (%v) and got a non-zero code from ContainerWait: %d", 563 cancelErr, ret) 564 } 565 // TODO: change error type, because jsonmessage.JSONError assumes HTTP 566 return &jsonmessage.JSONError{ 567 Message: fmt.Sprintf("The command '%s' returned a non-zero code: %d", strings.Join(b.runConfig.Cmd, " "), ret), 568 Code: ret, 569 } 570 } 571 close(finished) 572 return <-cancelErrCh 573 } 574 575 func (b *Builder) removeContainer(c string) error { 576 rmConfig := &types.ContainerRmConfig{ 577 ForceRemove: true, 578 RemoveVolume: true, 579 } 580 if err := b.docker.ContainerRm(c, rmConfig); err != nil { 581 fmt.Fprintf(b.Stdout, "Error removing intermediate container %s: %v\n", stringid.TruncateID(c), err) 582 return err 583 } 584 return nil 585 } 586 587 func (b *Builder) clearTmp() { 588 for c := range b.tmpContainers { 589 if err := b.removeContainer(c); err != nil { 590 return 591 } 592 delete(b.tmpContainers, c) 593 fmt.Fprintf(b.Stdout, "Removing intermediate container %s\n", stringid.TruncateID(c)) 594 } 595 } 596 597 // readDockerfile reads a Dockerfile from the current context. 598 func (b *Builder) readDockerfile() error { 599 // If no -f was specified then look for 'Dockerfile'. If we can't find 600 // that then look for 'dockerfile'. If neither are found then default 601 // back to 'Dockerfile' and use that in the error message. 602 if b.options.Dockerfile == "" { 603 b.options.Dockerfile = builder.DefaultDockerfileName 604 if _, _, err := b.context.Stat(b.options.Dockerfile); os.IsNotExist(err) { 605 lowercase := strings.ToLower(b.options.Dockerfile) 606 if _, _, err := b.context.Stat(lowercase); err == nil { 607 b.options.Dockerfile = lowercase 608 } 609 } 610 } 611 612 err := b.parseDockerfile() 613 614 if err != nil { 615 return err 616 } 617 618 // After the Dockerfile has been parsed, we need to check the .dockerignore 619 // file for either "Dockerfile" or ".dockerignore", and if either are 620 // present then erase them from the build context. These files should never 621 // have been sent from the client but we did send them to make sure that 622 // we had the Dockerfile to actually parse, and then we also need the 623 // .dockerignore file to know whether either file should be removed. 624 // Note that this assumes the Dockerfile has been read into memory and 625 // is now safe to be removed. 626 if dockerIgnore, ok := b.context.(builder.DockerIgnoreContext); ok { 627 dockerIgnore.Process([]string{b.options.Dockerfile}) 628 } 629 return nil 630 } 631 632 func (b *Builder) parseDockerfile() error { 633 f, err := b.context.Open(b.options.Dockerfile) 634 if err != nil { 635 if os.IsNotExist(err) { 636 return fmt.Errorf("Cannot locate specified Dockerfile: %s", b.options.Dockerfile) 637 } 638 return err 639 } 640 defer f.Close() 641 if f, ok := f.(*os.File); ok { 642 // ignoring error because Open already succeeded 643 fi, err := f.Stat() 644 if err != nil { 645 return fmt.Errorf("Unexpected error reading Dockerfile: %v", err) 646 } 647 if fi.Size() == 0 { 648 return fmt.Errorf("The Dockerfile (%s) cannot be empty", b.options.Dockerfile) 649 } 650 } 651 b.dockerfile, err = parser.Parse(f, &b.directive) 652 if err != nil { 653 return err 654 } 655 656 return nil 657 } 658 659 // determine if build arg is part of built-in args or user 660 // defined args in Dockerfile at any point in time. 661 func (b *Builder) isBuildArgAllowed(arg string) bool { 662 if _, ok := BuiltinAllowedBuildArgs[arg]; ok { 663 return true 664 } 665 if _, ok := b.allowedBuildArgs[arg]; ok { 666 return true 667 } 668 return false 669 }