github.com/shishir-a412ed/docker@v1.3.2-0.20180103180333-fda904911d87/builder/dockerfile/internals.go (about) 1 package dockerfile 2 3 // internals for handling commands. Covers many areas and a lot of 4 // non-contiguous functionality. Please read the comments. 5 6 import ( 7 "crypto/sha256" 8 "encoding/hex" 9 "fmt" 10 "io" 11 "os" 12 "path" 13 "path/filepath" 14 "strconv" 15 "strings" 16 17 "github.com/docker/docker/api/types" 18 "github.com/docker/docker/api/types/backend" 19 "github.com/docker/docker/api/types/container" 20 "github.com/docker/docker/image" 21 "github.com/docker/docker/pkg/archive" 22 "github.com/docker/docker/pkg/chrootarchive" 23 "github.com/docker/docker/pkg/containerfs" 24 "github.com/docker/docker/pkg/idtools" 25 "github.com/docker/docker/pkg/stringid" 26 "github.com/docker/docker/pkg/symlink" 27 "github.com/docker/docker/pkg/system" 28 "github.com/docker/go-connections/nat" 29 lcUser "github.com/opencontainers/runc/libcontainer/user" 30 "github.com/pkg/errors" 31 ) 32 33 // Archiver defines an interface for copying files from one destination to 34 // another using Tar/Untar. 35 type Archiver interface { 36 TarUntar(src, dst string) error 37 UntarPath(src, dst string) error 38 CopyWithTar(src, dst string) error 39 CopyFileWithTar(src, dst string) error 40 IDMappings() *idtools.IDMappings 41 } 42 43 // The builder will use the following interfaces if the container fs implements 44 // these for optimized copies to and from the container. 45 type extractor interface { 46 ExtractArchive(src io.Reader, dst string, opts *archive.TarOptions) error 47 } 48 49 type archiver interface { 50 ArchivePath(src string, opts *archive.TarOptions) (io.ReadCloser, error) 51 } 52 53 // helper functions to get tar/untar func 54 func untarFunc(i interface{}) containerfs.UntarFunc { 55 if ea, ok := i.(extractor); ok { 56 return ea.ExtractArchive 57 } 58 return chrootarchive.Untar 59 } 60 61 func tarFunc(i interface{}) containerfs.TarFunc { 62 if ap, ok := i.(archiver); ok { 63 return ap.ArchivePath 64 } 65 return archive.TarWithOptions 66 } 67 68 func (b *Builder) getArchiver(src, dst containerfs.Driver) Archiver { 69 t, u := tarFunc(src), untarFunc(dst) 70 return &containerfs.Archiver{ 71 SrcDriver: src, 72 DstDriver: dst, 73 Tar: t, 74 Untar: u, 75 IDMappingsVar: b.idMappings, 76 } 77 } 78 79 func (b *Builder) commit(dispatchState *dispatchState, comment string) error { 80 if b.disableCommit { 81 return nil 82 } 83 if !dispatchState.hasFromImage() { 84 return errors.New("Please provide a source image with `from` prior to commit") 85 } 86 87 optionsPlatform := system.ParsePlatform(b.options.Platform) 88 runConfigWithCommentCmd := copyRunConfig(dispatchState.runConfig, withCmdComment(comment, optionsPlatform.OS)) 89 hit, err := b.probeCache(dispatchState, runConfigWithCommentCmd) 90 if err != nil || hit { 91 return err 92 } 93 id, err := b.create(runConfigWithCommentCmd) 94 if err != nil { 95 return err 96 } 97 98 return b.commitContainer(dispatchState, id, runConfigWithCommentCmd) 99 } 100 101 func (b *Builder) commitContainer(dispatchState *dispatchState, id string, containerConfig *container.Config) error { 102 if b.disableCommit { 103 return nil 104 } 105 106 commitCfg := &backend.ContainerCommitConfig{ 107 ContainerCommitConfig: types.ContainerCommitConfig{ 108 Author: dispatchState.maintainer, 109 Pause: true, 110 // TODO: this should be done by Commit() 111 Config: copyRunConfig(dispatchState.runConfig), 112 }, 113 ContainerConfig: containerConfig, 114 } 115 116 // Commit the container 117 imageID, err := b.docker.Commit(id, commitCfg) 118 if err != nil { 119 return err 120 } 121 122 dispatchState.imageID = imageID 123 return nil 124 } 125 126 func (b *Builder) exportImage(state *dispatchState, imageMount *imageMount, runConfig *container.Config) error { 127 optionsPlatform := system.ParsePlatform(b.options.Platform) 128 newLayer, err := imageMount.Layer().Commit(optionsPlatform.OS) 129 if err != nil { 130 return err 131 } 132 133 // add an image mount without an image so the layer is properly unmounted 134 // if there is an error before we can add the full mount with image 135 b.imageSources.Add(newImageMount(nil, newLayer)) 136 137 parentImage, ok := imageMount.Image().(*image.Image) 138 if !ok { 139 return errors.Errorf("unexpected image type") 140 } 141 142 newImage := image.NewChildImage(parentImage, image.ChildConfig{ 143 Author: state.maintainer, 144 ContainerConfig: runConfig, 145 DiffID: newLayer.DiffID(), 146 Config: copyRunConfig(state.runConfig), 147 }, parentImage.OS) 148 149 // TODO: it seems strange to marshal this here instead of just passing in the 150 // image struct 151 config, err := newImage.MarshalJSON() 152 if err != nil { 153 return errors.Wrap(err, "failed to encode image config") 154 } 155 156 exportedImage, err := b.docker.CreateImage(config, state.imageID, parentImage.OS) 157 if err != nil { 158 return errors.Wrapf(err, "failed to export image") 159 } 160 161 state.imageID = exportedImage.ImageID() 162 b.imageSources.Add(newImageMount(exportedImage, newLayer)) 163 return nil 164 } 165 166 func (b *Builder) performCopy(state *dispatchState, inst copyInstruction) error { 167 srcHash := getSourceHashFromInfos(inst.infos) 168 169 var chownComment string 170 if inst.chownStr != "" { 171 chownComment = fmt.Sprintf("--chown=%s", inst.chownStr) 172 } 173 commentStr := fmt.Sprintf("%s %s%s in %s ", inst.cmdName, chownComment, srcHash, inst.dest) 174 175 // TODO: should this have been using origPaths instead of srcHash in the comment? 176 optionsPlatform := system.ParsePlatform(b.options.Platform) 177 runConfigWithCommentCmd := copyRunConfig( 178 state.runConfig, 179 withCmdCommentString(commentStr, optionsPlatform.OS)) 180 hit, err := b.probeCache(state, runConfigWithCommentCmd) 181 if err != nil || hit { 182 return err 183 } 184 185 imageMount, err := b.imageSources.Get(state.imageID, true) 186 if err != nil { 187 return errors.Wrapf(err, "failed to get destination image %q", state.imageID) 188 } 189 190 destInfo, err := createDestInfo(state.runConfig.WorkingDir, inst, imageMount, b.options.Platform) 191 if err != nil { 192 return err 193 } 194 195 chownPair := b.idMappings.RootPair() 196 // if a chown was requested, perform the steps to get the uid, gid 197 // translated (if necessary because of user namespaces), and replace 198 // the root pair with the chown pair for copy operations 199 if inst.chownStr != "" { 200 chownPair, err = parseChownFlag(inst.chownStr, destInfo.root.Path(), b.idMappings) 201 if err != nil { 202 return errors.Wrapf(err, "unable to convert uid/gid chown string to host mapping") 203 } 204 } 205 206 for _, info := range inst.infos { 207 opts := copyFileOptions{ 208 decompress: inst.allowLocalDecompression, 209 archiver: b.getArchiver(info.root, destInfo.root), 210 chownPair: chownPair, 211 } 212 if err := performCopyForInfo(destInfo, info, opts); err != nil { 213 return errors.Wrapf(err, "failed to copy files") 214 } 215 } 216 return b.exportImage(state, imageMount, runConfigWithCommentCmd) 217 } 218 219 func parseChownFlag(chown, ctrRootPath string, idMappings *idtools.IDMappings) (idtools.IDPair, error) { 220 var userStr, grpStr string 221 parts := strings.Split(chown, ":") 222 if len(parts) > 2 { 223 return idtools.IDPair{}, errors.New("invalid chown string format: " + chown) 224 } 225 if len(parts) == 1 { 226 // if no group specified, use the user spec as group as well 227 userStr, grpStr = parts[0], parts[0] 228 } else { 229 userStr, grpStr = parts[0], parts[1] 230 } 231 232 passwdPath, err := symlink.FollowSymlinkInScope(filepath.Join(ctrRootPath, "etc", "passwd"), ctrRootPath) 233 if err != nil { 234 return idtools.IDPair{}, errors.Wrapf(err, "can't resolve /etc/passwd path in container rootfs") 235 } 236 groupPath, err := symlink.FollowSymlinkInScope(filepath.Join(ctrRootPath, "etc", "group"), ctrRootPath) 237 if err != nil { 238 return idtools.IDPair{}, errors.Wrapf(err, "can't resolve /etc/group path in container rootfs") 239 } 240 uid, err := lookupUser(userStr, passwdPath) 241 if err != nil { 242 return idtools.IDPair{}, errors.Wrapf(err, "can't find uid for user "+userStr) 243 } 244 gid, err := lookupGroup(grpStr, groupPath) 245 if err != nil { 246 return idtools.IDPair{}, errors.Wrapf(err, "can't find gid for group "+grpStr) 247 } 248 249 // convert as necessary because of user namespaces 250 chownPair, err := idMappings.ToHost(idtools.IDPair{UID: uid, GID: gid}) 251 if err != nil { 252 return idtools.IDPair{}, errors.Wrapf(err, "unable to convert uid/gid to host mapping") 253 } 254 return chownPair, nil 255 } 256 257 func lookupUser(userStr, filepath string) (int, error) { 258 // if the string is actually a uid integer, parse to int and return 259 // as we don't need to translate with the help of files 260 uid, err := strconv.Atoi(userStr) 261 if err == nil { 262 return uid, nil 263 } 264 users, err := lcUser.ParsePasswdFileFilter(filepath, func(u lcUser.User) bool { 265 return u.Name == userStr 266 }) 267 if err != nil { 268 return 0, err 269 } 270 if len(users) == 0 { 271 return 0, errors.New("no such user: " + userStr) 272 } 273 return users[0].Uid, nil 274 } 275 276 func lookupGroup(groupStr, filepath string) (int, error) { 277 // if the string is actually a gid integer, parse to int and return 278 // as we don't need to translate with the help of files 279 gid, err := strconv.Atoi(groupStr) 280 if err == nil { 281 return gid, nil 282 } 283 groups, err := lcUser.ParseGroupFileFilter(filepath, func(g lcUser.Group) bool { 284 return g.Name == groupStr 285 }) 286 if err != nil { 287 return 0, err 288 } 289 if len(groups) == 0 { 290 return 0, errors.New("no such group: " + groupStr) 291 } 292 return groups[0].Gid, nil 293 } 294 295 func createDestInfo(workingDir string, inst copyInstruction, imageMount *imageMount, platform string) (copyInfo, error) { 296 // Twiddle the destination when it's a relative path - meaning, make it 297 // relative to the WORKINGDIR 298 dest, err := normalizeDest(workingDir, inst.dest, platform) 299 if err != nil { 300 return copyInfo{}, errors.Wrapf(err, "invalid %s", inst.cmdName) 301 } 302 303 destMount, err := imageMount.Source() 304 if err != nil { 305 return copyInfo{}, errors.Wrapf(err, "failed to mount copy source") 306 } 307 308 return newCopyInfoFromSource(destMount, dest, ""), nil 309 } 310 311 // normalizeDest normalises the destination of a COPY/ADD command in a 312 // platform semantically consistent way. 313 func normalizeDest(workingDir, requested string, platform string) (string, error) { 314 dest := fromSlash(requested, platform) 315 endsInSlash := strings.HasSuffix(dest, string(separator(platform))) 316 317 if platform != "windows" { 318 if !path.IsAbs(requested) { 319 dest = path.Join("/", filepath.ToSlash(workingDir), dest) 320 // Make sure we preserve any trailing slash 321 if endsInSlash { 322 dest += "/" 323 } 324 } 325 return dest, nil 326 } 327 328 // We are guaranteed that the working directory is already consistent, 329 // However, Windows also has, for now, the limitation that ADD/COPY can 330 // only be done to the system drive, not any drives that might be present 331 // as a result of a bind mount. 332 // 333 // So... if the path requested is Linux-style absolute (/foo or \\foo), 334 // we assume it is the system drive. If it is a Windows-style absolute 335 // (DRIVE:\\foo), error if DRIVE is not C. And finally, ensure we 336 // strip any configured working directories drive letter so that it 337 // can be subsequently legitimately converted to a Windows volume-style 338 // pathname. 339 340 // Not a typo - filepath.IsAbs, not system.IsAbs on this next check as 341 // we only want to validate where the DriveColon part has been supplied. 342 if filepath.IsAbs(dest) { 343 if strings.ToUpper(string(dest[0])) != "C" { 344 return "", fmt.Errorf("Windows does not support destinations not on the system drive (C:)") 345 } 346 dest = dest[2:] // Strip the drive letter 347 } 348 349 // Cannot handle relative where WorkingDir is not the system drive. 350 if len(workingDir) > 0 { 351 if ((len(workingDir) > 1) && !system.IsAbs(workingDir[2:])) || (len(workingDir) == 1) { 352 return "", fmt.Errorf("Current WorkingDir %s is not platform consistent", workingDir) 353 } 354 if !system.IsAbs(dest) { 355 if string(workingDir[0]) != "C" { 356 return "", fmt.Errorf("Windows does not support relative paths when WORKDIR is not the system drive") 357 } 358 dest = filepath.Join(string(os.PathSeparator), workingDir[2:], dest) 359 // Make sure we preserve any trailing slash 360 if endsInSlash { 361 dest += string(os.PathSeparator) 362 } 363 } 364 } 365 return dest, nil 366 } 367 368 // For backwards compat, if there's just one info then use it as the 369 // cache look-up string, otherwise hash 'em all into one 370 func getSourceHashFromInfos(infos []copyInfo) string { 371 if len(infos) == 1 { 372 return infos[0].hash 373 } 374 var hashs []string 375 for _, info := range infos { 376 hashs = append(hashs, info.hash) 377 } 378 return hashStringSlice("multi", hashs) 379 } 380 381 func hashStringSlice(prefix string, slice []string) string { 382 hasher := sha256.New() 383 hasher.Write([]byte(strings.Join(slice, ","))) 384 return prefix + ":" + hex.EncodeToString(hasher.Sum(nil)) 385 } 386 387 type runConfigModifier func(*container.Config) 388 389 func withCmd(cmd []string) runConfigModifier { 390 return func(runConfig *container.Config) { 391 runConfig.Cmd = cmd 392 } 393 } 394 395 // withCmdComment sets Cmd to a nop comment string. See withCmdCommentString for 396 // why there are two almost identical versions of this. 397 func withCmdComment(comment string, platform string) runConfigModifier { 398 return func(runConfig *container.Config) { 399 runConfig.Cmd = append(getShell(runConfig, platform), "#(nop) ", comment) 400 } 401 } 402 403 // withCmdCommentString exists to maintain compatibility with older versions. 404 // A few instructions (workdir, copy, add) used a nop comment that is a single arg 405 // where as all the other instructions used a two arg comment string. This 406 // function implements the single arg version. 407 func withCmdCommentString(comment string, platform string) runConfigModifier { 408 return func(runConfig *container.Config) { 409 runConfig.Cmd = append(getShell(runConfig, platform), "#(nop) "+comment) 410 } 411 } 412 413 func withEnv(env []string) runConfigModifier { 414 return func(runConfig *container.Config) { 415 runConfig.Env = env 416 } 417 } 418 419 // withEntrypointOverride sets an entrypoint on runConfig if the command is 420 // not empty. The entrypoint is left unmodified if command is empty. 421 // 422 // The dockerfile RUN instruction expect to run without an entrypoint 423 // so the runConfig entrypoint needs to be modified accordingly. ContainerCreate 424 // will change a []string{""} entrypoint to nil, so we probe the cache with the 425 // nil entrypoint. 426 func withEntrypointOverride(cmd []string, entrypoint []string) runConfigModifier { 427 return func(runConfig *container.Config) { 428 if len(cmd) > 0 { 429 runConfig.Entrypoint = entrypoint 430 } 431 } 432 } 433 434 func copyRunConfig(runConfig *container.Config, modifiers ...runConfigModifier) *container.Config { 435 copy := *runConfig 436 copy.Cmd = copyStringSlice(runConfig.Cmd) 437 copy.Env = copyStringSlice(runConfig.Env) 438 copy.Entrypoint = copyStringSlice(runConfig.Entrypoint) 439 copy.OnBuild = copyStringSlice(runConfig.OnBuild) 440 copy.Shell = copyStringSlice(runConfig.Shell) 441 442 if copy.Volumes != nil { 443 copy.Volumes = make(map[string]struct{}, len(runConfig.Volumes)) 444 for k, v := range runConfig.Volumes { 445 copy.Volumes[k] = v 446 } 447 } 448 449 if copy.ExposedPorts != nil { 450 copy.ExposedPorts = make(nat.PortSet, len(runConfig.ExposedPorts)) 451 for k, v := range runConfig.ExposedPorts { 452 copy.ExposedPorts[k] = v 453 } 454 } 455 456 if copy.Labels != nil { 457 copy.Labels = make(map[string]string, len(runConfig.Labels)) 458 for k, v := range runConfig.Labels { 459 copy.Labels[k] = v 460 } 461 } 462 463 for _, modifier := range modifiers { 464 modifier(©) 465 } 466 return © 467 } 468 469 func copyStringSlice(orig []string) []string { 470 if orig == nil { 471 return nil 472 } 473 return append([]string{}, orig...) 474 } 475 476 // getShell is a helper function which gets the right shell for prefixing the 477 // shell-form of RUN, ENTRYPOINT and CMD instructions 478 func getShell(c *container.Config, os string) []string { 479 if 0 == len(c.Shell) { 480 return append([]string{}, defaultShellForOS(os)[:]...) 481 } 482 return append([]string{}, c.Shell[:]...) 483 } 484 485 func (b *Builder) probeCache(dispatchState *dispatchState, runConfig *container.Config) (bool, error) { 486 cachedID, err := b.imageProber.Probe(dispatchState.imageID, runConfig) 487 if cachedID == "" || err != nil { 488 return false, err 489 } 490 fmt.Fprint(b.Stdout, " ---> Using cache\n") 491 492 dispatchState.imageID = cachedID 493 return true, nil 494 } 495 496 var defaultLogConfig = container.LogConfig{Type: "none"} 497 498 func (b *Builder) probeAndCreate(dispatchState *dispatchState, runConfig *container.Config) (string, error) { 499 if hit, err := b.probeCache(dispatchState, runConfig); err != nil || hit { 500 return "", err 501 } 502 // Set a log config to override any default value set on the daemon 503 hostConfig := &container.HostConfig{LogConfig: defaultLogConfig} 504 optionsPlatform := system.ParsePlatform(b.options.Platform) 505 container, err := b.containerManager.Create(runConfig, hostConfig, optionsPlatform.OS) 506 return container.ID, err 507 } 508 509 func (b *Builder) create(runConfig *container.Config) (string, error) { 510 hostConfig := hostConfigFromOptions(b.options) 511 optionsPlatform := system.ParsePlatform(b.options.Platform) 512 container, err := b.containerManager.Create(runConfig, hostConfig, optionsPlatform.OS) 513 if err != nil { 514 return "", err 515 } 516 // TODO: could this be moved into containerManager.Create() ? 517 for _, warning := range container.Warnings { 518 fmt.Fprintf(b.Stdout, " ---> [Warning] %s\n", warning) 519 } 520 fmt.Fprintf(b.Stdout, " ---> Running in %s\n", stringid.TruncateID(container.ID)) 521 return container.ID, nil 522 } 523 524 func hostConfigFromOptions(options *types.ImageBuildOptions) *container.HostConfig { 525 resources := container.Resources{ 526 CgroupParent: options.CgroupParent, 527 CPUShares: options.CPUShares, 528 CPUPeriod: options.CPUPeriod, 529 CPUQuota: options.CPUQuota, 530 CpusetCpus: options.CPUSetCPUs, 531 CpusetMems: options.CPUSetMems, 532 Memory: options.Memory, 533 MemorySwap: options.MemorySwap, 534 Ulimits: options.Ulimits, 535 } 536 537 return &container.HostConfig{ 538 SecurityOpt: options.SecurityOpt, 539 Isolation: options.Isolation, 540 ShmSize: options.ShmSize, 541 Resources: resources, 542 NetworkMode: container.NetworkMode(options.NetworkMode), 543 // Set a log config to override any default value set on the daemon 544 LogConfig: defaultLogConfig, 545 ExtraHosts: options.ExtraHosts, 546 } 547 } 548 549 // fromSlash works like filepath.FromSlash but with a given OS platform field 550 func fromSlash(path, platform string) string { 551 if platform == "windows" { 552 return strings.Replace(path, "/", "\\", -1) 553 } 554 return path 555 } 556 557 // separator returns a OS path separator for the given OS platform 558 func separator(platform string) byte { 559 if platform == "windows" { 560 return '\\' 561 } 562 return '/' 563 }