github.com/demonoid81/moby@v0.0.0-20200517203328-62dd8e17c460/builder/dockerfile/internals.go (about) 1 package dockerfile // import "github.com/demonoid81/moby/builder/dockerfile" 2 3 // internals for handling commands. Covers many areas and a lot of 4 // non-contiguous functionality. Please read the comments. 5 6 import ( 7 "crypto/sha256" 8 "encoding/hex" 9 "fmt" 10 "io" 11 "os" 12 "path" 13 "path/filepath" 14 "runtime" 15 "strings" 16 17 "github.com/demonoid81/moby/api/types" 18 "github.com/demonoid81/moby/api/types/backend" 19 "github.com/demonoid81/moby/api/types/container" 20 "github.com/demonoid81/moby/builder" 21 "github.com/demonoid81/moby/image" 22 "github.com/demonoid81/moby/pkg/archive" 23 "github.com/demonoid81/moby/pkg/chrootarchive" 24 "github.com/demonoid81/moby/pkg/containerfs" 25 "github.com/demonoid81/moby/pkg/idtools" 26 "github.com/demonoid81/moby/pkg/stringid" 27 "github.com/demonoid81/moby/pkg/system" 28 "github.com/docker/go-connections/nat" 29 specs "github.com/opencontainers/image-spec/specs-go/v1" 30 "github.com/pkg/errors" 31 "github.com/sirupsen/logrus" 32 ) 33 34 // Archiver defines an interface for copying files from one destination to 35 // another using Tar/Untar. 36 type Archiver interface { 37 TarUntar(src, dst string) error 38 UntarPath(src, dst string) error 39 CopyWithTar(src, dst string) error 40 CopyFileWithTar(src, dst string) error 41 IdentityMapping() *idtools.IdentityMapping 42 } 43 44 // The builder will use the following interfaces if the container fs implements 45 // these for optimized copies to and from the container. 46 type extractor interface { 47 ExtractArchive(src io.Reader, dst string, opts *archive.TarOptions) error 48 } 49 50 type archiver interface { 51 ArchivePath(src string, opts *archive.TarOptions) (io.ReadCloser, error) 52 } 53 54 // helper functions to get tar/untar func 55 func untarFunc(i interface{}) containerfs.UntarFunc { 56 if ea, ok := i.(extractor); ok { 57 return ea.ExtractArchive 58 } 59 return chrootarchive.Untar 60 } 61 62 func tarFunc(i interface{}) containerfs.TarFunc { 63 if ap, ok := i.(archiver); ok { 64 return ap.ArchivePath 65 } 66 return archive.TarWithOptions 67 } 68 69 func (b *Builder) getArchiver(src, dst containerfs.Driver) Archiver { 70 t, u := tarFunc(src), untarFunc(dst) 71 return &containerfs.Archiver{ 72 SrcDriver: src, 73 DstDriver: dst, 74 Tar: t, 75 Untar: u, 76 IDMapping: b.idMapping, 77 } 78 } 79 80 func (b *Builder) commit(dispatchState *dispatchState, comment string) error { 81 if b.disableCommit { 82 return nil 83 } 84 if !dispatchState.hasFromImage() { 85 return errors.New("Please provide a source image with `from` prior to commit") 86 } 87 88 runConfigWithCommentCmd := copyRunConfig(dispatchState.runConfig, withCmdComment(comment, dispatchState.operatingSystem)) 89 id, err := b.probeAndCreate(dispatchState, runConfigWithCommentCmd) 90 if err != nil || id == "" { 91 return err 92 } 93 94 return b.commitContainer(dispatchState, id, runConfigWithCommentCmd) 95 } 96 97 func (b *Builder) commitContainer(dispatchState *dispatchState, id string, containerConfig *container.Config) error { 98 if b.disableCommit { 99 return nil 100 } 101 102 commitCfg := backend.CommitConfig{ 103 Author: dispatchState.maintainer, 104 // TODO: this copy should be done by Commit() 105 Config: copyRunConfig(dispatchState.runConfig), 106 ContainerConfig: containerConfig, 107 ContainerID: id, 108 } 109 110 imageID, err := b.docker.CommitBuildStep(commitCfg) 111 dispatchState.imageID = string(imageID) 112 return err 113 } 114 115 func (b *Builder) exportImage(state *dispatchState, layer builder.RWLayer, parent builder.Image, runConfig *container.Config) error { 116 newLayer, err := layer.Commit() 117 if err != nil { 118 return err 119 } 120 121 parentImage, ok := parent.(*image.Image) 122 if !ok { 123 return errors.Errorf("unexpected image type") 124 } 125 126 platform := &specs.Platform{ 127 OS: parentImage.OS, 128 Architecture: parentImage.Architecture, 129 Variant: parentImage.Variant, 130 } 131 132 // add an image mount without an image so the layer is properly unmounted 133 // if there is an error before we can add the full mount with image 134 b.imageSources.Add(newImageMount(nil, newLayer), platform) 135 136 newImage := image.NewChildImage(parentImage, image.ChildConfig{ 137 Author: state.maintainer, 138 ContainerConfig: runConfig, 139 DiffID: newLayer.DiffID(), 140 Config: copyRunConfig(state.runConfig), 141 }, parentImage.OS) 142 143 // TODO: it seems strange to marshal this here instead of just passing in the 144 // image struct 145 config, err := newImage.MarshalJSON() 146 if err != nil { 147 return errors.Wrap(err, "failed to encode image config") 148 } 149 150 exportedImage, err := b.docker.CreateImage(config, state.imageID) 151 if err != nil { 152 return errors.Wrapf(err, "failed to export image") 153 } 154 155 state.imageID = exportedImage.ImageID() 156 b.imageSources.Add(newImageMount(exportedImage, newLayer), platform) 157 return nil 158 } 159 160 func (b *Builder) performCopy(req dispatchRequest, inst copyInstruction) error { 161 state := req.state 162 srcHash := getSourceHashFromInfos(inst.infos) 163 164 var chownComment string 165 if inst.chownStr != "" { 166 chownComment = fmt.Sprintf("--chown=%s", inst.chownStr) 167 } 168 commentStr := fmt.Sprintf("%s %s%s in %s ", inst.cmdName, chownComment, srcHash, inst.dest) 169 170 // TODO: should this have been using origPaths instead of srcHash in the comment? 171 runConfigWithCommentCmd := copyRunConfig( 172 state.runConfig, 173 withCmdCommentString(commentStr, state.operatingSystem)) 174 hit, err := b.probeCache(state, runConfigWithCommentCmd) 175 if err != nil || hit { 176 return err 177 } 178 179 imageMount, err := b.imageSources.Get(state.imageID, true, req.builder.platform) 180 if err != nil { 181 return errors.Wrapf(err, "failed to get destination image %q", state.imageID) 182 } 183 184 rwLayer, err := imageMount.NewRWLayer() 185 if err != nil { 186 return err 187 } 188 defer rwLayer.Release() 189 190 destInfo, err := createDestInfo(state.runConfig.WorkingDir, inst, rwLayer, state.operatingSystem) 191 if err != nil { 192 return err 193 } 194 195 identity := b.idMapping.RootPair() 196 // if a chown was requested, perform the steps to get the uid, gid 197 // translated (if necessary because of user namespaces), and replace 198 // the root pair with the chown pair for copy operations 199 if inst.chownStr != "" { 200 identity, err = parseChownFlag(b, state, inst.chownStr, destInfo.root.Path(), b.idMapping) 201 if err != nil { 202 if b.options.Platform != "windows" { 203 return errors.Wrapf(err, "unable to convert uid/gid chown string to host mapping") 204 } 205 206 return errors.Wrapf(err, "unable to map container user account name to SID") 207 } 208 } 209 210 for _, info := range inst.infos { 211 opts := copyFileOptions{ 212 decompress: inst.allowLocalDecompression, 213 archiver: b.getArchiver(info.root, destInfo.root), 214 } 215 if !inst.preserveOwnership { 216 opts.identity = &identity 217 } 218 if err := performCopyForInfo(destInfo, info, opts); err != nil { 219 return errors.Wrapf(err, "failed to copy files") 220 } 221 } 222 return b.exportImage(state, rwLayer, imageMount.Image(), runConfigWithCommentCmd) 223 } 224 225 func createDestInfo(workingDir string, inst copyInstruction, rwLayer builder.RWLayer, platform string) (copyInfo, error) { 226 // Twiddle the destination when it's a relative path - meaning, make it 227 // relative to the WORKINGDIR 228 dest, err := normalizeDest(workingDir, inst.dest, platform) 229 if err != nil { 230 return copyInfo{}, errors.Wrapf(err, "invalid %s", inst.cmdName) 231 } 232 233 return copyInfo{root: rwLayer.Root(), path: dest}, nil 234 } 235 236 // normalizeDest normalises the destination of a COPY/ADD command in a 237 // platform semantically consistent way. 238 func normalizeDest(workingDir, requested string, platform string) (string, error) { 239 dest := fromSlash(requested, platform) 240 endsInSlash := strings.HasSuffix(dest, string(separator(platform))) 241 242 if platform != "windows" { 243 if !path.IsAbs(requested) { 244 dest = path.Join("/", filepath.ToSlash(workingDir), dest) 245 // Make sure we preserve any trailing slash 246 if endsInSlash { 247 dest += "/" 248 } 249 } 250 return dest, nil 251 } 252 253 // We are guaranteed that the working directory is already consistent, 254 // However, Windows also has, for now, the limitation that ADD/COPY can 255 // only be done to the system drive, not any drives that might be present 256 // as a result of a bind mount. 257 // 258 // So... if the path requested is Linux-style absolute (/foo or \\foo), 259 // we assume it is the system drive. If it is a Windows-style absolute 260 // (DRIVE:\\foo), error if DRIVE is not C. And finally, ensure we 261 // strip any configured working directories drive letter so that it 262 // can be subsequently legitimately converted to a Windows volume-style 263 // pathname. 264 265 // Not a typo - filepath.IsAbs, not system.IsAbs on this next check as 266 // we only want to validate where the DriveColon part has been supplied. 267 if filepath.IsAbs(dest) { 268 if strings.ToUpper(string(dest[0])) != "C" { 269 return "", fmt.Errorf("Windows does not support destinations not on the system drive (C:)") 270 } 271 dest = dest[2:] // Strip the drive letter 272 } 273 274 // Cannot handle relative where WorkingDir is not the system drive. 275 if len(workingDir) > 0 { 276 if ((len(workingDir) > 1) && !system.IsAbs(workingDir[2:])) || (len(workingDir) == 1) { 277 return "", fmt.Errorf("Current WorkingDir %s is not platform consistent", workingDir) 278 } 279 if !system.IsAbs(dest) { 280 if string(workingDir[0]) != "C" { 281 return "", fmt.Errorf("Windows does not support relative paths when WORKDIR is not the system drive") 282 } 283 dest = filepath.Join(string(os.PathSeparator), workingDir[2:], dest) 284 // Make sure we preserve any trailing slash 285 if endsInSlash { 286 dest += string(os.PathSeparator) 287 } 288 } 289 } 290 return dest, nil 291 } 292 293 // For backwards compat, if there's just one info then use it as the 294 // cache look-up string, otherwise hash 'em all into one 295 func getSourceHashFromInfos(infos []copyInfo) string { 296 if len(infos) == 1 { 297 return infos[0].hash 298 } 299 var hashs []string 300 for _, info := range infos { 301 hashs = append(hashs, info.hash) 302 } 303 return hashStringSlice("multi", hashs) 304 } 305 306 func hashStringSlice(prefix string, slice []string) string { 307 hasher := sha256.New() 308 hasher.Write([]byte(strings.Join(slice, ","))) 309 return prefix + ":" + hex.EncodeToString(hasher.Sum(nil)) 310 } 311 312 type runConfigModifier func(*container.Config) 313 314 func withCmd(cmd []string) runConfigModifier { 315 return func(runConfig *container.Config) { 316 runConfig.Cmd = cmd 317 } 318 } 319 320 func withArgsEscaped(argsEscaped bool) runConfigModifier { 321 return func(runConfig *container.Config) { 322 runConfig.ArgsEscaped = argsEscaped 323 } 324 } 325 326 // withCmdComment sets Cmd to a nop comment string. See withCmdCommentString for 327 // why there are two almost identical versions of this. 328 func withCmdComment(comment string, platform string) runConfigModifier { 329 return func(runConfig *container.Config) { 330 runConfig.Cmd = append(getShell(runConfig, platform), "#(nop) ", comment) 331 } 332 } 333 334 // withCmdCommentString exists to maintain compatibility with older versions. 335 // A few instructions (workdir, copy, add) used a nop comment that is a single arg 336 // where as all the other instructions used a two arg comment string. This 337 // function implements the single arg version. 338 func withCmdCommentString(comment string, platform string) runConfigModifier { 339 return func(runConfig *container.Config) { 340 runConfig.Cmd = append(getShell(runConfig, platform), "#(nop) "+comment) 341 } 342 } 343 344 func withEnv(env []string) runConfigModifier { 345 return func(runConfig *container.Config) { 346 runConfig.Env = env 347 } 348 } 349 350 // withEntrypointOverride sets an entrypoint on runConfig if the command is 351 // not empty. The entrypoint is left unmodified if command is empty. 352 // 353 // The dockerfile RUN instruction expect to run without an entrypoint 354 // so the runConfig entrypoint needs to be modified accordingly. ContainerCreate 355 // will change a []string{""} entrypoint to nil, so we probe the cache with the 356 // nil entrypoint. 357 func withEntrypointOverride(cmd []string, entrypoint []string) runConfigModifier { 358 return func(runConfig *container.Config) { 359 if len(cmd) > 0 { 360 runConfig.Entrypoint = entrypoint 361 } 362 } 363 } 364 365 // withoutHealthcheck disables healthcheck. 366 // 367 // The dockerfile RUN instruction expect to run without healthcheck 368 // so the runConfig Healthcheck needs to be disabled. 369 func withoutHealthcheck() runConfigModifier { 370 return func(runConfig *container.Config) { 371 runConfig.Healthcheck = &container.HealthConfig{ 372 Test: []string{"NONE"}, 373 } 374 } 375 } 376 377 func copyRunConfig(runConfig *container.Config, modifiers ...runConfigModifier) *container.Config { 378 copy := *runConfig 379 copy.Cmd = copyStringSlice(runConfig.Cmd) 380 copy.Env = copyStringSlice(runConfig.Env) 381 copy.Entrypoint = copyStringSlice(runConfig.Entrypoint) 382 copy.OnBuild = copyStringSlice(runConfig.OnBuild) 383 copy.Shell = copyStringSlice(runConfig.Shell) 384 385 if copy.Volumes != nil { 386 copy.Volumes = make(map[string]struct{}, len(runConfig.Volumes)) 387 for k, v := range runConfig.Volumes { 388 copy.Volumes[k] = v 389 } 390 } 391 392 if copy.ExposedPorts != nil { 393 copy.ExposedPorts = make(nat.PortSet, len(runConfig.ExposedPorts)) 394 for k, v := range runConfig.ExposedPorts { 395 copy.ExposedPorts[k] = v 396 } 397 } 398 399 if copy.Labels != nil { 400 copy.Labels = make(map[string]string, len(runConfig.Labels)) 401 for k, v := range runConfig.Labels { 402 copy.Labels[k] = v 403 } 404 } 405 406 for _, modifier := range modifiers { 407 modifier(©) 408 } 409 return © 410 } 411 412 func copyStringSlice(orig []string) []string { 413 if orig == nil { 414 return nil 415 } 416 return append([]string{}, orig...) 417 } 418 419 // getShell is a helper function which gets the right shell for prefixing the 420 // shell-form of RUN, ENTRYPOINT and CMD instructions 421 func getShell(c *container.Config, os string) []string { 422 if 0 == len(c.Shell) { 423 return append([]string{}, defaultShellForOS(os)[:]...) 424 } 425 return append([]string{}, c.Shell[:]...) 426 } 427 428 func (b *Builder) probeCache(dispatchState *dispatchState, runConfig *container.Config) (bool, error) { 429 cachedID, err := b.imageProber.Probe(dispatchState.imageID, runConfig) 430 if cachedID == "" || err != nil { 431 return false, err 432 } 433 fmt.Fprint(b.Stdout, " ---> Using cache\n") 434 435 dispatchState.imageID = cachedID 436 return true, nil 437 } 438 439 var defaultLogConfig = container.LogConfig{Type: "none"} 440 441 func (b *Builder) probeAndCreate(dispatchState *dispatchState, runConfig *container.Config) (string, error) { 442 if hit, err := b.probeCache(dispatchState, runConfig); err != nil || hit { 443 return "", err 444 } 445 return b.create(runConfig) 446 } 447 448 func (b *Builder) create(runConfig *container.Config) (string, error) { 449 logrus.Debugf("[BUILDER] Command to be executed: %v", runConfig.Cmd) 450 451 isWCOW := runtime.GOOS == "windows" && b.platform != nil && b.platform.OS == "windows" 452 hostConfig := hostConfigFromOptions(b.options, isWCOW) 453 container, err := b.containerManager.Create(runConfig, hostConfig) 454 if err != nil { 455 return "", err 456 } 457 // TODO: could this be moved into containerManager.Create() ? 458 for _, warning := range container.Warnings { 459 fmt.Fprintf(b.Stdout, " ---> [Warning] %s\n", warning) 460 } 461 fmt.Fprintf(b.Stdout, " ---> Running in %s\n", stringid.TruncateID(container.ID)) 462 return container.ID, nil 463 } 464 465 func hostConfigFromOptions(options *types.ImageBuildOptions, isWCOW bool) *container.HostConfig { 466 resources := container.Resources{ 467 CgroupParent: options.CgroupParent, 468 CPUShares: options.CPUShares, 469 CPUPeriod: options.CPUPeriod, 470 CPUQuota: options.CPUQuota, 471 CpusetCpus: options.CPUSetCPUs, 472 CpusetMems: options.CPUSetMems, 473 Memory: options.Memory, 474 MemorySwap: options.MemorySwap, 475 Ulimits: options.Ulimits, 476 } 477 478 hc := &container.HostConfig{ 479 SecurityOpt: options.SecurityOpt, 480 Isolation: options.Isolation, 481 ShmSize: options.ShmSize, 482 Resources: resources, 483 NetworkMode: container.NetworkMode(options.NetworkMode), 484 // Set a log config to override any default value set on the daemon 485 LogConfig: defaultLogConfig, 486 ExtraHosts: options.ExtraHosts, 487 } 488 489 // For WCOW, the default of 20GB hard-coded in the platform 490 // is too small for builder scenarios where many users are 491 // using RUN statements to install large amounts of data. 492 // Use 127GB as that's the default size of a VHD in Hyper-V. 493 if isWCOW { 494 hc.StorageOpt = make(map[string]string) 495 hc.StorageOpt["size"] = "127GB" 496 } 497 498 return hc 499 } 500 501 // fromSlash works like filepath.FromSlash but with a given OS platform field 502 func fromSlash(path, platform string) string { 503 if platform == "windows" { 504 return strings.Replace(path, "/", "\\", -1) 505 } 506 return path 507 } 508 509 // separator returns a OS path separator for the given OS platform 510 func separator(platform string) byte { 511 if platform == "windows" { 512 return '\\' 513 } 514 return '/' 515 }