github.com/docker/engine@v22.0.0-20211208180946-d456264580cf+incompatible/builder/dockerfile/builder.go (about) 1 package dockerfile // import "github.com/docker/docker/builder/dockerfile" 2 3 import ( 4 "bytes" 5 "context" 6 "fmt" 7 "io" 8 "sort" 9 "strings" 10 11 "github.com/containerd/containerd/platforms" 12 "github.com/docker/docker/api/types" 13 "github.com/docker/docker/api/types/backend" 14 "github.com/docker/docker/api/types/container" 15 "github.com/docker/docker/builder" 16 "github.com/docker/docker/builder/remotecontext" 17 "github.com/docker/docker/errdefs" 18 "github.com/docker/docker/pkg/idtools" 19 "github.com/docker/docker/pkg/streamformatter" 20 "github.com/docker/docker/pkg/stringid" 21 "github.com/docker/docker/pkg/system" 22 "github.com/moby/buildkit/frontend/dockerfile/instructions" 23 "github.com/moby/buildkit/frontend/dockerfile/parser" 24 "github.com/moby/buildkit/frontend/dockerfile/shell" 25 specs "github.com/opencontainers/image-spec/specs-go/v1" 26 "github.com/pkg/errors" 27 "github.com/sirupsen/logrus" 28 "golang.org/x/sync/syncmap" 29 ) 30 31 var validCommitCommands = map[string]bool{ 32 "cmd": true, 33 "entrypoint": true, 34 "healthcheck": true, 35 "env": true, 36 "expose": true, 37 "label": true, 38 "onbuild": true, 39 "user": true, 40 "volume": true, 41 "workdir": true, 42 } 43 44 const ( 45 stepFormat = "Step %d/%d : %v" 46 ) 47 48 // BuildManager is shared across all Builder objects 49 type BuildManager struct { 50 idMapping *idtools.IdentityMapping 51 backend builder.Backend 52 pathCache pathCache // TODO: make this persistent 53 } 54 55 // NewBuildManager creates a BuildManager 56 func NewBuildManager(b builder.Backend, identityMapping *idtools.IdentityMapping) (*BuildManager, error) { 57 bm := &BuildManager{ 58 backend: b, 59 pathCache: &syncmap.Map{}, 60 idMapping: identityMapping, 61 } 62 return bm, nil 63 } 64 65 // Build starts a new build from a BuildConfig 66 func (bm *BuildManager) Build(ctx context.Context, config backend.BuildConfig) (*builder.Result, error) { 67 buildsTriggered.Inc() 68 if config.Options.Dockerfile == "" { 69 config.Options.Dockerfile = builder.DefaultDockerfileName 70 } 71 72 source, dockerfile, err := remotecontext.Detect(config) 73 if err != nil { 74 return nil, err 75 } 76 defer func() { 77 if source != nil { 78 if err := source.Close(); err != nil { 79 logrus.Debugf("[BUILDER] failed to remove temporary context: %v", err) 80 } 81 } 82 }() 83 84 ctx, cancel := context.WithCancel(ctx) 85 defer cancel() 86 87 builderOptions := builderOptions{ 88 Options: config.Options, 89 ProgressWriter: config.ProgressWriter, 90 Backend: bm.backend, 91 PathCache: bm.pathCache, 92 IDMapping: bm.idMapping, 93 } 94 b, err := newBuilder(ctx, builderOptions) 95 if err != nil { 96 return nil, err 97 } 98 return b.build(source, dockerfile) 99 } 100 101 // builderOptions are the dependencies required by the builder 102 type builderOptions struct { 103 Options *types.ImageBuildOptions 104 Backend builder.Backend 105 ProgressWriter backend.ProgressWriter 106 PathCache pathCache 107 IDMapping *idtools.IdentityMapping 108 } 109 110 // Builder is a Dockerfile builder 111 // It implements the builder.Backend interface. 112 type Builder struct { 113 options *types.ImageBuildOptions 114 115 Stdout io.Writer 116 Stderr io.Writer 117 Aux *streamformatter.AuxFormatter 118 Output io.Writer 119 120 docker builder.Backend 121 clientCtx context.Context 122 123 idMapping *idtools.IdentityMapping 124 disableCommit bool 125 imageSources *imageSources 126 pathCache pathCache 127 containerManager *containerManager 128 imageProber ImageProber 129 platform *specs.Platform 130 } 131 132 // newBuilder creates a new Dockerfile builder from an optional dockerfile and a Options. 133 func newBuilder(clientCtx context.Context, options builderOptions) (*Builder, error) { 134 config := options.Options 135 if config == nil { 136 config = new(types.ImageBuildOptions) 137 } 138 139 b := &Builder{ 140 clientCtx: clientCtx, 141 options: config, 142 Stdout: options.ProgressWriter.StdoutFormatter, 143 Stderr: options.ProgressWriter.StderrFormatter, 144 Aux: options.ProgressWriter.AuxFormatter, 145 Output: options.ProgressWriter.Output, 146 docker: options.Backend, 147 idMapping: options.IDMapping, 148 imageSources: newImageSources(clientCtx, options), 149 pathCache: options.PathCache, 150 imageProber: newImageProber(options.Backend, config.CacheFrom, config.NoCache), 151 containerManager: newContainerManager(options.Backend), 152 } 153 154 // same as in Builder.Build in builder/builder-next/builder.go 155 // TODO: remove once config.Platform is of type specs.Platform 156 if config.Platform != "" { 157 sp, err := platforms.Parse(config.Platform) 158 if err != nil { 159 return nil, err 160 } 161 b.platform = &sp 162 } 163 164 return b, nil 165 } 166 167 // Build 'LABEL' command(s) from '--label' options and add to the last stage 168 func buildLabelOptions(labels map[string]string, stages []instructions.Stage) { 169 keys := []string{} 170 for key := range labels { 171 keys = append(keys, key) 172 } 173 174 // Sort the label to have a repeatable order 175 sort.Strings(keys) 176 for _, key := range keys { 177 value := labels[key] 178 stages[len(stages)-1].AddCommand(instructions.NewLabelCommand(key, value, true)) 179 } 180 } 181 182 // Build runs the Dockerfile builder by parsing the Dockerfile and executing 183 // the instructions from the file. 184 func (b *Builder) build(source builder.Source, dockerfile *parser.Result) (*builder.Result, error) { 185 defer b.imageSources.Unmount() 186 187 stages, metaArgs, err := instructions.Parse(dockerfile.AST) 188 if err != nil { 189 var uiErr *instructions.UnknownInstruction 190 if errors.As(err, &uiErr) { 191 buildsFailed.WithValues(metricsUnknownInstructionError).Inc() 192 } 193 return nil, errdefs.InvalidParameter(err) 194 } 195 if b.options.Target != "" { 196 targetIx, found := instructions.HasStage(stages, b.options.Target) 197 if !found { 198 buildsFailed.WithValues(metricsBuildTargetNotReachableError).Inc() 199 return nil, errdefs.InvalidParameter(errors.Errorf("failed to reach build target %s in Dockerfile", b.options.Target)) 200 } 201 stages = stages[:targetIx+1] 202 } 203 204 // Add 'LABEL' command specified by '--label' option to the last stage 205 buildLabelOptions(b.options.Labels, stages) 206 207 dockerfile.PrintWarnings(b.Stderr) 208 dispatchState, err := b.dispatchDockerfileWithCancellation(stages, metaArgs, dockerfile.EscapeToken, source) 209 if err != nil { 210 return nil, err 211 } 212 if dispatchState.imageID == "" { 213 buildsFailed.WithValues(metricsDockerfileEmptyError).Inc() 214 return nil, errors.New("No image was generated. Is your Dockerfile empty?") 215 } 216 return &builder.Result{ImageID: dispatchState.imageID, FromImage: dispatchState.baseImage}, nil 217 } 218 219 func emitImageID(aux *streamformatter.AuxFormatter, state *dispatchState) error { 220 if aux == nil || state.imageID == "" { 221 return nil 222 } 223 return aux.Emit("", types.BuildResult{ID: state.imageID}) 224 } 225 226 func processMetaArg(meta instructions.ArgCommand, shlex *shell.Lex, args *BuildArgs) error { 227 // shell.Lex currently only support the concatenated string format 228 envs := convertMapToEnvList(args.GetAllAllowed()) 229 if err := meta.Expand(func(word string) (string, error) { 230 return shlex.ProcessWord(word, envs) 231 }); err != nil { 232 return err 233 } 234 for _, arg := range meta.Args { 235 args.AddArg(arg.Key, arg.Value) 236 args.AddMetaArg(arg.Key, arg.Value) 237 } 238 return nil 239 } 240 241 func printCommand(out io.Writer, currentCommandIndex int, totalCommands int, cmd interface{}) int { 242 fmt.Fprintf(out, stepFormat, currentCommandIndex, totalCommands, cmd) 243 fmt.Fprintln(out) 244 return currentCommandIndex + 1 245 } 246 247 func (b *Builder) dispatchDockerfileWithCancellation(parseResult []instructions.Stage, metaArgs []instructions.ArgCommand, escapeToken rune, source builder.Source) (*dispatchState, error) { 248 dispatchRequest := dispatchRequest{} 249 buildArgs := NewBuildArgs(b.options.BuildArgs) 250 totalCommands := len(metaArgs) + len(parseResult) 251 currentCommandIndex := 1 252 for _, stage := range parseResult { 253 totalCommands += len(stage.Commands) 254 } 255 shlex := shell.NewLex(escapeToken) 256 for i := range metaArgs { 257 currentCommandIndex = printCommand(b.Stdout, currentCommandIndex, totalCommands, &metaArgs[i]) 258 259 err := processMetaArg(metaArgs[i], shlex, buildArgs) 260 if err != nil { 261 return nil, err 262 } 263 } 264 265 stagesResults := newStagesBuildResults() 266 267 for _, s := range parseResult { 268 stage := s 269 if err := stagesResults.checkStageNameAvailable(stage.Name); err != nil { 270 return nil, err 271 } 272 dispatchRequest = newDispatchRequest(b, escapeToken, source, buildArgs, stagesResults) 273 274 currentCommandIndex = printCommand(b.Stdout, currentCommandIndex, totalCommands, stage.SourceCode) 275 if err := initializeStage(dispatchRequest, &stage); err != nil { 276 return nil, err 277 } 278 dispatchRequest.state.updateRunConfig() 279 fmt.Fprintf(b.Stdout, " ---> %s\n", stringid.TruncateID(dispatchRequest.state.imageID)) 280 for _, cmd := range stage.Commands { 281 select { 282 case <-b.clientCtx.Done(): 283 logrus.Debug("Builder: build cancelled!") 284 fmt.Fprint(b.Stdout, "Build cancelled\n") 285 buildsFailed.WithValues(metricsBuildCanceled).Inc() 286 return nil, errors.New("Build cancelled") 287 default: 288 // Not cancelled yet, keep going... 289 } 290 291 currentCommandIndex = printCommand(b.Stdout, currentCommandIndex, totalCommands, cmd) 292 293 if err := dispatch(dispatchRequest, cmd); err != nil { 294 return nil, err 295 } 296 dispatchRequest.state.updateRunConfig() 297 fmt.Fprintf(b.Stdout, " ---> %s\n", stringid.TruncateID(dispatchRequest.state.imageID)) 298 299 } 300 if err := emitImageID(b.Aux, dispatchRequest.state); err != nil { 301 return nil, err 302 } 303 buildArgs.MergeReferencedArgs(dispatchRequest.state.buildArgs) 304 if err := commitStage(dispatchRequest.state, stagesResults); err != nil { 305 return nil, err 306 } 307 } 308 buildArgs.WarnOnUnusedBuildArgs(b.Stdout) 309 return dispatchRequest.state, nil 310 } 311 312 // BuildFromConfig builds directly from `changes`, treating it as if it were the contents of a Dockerfile 313 // It will: 314 // - Call parse.Parse() to get an AST root for the concatenated Dockerfile entries. 315 // - Do build by calling builder.dispatch() to call all entries' handling routines 316 // 317 // BuildFromConfig is used by the /commit endpoint, with the changes 318 // coming from the query parameter of the same name. 319 // 320 // TODO: Remove? 321 func BuildFromConfig(config *container.Config, changes []string, os string) (*container.Config, error) { 322 if !system.IsOSSupported(os) { 323 return nil, errdefs.InvalidParameter(system.ErrNotSupportedOperatingSystem) 324 } 325 if len(changes) == 0 { 326 return config, nil 327 } 328 329 dockerfile, err := parser.Parse(bytes.NewBufferString(strings.Join(changes, "\n"))) 330 if err != nil { 331 return nil, errdefs.InvalidParameter(err) 332 } 333 334 b, err := newBuilder(context.Background(), builderOptions{ 335 Options: &types.ImageBuildOptions{NoCache: true}, 336 }) 337 if err != nil { 338 return nil, err 339 } 340 341 // ensure that the commands are valid 342 for _, n := range dockerfile.AST.Children { 343 if !validCommitCommands[n.Value] { 344 return nil, errdefs.InvalidParameter(errors.Errorf("%s is not a valid change command", n.Value)) 345 } 346 } 347 348 b.Stdout = io.Discard 349 b.Stderr = io.Discard 350 b.disableCommit = true 351 352 var commands []instructions.Command 353 for _, n := range dockerfile.AST.Children { 354 cmd, err := instructions.ParseCommand(n) 355 if err != nil { 356 return nil, errdefs.InvalidParameter(err) 357 } 358 commands = append(commands, cmd) 359 } 360 361 dispatchRequest := newDispatchRequest(b, dockerfile.EscapeToken, nil, NewBuildArgs(b.options.BuildArgs), newStagesBuildResults()) 362 // We make mutations to the configuration, ensure we have a copy 363 dispatchRequest.state.runConfig = copyRunConfig(config) 364 dispatchRequest.state.imageID = config.Image 365 dispatchRequest.state.operatingSystem = os 366 for _, cmd := range commands { 367 err := dispatch(dispatchRequest, cmd) 368 if err != nil { 369 return nil, errdefs.InvalidParameter(err) 370 } 371 dispatchRequest.state.updateRunConfig() 372 } 373 374 return dispatchRequest.state.runConfig, nil 375 } 376 377 func convertMapToEnvList(m map[string]string) []string { 378 result := []string{} 379 for k, v := range m { 380 result = append(result, k+"="+v) 381 } 382 return result 383 }