github.com/demonoid81/moby@v0.0.0-20200517203328-62dd8e17c460/builder/dockerfile/builder.go (about)

     1  package dockerfile // import "github.com/demonoid81/moby/builder/dockerfile"
     2  
     3  import (
     4  	"bytes"
     5  	"context"
     6  	"fmt"
     7  	"io"
     8  	"io/ioutil"
     9  	"sort"
    10  	"strings"
    11  
    12  	"github.com/containerd/containerd/platforms"
    13  	"github.com/demonoid81/moby/api/types"
    14  	"github.com/demonoid81/moby/api/types/backend"
    15  	"github.com/demonoid81/moby/api/types/container"
    16  	"github.com/demonoid81/moby/builder"
    17  	"github.com/demonoid81/moby/builder/remotecontext"
    18  	"github.com/demonoid81/moby/errdefs"
    19  	"github.com/demonoid81/moby/pkg/idtools"
    20  	"github.com/demonoid81/moby/pkg/streamformatter"
    21  	"github.com/demonoid81/moby/pkg/stringid"
    22  	"github.com/demonoid81/moby/pkg/system"
    23  	"github.com/moby/buildkit/frontend/dockerfile/instructions"
    24  	"github.com/moby/buildkit/frontend/dockerfile/parser"
    25  	"github.com/moby/buildkit/frontend/dockerfile/shell"
    26  	specs "github.com/opencontainers/image-spec/specs-go/v1"
    27  	"github.com/pkg/errors"
    28  	"github.com/sirupsen/logrus"
    29  	"golang.org/x/sync/syncmap"
    30  )
    31  
    32  var validCommitCommands = map[string]bool{
    33  	"cmd":         true,
    34  	"entrypoint":  true,
    35  	"healthcheck": true,
    36  	"env":         true,
    37  	"expose":      true,
    38  	"label":       true,
    39  	"onbuild":     true,
    40  	"user":        true,
    41  	"volume":      true,
    42  	"workdir":     true,
    43  }
    44  
    45  const (
    46  	stepFormat = "Step %d/%d : %v"
    47  )
    48  
    49  // BuildManager is shared across all Builder objects
    50  type BuildManager struct {
    51  	idMapping *idtools.IdentityMapping
    52  	backend   builder.Backend
    53  	pathCache pathCache // TODO: make this persistent
    54  }
    55  
    56  // NewBuildManager creates a BuildManager
    57  func NewBuildManager(b builder.Backend, identityMapping *idtools.IdentityMapping) (*BuildManager, error) {
    58  	bm := &BuildManager{
    59  		backend:   b,
    60  		pathCache: &syncmap.Map{},
    61  		idMapping: identityMapping,
    62  	}
    63  	return bm, nil
    64  }
    65  
    66  // Build starts a new build from a BuildConfig
    67  func (bm *BuildManager) Build(ctx context.Context, config backend.BuildConfig) (*builder.Result, error) {
    68  	buildsTriggered.Inc()
    69  	if config.Options.Dockerfile == "" {
    70  		config.Options.Dockerfile = builder.DefaultDockerfileName
    71  	}
    72  
    73  	source, dockerfile, err := remotecontext.Detect(config)
    74  	if err != nil {
    75  		return nil, err
    76  	}
    77  	defer func() {
    78  		if source != nil {
    79  			if err := source.Close(); err != nil {
    80  				logrus.Debugf("[BUILDER] failed to remove temporary context: %v", err)
    81  			}
    82  		}
    83  	}()
    84  
    85  	ctx, cancel := context.WithCancel(ctx)
    86  	defer cancel()
    87  
    88  	if config.Options.SessionID != "" {
    89  		return nil, errors.New("experimental session with v1 builder is no longer supported, use builder version v2 (BuildKit) instead")
    90  	}
    91  
    92  	builderOptions := builderOptions{
    93  		Options:        config.Options,
    94  		ProgressWriter: config.ProgressWriter,
    95  		Backend:        bm.backend,
    96  		PathCache:      bm.pathCache,
    97  		IDMapping:      bm.idMapping,
    98  	}
    99  	b, err := newBuilder(ctx, builderOptions)
   100  	if err != nil {
   101  		return nil, err
   102  	}
   103  	return b.build(source, dockerfile)
   104  }
   105  
   106  // builderOptions are the dependencies required by the builder
   107  type builderOptions struct {
   108  	Options        *types.ImageBuildOptions
   109  	Backend        builder.Backend
   110  	ProgressWriter backend.ProgressWriter
   111  	PathCache      pathCache
   112  	IDMapping      *idtools.IdentityMapping
   113  }
   114  
   115  // Builder is a Dockerfile builder
   116  // It implements the builder.Backend interface.
   117  type Builder struct {
   118  	options *types.ImageBuildOptions
   119  
   120  	Stdout io.Writer
   121  	Stderr io.Writer
   122  	Aux    *streamformatter.AuxFormatter
   123  	Output io.Writer
   124  
   125  	docker    builder.Backend
   126  	clientCtx context.Context
   127  
   128  	idMapping        *idtools.IdentityMapping
   129  	disableCommit    bool
   130  	imageSources     *imageSources
   131  	pathCache        pathCache
   132  	containerManager *containerManager
   133  	imageProber      ImageProber
   134  	platform         *specs.Platform
   135  }
   136  
   137  // newBuilder creates a new Dockerfile builder from an optional dockerfile and a Options.
   138  func newBuilder(clientCtx context.Context, options builderOptions) (*Builder, error) {
   139  	config := options.Options
   140  	if config == nil {
   141  		config = new(types.ImageBuildOptions)
   142  	}
   143  
   144  	b := &Builder{
   145  		clientCtx:        clientCtx,
   146  		options:          config,
   147  		Stdout:           options.ProgressWriter.StdoutFormatter,
   148  		Stderr:           options.ProgressWriter.StderrFormatter,
   149  		Aux:              options.ProgressWriter.AuxFormatter,
   150  		Output:           options.ProgressWriter.Output,
   151  		docker:           options.Backend,
   152  		idMapping:        options.IDMapping,
   153  		imageSources:     newImageSources(clientCtx, options),
   154  		pathCache:        options.PathCache,
   155  		imageProber:      newImageProber(options.Backend, config.CacheFrom, config.NoCache),
   156  		containerManager: newContainerManager(options.Backend),
   157  	}
   158  
   159  	// same as in Builder.Build in builder/builder-next/builder.go
   160  	// TODO: remove once config.Platform is of type specs.Platform
   161  	if config.Platform != "" {
   162  		sp, err := platforms.Parse(config.Platform)
   163  		if err != nil {
   164  			return nil, err
   165  		}
   166  		if err := system.ValidatePlatform(sp); err != nil {
   167  			return nil, err
   168  		}
   169  		b.platform = &sp
   170  	}
   171  
   172  	return b, nil
   173  }
   174  
   175  // Build 'LABEL' command(s) from '--label' options and add to the last stage
   176  func buildLabelOptions(labels map[string]string, stages []instructions.Stage) {
   177  	keys := []string{}
   178  	for key := range labels {
   179  		keys = append(keys, key)
   180  	}
   181  
   182  	// Sort the label to have a repeatable order
   183  	sort.Strings(keys)
   184  	for _, key := range keys {
   185  		value := labels[key]
   186  		stages[len(stages)-1].AddCommand(instructions.NewLabelCommand(key, value, true))
   187  	}
   188  }
   189  
   190  // Build runs the Dockerfile builder by parsing the Dockerfile and executing
   191  // the instructions from the file.
   192  func (b *Builder) build(source builder.Source, dockerfile *parser.Result) (*builder.Result, error) {
   193  	defer b.imageSources.Unmount()
   194  
   195  	stages, metaArgs, err := instructions.Parse(dockerfile.AST)
   196  	if err != nil {
   197  		if instructions.IsUnknownInstruction(err) {
   198  			buildsFailed.WithValues(metricsUnknownInstructionError).Inc()
   199  		}
   200  		return nil, errdefs.InvalidParameter(err)
   201  	}
   202  	if b.options.Target != "" {
   203  		targetIx, found := instructions.HasStage(stages, b.options.Target)
   204  		if !found {
   205  			buildsFailed.WithValues(metricsBuildTargetNotReachableError).Inc()
   206  			return nil, errdefs.InvalidParameter(errors.Errorf("failed to reach build target %s in Dockerfile", b.options.Target))
   207  		}
   208  		stages = stages[:targetIx+1]
   209  	}
   210  
   211  	// Add 'LABEL' command specified by '--label' option to the last stage
   212  	buildLabelOptions(b.options.Labels, stages)
   213  
   214  	dockerfile.PrintWarnings(b.Stderr)
   215  	dispatchState, err := b.dispatchDockerfileWithCancellation(stages, metaArgs, dockerfile.EscapeToken, source)
   216  	if err != nil {
   217  		return nil, err
   218  	}
   219  	if dispatchState.imageID == "" {
   220  		buildsFailed.WithValues(metricsDockerfileEmptyError).Inc()
   221  		return nil, errors.New("No image was generated. Is your Dockerfile empty?")
   222  	}
   223  	return &builder.Result{ImageID: dispatchState.imageID, FromImage: dispatchState.baseImage}, nil
   224  }
   225  
   226  func emitImageID(aux *streamformatter.AuxFormatter, state *dispatchState) error {
   227  	if aux == nil || state.imageID == "" {
   228  		return nil
   229  	}
   230  	return aux.Emit("", types.BuildResult{ID: state.imageID})
   231  }
   232  
   233  func processMetaArg(meta instructions.ArgCommand, shlex *shell.Lex, args *BuildArgs) error {
   234  	// shell.Lex currently only support the concatenated string format
   235  	envs := convertMapToEnvList(args.GetAllAllowed())
   236  	if err := meta.Expand(func(word string) (string, error) {
   237  		return shlex.ProcessWord(word, envs)
   238  	}); err != nil {
   239  		return err
   240  	}
   241  	args.AddArg(meta.Key, meta.Value)
   242  	args.AddMetaArg(meta.Key, meta.Value)
   243  	return nil
   244  }
   245  
   246  func printCommand(out io.Writer, currentCommandIndex int, totalCommands int, cmd interface{}) int {
   247  	fmt.Fprintf(out, stepFormat, currentCommandIndex, totalCommands, cmd)
   248  	fmt.Fprintln(out)
   249  	return currentCommandIndex + 1
   250  }
   251  
   252  func (b *Builder) dispatchDockerfileWithCancellation(parseResult []instructions.Stage, metaArgs []instructions.ArgCommand, escapeToken rune, source builder.Source) (*dispatchState, error) {
   253  	dispatchRequest := dispatchRequest{}
   254  	buildArgs := NewBuildArgs(b.options.BuildArgs)
   255  	totalCommands := len(metaArgs) + len(parseResult)
   256  	currentCommandIndex := 1
   257  	for _, stage := range parseResult {
   258  		totalCommands += len(stage.Commands)
   259  	}
   260  	shlex := shell.NewLex(escapeToken)
   261  	for _, meta := range metaArgs {
   262  		currentCommandIndex = printCommand(b.Stdout, currentCommandIndex, totalCommands, &meta)
   263  
   264  		err := processMetaArg(meta, shlex, buildArgs)
   265  		if err != nil {
   266  			return nil, err
   267  		}
   268  	}
   269  
   270  	stagesResults := newStagesBuildResults()
   271  
   272  	for _, stage := range parseResult {
   273  		if err := stagesResults.checkStageNameAvailable(stage.Name); err != nil {
   274  			return nil, err
   275  		}
   276  		dispatchRequest = newDispatchRequest(b, escapeToken, source, buildArgs, stagesResults)
   277  
   278  		currentCommandIndex = printCommand(b.Stdout, currentCommandIndex, totalCommands, stage.SourceCode)
   279  		if err := initializeStage(dispatchRequest, &stage); err != nil {
   280  			return nil, err
   281  		}
   282  		dispatchRequest.state.updateRunConfig()
   283  		fmt.Fprintf(b.Stdout, " ---> %s\n", stringid.TruncateID(dispatchRequest.state.imageID))
   284  		for _, cmd := range stage.Commands {
   285  			select {
   286  			case <-b.clientCtx.Done():
   287  				logrus.Debug("Builder: build cancelled!")
   288  				fmt.Fprint(b.Stdout, "Build cancelled\n")
   289  				buildsFailed.WithValues(metricsBuildCanceled).Inc()
   290  				return nil, errors.New("Build cancelled")
   291  			default:
   292  				// Not cancelled yet, keep going...
   293  			}
   294  
   295  			currentCommandIndex = printCommand(b.Stdout, currentCommandIndex, totalCommands, cmd)
   296  
   297  			if err := dispatch(dispatchRequest, cmd); err != nil {
   298  				return nil, err
   299  			}
   300  			dispatchRequest.state.updateRunConfig()
   301  			fmt.Fprintf(b.Stdout, " ---> %s\n", stringid.TruncateID(dispatchRequest.state.imageID))
   302  
   303  		}
   304  		if err := emitImageID(b.Aux, dispatchRequest.state); err != nil {
   305  			return nil, err
   306  		}
   307  		buildArgs.MergeReferencedArgs(dispatchRequest.state.buildArgs)
   308  		if err := commitStage(dispatchRequest.state, stagesResults); err != nil {
   309  			return nil, err
   310  		}
   311  	}
   312  	buildArgs.WarnOnUnusedBuildArgs(b.Stdout)
   313  	return dispatchRequest.state, nil
   314  }
   315  
   316  // BuildFromConfig builds directly from `changes`, treating it as if it were the contents of a Dockerfile
   317  // It will:
   318  // - Call parse.Parse() to get an AST root for the concatenated Dockerfile entries.
   319  // - Do build by calling builder.dispatch() to call all entries' handling routines
   320  //
   321  // BuildFromConfig is used by the /commit endpoint, with the changes
   322  // coming from the query parameter of the same name.
   323  //
   324  // TODO: Remove?
   325  func BuildFromConfig(config *container.Config, changes []string, os string) (*container.Config, error) {
   326  	if !system.IsOSSupported(os) {
   327  		return nil, errdefs.InvalidParameter(system.ErrNotSupportedOperatingSystem)
   328  	}
   329  	if len(changes) == 0 {
   330  		return config, nil
   331  	}
   332  
   333  	dockerfile, err := parser.Parse(bytes.NewBufferString(strings.Join(changes, "\n")))
   334  	if err != nil {
   335  		return nil, errdefs.InvalidParameter(err)
   336  	}
   337  
   338  	b, err := newBuilder(context.Background(), builderOptions{
   339  		Options: &types.ImageBuildOptions{NoCache: true},
   340  	})
   341  	if err != nil {
   342  		return nil, err
   343  	}
   344  
   345  	// ensure that the commands are valid
   346  	for _, n := range dockerfile.AST.Children {
   347  		if !validCommitCommands[n.Value] {
   348  			return nil, errdefs.InvalidParameter(errors.Errorf("%s is not a valid change command", n.Value))
   349  		}
   350  	}
   351  
   352  	b.Stdout = ioutil.Discard
   353  	b.Stderr = ioutil.Discard
   354  	b.disableCommit = true
   355  
   356  	var commands []instructions.Command
   357  	for _, n := range dockerfile.AST.Children {
   358  		cmd, err := instructions.ParseCommand(n)
   359  		if err != nil {
   360  			return nil, errdefs.InvalidParameter(err)
   361  		}
   362  		commands = append(commands, cmd)
   363  	}
   364  
   365  	dispatchRequest := newDispatchRequest(b, dockerfile.EscapeToken, nil, NewBuildArgs(b.options.BuildArgs), newStagesBuildResults())
   366  	// We make mutations to the configuration, ensure we have a copy
   367  	dispatchRequest.state.runConfig = copyRunConfig(config)
   368  	dispatchRequest.state.imageID = config.Image
   369  	dispatchRequest.state.operatingSystem = os
   370  	for _, cmd := range commands {
   371  		err := dispatch(dispatchRequest, cmd)
   372  		if err != nil {
   373  			return nil, errdefs.InvalidParameter(err)
   374  		}
   375  		dispatchRequest.state.updateRunConfig()
   376  	}
   377  
   378  	return dispatchRequest.state.runConfig, nil
   379  }
   380  
   381  func convertMapToEnvList(m map[string]string) []string {
   382  	result := []string{}
   383  	for k, v := range m {
   384  		result = append(result, k+"="+v)
   385  	}
   386  	return result
   387  }