github.com/tilt-dev/tilt@v0.33.15-0.20240515162809-0a22ed45d8a0/internal/dockercompose/client.go (about)

     1  package dockercompose
     2  
     3  import (
     4  	"bufio"
     5  	"bytes"
     6  	"context"
     7  	"fmt"
     8  	"io"
     9  	"os"
    10  	"os/exec"
    11  	"regexp"
    12  	"strings"
    13  	"sync"
    14  
    15  	"github.com/compose-spec/compose-go/loader"
    16  	"golang.org/x/mod/semver"
    17  
    18  	"github.com/compose-spec/compose-go/types"
    19  	"github.com/pkg/errors"
    20  
    21  	"github.com/tilt-dev/tilt/internal/container"
    22  	"github.com/tilt-dev/tilt/internal/docker"
    23  	"github.com/tilt-dev/tilt/pkg/apis/core/v1alpha1"
    24  	"github.com/tilt-dev/tilt/pkg/logger"
    25  
    26  	compose "github.com/compose-spec/compose-go/cli"
    27  )
    28  
    29  // versionRegex handles both v1 and v2 version outputs, which have several variations.
    30  // (See TestParseComposeVersionOutput for various cases.)
    31  var versionRegex = regexp.MustCompile(`(?mi)^docker[ -]compose(?: version)?:? v?([^\s,]+),?(?: build ([a-z0-9-]+))?`)
    32  
    33  // dcProjectOptions are used when loading Docker Compose projects via the Go library.
    34  //
    35  // See also: dcLoaderOption which is used for loading projects from the CLI fallback and for tests, which should
    36  // be kept in sync behavior-wise.
    37  var dcProjectOptions = []compose.ProjectOptionsFn{
    38  	compose.WithResolvedPaths(true),
    39  	compose.WithNormalization(true),
    40  	compose.WithOsEnv,
    41  }
    42  
    43  type DockerComposeClient interface {
    44  	Up(ctx context.Context, spec v1alpha1.DockerComposeServiceSpec, shouldBuild bool, stdout, stderr io.Writer) error
    45  	Down(ctx context.Context, spec v1alpha1.DockerComposeProject, stdout, stderr io.Writer) error
    46  	Rm(ctx context.Context, specs []v1alpha1.DockerComposeServiceSpec, stdout, stderr io.Writer) error
    47  	StreamEvents(ctx context.Context, spec v1alpha1.DockerComposeProject) (<-chan string, error)
    48  	Project(ctx context.Context, spec v1alpha1.DockerComposeProject) (*types.Project, error)
    49  	ContainerID(ctx context.Context, spec v1alpha1.DockerComposeServiceSpec) (container.ID, error)
    50  	Version(ctx context.Context) (canonicalVersion string, build string, err error)
    51  }
    52  
    53  type cmdDCClient struct {
    54  	env     docker.Env
    55  	mu      *sync.Mutex
    56  	initCmd *sync.Once
    57  
    58  	composeCmd []string
    59  	version    string
    60  	build      string
    61  	err        error
    62  }
    63  
    64  // TODO(dmiller): we might want to make this take a path to the docker-compose config so we don't
    65  // have to keep passing it in.
    66  func NewDockerComposeClient(lenv docker.LocalEnv) DockerComposeClient {
    67  	return &cmdDCClient{
    68  		env:     docker.Env(lenv),
    69  		mu:      &sync.Mutex{},
    70  		initCmd: &sync.Once{},
    71  	}
    72  }
    73  
    74  func (c *cmdDCClient) projectArgs(p v1alpha1.DockerComposeProject) []string {
    75  	result := []string{}
    76  
    77  	if p.Name != "" {
    78  		result = append(result, "--project-name", p.Name)
    79  	}
    80  
    81  	if p.ProjectPath != "" {
    82  		result = append(result, "--project-directory", p.ProjectPath)
    83  	}
    84  
    85  	if p.EnvFile != "" {
    86  		result = append(result, "--env-file", p.EnvFile)
    87  	}
    88  
    89  	if p.YAML != "" {
    90  		result = append(result, "-f", "-")
    91  	}
    92  
    93  	for _, cp := range p.ConfigPaths {
    94  		result = append(result, "-f", cp)
    95  	}
    96  
    97  	for _, p := range p.Profiles {
    98  		result = append(result, "--profile", p)
    99  	}
   100  
   101  	return result
   102  }
   103  
   104  func (c *cmdDCClient) Up(ctx context.Context, spec v1alpha1.DockerComposeServiceSpec, shouldBuild bool, stdout, stderr io.Writer) error {
   105  	genArgs := c.projectArgs(spec.Project)
   106  	// TODO(milas): this causes docker-compose to output a truly excessive amount of logging; it might
   107  	// 	make sense to hide it behind a special environment variable instead or something
   108  	if logger.Get(ctx).Level().ShouldDisplay(logger.VerboseLvl) {
   109  		genArgs = append(genArgs, "--verbose")
   110  	}
   111  
   112  	if shouldBuild {
   113  		var buildArgs = append([]string{}, genArgs...)
   114  		buildArgs = append(buildArgs, "build", spec.Service)
   115  		cmd := c.dcCommand(ctx, buildArgs)
   116  		cmd.Stdin = strings.NewReader(spec.Project.YAML)
   117  		cmd.Stdout = stdout
   118  		cmd.Stderr = stderr
   119  		err := cmd.Run()
   120  		if err != nil {
   121  			return FormatError(cmd, nil, err)
   122  		}
   123  	}
   124  
   125  	// docker-compose up is not thread-safe, because network operations are non-atomic. See:
   126  	// https://github.com/tilt-dev/tilt/issues/2817
   127  	//
   128  	// docker-compose build can run in parallel fine, so we only want the mutex on the 'up' call.
   129  	//
   130  	// TODO(nick): It might make sense to use a CondVar so that we can log a message
   131  	// when we're waiting on another build...
   132  	c.mu.Lock()
   133  	defer c.mu.Unlock()
   134  	runArgs := append([]string{}, genArgs...)
   135  	runArgs = append(runArgs, "up", "--no-deps", "--remove-orphans", "--no-build", "-d", spec.Service)
   136  
   137  	if spec.Project.Wait {
   138  		runArgs = append(runArgs, "--wait")
   139  	}
   140  
   141  	cmd := c.dcCommand(ctx, runArgs)
   142  	cmd.Stdin = strings.NewReader(spec.Project.YAML)
   143  	cmd.Stdout = stdout
   144  	cmd.Stderr = stderr
   145  
   146  	return FormatError(cmd, nil, cmd.Run())
   147  }
   148  
   149  func (c *cmdDCClient) Down(ctx context.Context, p v1alpha1.DockerComposeProject, stdout, stderr io.Writer) error {
   150  	// To be safe, we try not to run two docker-compose downs in parallel,
   151  	// because we know docker-compose up is not thread-safe.
   152  	c.mu.Lock()
   153  	defer c.mu.Unlock()
   154  
   155  	args := c.projectArgs(p)
   156  	if logger.Get(ctx).Level().ShouldDisplay(logger.VerboseLvl) {
   157  		args = append(args, "--verbose")
   158  	}
   159  
   160  	args = append(args, "down", "--remove-orphans")
   161  	cmd := c.dcCommand(ctx, args)
   162  	cmd.Stdin = strings.NewReader(p.YAML)
   163  	cmd.Stdout = stdout
   164  	cmd.Stderr = stderr
   165  
   166  	err := cmd.Run()
   167  	if err != nil {
   168  		return FormatError(cmd, nil, err)
   169  	}
   170  
   171  	return nil
   172  }
   173  
   174  func (c *cmdDCClient) Rm(ctx context.Context, specs []v1alpha1.DockerComposeServiceSpec, stdout, stderr io.Writer) error {
   175  	if len(specs) == 0 {
   176  		return nil
   177  	}
   178  
   179  	// To be safe, we try not to run two docker-compose downs in parallel,
   180  	// because we know docker-compose up is not thread-safe.
   181  	c.mu.Lock()
   182  	defer c.mu.Unlock()
   183  
   184  	p := specs[0].Project
   185  	args := c.projectArgs(p)
   186  	if logger.Get(ctx).Level().ShouldDisplay(logger.VerboseLvl) {
   187  		args = append(args, "--verbose")
   188  	}
   189  
   190  	var serviceNames []string
   191  	for _, s := range specs {
   192  		serviceNames = append(serviceNames, s.Service)
   193  	}
   194  
   195  	// `docker-compose rm` does not support a `--timeout` option, so it possibly defaults to 10,
   196  	// like `docker-compose stop` or `docker-compose down`.
   197  	// If it turns out this command's timeout is too long, we might want to change this to first
   198  	// call `docker-compose stop --timeout $NUM`, to do the presumably slow part under a smaller
   199  	// timeout.
   200  	args = append(args, []string{"rm", "--stop", "--force"}...)
   201  	args = append(args, serviceNames...)
   202  	cmd := c.dcCommand(ctx, args)
   203  	cmd.Stdin = strings.NewReader(p.YAML)
   204  	cmd.Stdout = stdout
   205  	cmd.Stderr = stderr
   206  
   207  	err := cmd.Run()
   208  	if err != nil {
   209  		return FormatError(cmd, nil, err)
   210  	}
   211  
   212  	return nil
   213  }
   214  
   215  func (c *cmdDCClient) StreamEvents(ctx context.Context, p v1alpha1.DockerComposeProject) (<-chan string, error) {
   216  	ch := make(chan string)
   217  
   218  	args := c.projectArgs(p)
   219  	args = append(args, "events", "--json")
   220  	cmd := c.dcCommand(ctx, args)
   221  	cmd.Stdin = strings.NewReader(p.YAML)
   222  	stdout, err := cmd.StdoutPipe()
   223  	if err != nil {
   224  		return ch, errors.Wrap(err, "making stdout pipe for `docker-compose events`")
   225  	}
   226  
   227  	err = cmd.Start()
   228  	if err != nil {
   229  		return ch, errors.Wrapf(err, "`docker-compose %s`",
   230  			strings.Join(args, " "))
   231  	}
   232  	go func() {
   233  		scanner := bufio.NewScanner(stdout)
   234  		for scanner.Scan() {
   235  			ch <- scanner.Text()
   236  		}
   237  
   238  		if err := scanner.Err(); err != nil {
   239  			logger.Get(ctx).Infof("[DOCKER-COMPOSE WATCHER] scanning `events` output: %v", err)
   240  		}
   241  
   242  		err = cmd.Wait()
   243  		if err != nil {
   244  			logger.Get(ctx).Infof("[DOCKER-COMPOSE WATCHER] exited with error: %v", err)
   245  		}
   246  	}()
   247  
   248  	return ch, nil
   249  }
   250  
   251  func (c *cmdDCClient) Project(ctx context.Context, spec v1alpha1.DockerComposeProject) (*types.Project, error) {
   252  	var proj *types.Project
   253  	var err error
   254  
   255  	// First, use compose-go to natively load the project.
   256  	if len(spec.ConfigPaths) > 0 {
   257  		parsed, err := c.loadProjectNative(spec)
   258  		if err == nil {
   259  			proj = parsed
   260  		}
   261  	}
   262  
   263  	// HACK(milas): compose-go has known regressions with resolving variables during YAML loading
   264  	// 	if it fails, attempt to fallback to using the CLI to resolve the YAML and then parse
   265  	// 	it with compose-go
   266  	// 	see https://github.com/tilt-dev/tilt/issues/4795
   267  	if proj == nil {
   268  		proj, err = c.loadProjectCLI(ctx, spec)
   269  		if err != nil {
   270  			return nil, err
   271  		}
   272  	}
   273  
   274  	return proj, nil
   275  }
   276  
   277  func (c *cmdDCClient) ContainerID(ctx context.Context, spec v1alpha1.DockerComposeServiceSpec) (container.ID, error) {
   278  	id, err := c.dcOutput(ctx, spec.Project, "ps", "-a", "-q", spec.Service)
   279  	if err != nil {
   280  		return container.ID(""), err
   281  	}
   282  
   283  	return container.ID(id), nil
   284  }
   285  
   286  // Version returns the parsed output of `docker compose version`, the canonical version and build (if present).
   287  //
   288  // NOTE: The version subcommand was added in Docker Compose v1.4.0 (released 2015-08-04), so this won't work for
   289  //
   290  //	truly ancient versions, but handles both v1 and v2.
   291  func (c *cmdDCClient) Version(ctx context.Context) (string, string, error) {
   292  	c.initDcCommand()
   293  	return c.version, c.build, c.err
   294  }
   295  
   296  func composeProjectOptions(modelProj v1alpha1.DockerComposeProject, env []string) (*compose.ProjectOptions, error) {
   297  	// NOTE: take care to keep behavior in sync with loadProjectCLI()
   298  	allProjectOptions := append(dcProjectOptions,
   299  		compose.WithWorkingDirectory(modelProj.ProjectPath),
   300  		compose.WithName(modelProj.Name),
   301  		compose.WithResolvedPaths(true),
   302  		compose.WithEnv(env),
   303  		compose.WithProfiles(modelProj.Profiles),
   304  	)
   305  	if modelProj.EnvFile != "" {
   306  		allProjectOptions = append(allProjectOptions, compose.WithEnvFiles(modelProj.EnvFile))
   307  	}
   308  	allProjectOptions = append(allProjectOptions, compose.WithDotEnv)
   309  	return compose.NewProjectOptions(modelProj.ConfigPaths, allProjectOptions...)
   310  }
   311  
   312  func (c *cmdDCClient) loadProjectNative(modelProj v1alpha1.DockerComposeProject) (*types.Project, error) {
   313  	opts, err := composeProjectOptions(modelProj, c.mergedEnv())
   314  	if err != nil {
   315  		return nil, err
   316  	}
   317  	proj, err := compose.ProjectFromOptions(opts)
   318  	if err != nil {
   319  		return nil, err
   320  	}
   321  	return proj, nil
   322  }
   323  
   324  func (c *cmdDCClient) loadProjectCLI(ctx context.Context, proj v1alpha1.DockerComposeProject) (*types.Project, error) {
   325  	resolvedYAML, err := c.dcOutput(ctx, proj, "config")
   326  	if err != nil {
   327  		return nil, err
   328  	}
   329  
   330  	// docker-compose is very inconsistent about whether it fully resolves paths or not via CLI, both between
   331  	// v1 and v2 as well as even different releases within v2, so set the workdir and force the loader to resolve
   332  	// any relative paths
   333  	return loader.LoadWithContext(ctx, types.ConfigDetails{
   334  		WorkingDir: proj.ProjectPath,
   335  		ConfigFiles: []types.ConfigFile{
   336  			{
   337  				Content: []byte(resolvedYAML),
   338  			},
   339  		},
   340  		// no environment specified because the CLI call will already have resolved all variables
   341  	}, dcLoaderOption(proj.Name), loader.WithProfiles(proj.Profiles))
   342  }
   343  
   344  // dcLoaderOption is used when loading Docker Compose projects via the CLI and fallback and for tests.
   345  //
   346  // See also: dcProjectOptions which is used for loading projects from the Go library, which should
   347  // be kept in sync behavior-wise.
   348  func dcLoaderOption(name string) func(opts *loader.Options) {
   349  	return func(opts *loader.Options) {
   350  		opts.SetProjectName(name, true)
   351  		opts.ResolvePaths = true
   352  		opts.SkipNormalization = false
   353  		opts.SkipInterpolation = false
   354  	}
   355  }
   356  
   357  func dcExecutableVersion(environ []string) ([]string, string, string, error) {
   358  	execVersion := func(names []string) (string, string, error) {
   359  		args := append(names, "version")
   360  		cmd := exec.Command(args[0], args[1:]...)
   361  		cmd.Env = append(os.Environ(), environ...)
   362  		stdout, err := cmd.Output()
   363  		if err != nil {
   364  			return "", "", FormatError(cmd, stdout, err)
   365  		}
   366  		ver, build, err := parseComposeVersionOutput(stdout)
   367  		return ver, build, err
   368  	}
   369  
   370  	var cmd []string
   371  	if cmdstr := os.Getenv("TILT_DOCKER_COMPOSE_CMD"); cmdstr != "" {
   372  		cmd = []string{cmdstr}
   373  		ver, build, err := execVersion(cmd)
   374  		return cmd, ver, build, err
   375  	}
   376  
   377  	cmd = []string{"docker", "compose"}
   378  	ver, build, err := execVersion(cmd)
   379  	if err != nil {
   380  		cmd = []string{"docker-compose"}
   381  		ver, build, err = execVersion(cmd)
   382  	}
   383  
   384  	return cmd, ver, build, err
   385  }
   386  
   387  func (c *cmdDCClient) initDcCommand() {
   388  	c.initCmd.Do(func() {
   389  		cmd, version, build, err := dcExecutableVersion(c.env.AsEnviron())
   390  		c.composeCmd = cmd
   391  		c.version = version
   392  		c.build = build
   393  		c.err = err
   394  	})
   395  }
   396  
   397  func (c *cmdDCClient) mergedEnv() []string {
   398  	return append(os.Environ(), c.env.AsEnviron()...)
   399  }
   400  
   401  func (c *cmdDCClient) dcCommand(ctx context.Context, args []string) *exec.Cmd {
   402  	c.initDcCommand()
   403  	composeCmd := c.composeCmd[0]
   404  	composeArgs := c.composeCmd[1:]
   405  	if len(composeArgs) > 0 {
   406  		args = append(composeArgs, args...)
   407  	}
   408  	cmd := exec.CommandContext(ctx, composeCmd, args...)
   409  	cmd.Env = c.mergedEnv()
   410  	return cmd
   411  }
   412  
   413  func (c *cmdDCClient) dcOutput(ctx context.Context, p v1alpha1.DockerComposeProject, args ...string) (string, error) {
   414  
   415  	tempArgs := c.projectArgs(p)
   416  	args = append(tempArgs, args...)
   417  	cmd := c.dcCommand(ctx, args)
   418  	cmd.Stdin = strings.NewReader(p.YAML)
   419  
   420  	output, err := cmd.Output()
   421  	if err != nil {
   422  		errorMessage := fmt.Sprintf("command %q failed.\nerror: %v\nstdout: %q", cmd.Args, err, string(output))
   423  		if err, ok := err.(*exec.ExitError); ok {
   424  			errorMessage += fmt.Sprintf("\nstderr: '%v'", string(err.Stderr))
   425  		}
   426  		err = fmt.Errorf("%s", errorMessage)
   427  	}
   428  	return strings.TrimSpace(string(output)), err
   429  }
   430  
   431  // parseComposeVersionOutput parses the raw output of `docker-compose version` for both v1.x + v2.x Compose
   432  // and returns the canonical semver + build (might be blank) or an error.
   433  func parseComposeVersionOutput(stdout []byte) (string, string, error) {
   434  	// match 0: raw output
   435  	// match 1: version w/o leading v (required)
   436  	// match 2: build (optional)
   437  	m := versionRegex.FindSubmatch(bytes.TrimSpace(stdout))
   438  	if len(m) < 2 {
   439  		return "", "", fmt.Errorf("could not parse version from output: %q", string(stdout))
   440  	}
   441  	rawVersion := "v" + string(m[1])
   442  	canonicalVersion := semver.Canonical(rawVersion)
   443  	if canonicalVersion == "" {
   444  		return "", "", fmt.Errorf("invalid version: %q", rawVersion)
   445  	}
   446  	build := semver.Build(rawVersion)
   447  	if build != "" {
   448  		// prefer semver build if present, but strip off the leading `+`
   449  		// (currently, Docker Compose has not made use of this, preferring to list the build independently if at all)
   450  		build = strings.TrimPrefix(build, "+")
   451  	} else if len(m) > 2 {
   452  		// otherwise, fall back to regex match if possible
   453  		build = string(m[2])
   454  	}
   455  	return canonicalVersion, build, nil
   456  }
   457  
   458  func FormatError(cmd *exec.Cmd, stdout []byte, err error) error {
   459  	if err == nil {
   460  		return nil
   461  	}
   462  	errorMessage := fmt.Sprintf("command %q failed.\nerror: %v\n", cmd.Args, err)
   463  	if len(stdout) > 0 {
   464  		errorMessage += fmt.Sprintf("\nstdout: '%v'", string(stdout))
   465  	}
   466  	if err, ok := err.(*exec.ExitError); ok && len(err.Stderr) > 0 {
   467  		errorMessage += fmt.Sprintf("\nstderr: '%v'", string(err.Stderr))
   468  	}
   469  	return fmt.Errorf("%s", errorMessage)
   470  }