github.com/tilt-dev/tilt@v0.33.15-0.20240515162809-0a22ed45d8a0/internal/dockercompose/client.go (about) 1 package dockercompose 2 3 import ( 4 "bufio" 5 "bytes" 6 "context" 7 "fmt" 8 "io" 9 "os" 10 "os/exec" 11 "regexp" 12 "strings" 13 "sync" 14 15 "github.com/compose-spec/compose-go/loader" 16 "golang.org/x/mod/semver" 17 18 "github.com/compose-spec/compose-go/types" 19 "github.com/pkg/errors" 20 21 "github.com/tilt-dev/tilt/internal/container" 22 "github.com/tilt-dev/tilt/internal/docker" 23 "github.com/tilt-dev/tilt/pkg/apis/core/v1alpha1" 24 "github.com/tilt-dev/tilt/pkg/logger" 25 26 compose "github.com/compose-spec/compose-go/cli" 27 ) 28 29 // versionRegex handles both v1 and v2 version outputs, which have several variations. 30 // (See TestParseComposeVersionOutput for various cases.) 31 var versionRegex = regexp.MustCompile(`(?mi)^docker[ -]compose(?: version)?:? v?([^\s,]+),?(?: build ([a-z0-9-]+))?`) 32 33 // dcProjectOptions are used when loading Docker Compose projects via the Go library. 34 // 35 // See also: dcLoaderOption which is used for loading projects from the CLI fallback and for tests, which should 36 // be kept in sync behavior-wise. 37 var dcProjectOptions = []compose.ProjectOptionsFn{ 38 compose.WithResolvedPaths(true), 39 compose.WithNormalization(true), 40 compose.WithOsEnv, 41 } 42 43 type DockerComposeClient interface { 44 Up(ctx context.Context, spec v1alpha1.DockerComposeServiceSpec, shouldBuild bool, stdout, stderr io.Writer) error 45 Down(ctx context.Context, spec v1alpha1.DockerComposeProject, stdout, stderr io.Writer) error 46 Rm(ctx context.Context, specs []v1alpha1.DockerComposeServiceSpec, stdout, stderr io.Writer) error 47 StreamEvents(ctx context.Context, spec v1alpha1.DockerComposeProject) (<-chan string, error) 48 Project(ctx context.Context, spec v1alpha1.DockerComposeProject) (*types.Project, error) 49 ContainerID(ctx context.Context, spec v1alpha1.DockerComposeServiceSpec) (container.ID, error) 50 Version(ctx context.Context) (canonicalVersion string, build string, err error) 51 } 52 53 type cmdDCClient struct { 54 env docker.Env 55 mu *sync.Mutex 56 initCmd *sync.Once 57 58 composeCmd []string 59 version string 60 build string 61 err error 62 } 63 64 // TODO(dmiller): we might want to make this take a path to the docker-compose config so we don't 65 // have to keep passing it in. 66 func NewDockerComposeClient(lenv docker.LocalEnv) DockerComposeClient { 67 return &cmdDCClient{ 68 env: docker.Env(lenv), 69 mu: &sync.Mutex{}, 70 initCmd: &sync.Once{}, 71 } 72 } 73 74 func (c *cmdDCClient) projectArgs(p v1alpha1.DockerComposeProject) []string { 75 result := []string{} 76 77 if p.Name != "" { 78 result = append(result, "--project-name", p.Name) 79 } 80 81 if p.ProjectPath != "" { 82 result = append(result, "--project-directory", p.ProjectPath) 83 } 84 85 if p.EnvFile != "" { 86 result = append(result, "--env-file", p.EnvFile) 87 } 88 89 if p.YAML != "" { 90 result = append(result, "-f", "-") 91 } 92 93 for _, cp := range p.ConfigPaths { 94 result = append(result, "-f", cp) 95 } 96 97 for _, p := range p.Profiles { 98 result = append(result, "--profile", p) 99 } 100 101 return result 102 } 103 104 func (c *cmdDCClient) Up(ctx context.Context, spec v1alpha1.DockerComposeServiceSpec, shouldBuild bool, stdout, stderr io.Writer) error { 105 genArgs := c.projectArgs(spec.Project) 106 // TODO(milas): this causes docker-compose to output a truly excessive amount of logging; it might 107 // make sense to hide it behind a special environment variable instead or something 108 if logger.Get(ctx).Level().ShouldDisplay(logger.VerboseLvl) { 109 genArgs = append(genArgs, "--verbose") 110 } 111 112 if shouldBuild { 113 var buildArgs = append([]string{}, genArgs...) 114 buildArgs = append(buildArgs, "build", spec.Service) 115 cmd := c.dcCommand(ctx, buildArgs) 116 cmd.Stdin = strings.NewReader(spec.Project.YAML) 117 cmd.Stdout = stdout 118 cmd.Stderr = stderr 119 err := cmd.Run() 120 if err != nil { 121 return FormatError(cmd, nil, err) 122 } 123 } 124 125 // docker-compose up is not thread-safe, because network operations are non-atomic. See: 126 // https://github.com/tilt-dev/tilt/issues/2817 127 // 128 // docker-compose build can run in parallel fine, so we only want the mutex on the 'up' call. 129 // 130 // TODO(nick): It might make sense to use a CondVar so that we can log a message 131 // when we're waiting on another build... 132 c.mu.Lock() 133 defer c.mu.Unlock() 134 runArgs := append([]string{}, genArgs...) 135 runArgs = append(runArgs, "up", "--no-deps", "--remove-orphans", "--no-build", "-d", spec.Service) 136 137 if spec.Project.Wait { 138 runArgs = append(runArgs, "--wait") 139 } 140 141 cmd := c.dcCommand(ctx, runArgs) 142 cmd.Stdin = strings.NewReader(spec.Project.YAML) 143 cmd.Stdout = stdout 144 cmd.Stderr = stderr 145 146 return FormatError(cmd, nil, cmd.Run()) 147 } 148 149 func (c *cmdDCClient) Down(ctx context.Context, p v1alpha1.DockerComposeProject, stdout, stderr io.Writer) error { 150 // To be safe, we try not to run two docker-compose downs in parallel, 151 // because we know docker-compose up is not thread-safe. 152 c.mu.Lock() 153 defer c.mu.Unlock() 154 155 args := c.projectArgs(p) 156 if logger.Get(ctx).Level().ShouldDisplay(logger.VerboseLvl) { 157 args = append(args, "--verbose") 158 } 159 160 args = append(args, "down", "--remove-orphans") 161 cmd := c.dcCommand(ctx, args) 162 cmd.Stdin = strings.NewReader(p.YAML) 163 cmd.Stdout = stdout 164 cmd.Stderr = stderr 165 166 err := cmd.Run() 167 if err != nil { 168 return FormatError(cmd, nil, err) 169 } 170 171 return nil 172 } 173 174 func (c *cmdDCClient) Rm(ctx context.Context, specs []v1alpha1.DockerComposeServiceSpec, stdout, stderr io.Writer) error { 175 if len(specs) == 0 { 176 return nil 177 } 178 179 // To be safe, we try not to run two docker-compose downs in parallel, 180 // because we know docker-compose up is not thread-safe. 181 c.mu.Lock() 182 defer c.mu.Unlock() 183 184 p := specs[0].Project 185 args := c.projectArgs(p) 186 if logger.Get(ctx).Level().ShouldDisplay(logger.VerboseLvl) { 187 args = append(args, "--verbose") 188 } 189 190 var serviceNames []string 191 for _, s := range specs { 192 serviceNames = append(serviceNames, s.Service) 193 } 194 195 // `docker-compose rm` does not support a `--timeout` option, so it possibly defaults to 10, 196 // like `docker-compose stop` or `docker-compose down`. 197 // If it turns out this command's timeout is too long, we might want to change this to first 198 // call `docker-compose stop --timeout $NUM`, to do the presumably slow part under a smaller 199 // timeout. 200 args = append(args, []string{"rm", "--stop", "--force"}...) 201 args = append(args, serviceNames...) 202 cmd := c.dcCommand(ctx, args) 203 cmd.Stdin = strings.NewReader(p.YAML) 204 cmd.Stdout = stdout 205 cmd.Stderr = stderr 206 207 err := cmd.Run() 208 if err != nil { 209 return FormatError(cmd, nil, err) 210 } 211 212 return nil 213 } 214 215 func (c *cmdDCClient) StreamEvents(ctx context.Context, p v1alpha1.DockerComposeProject) (<-chan string, error) { 216 ch := make(chan string) 217 218 args := c.projectArgs(p) 219 args = append(args, "events", "--json") 220 cmd := c.dcCommand(ctx, args) 221 cmd.Stdin = strings.NewReader(p.YAML) 222 stdout, err := cmd.StdoutPipe() 223 if err != nil { 224 return ch, errors.Wrap(err, "making stdout pipe for `docker-compose events`") 225 } 226 227 err = cmd.Start() 228 if err != nil { 229 return ch, errors.Wrapf(err, "`docker-compose %s`", 230 strings.Join(args, " ")) 231 } 232 go func() { 233 scanner := bufio.NewScanner(stdout) 234 for scanner.Scan() { 235 ch <- scanner.Text() 236 } 237 238 if err := scanner.Err(); err != nil { 239 logger.Get(ctx).Infof("[DOCKER-COMPOSE WATCHER] scanning `events` output: %v", err) 240 } 241 242 err = cmd.Wait() 243 if err != nil { 244 logger.Get(ctx).Infof("[DOCKER-COMPOSE WATCHER] exited with error: %v", err) 245 } 246 }() 247 248 return ch, nil 249 } 250 251 func (c *cmdDCClient) Project(ctx context.Context, spec v1alpha1.DockerComposeProject) (*types.Project, error) { 252 var proj *types.Project 253 var err error 254 255 // First, use compose-go to natively load the project. 256 if len(spec.ConfigPaths) > 0 { 257 parsed, err := c.loadProjectNative(spec) 258 if err == nil { 259 proj = parsed 260 } 261 } 262 263 // HACK(milas): compose-go has known regressions with resolving variables during YAML loading 264 // if it fails, attempt to fallback to using the CLI to resolve the YAML and then parse 265 // it with compose-go 266 // see https://github.com/tilt-dev/tilt/issues/4795 267 if proj == nil { 268 proj, err = c.loadProjectCLI(ctx, spec) 269 if err != nil { 270 return nil, err 271 } 272 } 273 274 return proj, nil 275 } 276 277 func (c *cmdDCClient) ContainerID(ctx context.Context, spec v1alpha1.DockerComposeServiceSpec) (container.ID, error) { 278 id, err := c.dcOutput(ctx, spec.Project, "ps", "-a", "-q", spec.Service) 279 if err != nil { 280 return container.ID(""), err 281 } 282 283 return container.ID(id), nil 284 } 285 286 // Version returns the parsed output of `docker compose version`, the canonical version and build (if present). 287 // 288 // NOTE: The version subcommand was added in Docker Compose v1.4.0 (released 2015-08-04), so this won't work for 289 // 290 // truly ancient versions, but handles both v1 and v2. 291 func (c *cmdDCClient) Version(ctx context.Context) (string, string, error) { 292 c.initDcCommand() 293 return c.version, c.build, c.err 294 } 295 296 func composeProjectOptions(modelProj v1alpha1.DockerComposeProject, env []string) (*compose.ProjectOptions, error) { 297 // NOTE: take care to keep behavior in sync with loadProjectCLI() 298 allProjectOptions := append(dcProjectOptions, 299 compose.WithWorkingDirectory(modelProj.ProjectPath), 300 compose.WithName(modelProj.Name), 301 compose.WithResolvedPaths(true), 302 compose.WithEnv(env), 303 compose.WithProfiles(modelProj.Profiles), 304 ) 305 if modelProj.EnvFile != "" { 306 allProjectOptions = append(allProjectOptions, compose.WithEnvFiles(modelProj.EnvFile)) 307 } 308 allProjectOptions = append(allProjectOptions, compose.WithDotEnv) 309 return compose.NewProjectOptions(modelProj.ConfigPaths, allProjectOptions...) 310 } 311 312 func (c *cmdDCClient) loadProjectNative(modelProj v1alpha1.DockerComposeProject) (*types.Project, error) { 313 opts, err := composeProjectOptions(modelProj, c.mergedEnv()) 314 if err != nil { 315 return nil, err 316 } 317 proj, err := compose.ProjectFromOptions(opts) 318 if err != nil { 319 return nil, err 320 } 321 return proj, nil 322 } 323 324 func (c *cmdDCClient) loadProjectCLI(ctx context.Context, proj v1alpha1.DockerComposeProject) (*types.Project, error) { 325 resolvedYAML, err := c.dcOutput(ctx, proj, "config") 326 if err != nil { 327 return nil, err 328 } 329 330 // docker-compose is very inconsistent about whether it fully resolves paths or not via CLI, both between 331 // v1 and v2 as well as even different releases within v2, so set the workdir and force the loader to resolve 332 // any relative paths 333 return loader.LoadWithContext(ctx, types.ConfigDetails{ 334 WorkingDir: proj.ProjectPath, 335 ConfigFiles: []types.ConfigFile{ 336 { 337 Content: []byte(resolvedYAML), 338 }, 339 }, 340 // no environment specified because the CLI call will already have resolved all variables 341 }, dcLoaderOption(proj.Name), loader.WithProfiles(proj.Profiles)) 342 } 343 344 // dcLoaderOption is used when loading Docker Compose projects via the CLI and fallback and for tests. 345 // 346 // See also: dcProjectOptions which is used for loading projects from the Go library, which should 347 // be kept in sync behavior-wise. 348 func dcLoaderOption(name string) func(opts *loader.Options) { 349 return func(opts *loader.Options) { 350 opts.SetProjectName(name, true) 351 opts.ResolvePaths = true 352 opts.SkipNormalization = false 353 opts.SkipInterpolation = false 354 } 355 } 356 357 func dcExecutableVersion(environ []string) ([]string, string, string, error) { 358 execVersion := func(names []string) (string, string, error) { 359 args := append(names, "version") 360 cmd := exec.Command(args[0], args[1:]...) 361 cmd.Env = append(os.Environ(), environ...) 362 stdout, err := cmd.Output() 363 if err != nil { 364 return "", "", FormatError(cmd, stdout, err) 365 } 366 ver, build, err := parseComposeVersionOutput(stdout) 367 return ver, build, err 368 } 369 370 var cmd []string 371 if cmdstr := os.Getenv("TILT_DOCKER_COMPOSE_CMD"); cmdstr != "" { 372 cmd = []string{cmdstr} 373 ver, build, err := execVersion(cmd) 374 return cmd, ver, build, err 375 } 376 377 cmd = []string{"docker", "compose"} 378 ver, build, err := execVersion(cmd) 379 if err != nil { 380 cmd = []string{"docker-compose"} 381 ver, build, err = execVersion(cmd) 382 } 383 384 return cmd, ver, build, err 385 } 386 387 func (c *cmdDCClient) initDcCommand() { 388 c.initCmd.Do(func() { 389 cmd, version, build, err := dcExecutableVersion(c.env.AsEnviron()) 390 c.composeCmd = cmd 391 c.version = version 392 c.build = build 393 c.err = err 394 }) 395 } 396 397 func (c *cmdDCClient) mergedEnv() []string { 398 return append(os.Environ(), c.env.AsEnviron()...) 399 } 400 401 func (c *cmdDCClient) dcCommand(ctx context.Context, args []string) *exec.Cmd { 402 c.initDcCommand() 403 composeCmd := c.composeCmd[0] 404 composeArgs := c.composeCmd[1:] 405 if len(composeArgs) > 0 { 406 args = append(composeArgs, args...) 407 } 408 cmd := exec.CommandContext(ctx, composeCmd, args...) 409 cmd.Env = c.mergedEnv() 410 return cmd 411 } 412 413 func (c *cmdDCClient) dcOutput(ctx context.Context, p v1alpha1.DockerComposeProject, args ...string) (string, error) { 414 415 tempArgs := c.projectArgs(p) 416 args = append(tempArgs, args...) 417 cmd := c.dcCommand(ctx, args) 418 cmd.Stdin = strings.NewReader(p.YAML) 419 420 output, err := cmd.Output() 421 if err != nil { 422 errorMessage := fmt.Sprintf("command %q failed.\nerror: %v\nstdout: %q", cmd.Args, err, string(output)) 423 if err, ok := err.(*exec.ExitError); ok { 424 errorMessage += fmt.Sprintf("\nstderr: '%v'", string(err.Stderr)) 425 } 426 err = fmt.Errorf("%s", errorMessage) 427 } 428 return strings.TrimSpace(string(output)), err 429 } 430 431 // parseComposeVersionOutput parses the raw output of `docker-compose version` for both v1.x + v2.x Compose 432 // and returns the canonical semver + build (might be blank) or an error. 433 func parseComposeVersionOutput(stdout []byte) (string, string, error) { 434 // match 0: raw output 435 // match 1: version w/o leading v (required) 436 // match 2: build (optional) 437 m := versionRegex.FindSubmatch(bytes.TrimSpace(stdout)) 438 if len(m) < 2 { 439 return "", "", fmt.Errorf("could not parse version from output: %q", string(stdout)) 440 } 441 rawVersion := "v" + string(m[1]) 442 canonicalVersion := semver.Canonical(rawVersion) 443 if canonicalVersion == "" { 444 return "", "", fmt.Errorf("invalid version: %q", rawVersion) 445 } 446 build := semver.Build(rawVersion) 447 if build != "" { 448 // prefer semver build if present, but strip off the leading `+` 449 // (currently, Docker Compose has not made use of this, preferring to list the build independently if at all) 450 build = strings.TrimPrefix(build, "+") 451 } else if len(m) > 2 { 452 // otherwise, fall back to regex match if possible 453 build = string(m[2]) 454 } 455 return canonicalVersion, build, nil 456 } 457 458 func FormatError(cmd *exec.Cmd, stdout []byte, err error) error { 459 if err == nil { 460 return nil 461 } 462 errorMessage := fmt.Sprintf("command %q failed.\nerror: %v\n", cmd.Args, err) 463 if len(stdout) > 0 { 464 errorMessage += fmt.Sprintf("\nstdout: '%v'", string(stdout)) 465 } 466 if err, ok := err.(*exec.ExitError); ok && len(err.Stderr) > 0 { 467 errorMessage += fmt.Sprintf("\nstderr: '%v'", string(err.Stderr)) 468 } 469 return fmt.Errorf("%s", errorMessage) 470 }