github.com/tilt-dev/tilt@v0.33.15-0.20240515162809-0a22ed45d8a0/internal/tiltfile/docker_compose.go (about)

     1  package tiltfile
     2  
     3  import (
     4  	"context"
     5  	"crypto/sha256"
     6  	"fmt"
     7  	"os"
     8  	"path/filepath"
     9  	"reflect"
    10  	"strconv"
    11  	"strings"
    12  
    13  	"golang.org/x/exp/slices"
    14  
    15  	"github.com/compose-spec/compose-go/consts"
    16  	"github.com/compose-spec/compose-go/loader"
    17  	"github.com/compose-spec/compose-go/types"
    18  	"github.com/distribution/reference"
    19  	"github.com/pkg/errors"
    20  	"go.starlark.net/starlark"
    21  	composeyaml "gopkg.in/yaml.v3"
    22  
    23  	"github.com/tilt-dev/tilt/internal/container"
    24  	"github.com/tilt-dev/tilt/internal/controllers/apis/liveupdate"
    25  	"github.com/tilt-dev/tilt/internal/dockercompose"
    26  	"github.com/tilt-dev/tilt/internal/sliceutils"
    27  	"github.com/tilt-dev/tilt/internal/tiltfile/io"
    28  	"github.com/tilt-dev/tilt/internal/tiltfile/links"
    29  	"github.com/tilt-dev/tilt/internal/tiltfile/starkit"
    30  	"github.com/tilt-dev/tilt/internal/tiltfile/value"
    31  	"github.com/tilt-dev/tilt/pkg/apis/core/v1alpha1"
    32  	"github.com/tilt-dev/tilt/pkg/logger"
    33  	"github.com/tilt-dev/tilt/pkg/model"
    34  )
    35  
    36  // dcResourceSet represents a single docker-compose config file and all its associated services
    37  type dcResourceSet struct {
    38  	Project v1alpha1.DockerComposeProject
    39  
    40  	configPaths  []string
    41  	tiltfilePath string
    42  	services     map[string]*dcService
    43  	serviceNames []string
    44  	resOptions   map[string]*dcResourceOptions
    45  }
    46  
    47  type dcResourceMap map[string]*dcResourceSet
    48  
    49  func (dc dcResourceSet) Empty() bool { return reflect.DeepEqual(dc, dcResourceSet{}) }
    50  
    51  func (dc dcResourceSet) ServiceCount() int { return len(dc.services) }
    52  
    53  func (dcm dcResourceMap) ServiceCount() int {
    54  	svcCount := 0
    55  	for _, dc := range dcm {
    56  		svcCount += dc.ServiceCount()
    57  	}
    58  	return svcCount
    59  }
    60  
    61  func (s *tiltfileState) dockerCompose(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
    62  	var configPaths starlark.Value
    63  	var projectName string
    64  	var profiles value.StringOrStringList
    65  	var wait = value.Optional[starlark.Bool]{Value: false}
    66  	envFile := value.NewLocalPathUnpacker(thread)
    67  
    68  	err := s.unpackArgs(fn.Name(), args, kwargs,
    69  		"configPaths", &configPaths,
    70  		"env_file?", &envFile,
    71  		"project_name?", &projectName,
    72  		"profiles?", &profiles,
    73  		"wait?", &wait,
    74  	)
    75  	if err != nil {
    76  		return nil, err
    77  	}
    78  
    79  	paths := starlarkValueOrSequenceToSlice(configPaths)
    80  
    81  	if len(paths) == 0 {
    82  		return nil, fmt.Errorf("Nothing to compose")
    83  	}
    84  
    85  	project := v1alpha1.DockerComposeProject{
    86  		Name:     projectName,
    87  		EnvFile:  envFile.Value,
    88  		Profiles: profiles.Values,
    89  		Wait:     bool(wait.Value),
    90  	}
    91  
    92  	if project.EnvFile != "" {
    93  		err = io.RecordReadPath(thread, io.WatchFileOnly, project.EnvFile)
    94  		if err != nil {
    95  			return nil, err
    96  		}
    97  	}
    98  
    99  	for _, val := range paths {
   100  		switch v := val.(type) {
   101  		case nil:
   102  			continue
   103  		case io.Blob:
   104  			yaml := v.String()
   105  			message := "unable to store yaml blob"
   106  			tmpdir, err := s.tempDir()
   107  			if err != nil {
   108  				return nil, errors.Wrap(err, message)
   109  			}
   110  			tmpfile, err := os.Create(filepath.Join(tmpdir.Path(), fmt.Sprintf("%x.yml", sha256.Sum256([]byte(yaml)))))
   111  			if err != nil {
   112  				return nil, errors.Wrap(err, message)
   113  			}
   114  			_, err = tmpfile.WriteString(yaml)
   115  			if err != nil {
   116  				tmpfile.Close()
   117  				return nil, errors.Wrap(err, message)
   118  			}
   119  			err = tmpfile.Close()
   120  			if err != nil {
   121  				return nil, errors.Wrap(err, message)
   122  			}
   123  			project.ConfigPaths = append(project.ConfigPaths, tmpfile.Name())
   124  		default:
   125  			path, err := value.ValueToAbsPath(thread, val)
   126  			if err != nil {
   127  				return starlark.None, fmt.Errorf("expected blob | path (string). Actual type: %T", val)
   128  			}
   129  
   130  			// Set project path/name to dir of first compose file, like DC CLI does
   131  			if project.ProjectPath == "" {
   132  				project.ProjectPath = filepath.Dir(path)
   133  			}
   134  			if project.Name == "" {
   135  				project.Name = loader.NormalizeProjectName(filepath.Base(filepath.Dir(path)))
   136  			}
   137  
   138  			project.ConfigPaths = append(project.ConfigPaths, path)
   139  			err = io.RecordReadPath(thread, io.WatchFileOnly, path)
   140  			if err != nil {
   141  				return nil, err
   142  			}
   143  		}
   144  	}
   145  
   146  	currentTiltfilePath := starkit.CurrentExecPath(thread)
   147  
   148  	if project.Name == "" {
   149  		project.Name = loader.NormalizeProjectName(filepath.Base(filepath.Dir(currentTiltfilePath)))
   150  	}
   151  
   152  	// Set to tiltfile directory for YAML blob tempfiles
   153  	if project.ProjectPath == "" {
   154  		project.ProjectPath = filepath.Dir(currentTiltfilePath)
   155  	}
   156  
   157  	if !profiles.IsSet && os.Getenv(consts.ComposeProfiles) != "" {
   158  		logger.Get(s.ctx).Infof("Compose project %q loading profiles from environment: %s",
   159  			project.Name, os.Getenv(consts.ComposeProfiles))
   160  		project.Profiles = strings.Split(os.Getenv(consts.ComposeProfiles), ",")
   161  	}
   162  
   163  	dc := s.dc[project.Name]
   164  	if dc == nil {
   165  		dc = &dcResourceSet{
   166  			Project:      project,
   167  			services:     make(map[string]*dcService),
   168  			resOptions:   make(map[string]*dcResourceOptions),
   169  			configPaths:  project.ConfigPaths,
   170  			tiltfilePath: currentTiltfilePath,
   171  		}
   172  		s.dc[project.Name] = dc
   173  	} else {
   174  		dc.configPaths = sliceutils.AppendWithoutDupes(dc.configPaths, project.ConfigPaths...)
   175  		dc.Project.ConfigPaths = dc.configPaths
   176  		if project.EnvFile != "" {
   177  			dc.Project.EnvFile = project.EnvFile
   178  		}
   179  		project = dc.Project
   180  	}
   181  
   182  	services, err := parseDCConfig(s.ctx, s.dcCli, dc)
   183  	if err != nil {
   184  		return nil, err
   185  	}
   186  
   187  	dc.services = make(map[string]*dcService)
   188  	dc.serviceNames = []string{}
   189  	for _, svc := range services {
   190  		err := s.checkResourceConflict(svc.Name)
   191  		if err != nil {
   192  			return nil, err
   193  		}
   194  
   195  		dc.serviceNames = append(dc.serviceNames, svc.Name)
   196  		for _, f := range svc.ServiceConfig.EnvFile {
   197  			if !filepath.IsAbs(f) {
   198  				f = filepath.Join(project.ProjectPath, f)
   199  			}
   200  			err = io.RecordReadPath(thread, io.WatchFileOnly, f)
   201  			if err != nil {
   202  				return nil, err
   203  			}
   204  		}
   205  		dc.services[svc.Name] = svc
   206  	}
   207  
   208  	return starlark.None, nil
   209  }
   210  
   211  // DCResource allows you to adjust specific settings on a DC resource that we assume
   212  // to be defined in a `docker_compose.yml`
   213  func (s *tiltfileState) dcResource(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
   214  	var name string
   215  	var projectName string
   216  	var newName string
   217  	var imageVal starlark.Value
   218  	var triggerMode triggerMode
   219  	var resourceDepsVal starlark.Sequence
   220  	var inferLinks = value.Optional[starlark.Bool]{Value: true}
   221  	var links links.LinkList
   222  	var labels value.LabelSet
   223  	var autoInit = value.Optional[starlark.Bool]{Value: true}
   224  
   225  	if err := s.unpackArgs(fn.Name(), args, kwargs,
   226  		"name", &name,
   227  		// TODO(milas): this argument is undocumented and arguably unnecessary
   228  		// 	now that Tilt correctly infers the Docker Compose image ref format
   229  		"image?", &imageVal,
   230  		"trigger_mode?", &triggerMode,
   231  		"resource_deps?", &resourceDepsVal,
   232  		"infer_links?", &inferLinks,
   233  		"links?", &links,
   234  		"labels?", &labels,
   235  		"auto_init?", &autoInit,
   236  		"project_name?", &projectName,
   237  		"new_name?", &newName,
   238  	); err != nil {
   239  		return nil, err
   240  	}
   241  
   242  	if name == "" {
   243  		return nil, fmt.Errorf("dc_resource: `name` must not be empty")
   244  	}
   245  
   246  	var imageRefAsStr *string
   247  	switch imageVal := imageVal.(type) {
   248  	case nil: // optional arg, this is fine
   249  	case starlark.String:
   250  		s := string(imageVal)
   251  		imageRefAsStr = &s
   252  	default:
   253  		return nil, fmt.Errorf("image arg must be a string; got %T", imageVal)
   254  	}
   255  
   256  	projectName, svc, err := s.getDCService(name, projectName)
   257  	if err != nil {
   258  		return nil, err
   259  	}
   260  
   261  	if newName != "" {
   262  		name, err = s.renameDCService(projectName, name, newName, svc)
   263  		if err != nil {
   264  			return nil, err
   265  		}
   266  	}
   267  
   268  	options := s.dc[projectName].resOptions[name]
   269  	if options == nil {
   270  		options = newDcResourceOptions()
   271  	}
   272  
   273  	if triggerMode != TriggerModeUnset {
   274  		options.TriggerMode = triggerMode
   275  	}
   276  
   277  	if inferLinks.IsSet {
   278  		options.InferLinks = inferLinks
   279  	}
   280  
   281  	options.Links = append(options.Links, links.Links...)
   282  
   283  	for key, val := range labels.Values {
   284  		options.Labels[key] = val
   285  	}
   286  
   287  	if imageRefAsStr != nil {
   288  		normalized, err := container.ParseNamed(*imageRefAsStr)
   289  		if err != nil {
   290  			return nil, err
   291  		}
   292  		options.imageRefFromUser = normalized
   293  	}
   294  
   295  	rds, err := value.SequenceToStringSlice(resourceDepsVal)
   296  	if err != nil {
   297  		return nil, errors.Wrapf(err, "%s: resource_deps", fn.Name())
   298  	}
   299  	for _, dep := range rds {
   300  		options.resourceDeps = sliceutils.AppendWithoutDupes(options.resourceDeps, dep)
   301  	}
   302  
   303  	if autoInit.IsSet {
   304  		options.AutoInit = autoInit
   305  	}
   306  
   307  	s.dc[projectName].resOptions[name] = options
   308  	svc.Options = options
   309  	return starlark.None, nil
   310  }
   311  
   312  func (s *tiltfileState) getDCService(svcName, projName string) (string, *dcService, error) {
   313  	allNames := []string{}
   314  	foundProjName := ""
   315  	var foundSvc *dcService
   316  
   317  	for _, dc := range s.dc {
   318  		if projName != "" && dc.Project.Name != projName {
   319  			continue
   320  		}
   321  
   322  		for key, svc := range dc.services {
   323  			if key == svcName {
   324  				if foundSvc != nil {
   325  					return "", nil, fmt.Errorf("found multiple resources named %q, "+
   326  						"please specify which one with project_name= argument", svcName)
   327  				}
   328  				foundProjName = dc.Project.Name
   329  				foundSvc = svc
   330  			}
   331  			allNames = append(allNames, key)
   332  		}
   333  	}
   334  
   335  	if foundSvc == nil {
   336  		return "", nil, fmt.Errorf("no Docker Compose service found with name %q. "+
   337  			"Found these instead:\n\t%s", svcName, strings.Join(allNames, "; "))
   338  	}
   339  
   340  	return foundProjName, foundSvc, nil
   341  }
   342  
   343  func (s *tiltfileState) renameDCService(projectName, name, newName string, svc *dcService) (string, error) {
   344  	err := s.checkResourceConflict(newName)
   345  	if err != nil {
   346  		return "", err
   347  	}
   348  
   349  	project := s.dc[projectName]
   350  	services := project.services
   351  
   352  	services[newName] = svc
   353  	delete(services, name)
   354  	if opts, exists := project.resOptions[name]; exists {
   355  		project.resOptions[newName] = opts
   356  		delete(project.resOptions, name)
   357  	}
   358  	index := -1
   359  	for i, n := range project.serviceNames {
   360  		if n == name && index == -1 {
   361  			index = i
   362  		} else if sd, ok := services[n].ServiceConfig.DependsOn[name]; ok {
   363  			services[n].ServiceConfig.DependsOn[newName] = sd
   364  			if rdIndex := slices.Index(services[n].Options.resourceDeps, name); rdIndex != -1 {
   365  				services[n].Options.resourceDeps[rdIndex] = newName
   366  			}
   367  			delete(services[n].ServiceConfig.DependsOn, name)
   368  		}
   369  	}
   370  	project.serviceNames[index] = newName
   371  	svc.Name = newName
   372  	return newName, nil
   373  }
   374  
   375  // A docker-compose service, according to Tilt.
   376  type dcService struct {
   377  	Name string
   378  
   379  	// Contains the name of the service as referenced in the compose file where it was loaded.
   380  	ServiceName string
   381  
   382  	// these are the host machine paths that DC will sync from the local volume into the container
   383  	// https://docs.docker.com/compose/compose-file/#volumes
   384  	MountedLocalDirs []string
   385  
   386  	// RefSelector of the image associated with this service
   387  	// The user-provided image ref overrides the config-provided image ref
   388  	imageRefFromConfig reference.Named // from docker-compose.yml `Image` field
   389  
   390  	ServiceConfig types.ServiceConfig
   391  
   392  	// Currently just use this to diff against when config files are edited to see if manifest has changed
   393  	ServiceYAML []byte
   394  
   395  	ImageMapDeps   []string
   396  	PublishedPorts []int
   397  
   398  	Options *dcResourceOptions
   399  }
   400  
   401  // Options set via dc_resource
   402  type dcResourceOptions struct {
   403  	imageRefFromUser reference.Named
   404  	TriggerMode      triggerMode
   405  	InferLinks       value.Optional[starlark.Bool]
   406  	Links            []model.Link
   407  	AutoInit         value.Optional[starlark.Bool]
   408  
   409  	Labels map[string]string
   410  
   411  	resourceDeps []string
   412  }
   413  
   414  func newDcResourceOptions() *dcResourceOptions {
   415  	return &dcResourceOptions{
   416  		Labels: make(map[string]string),
   417  	}
   418  }
   419  
   420  func (svc dcService) ImageRef() reference.Named {
   421  	if svc.Options != nil && svc.Options.imageRefFromUser != nil {
   422  		return svc.Options.imageRefFromUser
   423  	}
   424  	return svc.imageRefFromConfig
   425  }
   426  
   427  func dockerComposeConfigToService(dcrs *dcResourceSet, projectName string, svcConfig types.ServiceConfig) (dcService, error) {
   428  	var mountedLocalDirs []string
   429  	for _, v := range svcConfig.Volumes {
   430  		mountedLocalDirs = append(mountedLocalDirs, v.Source)
   431  	}
   432  
   433  	var publishedPorts []int
   434  	for _, portSpec := range svcConfig.Ports {
   435  		// a published port can be a string range of ports (e.g. "80-90")
   436  		// this case is unusual and unsupported/ignored by Tilt for now
   437  		publishedPort, err := strconv.Atoi(portSpec.Published)
   438  		if err == nil && publishedPort != 0 {
   439  			publishedPorts = append(publishedPorts, publishedPort)
   440  		}
   441  	}
   442  
   443  	rawConfig, err := composeyaml.Marshal(svcConfig)
   444  	if err != nil {
   445  		return dcService{}, err
   446  	}
   447  
   448  	imageName := svcConfig.Image
   449  	if imageName == "" {
   450  		// see https://github.com/docker/compose/blob/7b84f2c2a538a1241dcf65f4b2828232189ef0ad/pkg/compose/create.go#L221-L227
   451  		imageName = fmt.Sprintf("%s_%s", projectName, svcConfig.Name)
   452  	}
   453  
   454  	imageRef, err := container.ParseNamed(imageName)
   455  	if err != nil {
   456  		// TODO(nick): This doesn't seem like the right place to report this
   457  		// error, but we don't really have a better way right now.
   458  		return dcService{}, fmt.Errorf("Error parsing image name %q: %v", imageName, err)
   459  	}
   460  
   461  	options, exists := dcrs.resOptions[svcConfig.Name]
   462  	if !exists {
   463  		options = newDcResourceOptions()
   464  		dcrs.resOptions[svcConfig.Name] = options
   465  	}
   466  
   467  	options.resourceDeps = sliceutils.DedupedAndSorted(
   468  		append(options.resourceDeps, svcConfig.GetDependencies()...))
   469  
   470  	svc := dcService{
   471  		Name:               svcConfig.Name,
   472  		ServiceName:        svcConfig.Name,
   473  		ServiceConfig:      svcConfig,
   474  		MountedLocalDirs:   mountedLocalDirs,
   475  		ServiceYAML:        rawConfig,
   476  		PublishedPorts:     publishedPorts,
   477  		Options:            options,
   478  		imageRefFromConfig: imageRef,
   479  	}
   480  
   481  	return svc, nil
   482  }
   483  
   484  func parseDCConfig(ctx context.Context, dcc dockercompose.DockerComposeClient, dcrs *dcResourceSet) ([]*dcService, error) {
   485  	proj, err := dcc.Project(ctx, dcrs.Project)
   486  	if err != nil {
   487  		return nil, err
   488  	}
   489  
   490  	var services []*dcService
   491  	err = proj.WithServices(proj.ServiceNames(), func(svcConfig types.ServiceConfig) error {
   492  		svc, err := dockerComposeConfigToService(dcrs, proj.Name, svcConfig)
   493  		if err != nil {
   494  			return errors.Wrapf(err, "getting service %s", svcConfig.Name)
   495  		}
   496  		services = append(services, &svc)
   497  		return nil
   498  	})
   499  	if err != nil {
   500  		return nil, err
   501  	}
   502  
   503  	return services, nil
   504  }
   505  
   506  func (s *tiltfileState) dcServiceToManifest(service *dcService, dcSet *dcResourceSet, iTargets []model.ImageTarget) (model.Manifest, error) {
   507  	options := service.Options
   508  	if options == nil {
   509  		options = newDcResourceOptions()
   510  	}
   511  
   512  	dcInfo := model.DockerComposeTarget{
   513  		Name: model.TargetName(service.Name),
   514  		Spec: v1alpha1.DockerComposeServiceSpec{
   515  			Service: service.ServiceName,
   516  			Project: dcSet.Project,
   517  		},
   518  		ServiceYAML: string(service.ServiceYAML),
   519  		Links:       options.Links,
   520  	}.WithImageMapDeps(model.FilterLiveUpdateOnly(service.ImageMapDeps, iTargets)).
   521  		WithPublishedPorts(service.PublishedPorts)
   522  
   523  	if options.InferLinks.IsSet {
   524  		dcInfo = dcInfo.WithInferLinks(bool(options.InferLinks.Value))
   525  	}
   526  
   527  	autoInit := true
   528  	if options.AutoInit.IsSet {
   529  		autoInit = bool(options.AutoInit.Value)
   530  	}
   531  	um, err := starlarkTriggerModeToModel(s.triggerModeForResource(options.TriggerMode), autoInit)
   532  	if err != nil {
   533  		return model.Manifest{}, err
   534  	}
   535  
   536  	var mds []model.ManifestName
   537  	for _, md := range options.resourceDeps {
   538  		mds = append(mds, model.ManifestName(md))
   539  	}
   540  
   541  	for i, iTarget := range iTargets {
   542  		if liveupdate.IsEmptySpec(iTarget.LiveUpdateSpec) {
   543  			continue
   544  		}
   545  		iTarget.LiveUpdateReconciler = true
   546  		iTargets[i] = iTarget
   547  	}
   548  
   549  	m := model.Manifest{
   550  		Name:                 model.ManifestName(service.Name),
   551  		TriggerMode:          um,
   552  		ResourceDependencies: mds,
   553  	}.WithDeployTarget(dcInfo).
   554  		WithLabels(options.Labels).
   555  		WithImageTargets(iTargets)
   556  
   557  	return m, nil
   558  }