github.com/tilt-dev/tilt@v0.33.15-0.20240515162809-0a22ed45d8a0/internal/tiltfile/docker_compose.go (about) 1 package tiltfile 2 3 import ( 4 "context" 5 "crypto/sha256" 6 "fmt" 7 "os" 8 "path/filepath" 9 "reflect" 10 "strconv" 11 "strings" 12 13 "golang.org/x/exp/slices" 14 15 "github.com/compose-spec/compose-go/consts" 16 "github.com/compose-spec/compose-go/loader" 17 "github.com/compose-spec/compose-go/types" 18 "github.com/distribution/reference" 19 "github.com/pkg/errors" 20 "go.starlark.net/starlark" 21 composeyaml "gopkg.in/yaml.v3" 22 23 "github.com/tilt-dev/tilt/internal/container" 24 "github.com/tilt-dev/tilt/internal/controllers/apis/liveupdate" 25 "github.com/tilt-dev/tilt/internal/dockercompose" 26 "github.com/tilt-dev/tilt/internal/sliceutils" 27 "github.com/tilt-dev/tilt/internal/tiltfile/io" 28 "github.com/tilt-dev/tilt/internal/tiltfile/links" 29 "github.com/tilt-dev/tilt/internal/tiltfile/starkit" 30 "github.com/tilt-dev/tilt/internal/tiltfile/value" 31 "github.com/tilt-dev/tilt/pkg/apis/core/v1alpha1" 32 "github.com/tilt-dev/tilt/pkg/logger" 33 "github.com/tilt-dev/tilt/pkg/model" 34 ) 35 36 // dcResourceSet represents a single docker-compose config file and all its associated services 37 type dcResourceSet struct { 38 Project v1alpha1.DockerComposeProject 39 40 configPaths []string 41 tiltfilePath string 42 services map[string]*dcService 43 serviceNames []string 44 resOptions map[string]*dcResourceOptions 45 } 46 47 type dcResourceMap map[string]*dcResourceSet 48 49 func (dc dcResourceSet) Empty() bool { return reflect.DeepEqual(dc, dcResourceSet{}) } 50 51 func (dc dcResourceSet) ServiceCount() int { return len(dc.services) } 52 53 func (dcm dcResourceMap) ServiceCount() int { 54 svcCount := 0 55 for _, dc := range dcm { 56 svcCount += dc.ServiceCount() 57 } 58 return svcCount 59 } 60 61 func (s *tiltfileState) dockerCompose(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { 62 var configPaths starlark.Value 63 var projectName string 64 var profiles value.StringOrStringList 65 var wait = value.Optional[starlark.Bool]{Value: false} 66 envFile := value.NewLocalPathUnpacker(thread) 67 68 err := s.unpackArgs(fn.Name(), args, kwargs, 69 "configPaths", &configPaths, 70 "env_file?", &envFile, 71 "project_name?", &projectName, 72 "profiles?", &profiles, 73 "wait?", &wait, 74 ) 75 if err != nil { 76 return nil, err 77 } 78 79 paths := starlarkValueOrSequenceToSlice(configPaths) 80 81 if len(paths) == 0 { 82 return nil, fmt.Errorf("Nothing to compose") 83 } 84 85 project := v1alpha1.DockerComposeProject{ 86 Name: projectName, 87 EnvFile: envFile.Value, 88 Profiles: profiles.Values, 89 Wait: bool(wait.Value), 90 } 91 92 if project.EnvFile != "" { 93 err = io.RecordReadPath(thread, io.WatchFileOnly, project.EnvFile) 94 if err != nil { 95 return nil, err 96 } 97 } 98 99 for _, val := range paths { 100 switch v := val.(type) { 101 case nil: 102 continue 103 case io.Blob: 104 yaml := v.String() 105 message := "unable to store yaml blob" 106 tmpdir, err := s.tempDir() 107 if err != nil { 108 return nil, errors.Wrap(err, message) 109 } 110 tmpfile, err := os.Create(filepath.Join(tmpdir.Path(), fmt.Sprintf("%x.yml", sha256.Sum256([]byte(yaml))))) 111 if err != nil { 112 return nil, errors.Wrap(err, message) 113 } 114 _, err = tmpfile.WriteString(yaml) 115 if err != nil { 116 tmpfile.Close() 117 return nil, errors.Wrap(err, message) 118 } 119 err = tmpfile.Close() 120 if err != nil { 121 return nil, errors.Wrap(err, message) 122 } 123 project.ConfigPaths = append(project.ConfigPaths, tmpfile.Name()) 124 default: 125 path, err := value.ValueToAbsPath(thread, val) 126 if err != nil { 127 return starlark.None, fmt.Errorf("expected blob | path (string). Actual type: %T", val) 128 } 129 130 // Set project path/name to dir of first compose file, like DC CLI does 131 if project.ProjectPath == "" { 132 project.ProjectPath = filepath.Dir(path) 133 } 134 if project.Name == "" { 135 project.Name = loader.NormalizeProjectName(filepath.Base(filepath.Dir(path))) 136 } 137 138 project.ConfigPaths = append(project.ConfigPaths, path) 139 err = io.RecordReadPath(thread, io.WatchFileOnly, path) 140 if err != nil { 141 return nil, err 142 } 143 } 144 } 145 146 currentTiltfilePath := starkit.CurrentExecPath(thread) 147 148 if project.Name == "" { 149 project.Name = loader.NormalizeProjectName(filepath.Base(filepath.Dir(currentTiltfilePath))) 150 } 151 152 // Set to tiltfile directory for YAML blob tempfiles 153 if project.ProjectPath == "" { 154 project.ProjectPath = filepath.Dir(currentTiltfilePath) 155 } 156 157 if !profiles.IsSet && os.Getenv(consts.ComposeProfiles) != "" { 158 logger.Get(s.ctx).Infof("Compose project %q loading profiles from environment: %s", 159 project.Name, os.Getenv(consts.ComposeProfiles)) 160 project.Profiles = strings.Split(os.Getenv(consts.ComposeProfiles), ",") 161 } 162 163 dc := s.dc[project.Name] 164 if dc == nil { 165 dc = &dcResourceSet{ 166 Project: project, 167 services: make(map[string]*dcService), 168 resOptions: make(map[string]*dcResourceOptions), 169 configPaths: project.ConfigPaths, 170 tiltfilePath: currentTiltfilePath, 171 } 172 s.dc[project.Name] = dc 173 } else { 174 dc.configPaths = sliceutils.AppendWithoutDupes(dc.configPaths, project.ConfigPaths...) 175 dc.Project.ConfigPaths = dc.configPaths 176 if project.EnvFile != "" { 177 dc.Project.EnvFile = project.EnvFile 178 } 179 project = dc.Project 180 } 181 182 services, err := parseDCConfig(s.ctx, s.dcCli, dc) 183 if err != nil { 184 return nil, err 185 } 186 187 dc.services = make(map[string]*dcService) 188 dc.serviceNames = []string{} 189 for _, svc := range services { 190 err := s.checkResourceConflict(svc.Name) 191 if err != nil { 192 return nil, err 193 } 194 195 dc.serviceNames = append(dc.serviceNames, svc.Name) 196 for _, f := range svc.ServiceConfig.EnvFile { 197 if !filepath.IsAbs(f) { 198 f = filepath.Join(project.ProjectPath, f) 199 } 200 err = io.RecordReadPath(thread, io.WatchFileOnly, f) 201 if err != nil { 202 return nil, err 203 } 204 } 205 dc.services[svc.Name] = svc 206 } 207 208 return starlark.None, nil 209 } 210 211 // DCResource allows you to adjust specific settings on a DC resource that we assume 212 // to be defined in a `docker_compose.yml` 213 func (s *tiltfileState) dcResource(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { 214 var name string 215 var projectName string 216 var newName string 217 var imageVal starlark.Value 218 var triggerMode triggerMode 219 var resourceDepsVal starlark.Sequence 220 var inferLinks = value.Optional[starlark.Bool]{Value: true} 221 var links links.LinkList 222 var labels value.LabelSet 223 var autoInit = value.Optional[starlark.Bool]{Value: true} 224 225 if err := s.unpackArgs(fn.Name(), args, kwargs, 226 "name", &name, 227 // TODO(milas): this argument is undocumented and arguably unnecessary 228 // now that Tilt correctly infers the Docker Compose image ref format 229 "image?", &imageVal, 230 "trigger_mode?", &triggerMode, 231 "resource_deps?", &resourceDepsVal, 232 "infer_links?", &inferLinks, 233 "links?", &links, 234 "labels?", &labels, 235 "auto_init?", &autoInit, 236 "project_name?", &projectName, 237 "new_name?", &newName, 238 ); err != nil { 239 return nil, err 240 } 241 242 if name == "" { 243 return nil, fmt.Errorf("dc_resource: `name` must not be empty") 244 } 245 246 var imageRefAsStr *string 247 switch imageVal := imageVal.(type) { 248 case nil: // optional arg, this is fine 249 case starlark.String: 250 s := string(imageVal) 251 imageRefAsStr = &s 252 default: 253 return nil, fmt.Errorf("image arg must be a string; got %T", imageVal) 254 } 255 256 projectName, svc, err := s.getDCService(name, projectName) 257 if err != nil { 258 return nil, err 259 } 260 261 if newName != "" { 262 name, err = s.renameDCService(projectName, name, newName, svc) 263 if err != nil { 264 return nil, err 265 } 266 } 267 268 options := s.dc[projectName].resOptions[name] 269 if options == nil { 270 options = newDcResourceOptions() 271 } 272 273 if triggerMode != TriggerModeUnset { 274 options.TriggerMode = triggerMode 275 } 276 277 if inferLinks.IsSet { 278 options.InferLinks = inferLinks 279 } 280 281 options.Links = append(options.Links, links.Links...) 282 283 for key, val := range labels.Values { 284 options.Labels[key] = val 285 } 286 287 if imageRefAsStr != nil { 288 normalized, err := container.ParseNamed(*imageRefAsStr) 289 if err != nil { 290 return nil, err 291 } 292 options.imageRefFromUser = normalized 293 } 294 295 rds, err := value.SequenceToStringSlice(resourceDepsVal) 296 if err != nil { 297 return nil, errors.Wrapf(err, "%s: resource_deps", fn.Name()) 298 } 299 for _, dep := range rds { 300 options.resourceDeps = sliceutils.AppendWithoutDupes(options.resourceDeps, dep) 301 } 302 303 if autoInit.IsSet { 304 options.AutoInit = autoInit 305 } 306 307 s.dc[projectName].resOptions[name] = options 308 svc.Options = options 309 return starlark.None, nil 310 } 311 312 func (s *tiltfileState) getDCService(svcName, projName string) (string, *dcService, error) { 313 allNames := []string{} 314 foundProjName := "" 315 var foundSvc *dcService 316 317 for _, dc := range s.dc { 318 if projName != "" && dc.Project.Name != projName { 319 continue 320 } 321 322 for key, svc := range dc.services { 323 if key == svcName { 324 if foundSvc != nil { 325 return "", nil, fmt.Errorf("found multiple resources named %q, "+ 326 "please specify which one with project_name= argument", svcName) 327 } 328 foundProjName = dc.Project.Name 329 foundSvc = svc 330 } 331 allNames = append(allNames, key) 332 } 333 } 334 335 if foundSvc == nil { 336 return "", nil, fmt.Errorf("no Docker Compose service found with name %q. "+ 337 "Found these instead:\n\t%s", svcName, strings.Join(allNames, "; ")) 338 } 339 340 return foundProjName, foundSvc, nil 341 } 342 343 func (s *tiltfileState) renameDCService(projectName, name, newName string, svc *dcService) (string, error) { 344 err := s.checkResourceConflict(newName) 345 if err != nil { 346 return "", err 347 } 348 349 project := s.dc[projectName] 350 services := project.services 351 352 services[newName] = svc 353 delete(services, name) 354 if opts, exists := project.resOptions[name]; exists { 355 project.resOptions[newName] = opts 356 delete(project.resOptions, name) 357 } 358 index := -1 359 for i, n := range project.serviceNames { 360 if n == name && index == -1 { 361 index = i 362 } else if sd, ok := services[n].ServiceConfig.DependsOn[name]; ok { 363 services[n].ServiceConfig.DependsOn[newName] = sd 364 if rdIndex := slices.Index(services[n].Options.resourceDeps, name); rdIndex != -1 { 365 services[n].Options.resourceDeps[rdIndex] = newName 366 } 367 delete(services[n].ServiceConfig.DependsOn, name) 368 } 369 } 370 project.serviceNames[index] = newName 371 svc.Name = newName 372 return newName, nil 373 } 374 375 // A docker-compose service, according to Tilt. 376 type dcService struct { 377 Name string 378 379 // Contains the name of the service as referenced in the compose file where it was loaded. 380 ServiceName string 381 382 // these are the host machine paths that DC will sync from the local volume into the container 383 // https://docs.docker.com/compose/compose-file/#volumes 384 MountedLocalDirs []string 385 386 // RefSelector of the image associated with this service 387 // The user-provided image ref overrides the config-provided image ref 388 imageRefFromConfig reference.Named // from docker-compose.yml `Image` field 389 390 ServiceConfig types.ServiceConfig 391 392 // Currently just use this to diff against when config files are edited to see if manifest has changed 393 ServiceYAML []byte 394 395 ImageMapDeps []string 396 PublishedPorts []int 397 398 Options *dcResourceOptions 399 } 400 401 // Options set via dc_resource 402 type dcResourceOptions struct { 403 imageRefFromUser reference.Named 404 TriggerMode triggerMode 405 InferLinks value.Optional[starlark.Bool] 406 Links []model.Link 407 AutoInit value.Optional[starlark.Bool] 408 409 Labels map[string]string 410 411 resourceDeps []string 412 } 413 414 func newDcResourceOptions() *dcResourceOptions { 415 return &dcResourceOptions{ 416 Labels: make(map[string]string), 417 } 418 } 419 420 func (svc dcService) ImageRef() reference.Named { 421 if svc.Options != nil && svc.Options.imageRefFromUser != nil { 422 return svc.Options.imageRefFromUser 423 } 424 return svc.imageRefFromConfig 425 } 426 427 func dockerComposeConfigToService(dcrs *dcResourceSet, projectName string, svcConfig types.ServiceConfig) (dcService, error) { 428 var mountedLocalDirs []string 429 for _, v := range svcConfig.Volumes { 430 mountedLocalDirs = append(mountedLocalDirs, v.Source) 431 } 432 433 var publishedPorts []int 434 for _, portSpec := range svcConfig.Ports { 435 // a published port can be a string range of ports (e.g. "80-90") 436 // this case is unusual and unsupported/ignored by Tilt for now 437 publishedPort, err := strconv.Atoi(portSpec.Published) 438 if err == nil && publishedPort != 0 { 439 publishedPorts = append(publishedPorts, publishedPort) 440 } 441 } 442 443 rawConfig, err := composeyaml.Marshal(svcConfig) 444 if err != nil { 445 return dcService{}, err 446 } 447 448 imageName := svcConfig.Image 449 if imageName == "" { 450 // see https://github.com/docker/compose/blob/7b84f2c2a538a1241dcf65f4b2828232189ef0ad/pkg/compose/create.go#L221-L227 451 imageName = fmt.Sprintf("%s_%s", projectName, svcConfig.Name) 452 } 453 454 imageRef, err := container.ParseNamed(imageName) 455 if err != nil { 456 // TODO(nick): This doesn't seem like the right place to report this 457 // error, but we don't really have a better way right now. 458 return dcService{}, fmt.Errorf("Error parsing image name %q: %v", imageName, err) 459 } 460 461 options, exists := dcrs.resOptions[svcConfig.Name] 462 if !exists { 463 options = newDcResourceOptions() 464 dcrs.resOptions[svcConfig.Name] = options 465 } 466 467 options.resourceDeps = sliceutils.DedupedAndSorted( 468 append(options.resourceDeps, svcConfig.GetDependencies()...)) 469 470 svc := dcService{ 471 Name: svcConfig.Name, 472 ServiceName: svcConfig.Name, 473 ServiceConfig: svcConfig, 474 MountedLocalDirs: mountedLocalDirs, 475 ServiceYAML: rawConfig, 476 PublishedPorts: publishedPorts, 477 Options: options, 478 imageRefFromConfig: imageRef, 479 } 480 481 return svc, nil 482 } 483 484 func parseDCConfig(ctx context.Context, dcc dockercompose.DockerComposeClient, dcrs *dcResourceSet) ([]*dcService, error) { 485 proj, err := dcc.Project(ctx, dcrs.Project) 486 if err != nil { 487 return nil, err 488 } 489 490 var services []*dcService 491 err = proj.WithServices(proj.ServiceNames(), func(svcConfig types.ServiceConfig) error { 492 svc, err := dockerComposeConfigToService(dcrs, proj.Name, svcConfig) 493 if err != nil { 494 return errors.Wrapf(err, "getting service %s", svcConfig.Name) 495 } 496 services = append(services, &svc) 497 return nil 498 }) 499 if err != nil { 500 return nil, err 501 } 502 503 return services, nil 504 } 505 506 func (s *tiltfileState) dcServiceToManifest(service *dcService, dcSet *dcResourceSet, iTargets []model.ImageTarget) (model.Manifest, error) { 507 options := service.Options 508 if options == nil { 509 options = newDcResourceOptions() 510 } 511 512 dcInfo := model.DockerComposeTarget{ 513 Name: model.TargetName(service.Name), 514 Spec: v1alpha1.DockerComposeServiceSpec{ 515 Service: service.ServiceName, 516 Project: dcSet.Project, 517 }, 518 ServiceYAML: string(service.ServiceYAML), 519 Links: options.Links, 520 }.WithImageMapDeps(model.FilterLiveUpdateOnly(service.ImageMapDeps, iTargets)). 521 WithPublishedPorts(service.PublishedPorts) 522 523 if options.InferLinks.IsSet { 524 dcInfo = dcInfo.WithInferLinks(bool(options.InferLinks.Value)) 525 } 526 527 autoInit := true 528 if options.AutoInit.IsSet { 529 autoInit = bool(options.AutoInit.Value) 530 } 531 um, err := starlarkTriggerModeToModel(s.triggerModeForResource(options.TriggerMode), autoInit) 532 if err != nil { 533 return model.Manifest{}, err 534 } 535 536 var mds []model.ManifestName 537 for _, md := range options.resourceDeps { 538 mds = append(mds, model.ManifestName(md)) 539 } 540 541 for i, iTarget := range iTargets { 542 if liveupdate.IsEmptySpec(iTarget.LiveUpdateSpec) { 543 continue 544 } 545 iTarget.LiveUpdateReconciler = true 546 iTargets[i] = iTarget 547 } 548 549 m := model.Manifest{ 550 Name: model.ManifestName(service.Name), 551 TriggerMode: um, 552 ResourceDependencies: mds, 553 }.WithDeployTarget(dcInfo). 554 WithLabels(options.Labels). 555 WithImageTargets(iTargets) 556 557 return m, nil 558 }