github.com/pdmccormick/importable-docker-buildx@v0.0.0-20240426161518-e47091289030/bake/bake.go (about) 1 package bake 2 3 import ( 4 "context" 5 "encoding/csv" 6 "io" 7 "os" 8 "path" 9 "path/filepath" 10 "regexp" 11 "sort" 12 "strconv" 13 "strings" 14 "time" 15 16 composecli "github.com/compose-spec/compose-go/v2/cli" 17 "github.com/docker/buildx/bake/hclparser" 18 "github.com/docker/buildx/build" 19 controllerapi "github.com/docker/buildx/controller/pb" 20 "github.com/docker/buildx/util/buildflags" 21 "github.com/docker/buildx/util/platformutil" 22 "github.com/docker/buildx/util/progress" 23 "github.com/docker/cli/cli/config" 24 dockeropts "github.com/docker/cli/opts" 25 hcl "github.com/hashicorp/hcl/v2" 26 "github.com/moby/buildkit/client" 27 "github.com/moby/buildkit/client/llb" 28 "github.com/moby/buildkit/session/auth/authprovider" 29 "github.com/pkg/errors" 30 "github.com/zclconf/go-cty/cty" 31 "github.com/zclconf/go-cty/cty/convert" 32 ) 33 34 var ( 35 validTargetNameChars = `[a-zA-Z0-9_-]+` 36 targetNamePattern = regexp.MustCompile(`^` + validTargetNameChars + `$`) 37 ) 38 39 type File struct { 40 Name string 41 Data []byte 42 } 43 44 type Override struct { 45 Value string 46 ArrValue []string 47 } 48 49 func defaultFilenames() []string { 50 names := []string{} 51 names = append(names, composecli.DefaultFileNames...) 52 names = append(names, []string{ 53 "docker-bake.json", 54 "docker-bake.override.json", 55 "docker-bake.hcl", 56 "docker-bake.override.hcl", 57 }...) 58 return names 59 } 60 61 func ReadLocalFiles(names []string, stdin io.Reader, l progress.SubLogger) ([]File, error) { 62 isDefault := false 63 if len(names) == 0 { 64 isDefault = true 65 names = defaultFilenames() 66 } 67 out := make([]File, 0, len(names)) 68 69 setStatus := func(st *client.VertexStatus) { 70 if l != nil { 71 l.SetStatus(st) 72 } 73 } 74 75 for _, n := range names { 76 var dt []byte 77 var err error 78 if n == "-" { 79 dt, err = readWithProgress(stdin, setStatus) 80 if err != nil { 81 return nil, err 82 } 83 } else { 84 dt, err = readFileWithProgress(n, isDefault, setStatus) 85 if dt == nil && err == nil { 86 continue 87 } 88 if err != nil { 89 return nil, err 90 } 91 } 92 out = append(out, File{Name: n, Data: dt}) 93 } 94 return out, nil 95 } 96 97 func readFileWithProgress(fname string, isDefault bool, setStatus func(st *client.VertexStatus)) (dt []byte, err error) { 98 st := &client.VertexStatus{ 99 ID: "reading " + fname, 100 } 101 102 defer func() { 103 now := time.Now() 104 st.Completed = &now 105 if dt != nil || err != nil { 106 setStatus(st) 107 } 108 }() 109 110 now := time.Now() 111 st.Started = &now 112 113 f, err := os.Open(fname) 114 if err != nil { 115 if isDefault && errors.Is(err, os.ErrNotExist) { 116 return nil, nil 117 } 118 return nil, err 119 } 120 defer f.Close() 121 setStatus(st) 122 123 info, err := f.Stat() 124 if err != nil { 125 return nil, err 126 } 127 st.Total = info.Size() 128 setStatus(st) 129 130 buf := make([]byte, 1024) 131 for { 132 n, err := f.Read(buf) 133 if err == io.EOF { 134 break 135 } 136 if err != nil { 137 return nil, err 138 } 139 dt = append(dt, buf[:n]...) 140 st.Current += int64(n) 141 setStatus(st) 142 } 143 144 return dt, nil 145 } 146 147 func readWithProgress(r io.Reader, setStatus func(st *client.VertexStatus)) (dt []byte, err error) { 148 st := &client.VertexStatus{ 149 ID: "reading from stdin", 150 } 151 152 defer func() { 153 now := time.Now() 154 st.Completed = &now 155 setStatus(st) 156 }() 157 158 now := time.Now() 159 st.Started = &now 160 setStatus(st) 161 162 buf := make([]byte, 1024) 163 for { 164 n, err := r.Read(buf) 165 if err == io.EOF { 166 break 167 } 168 if err != nil { 169 return nil, err 170 } 171 dt = append(dt, buf[:n]...) 172 st.Current += int64(n) 173 setStatus(st) 174 } 175 176 return dt, nil 177 } 178 179 func ListTargets(files []File) ([]string, error) { 180 c, err := ParseFiles(files, nil) 181 if err != nil { 182 return nil, err 183 } 184 var targets []string 185 for _, g := range c.Groups { 186 targets = append(targets, g.Name) 187 } 188 for _, t := range c.Targets { 189 targets = append(targets, t.Name) 190 } 191 return dedupSlice(targets), nil 192 } 193 194 func ReadTargets(ctx context.Context, files []File, targets, overrides []string, defaults map[string]string) (map[string]*Target, map[string]*Group, error) { 195 c, err := ParseFiles(files, defaults) 196 if err != nil { 197 return nil, nil, err 198 } 199 200 for i, t := range targets { 201 targets[i] = sanitizeTargetName(t) 202 } 203 204 o, err := c.newOverrides(overrides) 205 if err != nil { 206 return nil, nil, err 207 } 208 m := map[string]*Target{} 209 n := map[string]*Group{} 210 for _, target := range targets { 211 ts, gs := c.ResolveGroup(target) 212 for _, tname := range ts { 213 t, err := c.ResolveTarget(tname, o) 214 if err != nil { 215 return nil, nil, err 216 } 217 if t != nil { 218 m[tname] = t 219 } 220 } 221 for _, gname := range gs { 222 for _, group := range c.Groups { 223 if group.Name == gname { 224 n[gname] = group 225 break 226 } 227 } 228 } 229 } 230 231 for _, target := range targets { 232 if target == "default" { 233 continue 234 } 235 if _, ok := n["default"]; !ok { 236 n["default"] = &Group{Name: "default"} 237 } 238 n["default"].Targets = append(n["default"].Targets, target) 239 } 240 if g, ok := n["default"]; ok { 241 g.Targets = dedupSlice(g.Targets) 242 } 243 244 for name, t := range m { 245 if err := c.loadLinks(name, t, m, o, nil); err != nil { 246 return nil, nil, err 247 } 248 } 249 250 return m, n, nil 251 } 252 253 func dedupSlice(s []string) []string { 254 if len(s) == 0 { 255 return s 256 } 257 var res []string 258 seen := make(map[string]struct{}) 259 for _, val := range s { 260 if _, ok := seen[val]; !ok { 261 res = append(res, val) 262 seen[val] = struct{}{} 263 } 264 } 265 return res 266 } 267 268 func dedupMap(ms ...map[string]string) map[string]string { 269 if len(ms) == 0 { 270 return nil 271 } 272 res := map[string]string{} 273 for _, m := range ms { 274 if len(m) == 0 { 275 continue 276 } 277 for k, v := range m { 278 if _, ok := res[k]; !ok { 279 res[k] = v 280 } 281 } 282 } 283 return res 284 } 285 286 func sliceToMap(env []string) (res map[string]string) { 287 res = make(map[string]string) 288 for _, s := range env { 289 kv := strings.SplitN(s, "=", 2) 290 key := kv[0] 291 switch { 292 case len(kv) == 1: 293 res[key] = "" 294 default: 295 res[key] = kv[1] 296 } 297 } 298 return 299 } 300 301 func ParseFiles(files []File, defaults map[string]string) (_ *Config, err error) { 302 defer func() { 303 err = formatHCLError(err, files) 304 }() 305 306 var c Config 307 var composeFiles []File 308 var hclFiles []*hcl.File 309 for _, f := range files { 310 isCompose, composeErr := validateComposeFile(f.Data, f.Name) 311 if isCompose { 312 if composeErr != nil { 313 return nil, composeErr 314 } 315 composeFiles = append(composeFiles, f) 316 } 317 if !isCompose { 318 hf, isHCL, err := ParseHCLFile(f.Data, f.Name) 319 if isHCL { 320 if err != nil { 321 return nil, err 322 } 323 hclFiles = append(hclFiles, hf) 324 } else if composeErr != nil { 325 return nil, errors.Wrapf(err, "failed to parse %s: parsing yaml: %v, parsing hcl", f.Name, composeErr) 326 } else { 327 return nil, err 328 } 329 } 330 } 331 332 if len(composeFiles) > 0 { 333 cfg, cmperr := ParseComposeFiles(composeFiles) 334 if cmperr != nil { 335 return nil, errors.Wrap(cmperr, "failed to parse compose file") 336 } 337 c = mergeConfig(c, *cfg) 338 c = dedupeConfig(c) 339 } 340 341 if len(hclFiles) > 0 { 342 renamed, err := hclparser.Parse(hclparser.MergeFiles(hclFiles), hclparser.Opt{ 343 LookupVar: os.LookupEnv, 344 Vars: defaults, 345 ValidateLabel: validateTargetName, 346 }, &c) 347 if err.HasErrors() { 348 return nil, err 349 } 350 351 for _, renamed := range renamed { 352 for oldName, newNames := range renamed { 353 newNames = dedupSlice(newNames) 354 if len(newNames) == 1 && oldName == newNames[0] { 355 continue 356 } 357 c.Groups = append(c.Groups, &Group{ 358 Name: oldName, 359 Targets: newNames, 360 }) 361 } 362 } 363 c = dedupeConfig(c) 364 } 365 366 return &c, nil 367 } 368 369 func dedupeConfig(c Config) Config { 370 c2 := c 371 c2.Groups = make([]*Group, 0, len(c2.Groups)) 372 for _, g := range c.Groups { 373 g1 := *g 374 g1.Targets = dedupSlice(g1.Targets) 375 c2.Groups = append(c2.Groups, &g1) 376 } 377 c2.Targets = make([]*Target, 0, len(c2.Targets)) 378 mt := map[string]*Target{} 379 for _, t := range c.Targets { 380 if t2, ok := mt[t.Name]; ok { 381 t2.Merge(t) 382 } else { 383 mt[t.Name] = t 384 c2.Targets = append(c2.Targets, t) 385 } 386 } 387 return c2 388 } 389 390 func ParseFile(dt []byte, fn string) (*Config, error) { 391 return ParseFiles([]File{{Data: dt, Name: fn}}, nil) 392 } 393 394 type Config struct { 395 Groups []*Group `json:"group" hcl:"group,block" cty:"group"` 396 Targets []*Target `json:"target" hcl:"target,block" cty:"target"` 397 } 398 399 func mergeConfig(c1, c2 Config) Config { 400 if c1.Groups == nil { 401 c1.Groups = []*Group{} 402 } 403 404 for _, g2 := range c2.Groups { 405 var g1 *Group 406 for _, g := range c1.Groups { 407 if g2.Name == g.Name { 408 g1 = g 409 break 410 } 411 } 412 if g1 == nil { 413 c1.Groups = append(c1.Groups, g2) 414 continue 415 } 416 417 nextTarget: 418 for _, t2 := range g2.Targets { 419 for _, t1 := range g1.Targets { 420 if t1 == t2 { 421 continue nextTarget 422 } 423 } 424 g1.Targets = append(g1.Targets, t2) 425 } 426 c1.Groups = append(c1.Groups, g1) 427 } 428 429 if c1.Targets == nil { 430 c1.Targets = []*Target{} 431 } 432 433 for _, t2 := range c2.Targets { 434 var t1 *Target 435 for _, t := range c1.Targets { 436 if t2.Name == t.Name { 437 t1 = t 438 break 439 } 440 } 441 if t1 != nil { 442 t1.Merge(t2) 443 t2 = t1 444 } 445 c1.Targets = append(c1.Targets, t2) 446 } 447 448 return c1 449 } 450 451 func (c Config) expandTargets(pattern string) ([]string, error) { 452 for _, target := range c.Targets { 453 if target.Name == pattern { 454 return []string{pattern}, nil 455 } 456 } 457 458 var names []string 459 for _, target := range c.Targets { 460 ok, err := path.Match(pattern, target.Name) 461 if err != nil { 462 return nil, errors.Wrapf(err, "could not match targets with '%s'", pattern) 463 } 464 if ok { 465 names = append(names, target.Name) 466 } 467 } 468 if len(names) == 0 { 469 return nil, errors.Errorf("could not find any target matching '%s'", pattern) 470 } 471 return names, nil 472 } 473 474 func (c Config) loadLinks(name string, t *Target, m map[string]*Target, o map[string]map[string]Override, visited []string) error { 475 visited = append(visited, name) 476 for _, v := range t.Contexts { 477 if strings.HasPrefix(v, "target:") { 478 target := strings.TrimPrefix(v, "target:") 479 if target == t.Name { 480 return errors.Errorf("target %s cannot link to itself", target) 481 } 482 for _, v := range visited { 483 if v == target { 484 return errors.Errorf("infinite loop from %s to %s", name, target) 485 } 486 } 487 t2, ok := m[target] 488 if !ok { 489 var err error 490 t2, err = c.ResolveTarget(target, o) 491 if err != nil { 492 return err 493 } 494 t2.Outputs = nil 495 t2.linked = true 496 m[target] = t2 497 } 498 if err := c.loadLinks(target, t2, m, o, visited); err != nil { 499 return err 500 } 501 if len(t.Platforms) > 1 && len(t2.Platforms) > 1 { 502 if !sliceEqual(t.Platforms, t2.Platforms) { 503 return errors.Errorf("target %s can't be used by %s because it is defined for different platforms %v and %v", target, name, t2.Platforms, t.Platforms) 504 } 505 } 506 } 507 } 508 return nil 509 } 510 511 func (c Config) newOverrides(v []string) (map[string]map[string]Override, error) { 512 m := map[string]map[string]Override{} 513 for _, v := range v { 514 parts := strings.SplitN(v, "=", 2) 515 keys := strings.SplitN(parts[0], ".", 3) 516 if len(keys) < 2 { 517 return nil, errors.Errorf("invalid override key %s, expected target.name", parts[0]) 518 } 519 520 pattern := keys[0] 521 if len(parts) != 2 && keys[1] != "args" { 522 return nil, errors.Errorf("invalid override %s, expected target.name=value", v) 523 } 524 525 names, err := c.expandTargets(pattern) 526 if err != nil { 527 return nil, err 528 } 529 530 kk := strings.SplitN(parts[0], ".", 2) 531 532 for _, name := range names { 533 t, ok := m[name] 534 if !ok { 535 t = map[string]Override{} 536 m[name] = t 537 } 538 539 o := t[kk[1]] 540 541 switch keys[1] { 542 case "output", "cache-to", "cache-from", "tags", "platform", "secrets", "ssh", "attest": 543 if len(parts) == 2 { 544 o.ArrValue = append(o.ArrValue, parts[1]) 545 } 546 case "args": 547 if len(keys) != 3 { 548 return nil, errors.Errorf("invalid key %s, args requires name", parts[0]) 549 } 550 if len(parts) < 2 { 551 v, ok := os.LookupEnv(keys[2]) 552 if !ok { 553 continue 554 } 555 o.Value = v 556 } 557 fallthrough 558 case "contexts": 559 if len(keys) != 3 { 560 return nil, errors.Errorf("invalid key %s, contexts requires name", parts[0]) 561 } 562 fallthrough 563 default: 564 if len(parts) == 2 { 565 o.Value = parts[1] 566 } 567 } 568 569 t[kk[1]] = o 570 } 571 } 572 return m, nil 573 } 574 575 func (c Config) ResolveGroup(name string) ([]string, []string) { 576 targets, groups := c.group(name, map[string]visit{}) 577 return dedupSlice(targets), dedupSlice(groups) 578 } 579 580 type visit struct { 581 target []string 582 group []string 583 } 584 585 func (c Config) group(name string, visited map[string]visit) ([]string, []string) { 586 if v, ok := visited[name]; ok { 587 return v.target, v.group 588 } 589 var g *Group 590 for _, group := range c.Groups { 591 if group.Name == name { 592 g = group 593 break 594 } 595 } 596 if g == nil { 597 return []string{name}, nil 598 } 599 visited[name] = visit{} 600 targets := make([]string, 0, len(g.Targets)) 601 groups := []string{name} 602 for _, t := range g.Targets { 603 ttarget, tgroup := c.group(t, visited) 604 if len(ttarget) > 0 { 605 targets = append(targets, ttarget...) 606 } else { 607 targets = append(targets, t) 608 } 609 if len(tgroup) > 0 { 610 groups = append(groups, tgroup...) 611 } 612 } 613 visited[name] = visit{target: targets, group: groups} 614 return targets, groups 615 } 616 617 func (c Config) ResolveTarget(name string, overrides map[string]map[string]Override) (*Target, error) { 618 t, err := c.target(name, map[string]*Target{}, overrides) 619 if err != nil { 620 return nil, err 621 } 622 t.Inherits = nil 623 if t.Context == nil { 624 s := "." 625 t.Context = &s 626 } 627 if t.Dockerfile == nil { 628 s := "Dockerfile" 629 t.Dockerfile = &s 630 } 631 return t, nil 632 } 633 634 func (c Config) target(name string, visited map[string]*Target, overrides map[string]map[string]Override) (*Target, error) { 635 if t, ok := visited[name]; ok { 636 return t, nil 637 } 638 visited[name] = nil 639 var t *Target 640 for _, target := range c.Targets { 641 if target.Name == name { 642 t = target 643 break 644 } 645 } 646 if t == nil { 647 return nil, errors.Errorf("failed to find target %s", name) 648 } 649 tt := &Target{} 650 for _, name := range t.Inherits { 651 t, err := c.target(name, visited, overrides) 652 if err != nil { 653 return nil, err 654 } 655 if t != nil { 656 tt.Merge(t) 657 } 658 } 659 m := defaultTarget() 660 m.Merge(tt) 661 m.Merge(t) 662 tt = m 663 if err := tt.AddOverrides(overrides[name]); err != nil { 664 return nil, err 665 } 666 tt.normalize() 667 visited[name] = tt 668 return tt, nil 669 } 670 671 type Group struct { 672 Name string `json:"-" hcl:"name,label" cty:"name"` 673 Targets []string `json:"targets" hcl:"targets" cty:"targets"` 674 // Target // TODO? 675 } 676 677 type Target struct { 678 Name string `json:"-" hcl:"name,label" cty:"name"` 679 680 // Inherits is the only field that cannot be overridden with --set 681 Inherits []string `json:"inherits,omitempty" hcl:"inherits,optional" cty:"inherits"` 682 683 Annotations []string `json:"annotations,omitempty" hcl:"annotations,optional" cty:"annotations"` 684 Attest []string `json:"attest,omitempty" hcl:"attest,optional" cty:"attest"` 685 Context *string `json:"context,omitempty" hcl:"context,optional" cty:"context"` 686 Contexts map[string]string `json:"contexts,omitempty" hcl:"contexts,optional" cty:"contexts"` 687 Dockerfile *string `json:"dockerfile,omitempty" hcl:"dockerfile,optional" cty:"dockerfile"` 688 DockerfileInline *string `json:"dockerfile-inline,omitempty" hcl:"dockerfile-inline,optional" cty:"dockerfile-inline"` 689 Args map[string]*string `json:"args,omitempty" hcl:"args,optional" cty:"args"` 690 Labels map[string]*string `json:"labels,omitempty" hcl:"labels,optional" cty:"labels"` 691 Tags []string `json:"tags,omitempty" hcl:"tags,optional" cty:"tags"` 692 CacheFrom []string `json:"cache-from,omitempty" hcl:"cache-from,optional" cty:"cache-from"` 693 CacheTo []string `json:"cache-to,omitempty" hcl:"cache-to,optional" cty:"cache-to"` 694 Target *string `json:"target,omitempty" hcl:"target,optional" cty:"target"` 695 Secrets []string `json:"secret,omitempty" hcl:"secret,optional" cty:"secret"` 696 SSH []string `json:"ssh,omitempty" hcl:"ssh,optional" cty:"ssh"` 697 Platforms []string `json:"platforms,omitempty" hcl:"platforms,optional" cty:"platforms"` 698 Outputs []string `json:"output,omitempty" hcl:"output,optional" cty:"output"` 699 Pull *bool `json:"pull,omitempty" hcl:"pull,optional" cty:"pull"` 700 NoCache *bool `json:"no-cache,omitempty" hcl:"no-cache,optional" cty:"no-cache"` 701 NetworkMode *string `json:"-" hcl:"-" cty:"-"` 702 NoCacheFilter []string `json:"no-cache-filter,omitempty" hcl:"no-cache-filter,optional" cty:"no-cache-filter"` 703 ShmSize *string `json:"shm-size,omitempty" hcl:"shm-size,optional"` 704 Ulimits []string `json:"ulimits,omitempty" hcl:"ulimits,optional"` 705 // IMPORTANT: if you add more fields here, do not forget to update newOverrides and docs/bake-reference.md. 706 707 // linked is a private field to mark a target used as a linked one 708 linked bool 709 } 710 711 var _ hclparser.WithEvalContexts = &Target{} 712 var _ hclparser.WithGetName = &Target{} 713 var _ hclparser.WithEvalContexts = &Group{} 714 var _ hclparser.WithGetName = &Group{} 715 716 func (t *Target) normalize() { 717 t.Annotations = removeDupes(t.Annotations) 718 t.Attest = removeAttestDupes(t.Attest) 719 t.Tags = removeDupes(t.Tags) 720 t.Secrets = removeDupes(t.Secrets) 721 t.SSH = removeDupes(t.SSH) 722 t.Platforms = removeDupes(t.Platforms) 723 t.CacheFrom = removeDupes(t.CacheFrom) 724 t.CacheTo = removeDupes(t.CacheTo) 725 t.Outputs = removeDupes(t.Outputs) 726 t.NoCacheFilter = removeDupes(t.NoCacheFilter) 727 t.Ulimits = removeDupes(t.Ulimits) 728 729 for k, v := range t.Contexts { 730 if v == "" { 731 delete(t.Contexts, k) 732 } 733 } 734 if len(t.Contexts) == 0 { 735 t.Contexts = nil 736 } 737 } 738 739 func (t *Target) Merge(t2 *Target) { 740 if t2.Context != nil { 741 t.Context = t2.Context 742 } 743 if t2.Dockerfile != nil { 744 t.Dockerfile = t2.Dockerfile 745 } 746 if t2.DockerfileInline != nil { 747 t.DockerfileInline = t2.DockerfileInline 748 } 749 for k, v := range t2.Args { 750 if v == nil { 751 continue 752 } 753 if t.Args == nil { 754 t.Args = map[string]*string{} 755 } 756 t.Args[k] = v 757 } 758 for k, v := range t2.Contexts { 759 if t.Contexts == nil { 760 t.Contexts = map[string]string{} 761 } 762 t.Contexts[k] = v 763 } 764 for k, v := range t2.Labels { 765 if v == nil { 766 continue 767 } 768 if t.Labels == nil { 769 t.Labels = map[string]*string{} 770 } 771 t.Labels[k] = v 772 } 773 if t2.Tags != nil { // no merge 774 t.Tags = t2.Tags 775 } 776 if t2.Target != nil { 777 t.Target = t2.Target 778 } 779 if t2.Annotations != nil { // merge 780 t.Annotations = append(t.Annotations, t2.Annotations...) 781 } 782 if t2.Attest != nil { // merge 783 t.Attest = append(t.Attest, t2.Attest...) 784 t.Attest = removeAttestDupes(t.Attest) 785 } 786 if t2.Secrets != nil { // merge 787 t.Secrets = append(t.Secrets, t2.Secrets...) 788 } 789 if t2.SSH != nil { // merge 790 t.SSH = append(t.SSH, t2.SSH...) 791 } 792 if t2.Platforms != nil { // no merge 793 t.Platforms = t2.Platforms 794 } 795 if t2.CacheFrom != nil { // merge 796 t.CacheFrom = append(t.CacheFrom, t2.CacheFrom...) 797 } 798 if t2.CacheTo != nil { // no merge 799 t.CacheTo = t2.CacheTo 800 } 801 if t2.Outputs != nil { // no merge 802 t.Outputs = t2.Outputs 803 } 804 if t2.Pull != nil { 805 t.Pull = t2.Pull 806 } 807 if t2.NoCache != nil { 808 t.NoCache = t2.NoCache 809 } 810 if t2.NetworkMode != nil { 811 t.NetworkMode = t2.NetworkMode 812 } 813 if t2.NoCacheFilter != nil { // merge 814 t.NoCacheFilter = append(t.NoCacheFilter, t2.NoCacheFilter...) 815 } 816 if t2.ShmSize != nil { // no merge 817 t.ShmSize = t2.ShmSize 818 } 819 if t2.Ulimits != nil { // merge 820 t.Ulimits = append(t.Ulimits, t2.Ulimits...) 821 } 822 t.Inherits = append(t.Inherits, t2.Inherits...) 823 } 824 825 func (t *Target) AddOverrides(overrides map[string]Override) error { 826 for key, o := range overrides { 827 value := o.Value 828 keys := strings.SplitN(key, ".", 2) 829 switch keys[0] { 830 case "context": 831 t.Context = &value 832 case "dockerfile": 833 t.Dockerfile = &value 834 case "args": 835 if len(keys) != 2 { 836 return errors.Errorf("args require name") 837 } 838 if t.Args == nil { 839 t.Args = map[string]*string{} 840 } 841 t.Args[keys[1]] = &value 842 case "contexts": 843 if len(keys) != 2 { 844 return errors.Errorf("contexts require name") 845 } 846 if t.Contexts == nil { 847 t.Contexts = map[string]string{} 848 } 849 t.Contexts[keys[1]] = value 850 case "labels": 851 if len(keys) != 2 { 852 return errors.Errorf("labels require name") 853 } 854 if t.Labels == nil { 855 t.Labels = map[string]*string{} 856 } 857 t.Labels[keys[1]] = &value 858 case "tags": 859 t.Tags = o.ArrValue 860 case "cache-from": 861 t.CacheFrom = o.ArrValue 862 case "cache-to": 863 t.CacheTo = o.ArrValue 864 case "target": 865 t.Target = &value 866 case "secrets": 867 t.Secrets = o.ArrValue 868 case "ssh": 869 t.SSH = o.ArrValue 870 case "platform": 871 t.Platforms = o.ArrValue 872 case "output": 873 t.Outputs = o.ArrValue 874 case "annotations": 875 t.Annotations = append(t.Annotations, o.ArrValue...) 876 case "attest": 877 t.Attest = append(t.Attest, o.ArrValue...) 878 case "no-cache": 879 noCache, err := strconv.ParseBool(value) 880 if err != nil { 881 return errors.Errorf("invalid value %s for boolean key no-cache", value) 882 } 883 t.NoCache = &noCache 884 case "no-cache-filter": 885 t.NoCacheFilter = o.ArrValue 886 case "shm-size": 887 t.ShmSize = &value 888 case "ulimits": 889 t.Ulimits = o.ArrValue 890 case "pull": 891 pull, err := strconv.ParseBool(value) 892 if err != nil { 893 return errors.Errorf("invalid value %s for boolean key pull", value) 894 } 895 t.Pull = &pull 896 case "push": 897 push, err := strconv.ParseBool(value) 898 if err != nil { 899 return errors.Errorf("invalid value %s for boolean key push", value) 900 } 901 t.Outputs = setPushOverride(t.Outputs, push) 902 case "load": 903 load, err := strconv.ParseBool(value) 904 if err != nil { 905 return errors.Errorf("invalid value %s for boolean key load", value) 906 } 907 t.Outputs = setLoadOverride(t.Outputs, load) 908 default: 909 return errors.Errorf("unknown key: %s", keys[0]) 910 } 911 } 912 return nil 913 } 914 915 func (g *Group) GetEvalContexts(ectx *hcl.EvalContext, block *hcl.Block, loadDeps func(hcl.Expression) hcl.Diagnostics) ([]*hcl.EvalContext, error) { 916 content, _, err := block.Body.PartialContent(&hcl.BodySchema{ 917 Attributes: []hcl.AttributeSchema{{Name: "matrix"}}, 918 }) 919 if err != nil { 920 return nil, err 921 } 922 if _, ok := content.Attributes["matrix"]; ok { 923 return nil, errors.Errorf("matrix is not supported for groups") 924 } 925 return []*hcl.EvalContext{ectx}, nil 926 } 927 928 func (t *Target) GetEvalContexts(ectx *hcl.EvalContext, block *hcl.Block, loadDeps func(hcl.Expression) hcl.Diagnostics) ([]*hcl.EvalContext, error) { 929 content, _, err := block.Body.PartialContent(&hcl.BodySchema{ 930 Attributes: []hcl.AttributeSchema{{Name: "matrix"}}, 931 }) 932 if err != nil { 933 return nil, err 934 } 935 936 attr, ok := content.Attributes["matrix"] 937 if !ok { 938 return []*hcl.EvalContext{ectx}, nil 939 } 940 if diags := loadDeps(attr.Expr); diags.HasErrors() { 941 return nil, diags 942 } 943 value, err := attr.Expr.Value(ectx) 944 if err != nil { 945 return nil, err 946 } 947 948 if !value.Type().IsMapType() && !value.Type().IsObjectType() { 949 return nil, errors.Errorf("matrix must be a map") 950 } 951 matrix := value.AsValueMap() 952 953 ectxs := []*hcl.EvalContext{ectx} 954 for k, expr := range matrix { 955 if !expr.CanIterateElements() { 956 return nil, errors.Errorf("matrix values must be a list") 957 } 958 959 ectxs2 := []*hcl.EvalContext{} 960 for _, v := range expr.AsValueSlice() { 961 for _, e := range ectxs { 962 e2 := ectx.NewChild() 963 e2.Variables = make(map[string]cty.Value) 964 if e != ectx { 965 for k, v := range e.Variables { 966 e2.Variables[k] = v 967 } 968 } 969 e2.Variables[k] = v 970 ectxs2 = append(ectxs2, e2) 971 } 972 } 973 ectxs = ectxs2 974 } 975 return ectxs, nil 976 } 977 978 func (g *Group) GetName(ectx *hcl.EvalContext, block *hcl.Block, loadDeps func(hcl.Expression) hcl.Diagnostics) (string, error) { 979 content, _, diags := block.Body.PartialContent(&hcl.BodySchema{ 980 Attributes: []hcl.AttributeSchema{{Name: "name"}, {Name: "matrix"}}, 981 }) 982 if diags != nil { 983 return "", diags 984 } 985 986 if _, ok := content.Attributes["name"]; ok { 987 return "", errors.Errorf("name is not supported for groups") 988 } 989 if _, ok := content.Attributes["matrix"]; ok { 990 return "", errors.Errorf("matrix is not supported for groups") 991 } 992 return block.Labels[0], nil 993 } 994 995 func (t *Target) GetName(ectx *hcl.EvalContext, block *hcl.Block, loadDeps func(hcl.Expression) hcl.Diagnostics) (string, error) { 996 content, _, diags := block.Body.PartialContent(&hcl.BodySchema{ 997 Attributes: []hcl.AttributeSchema{{Name: "name"}, {Name: "matrix"}}, 998 }) 999 if diags != nil { 1000 return "", diags 1001 } 1002 1003 attr, ok := content.Attributes["name"] 1004 if !ok { 1005 return block.Labels[0], nil 1006 } 1007 if _, ok := content.Attributes["matrix"]; !ok { 1008 return "", errors.Errorf("name requires matrix") 1009 } 1010 if diags := loadDeps(attr.Expr); diags.HasErrors() { 1011 return "", diags 1012 } 1013 value, diags := attr.Expr.Value(ectx) 1014 if diags != nil { 1015 return "", diags 1016 } 1017 1018 value, err := convert.Convert(value, cty.String) 1019 if err != nil { 1020 return "", err 1021 } 1022 return value.AsString(), nil 1023 } 1024 1025 func TargetsToBuildOpt(m map[string]*Target, inp *Input) (map[string]build.Options, error) { 1026 // make sure local credentials are loaded multiple times for different targets 1027 dockerConfig := config.LoadDefaultConfigFile(os.Stderr) 1028 authProvider := authprovider.NewDockerAuthProvider(dockerConfig, nil) 1029 1030 m2 := make(map[string]build.Options, len(m)) 1031 for k, v := range m { 1032 bo, err := toBuildOpt(v, inp) 1033 if err != nil { 1034 return nil, err 1035 } 1036 bo.Session = append(bo.Session, authProvider) 1037 m2[k] = *bo 1038 } 1039 return m2, nil 1040 } 1041 1042 func updateContext(t *build.Inputs, inp *Input) { 1043 if inp == nil || inp.State == nil { 1044 return 1045 } 1046 1047 for k, v := range t.NamedContexts { 1048 if v.Path == "." { 1049 t.NamedContexts[k] = build.NamedContext{Path: inp.URL} 1050 } 1051 if strings.HasPrefix(v.Path, "cwd://") || strings.HasPrefix(v.Path, "target:") || strings.HasPrefix(v.Path, "docker-image:") { 1052 continue 1053 } 1054 if build.IsRemoteURL(v.Path) { 1055 continue 1056 } 1057 st := llb.Scratch().File(llb.Copy(*inp.State, v.Path, "/"), llb.WithCustomNamef("set context %s to %s", k, v.Path)) 1058 t.NamedContexts[k] = build.NamedContext{State: &st} 1059 } 1060 1061 if t.ContextPath == "." { 1062 t.ContextPath = inp.URL 1063 return 1064 } 1065 if strings.HasPrefix(t.ContextPath, "cwd://") { 1066 return 1067 } 1068 if build.IsRemoteURL(t.ContextPath) { 1069 return 1070 } 1071 st := llb.Scratch().File( 1072 llb.Copy(*inp.State, t.ContextPath, "/", &llb.CopyInfo{ 1073 CopyDirContentsOnly: true, 1074 }), 1075 llb.WithCustomNamef("set context to %s", t.ContextPath), 1076 ) 1077 t.ContextState = &st 1078 } 1079 1080 // validateContextsEntitlements is a basic check to ensure contexts do not 1081 // escape local directories when loaded from remote sources. This is to be 1082 // replaced with proper entitlements support in the future. 1083 func validateContextsEntitlements(t build.Inputs, inp *Input) error { 1084 if inp == nil || inp.State == nil { 1085 return nil 1086 } 1087 if v, ok := os.LookupEnv("BAKE_ALLOW_REMOTE_FS_ACCESS"); ok { 1088 if vv, _ := strconv.ParseBool(v); vv { 1089 return nil 1090 } 1091 } 1092 if t.ContextState == nil { 1093 if err := checkPath(t.ContextPath); err != nil { 1094 return err 1095 } 1096 } 1097 for _, v := range t.NamedContexts { 1098 if v.State != nil { 1099 continue 1100 } 1101 if err := checkPath(v.Path); err != nil { 1102 return err 1103 } 1104 } 1105 return nil 1106 } 1107 1108 func checkPath(p string) error { 1109 if build.IsRemoteURL(p) || strings.HasPrefix(p, "target:") || strings.HasPrefix(p, "docker-image:") { 1110 return nil 1111 } 1112 p, err := filepath.EvalSymlinks(p) 1113 if err != nil { 1114 if os.IsNotExist(err) { 1115 return nil 1116 } 1117 return err 1118 } 1119 p, err = filepath.Abs(p) 1120 if err != nil { 1121 return err 1122 } 1123 wd, err := os.Getwd() 1124 if err != nil { 1125 return err 1126 } 1127 rel, err := filepath.Rel(wd, p) 1128 if err != nil { 1129 return err 1130 } 1131 parts := strings.Split(rel, string(os.PathSeparator)) 1132 if parts[0] == ".." { 1133 return errors.Errorf("path %s is outside of the working directory, please set BAKE_ALLOW_REMOTE_FS_ACCESS=1", p) 1134 } 1135 return nil 1136 } 1137 1138 func toBuildOpt(t *Target, inp *Input) (*build.Options, error) { 1139 if v := t.Context; v != nil && *v == "-" { 1140 return nil, errors.Errorf("context from stdin not allowed in bake") 1141 } 1142 if v := t.Dockerfile; v != nil && *v == "-" { 1143 return nil, errors.Errorf("dockerfile from stdin not allowed in bake") 1144 } 1145 1146 contextPath := "." 1147 if t.Context != nil { 1148 contextPath = *t.Context 1149 } 1150 if !strings.HasPrefix(contextPath, "cwd://") && !build.IsRemoteURL(contextPath) { 1151 contextPath = path.Clean(contextPath) 1152 } 1153 dockerfilePath := "Dockerfile" 1154 if t.Dockerfile != nil { 1155 dockerfilePath = *t.Dockerfile 1156 } 1157 if !strings.HasPrefix(dockerfilePath, "cwd://") { 1158 dockerfilePath = path.Clean(dockerfilePath) 1159 } 1160 1161 bi := build.Inputs{ 1162 ContextPath: contextPath, 1163 DockerfilePath: dockerfilePath, 1164 NamedContexts: toNamedContexts(t.Contexts), 1165 } 1166 if t.DockerfileInline != nil { 1167 bi.DockerfileInline = *t.DockerfileInline 1168 } 1169 updateContext(&bi, inp) 1170 if strings.HasPrefix(bi.DockerfilePath, "cwd://") { 1171 // If Dockerfile is local for a remote invocation, we first check if 1172 // it's not outside the working directory and then resolve it to an 1173 // absolute path. 1174 bi.DockerfilePath = path.Clean(strings.TrimPrefix(bi.DockerfilePath, "cwd://")) 1175 if err := checkPath(bi.DockerfilePath); err != nil { 1176 return nil, err 1177 } 1178 var err error 1179 bi.DockerfilePath, err = filepath.Abs(bi.DockerfilePath) 1180 if err != nil { 1181 return nil, err 1182 } 1183 } else if !build.IsRemoteURL(bi.DockerfilePath) && strings.HasPrefix(bi.ContextPath, "cwd://") && (inp != nil && build.IsRemoteURL(inp.URL)) { 1184 // We don't currently support reading a remote Dockerfile with a local 1185 // context when doing a remote invocation because we automatically 1186 // derive the dockerfile from the context atm: 1187 // 1188 // target "default" { 1189 // context = BAKE_CMD_CONTEXT 1190 // dockerfile = "Dockerfile.app" 1191 // } 1192 // 1193 // > docker buildx bake https://github.com/foo/bar.git 1194 // failed to solve: failed to read dockerfile: open /var/lib/docker/tmp/buildkit-mount3004544897/Dockerfile.app: no such file or directory 1195 // 1196 // To avoid mistakenly reading a local Dockerfile, we check if the 1197 // Dockerfile exists locally and if so, we error out. 1198 if _, err := os.Stat(filepath.Join(path.Clean(strings.TrimPrefix(bi.ContextPath, "cwd://")), bi.DockerfilePath)); err == nil { 1199 return nil, errors.Errorf("reading a dockerfile for a remote build invocation is currently not supported") 1200 } 1201 } 1202 if strings.HasPrefix(bi.ContextPath, "cwd://") { 1203 bi.ContextPath = path.Clean(strings.TrimPrefix(bi.ContextPath, "cwd://")) 1204 } 1205 if !build.IsRemoteURL(bi.ContextPath) && bi.ContextState == nil && !path.IsAbs(bi.DockerfilePath) { 1206 bi.DockerfilePath = path.Join(bi.ContextPath, bi.DockerfilePath) 1207 } 1208 for k, v := range bi.NamedContexts { 1209 if strings.HasPrefix(v.Path, "cwd://") { 1210 bi.NamedContexts[k] = build.NamedContext{Path: path.Clean(strings.TrimPrefix(v.Path, "cwd://"))} 1211 } 1212 } 1213 1214 if err := validateContextsEntitlements(bi, inp); err != nil { 1215 return nil, err 1216 } 1217 1218 t.Context = &bi.ContextPath 1219 1220 args := map[string]string{} 1221 for k, v := range t.Args { 1222 if v == nil { 1223 continue 1224 } 1225 args[k] = *v 1226 } 1227 1228 labels := map[string]string{} 1229 for k, v := range t.Labels { 1230 if v == nil { 1231 continue 1232 } 1233 labels[k] = *v 1234 } 1235 1236 noCache := false 1237 if t.NoCache != nil { 1238 noCache = *t.NoCache 1239 } 1240 pull := false 1241 if t.Pull != nil { 1242 pull = *t.Pull 1243 } 1244 networkMode := "" 1245 if t.NetworkMode != nil { 1246 networkMode = *t.NetworkMode 1247 } 1248 shmSize := new(dockeropts.MemBytes) 1249 if t.ShmSize != nil { 1250 if err := shmSize.Set(*t.ShmSize); err != nil { 1251 return nil, errors.Errorf("invalid value %s for membytes key shm-size", *t.ShmSize) 1252 } 1253 } 1254 1255 bo := &build.Options{ 1256 Inputs: bi, 1257 Tags: t.Tags, 1258 BuildArgs: args, 1259 Labels: labels, 1260 NoCache: noCache, 1261 NoCacheFilter: t.NoCacheFilter, 1262 Pull: pull, 1263 NetworkMode: networkMode, 1264 Linked: t.linked, 1265 ShmSize: *shmSize, 1266 } 1267 1268 platforms, err := platformutil.Parse(t.Platforms) 1269 if err != nil { 1270 return nil, err 1271 } 1272 bo.Platforms = platforms 1273 1274 secrets, err := buildflags.ParseSecretSpecs(t.Secrets) 1275 if err != nil { 1276 return nil, err 1277 } 1278 secretAttachment, err := controllerapi.CreateSecrets(secrets) 1279 if err != nil { 1280 return nil, err 1281 } 1282 bo.Session = append(bo.Session, secretAttachment) 1283 1284 sshSpecs, err := buildflags.ParseSSHSpecs(t.SSH) 1285 if err != nil { 1286 return nil, err 1287 } 1288 if len(sshSpecs) == 0 && (buildflags.IsGitSSH(bi.ContextPath) || (inp != nil && buildflags.IsGitSSH(inp.URL))) { 1289 sshSpecs = append(sshSpecs, &controllerapi.SSH{ID: "default"}) 1290 } 1291 sshAttachment, err := controllerapi.CreateSSH(sshSpecs) 1292 if err != nil { 1293 return nil, err 1294 } 1295 bo.Session = append(bo.Session, sshAttachment) 1296 1297 if t.Target != nil { 1298 bo.Target = *t.Target 1299 } 1300 1301 cacheImports, err := buildflags.ParseCacheEntry(t.CacheFrom) 1302 if err != nil { 1303 return nil, err 1304 } 1305 bo.CacheFrom = controllerapi.CreateCaches(cacheImports) 1306 1307 cacheExports, err := buildflags.ParseCacheEntry(t.CacheTo) 1308 if err != nil { 1309 return nil, err 1310 } 1311 bo.CacheTo = controllerapi.CreateCaches(cacheExports) 1312 1313 outputs, err := buildflags.ParseExports(t.Outputs) 1314 if err != nil { 1315 return nil, err 1316 } 1317 bo.Exports, err = controllerapi.CreateExports(outputs) 1318 if err != nil { 1319 return nil, err 1320 } 1321 1322 annotations, err := buildflags.ParseAnnotations(t.Annotations) 1323 if err != nil { 1324 return nil, err 1325 } 1326 for _, e := range bo.Exports { 1327 for k, v := range annotations { 1328 e.Attrs[k.String()] = v 1329 } 1330 } 1331 1332 attests, err := buildflags.ParseAttests(t.Attest) 1333 if err != nil { 1334 return nil, err 1335 } 1336 bo.Attests = controllerapi.CreateAttestations(attests) 1337 1338 bo.SourcePolicy, err = build.ReadSourcePolicy() 1339 if err != nil { 1340 return nil, err 1341 } 1342 1343 ulimits := dockeropts.NewUlimitOpt(nil) 1344 for _, field := range t.Ulimits { 1345 if err := ulimits.Set(field); err != nil { 1346 return nil, err 1347 } 1348 } 1349 bo.Ulimits = ulimits 1350 1351 return bo, nil 1352 } 1353 1354 func defaultTarget() *Target { 1355 return &Target{} 1356 } 1357 1358 func removeDupes(s []string) []string { 1359 i := 0 1360 seen := make(map[string]struct{}, len(s)) 1361 for _, v := range s { 1362 if _, ok := seen[v]; ok { 1363 continue 1364 } 1365 if v == "" { 1366 continue 1367 } 1368 seen[v] = struct{}{} 1369 s[i] = v 1370 i++ 1371 } 1372 return s[:i] 1373 } 1374 1375 func removeAttestDupes(s []string) []string { 1376 res := []string{} 1377 m := map[string]int{} 1378 for _, v := range s { 1379 att, err := buildflags.ParseAttest(v) 1380 if err != nil { 1381 res = append(res, v) 1382 continue 1383 } 1384 1385 if i, ok := m[att.Type]; ok { 1386 res[i] = v 1387 } else { 1388 m[att.Type] = len(res) 1389 res = append(res, v) 1390 } 1391 } 1392 return res 1393 } 1394 1395 func parseOutput(str string) map[string]string { 1396 csvReader := csv.NewReader(strings.NewReader(str)) 1397 fields, err := csvReader.Read() 1398 if err != nil { 1399 return nil 1400 } 1401 res := map[string]string{} 1402 for _, field := range fields { 1403 parts := strings.SplitN(field, "=", 2) 1404 if len(parts) == 2 { 1405 res[parts[0]] = parts[1] 1406 } 1407 } 1408 return res 1409 } 1410 1411 func parseOutputType(str string) string { 1412 if out := parseOutput(str); out != nil { 1413 if v, ok := out["type"]; ok { 1414 return v 1415 } 1416 } 1417 return "" 1418 } 1419 1420 func setPushOverride(outputs []string, push bool) []string { 1421 var out []string 1422 setPush := true 1423 for _, output := range outputs { 1424 typ := parseOutputType(output) 1425 if typ == "image" || typ == "registry" { 1426 // no need to set push if image or registry types already defined 1427 setPush = false 1428 if typ == "registry" { 1429 if !push { 1430 // don't set registry output if "push" is false 1431 continue 1432 } 1433 // no need to set "push" attribute to true for registry 1434 out = append(out, output) 1435 continue 1436 } 1437 out = append(out, output+",push="+strconv.FormatBool(push)) 1438 } else { 1439 if typ != "docker" { 1440 // if there is any output that is not docker, don't set "push" 1441 setPush = false 1442 } 1443 out = append(out, output) 1444 } 1445 } 1446 if push && setPush { 1447 out = append(out, "type=image,push=true") 1448 } 1449 return out 1450 } 1451 1452 func setLoadOverride(outputs []string, load bool) []string { 1453 if !load { 1454 return outputs 1455 } 1456 setLoad := true 1457 for _, output := range outputs { 1458 if typ := parseOutputType(output); typ == "docker" { 1459 if v := parseOutput(output); v != nil { 1460 // dest set means we want to output as tar so don't set load 1461 if _, ok := v["dest"]; !ok { 1462 setLoad = false 1463 break 1464 } 1465 } 1466 } else if typ != "image" && typ != "registry" && typ != "oci" { 1467 // if there is any output that is not an image, registry 1468 // or oci, don't set "load" similar to push override 1469 setLoad = false 1470 break 1471 } 1472 } 1473 if setLoad { 1474 outputs = append(outputs, "type=docker") 1475 } 1476 return outputs 1477 } 1478 1479 func validateTargetName(name string) error { 1480 if !targetNamePattern.MatchString(name) { 1481 return errors.Errorf("only %q are allowed", validTargetNameChars) 1482 } 1483 return nil 1484 } 1485 1486 func sanitizeTargetName(target string) string { 1487 // as stipulated in compose spec, service name can contain a dot so as 1488 // best-effort and to avoid any potential ambiguity, we replace the dot 1489 // with an underscore. 1490 return strings.ReplaceAll(target, ".", "_") 1491 } 1492 1493 func sliceEqual(s1, s2 []string) bool { 1494 if len(s1) != len(s2) { 1495 return false 1496 } 1497 sort.Strings(s1) 1498 sort.Strings(s2) 1499 for i := range s1 { 1500 if s1[i] != s2[i] { 1501 return false 1502 } 1503 } 1504 return true 1505 } 1506 1507 func toNamedContexts(m map[string]string) map[string]build.NamedContext { 1508 m2 := make(map[string]build.NamedContext, len(m)) 1509 for k, v := range m { 1510 m2[k] = build.NamedContext{Path: v} 1511 } 1512 return m2 1513 }