github.com/ali-iotechsys/cli@v20.10.0+incompatible/cli/command/image/build_buildkit.go (about) 1 package image 2 3 import ( 4 "bytes" 5 "context" 6 "encoding/csv" 7 "encoding/json" 8 "fmt" 9 "io" 10 "io/ioutil" 11 "net" 12 "os" 13 "path/filepath" 14 "strings" 15 16 "github.com/containerd/console" 17 "github.com/containerd/containerd/platforms" 18 "github.com/docker/cli/cli" 19 "github.com/docker/cli/cli/command" 20 "github.com/docker/cli/cli/command/image/build" 21 "github.com/docker/cli/opts" 22 "github.com/docker/docker/api/types" 23 "github.com/docker/docker/pkg/jsonmessage" 24 "github.com/docker/docker/pkg/stringid" 25 "github.com/docker/docker/pkg/urlutil" 26 controlapi "github.com/moby/buildkit/api/services/control" 27 "github.com/moby/buildkit/client" 28 "github.com/moby/buildkit/session" 29 "github.com/moby/buildkit/session/auth/authprovider" 30 "github.com/moby/buildkit/session/filesync" 31 "github.com/moby/buildkit/session/secrets/secretsprovider" 32 "github.com/moby/buildkit/session/sshforward/sshprovider" 33 "github.com/moby/buildkit/util/appcontext" 34 "github.com/moby/buildkit/util/progress/progressui" 35 "github.com/pkg/errors" 36 fsutiltypes "github.com/tonistiigi/fsutil/types" 37 "github.com/tonistiigi/go-rosetta" 38 "golang.org/x/sync/errgroup" 39 ) 40 41 const uploadRequestRemote = "upload-request" 42 43 var errDockerfileConflict = errors.New("ambiguous Dockerfile source: both stdin and flag correspond to Dockerfiles") 44 45 //nolint: gocyclo 46 func runBuildBuildKit(dockerCli command.Cli, options buildOptions) error { 47 ctx := appcontext.Context() 48 49 s, err := trySession(dockerCli, options.context, false) 50 if err != nil { 51 return err 52 } 53 if s == nil { 54 return errors.Errorf("buildkit not supported by daemon") 55 } 56 57 if options.imageIDFile != "" { 58 // Avoid leaving a stale file if we eventually fail 59 if err := os.Remove(options.imageIDFile); err != nil && !os.IsNotExist(err) { 60 return errors.Wrap(err, "removing image ID file") 61 } 62 } 63 64 var ( 65 remote string 66 body io.Reader 67 dockerfileName = options.dockerfileName 68 dockerfileReader io.ReadCloser 69 dockerfileDir string 70 contextDir string 71 ) 72 73 stdoutUsed := false 74 75 switch { 76 case options.contextFromStdin(): 77 if options.dockerfileFromStdin() { 78 return errStdinConflict 79 } 80 rc, isArchive, err := build.DetectArchiveReader(os.Stdin) 81 if err != nil { 82 return err 83 } 84 if isArchive { 85 body = rc 86 remote = uploadRequestRemote 87 } else { 88 if options.dockerfileName != "" { 89 return errDockerfileConflict 90 } 91 dockerfileReader = rc 92 remote = clientSessionRemote 93 // TODO: make fssync handle empty contextdir 94 contextDir, _ = ioutil.TempDir("", "empty-dir") 95 defer os.RemoveAll(contextDir) 96 } 97 case isLocalDir(options.context): 98 contextDir = options.context 99 if options.dockerfileFromStdin() { 100 dockerfileReader = os.Stdin 101 } else if options.dockerfileName != "" { 102 dockerfileName = filepath.Base(options.dockerfileName) 103 dockerfileDir = filepath.Dir(options.dockerfileName) 104 } else { 105 dockerfileDir = options.context 106 } 107 remote = clientSessionRemote 108 case urlutil.IsGitURL(options.context): 109 remote = options.context 110 case urlutil.IsURL(options.context): 111 remote = options.context 112 default: 113 return errors.Errorf("unable to prepare context: path %q not found", options.context) 114 } 115 116 if dockerfileReader != nil { 117 dockerfileName = build.DefaultDockerfileName 118 dockerfileDir, err = build.WriteTempDockerfile(dockerfileReader) 119 if err != nil { 120 return err 121 } 122 defer os.RemoveAll(dockerfileDir) 123 } 124 125 outputs, err := parseOutputs(options.outputs) 126 if err != nil { 127 return errors.Wrapf(err, "failed to parse outputs") 128 } 129 130 for _, out := range outputs { 131 switch out.Type { 132 case "local": 133 // dest is handled on client side for local exporter 134 outDir, ok := out.Attrs["dest"] 135 if !ok { 136 return errors.Errorf("dest is required for local output") 137 } 138 delete(out.Attrs, "dest") 139 s.Allow(filesync.NewFSSyncTargetDir(outDir)) 140 case "tar": 141 // dest is handled on client side for tar exporter 142 outFile, ok := out.Attrs["dest"] 143 if !ok { 144 return errors.Errorf("dest is required for tar output") 145 } 146 var w io.WriteCloser 147 if outFile == "-" { 148 if _, err := console.ConsoleFromFile(os.Stdout); err == nil { 149 return errors.Errorf("refusing to write output to console") 150 } 151 w = os.Stdout 152 stdoutUsed = true 153 } else { 154 f, err := os.Create(outFile) 155 if err != nil { 156 return errors.Wrapf(err, "failed to open %s", outFile) 157 } 158 w = f 159 } 160 output := func(map[string]string) (io.WriteCloser, error) { return w, nil } 161 s.Allow(filesync.NewFSSyncTarget(output)) 162 } 163 } 164 165 if dockerfileDir != "" { 166 s.Allow(filesync.NewFSSyncProvider([]filesync.SyncedDir{ 167 { 168 Name: "context", 169 Dir: contextDir, 170 Map: resetUIDAndGID, 171 }, 172 { 173 Name: "dockerfile", 174 Dir: dockerfileDir, 175 }, 176 })) 177 } 178 179 s.Allow(authprovider.NewDockerAuthProvider(os.Stderr)) 180 if len(options.secrets) > 0 { 181 sp, err := parseSecretSpecs(options.secrets) 182 if err != nil { 183 return errors.Wrapf(err, "could not parse secrets: %v", options.secrets) 184 } 185 s.Allow(sp) 186 } 187 if len(options.ssh) > 0 { 188 sshp, err := parseSSHSpecs(options.ssh) 189 if err != nil { 190 return errors.Wrapf(err, "could not parse ssh: %v", options.ssh) 191 } 192 s.Allow(sshp) 193 } 194 195 eg, ctx := errgroup.WithContext(ctx) 196 197 dialSession := func(ctx context.Context, proto string, meta map[string][]string) (net.Conn, error) { 198 return dockerCli.Client().DialHijack(ctx, "/session", proto, meta) 199 } 200 eg.Go(func() error { 201 return s.Run(context.TODO(), dialSession) 202 }) 203 204 buildID := stringid.GenerateRandomID() 205 if body != nil { 206 eg.Go(func() error { 207 buildOptions := types.ImageBuildOptions{ 208 Version: types.BuilderBuildKit, 209 BuildID: uploadRequestRemote + ":" + buildID, 210 } 211 212 response, err := dockerCli.Client().ImageBuild(context.Background(), body, buildOptions) 213 if err != nil { 214 return err 215 } 216 defer response.Body.Close() 217 return nil 218 }) 219 } 220 221 if v := os.Getenv("BUILDKIT_PROGRESS"); v != "" && options.progress == "auto" { 222 options.progress = v 223 } 224 225 if strings.EqualFold(options.platform, "local") { 226 p := platforms.DefaultSpec() 227 p.Architecture = rosetta.NativeArch() // current binary architecture might be emulated 228 options.platform = platforms.Format(p) 229 } 230 231 eg.Go(func() error { 232 defer func() { // make sure the Status ends cleanly on build errors 233 s.Close() 234 }() 235 236 buildOptions := imageBuildOptions(dockerCli, options) 237 buildOptions.Version = types.BuilderBuildKit 238 buildOptions.Dockerfile = dockerfileName 239 // buildOptions.AuthConfigs = authConfigs // handled by session 240 buildOptions.RemoteContext = remote 241 buildOptions.SessionID = s.ID() 242 buildOptions.BuildID = buildID 243 buildOptions.Outputs = outputs 244 return doBuild(ctx, eg, dockerCli, stdoutUsed, options, buildOptions) 245 }) 246 247 return eg.Wait() 248 } 249 250 //nolint: gocyclo 251 func doBuild(ctx context.Context, eg *errgroup.Group, dockerCli command.Cli, stdoutUsed bool, options buildOptions, buildOptions types.ImageBuildOptions) (finalErr error) { 252 response, err := dockerCli.Client().ImageBuild(context.Background(), nil, buildOptions) 253 if err != nil { 254 return err 255 } 256 defer response.Body.Close() 257 258 done := make(chan struct{}) 259 defer close(done) 260 eg.Go(func() error { 261 select { 262 case <-ctx.Done(): 263 return dockerCli.Client().BuildCancel(context.TODO(), buildOptions.BuildID) 264 case <-done: 265 } 266 return nil 267 }) 268 269 t := newTracer() 270 ssArr := []*client.SolveStatus{} 271 272 if err := opts.ValidateProgressOutput(options.progress); err != nil { 273 return err 274 } 275 276 displayStatus := func(out *os.File, displayCh chan *client.SolveStatus) { 277 var c console.Console 278 // TODO: Handle tty output in non-tty environment. 279 if cons, err := console.ConsoleFromFile(out); err == nil && (options.progress == "auto" || options.progress == "tty") { 280 c = cons 281 } 282 // not using shared context to not disrupt display but let is finish reporting errors 283 eg.Go(func() error { 284 return progressui.DisplaySolveStatus(context.TODO(), "", c, out, displayCh) 285 }) 286 } 287 288 if options.quiet { 289 eg.Go(func() error { 290 // TODO: make sure t.displayCh closes 291 for ss := range t.displayCh { 292 ssArr = append(ssArr, ss) 293 } 294 <-done 295 // TODO: verify that finalErr is indeed set when error occurs 296 if finalErr != nil { 297 displayCh := make(chan *client.SolveStatus) 298 go func() { 299 for _, ss := range ssArr { 300 displayCh <- ss 301 } 302 close(displayCh) 303 }() 304 displayStatus(os.Stderr, displayCh) 305 } 306 return nil 307 }) 308 } else { 309 displayStatus(os.Stderr, t.displayCh) 310 } 311 defer close(t.displayCh) 312 313 buf := bytes.NewBuffer(nil) 314 315 imageID := "" 316 writeAux := func(msg jsonmessage.JSONMessage) { 317 if msg.ID == "moby.image.id" { 318 var result types.BuildResult 319 if err := json.Unmarshal(*msg.Aux, &result); err != nil { 320 fmt.Fprintf(dockerCli.Err(), "failed to parse aux message: %v", err) 321 } 322 imageID = result.ID 323 return 324 } 325 t.write(msg) 326 } 327 328 err = jsonmessage.DisplayJSONMessagesStream(response.Body, buf, dockerCli.Out().FD(), dockerCli.Out().IsTerminal(), writeAux) 329 if err != nil { 330 if jerr, ok := err.(*jsonmessage.JSONError); ok { 331 // If no error code is set, default to 1 332 if jerr.Code == 0 { 333 jerr.Code = 1 334 } 335 return cli.StatusError{Status: jerr.Message, StatusCode: jerr.Code} 336 } 337 } 338 339 // Everything worked so if -q was provided the output from the daemon 340 // should be just the image ID and we'll print that to stdout. 341 // 342 // TODO: we may want to use Aux messages with ID "moby.image.id" regardless of options.quiet (i.e. don't send HTTP param q=1) 343 // instead of assuming that output is image ID if options.quiet. 344 if options.quiet && !stdoutUsed { 345 imageID = buf.String() 346 fmt.Fprint(dockerCli.Out(), imageID) 347 } 348 349 if options.imageIDFile != "" { 350 if imageID == "" { 351 return errors.Errorf("cannot write %s because server did not provide an image ID", options.imageIDFile) 352 } 353 imageID = strings.TrimSpace(imageID) 354 if err := ioutil.WriteFile(options.imageIDFile, []byte(imageID), 0666); err != nil { 355 return errors.Wrap(err, "cannot write image ID file") 356 } 357 } 358 return err 359 } 360 361 func resetUIDAndGID(_ string, s *fsutiltypes.Stat) bool { 362 s.Uid = 0 363 s.Gid = 0 364 return true 365 } 366 367 type tracer struct { 368 displayCh chan *client.SolveStatus 369 } 370 371 func newTracer() *tracer { 372 return &tracer{ 373 displayCh: make(chan *client.SolveStatus), 374 } 375 } 376 377 func (t *tracer) write(msg jsonmessage.JSONMessage) { 378 var resp controlapi.StatusResponse 379 380 if msg.ID != "moby.buildkit.trace" { 381 return 382 } 383 384 var dt []byte 385 // ignoring all messages that are not understood 386 if err := json.Unmarshal(*msg.Aux, &dt); err != nil { 387 return 388 } 389 if err := (&resp).Unmarshal(dt); err != nil { 390 return 391 } 392 393 s := client.SolveStatus{} 394 for _, v := range resp.Vertexes { 395 s.Vertexes = append(s.Vertexes, &client.Vertex{ 396 Digest: v.Digest, 397 Inputs: v.Inputs, 398 Name: v.Name, 399 Started: v.Started, 400 Completed: v.Completed, 401 Error: v.Error, 402 Cached: v.Cached, 403 }) 404 } 405 for _, v := range resp.Statuses { 406 s.Statuses = append(s.Statuses, &client.VertexStatus{ 407 ID: v.ID, 408 Vertex: v.Vertex, 409 Name: v.Name, 410 Total: v.Total, 411 Current: v.Current, 412 Timestamp: v.Timestamp, 413 Started: v.Started, 414 Completed: v.Completed, 415 }) 416 } 417 for _, v := range resp.Logs { 418 s.Logs = append(s.Logs, &client.VertexLog{ 419 Vertex: v.Vertex, 420 Stream: int(v.Stream), 421 Data: v.Msg, 422 Timestamp: v.Timestamp, 423 }) 424 } 425 426 t.displayCh <- &s 427 } 428 429 func parseSecretSpecs(sl []string) (session.Attachable, error) { 430 fs := make([]secretsprovider.Source, 0, len(sl)) 431 for _, v := range sl { 432 s, err := parseSecret(v) 433 if err != nil { 434 return nil, err 435 } 436 fs = append(fs, *s) 437 } 438 store, err := secretsprovider.NewStore(fs) 439 if err != nil { 440 return nil, err 441 } 442 return secretsprovider.NewSecretProvider(store), nil 443 } 444 445 func parseSecret(value string) (*secretsprovider.Source, error) { 446 csvReader := csv.NewReader(strings.NewReader(value)) 447 fields, err := csvReader.Read() 448 if err != nil { 449 return nil, errors.Wrap(err, "failed to parse csv secret") 450 } 451 452 fs := secretsprovider.Source{} 453 454 var typ string 455 for _, field := range fields { 456 parts := strings.SplitN(field, "=", 2) 457 key := strings.ToLower(parts[0]) 458 459 if len(parts) != 2 { 460 return nil, errors.Errorf("invalid field '%s' must be a key=value pair", field) 461 } 462 463 value := parts[1] 464 switch key { 465 case "type": 466 if value != "file" && value != "env" { 467 return nil, errors.Errorf("unsupported secret type %q", value) 468 } 469 typ = value 470 case "id": 471 fs.ID = value 472 case "source", "src": 473 fs.FilePath = value 474 case "env": 475 fs.Env = value 476 default: 477 return nil, errors.Errorf("unexpected key '%s' in '%s'", key, field) 478 } 479 } 480 if typ == "env" && fs.Env == "" { 481 fs.Env = fs.FilePath 482 fs.FilePath = "" 483 } 484 return &fs, nil 485 } 486 487 func parseSSHSpecs(sl []string) (session.Attachable, error) { 488 configs := make([]sshprovider.AgentConfig, 0, len(sl)) 489 for _, v := range sl { 490 c := parseSSH(v) 491 configs = append(configs, *c) 492 } 493 return sshprovider.NewSSHAgentProvider(configs) 494 } 495 496 func parseSSH(value string) *sshprovider.AgentConfig { 497 parts := strings.SplitN(value, "=", 2) 498 cfg := sshprovider.AgentConfig{ 499 ID: parts[0], 500 } 501 if len(parts) > 1 { 502 cfg.Paths = strings.Split(parts[1], ",") 503 } 504 return &cfg 505 }