github.com/OpenFlowLabs/moby@v17.12.1-ce-rc2+incompatible/integration-cli/docker_api_build_test.go (about) 1 package main 2 3 import ( 4 "archive/tar" 5 "bytes" 6 "encoding/json" 7 "fmt" 8 "io" 9 "io/ioutil" 10 "net/http" 11 "regexp" 12 "strings" 13 14 "github.com/docker/docker/api/types" 15 "github.com/docker/docker/integration-cli/checker" 16 "github.com/docker/docker/integration-cli/cli/build/fakecontext" 17 "github.com/docker/docker/integration-cli/cli/build/fakegit" 18 "github.com/docker/docker/integration-cli/cli/build/fakestorage" 19 "github.com/docker/docker/integration-cli/request" 20 "github.com/go-check/check" 21 "github.com/moby/buildkit/session" 22 "github.com/moby/buildkit/session/filesync" 23 "github.com/stretchr/testify/assert" 24 "github.com/stretchr/testify/require" 25 "golang.org/x/net/context" 26 "golang.org/x/sync/errgroup" 27 ) 28 29 func (s *DockerSuite) TestBuildAPIDockerFileRemote(c *check.C) { 30 testRequires(c, NotUserNamespace) 31 32 var testD string 33 if testEnv.DaemonPlatform() == "windows" { 34 testD = `FROM busybox 35 RUN find / -name ba* 36 RUN find /tmp/` 37 } else { 38 // -xdev is required because sysfs can cause EPERM 39 testD = `FROM busybox 40 RUN find / -xdev -name ba* 41 RUN find /tmp/` 42 } 43 server := fakestorage.New(c, "", fakecontext.WithFiles(map[string]string{"testD": testD})) 44 defer server.Close() 45 46 res, body, err := request.Post("/build?dockerfile=baz&remote="+server.URL()+"/testD", request.JSON) 47 c.Assert(err, checker.IsNil) 48 c.Assert(res.StatusCode, checker.Equals, http.StatusOK) 49 50 buf, err := request.ReadBody(body) 51 c.Assert(err, checker.IsNil) 52 53 // Make sure Dockerfile exists. 54 // Make sure 'baz' doesn't exist ANYWHERE despite being mentioned in the URL 55 out := string(buf) 56 c.Assert(out, checker.Contains, "RUN find /tmp") 57 c.Assert(out, checker.Not(checker.Contains), "baz") 58 } 59 60 func (s *DockerSuite) TestBuildAPIRemoteTarballContext(c *check.C) { 61 buffer := new(bytes.Buffer) 62 tw := tar.NewWriter(buffer) 63 defer tw.Close() 64 65 dockerfile := []byte("FROM busybox") 66 err := tw.WriteHeader(&tar.Header{ 67 Name: "Dockerfile", 68 Size: int64(len(dockerfile)), 69 }) 70 // failed to write tar file header 71 c.Assert(err, checker.IsNil) 72 73 _, err = tw.Write(dockerfile) 74 // failed to write tar file content 75 c.Assert(err, checker.IsNil) 76 77 // failed to close tar archive 78 c.Assert(tw.Close(), checker.IsNil) 79 80 server := fakestorage.New(c, "", fakecontext.WithBinaryFiles(map[string]*bytes.Buffer{ 81 "testT.tar": buffer, 82 })) 83 defer server.Close() 84 85 res, b, err := request.Post("/build?remote="+server.URL()+"/testT.tar", request.ContentType("application/tar")) 86 c.Assert(err, checker.IsNil) 87 c.Assert(res.StatusCode, checker.Equals, http.StatusOK) 88 b.Close() 89 } 90 91 func (s *DockerSuite) TestBuildAPIRemoteTarballContextWithCustomDockerfile(c *check.C) { 92 buffer := new(bytes.Buffer) 93 tw := tar.NewWriter(buffer) 94 defer tw.Close() 95 96 dockerfile := []byte(`FROM busybox 97 RUN echo 'wrong'`) 98 err := tw.WriteHeader(&tar.Header{ 99 Name: "Dockerfile", 100 Size: int64(len(dockerfile)), 101 }) 102 // failed to write tar file header 103 c.Assert(err, checker.IsNil) 104 105 _, err = tw.Write(dockerfile) 106 // failed to write tar file content 107 c.Assert(err, checker.IsNil) 108 109 custom := []byte(`FROM busybox 110 RUN echo 'right' 111 `) 112 err = tw.WriteHeader(&tar.Header{ 113 Name: "custom", 114 Size: int64(len(custom)), 115 }) 116 117 // failed to write tar file header 118 c.Assert(err, checker.IsNil) 119 120 _, err = tw.Write(custom) 121 // failed to write tar file content 122 c.Assert(err, checker.IsNil) 123 124 // failed to close tar archive 125 c.Assert(tw.Close(), checker.IsNil) 126 127 server := fakestorage.New(c, "", fakecontext.WithBinaryFiles(map[string]*bytes.Buffer{ 128 "testT.tar": buffer, 129 })) 130 defer server.Close() 131 132 url := "/build?dockerfile=custom&remote=" + server.URL() + "/testT.tar" 133 res, body, err := request.Post(url, request.ContentType("application/tar")) 134 c.Assert(err, checker.IsNil) 135 c.Assert(res.StatusCode, checker.Equals, http.StatusOK) 136 137 defer body.Close() 138 content, err := request.ReadBody(body) 139 c.Assert(err, checker.IsNil) 140 141 // Build used the wrong dockerfile. 142 c.Assert(string(content), checker.Not(checker.Contains), "wrong") 143 } 144 145 func (s *DockerSuite) TestBuildAPILowerDockerfile(c *check.C) { 146 git := fakegit.New(c, "repo", map[string]string{ 147 "dockerfile": `FROM busybox 148 RUN echo from dockerfile`, 149 }, false) 150 defer git.Close() 151 152 res, body, err := request.Post("/build?remote="+git.RepoURL, request.JSON) 153 c.Assert(err, checker.IsNil) 154 c.Assert(res.StatusCode, checker.Equals, http.StatusOK) 155 156 buf, err := request.ReadBody(body) 157 c.Assert(err, checker.IsNil) 158 159 out := string(buf) 160 c.Assert(out, checker.Contains, "from dockerfile") 161 } 162 163 func (s *DockerSuite) TestBuildAPIBuildGitWithF(c *check.C) { 164 git := fakegit.New(c, "repo", map[string]string{ 165 "baz": `FROM busybox 166 RUN echo from baz`, 167 "Dockerfile": `FROM busybox 168 RUN echo from Dockerfile`, 169 }, false) 170 defer git.Close() 171 172 // Make sure it tries to 'dockerfile' query param value 173 res, body, err := request.Post("/build?dockerfile=baz&remote="+git.RepoURL, request.JSON) 174 c.Assert(err, checker.IsNil) 175 c.Assert(res.StatusCode, checker.Equals, http.StatusOK) 176 177 buf, err := request.ReadBody(body) 178 c.Assert(err, checker.IsNil) 179 180 out := string(buf) 181 c.Assert(out, checker.Contains, "from baz") 182 } 183 184 func (s *DockerSuite) TestBuildAPIDoubleDockerfile(c *check.C) { 185 testRequires(c, UnixCli) // dockerfile overwrites Dockerfile on Windows 186 git := fakegit.New(c, "repo", map[string]string{ 187 "Dockerfile": `FROM busybox 188 RUN echo from Dockerfile`, 189 "dockerfile": `FROM busybox 190 RUN echo from dockerfile`, 191 }, false) 192 defer git.Close() 193 194 // Make sure it tries to 'dockerfile' query param value 195 res, body, err := request.Post("/build?remote="+git.RepoURL, request.JSON) 196 c.Assert(err, checker.IsNil) 197 c.Assert(res.StatusCode, checker.Equals, http.StatusOK) 198 199 buf, err := request.ReadBody(body) 200 c.Assert(err, checker.IsNil) 201 202 out := string(buf) 203 c.Assert(out, checker.Contains, "from Dockerfile") 204 } 205 206 func (s *DockerSuite) TestBuildAPIUnnormalizedTarPaths(c *check.C) { 207 // Make sure that build context tars with entries of the form 208 // x/./y don't cause caching false positives. 209 210 buildFromTarContext := func(fileContents []byte) string { 211 buffer := new(bytes.Buffer) 212 tw := tar.NewWriter(buffer) 213 defer tw.Close() 214 215 dockerfile := []byte(`FROM busybox 216 COPY dir /dir/`) 217 err := tw.WriteHeader(&tar.Header{ 218 Name: "Dockerfile", 219 Size: int64(len(dockerfile)), 220 }) 221 //failed to write tar file header 222 c.Assert(err, checker.IsNil) 223 224 _, err = tw.Write(dockerfile) 225 // failed to write Dockerfile in tar file content 226 c.Assert(err, checker.IsNil) 227 228 err = tw.WriteHeader(&tar.Header{ 229 Name: "dir/./file", 230 Size: int64(len(fileContents)), 231 }) 232 //failed to write tar file header 233 c.Assert(err, checker.IsNil) 234 235 _, err = tw.Write(fileContents) 236 // failed to write file contents in tar file content 237 c.Assert(err, checker.IsNil) 238 239 // failed to close tar archive 240 c.Assert(tw.Close(), checker.IsNil) 241 242 res, body, err := request.Post("/build", request.RawContent(ioutil.NopCloser(buffer)), request.ContentType("application/x-tar")) 243 c.Assert(err, checker.IsNil) 244 c.Assert(res.StatusCode, checker.Equals, http.StatusOK) 245 246 out, err := request.ReadBody(body) 247 c.Assert(err, checker.IsNil) 248 lines := strings.Split(string(out), "\n") 249 c.Assert(len(lines), checker.GreaterThan, 1) 250 c.Assert(lines[len(lines)-2], checker.Matches, ".*Successfully built [0-9a-f]{12}.*") 251 252 re := regexp.MustCompile("Successfully built ([0-9a-f]{12})") 253 matches := re.FindStringSubmatch(lines[len(lines)-2]) 254 return matches[1] 255 } 256 257 imageA := buildFromTarContext([]byte("abc")) 258 imageB := buildFromTarContext([]byte("def")) 259 260 c.Assert(imageA, checker.Not(checker.Equals), imageB) 261 } 262 263 func (s *DockerSuite) TestBuildOnBuildWithCopy(c *check.C) { 264 dockerfile := ` 265 FROM ` + minimalBaseImage() + ` as onbuildbase 266 ONBUILD COPY file /file 267 268 FROM onbuildbase 269 ` 270 ctx := fakecontext.New(c, "", 271 fakecontext.WithDockerfile(dockerfile), 272 fakecontext.WithFile("file", "some content"), 273 ) 274 defer ctx.Close() 275 276 res, body, err := request.Post( 277 "/build", 278 request.RawContent(ctx.AsTarReader(c)), 279 request.ContentType("application/x-tar")) 280 c.Assert(err, checker.IsNil) 281 c.Assert(res.StatusCode, checker.Equals, http.StatusOK) 282 283 out, err := request.ReadBody(body) 284 c.Assert(err, checker.IsNil) 285 c.Assert(string(out), checker.Contains, "Successfully built") 286 } 287 288 func (s *DockerSuite) TestBuildOnBuildCache(c *check.C) { 289 build := func(dockerfile string) []byte { 290 ctx := fakecontext.New(c, "", 291 fakecontext.WithDockerfile(dockerfile), 292 ) 293 defer ctx.Close() 294 295 res, body, err := request.Post( 296 "/build", 297 request.RawContent(ctx.AsTarReader(c)), 298 request.ContentType("application/x-tar")) 299 require.NoError(c, err) 300 assert.Equal(c, http.StatusOK, res.StatusCode) 301 302 out, err := request.ReadBody(body) 303 require.NoError(c, err) 304 assert.Contains(c, string(out), "Successfully built") 305 return out 306 } 307 308 dockerfile := ` 309 FROM ` + minimalBaseImage() + ` as onbuildbase 310 ENV something=bar 311 ONBUILD ENV foo=bar 312 ` 313 build(dockerfile) 314 315 dockerfile += "FROM onbuildbase" 316 out := build(dockerfile) 317 318 imageIDs := getImageIDsFromBuild(c, out) 319 assert.Len(c, imageIDs, 2) 320 parentID, childID := imageIDs[0], imageIDs[1] 321 322 client, err := request.NewClient() 323 require.NoError(c, err) 324 325 // check parentID is correct 326 image, _, err := client.ImageInspectWithRaw(context.Background(), childID) 327 require.NoError(c, err) 328 assert.Equal(c, parentID, image.Parent) 329 } 330 331 func (s *DockerRegistrySuite) TestBuildCopyFromForcePull(c *check.C) { 332 client, err := request.NewClient() 333 require.NoError(c, err) 334 335 repoName := fmt.Sprintf("%v/dockercli/busybox", privateRegistryURL) 336 // tag the image to upload it to the private registry 337 err = client.ImageTag(context.TODO(), "busybox", repoName) 338 assert.Nil(c, err) 339 // push the image to the registry 340 rc, err := client.ImagePush(context.TODO(), repoName, types.ImagePushOptions{RegistryAuth: "{}"}) 341 assert.Nil(c, err) 342 _, err = io.Copy(ioutil.Discard, rc) 343 assert.Nil(c, err) 344 345 dockerfile := fmt.Sprintf(` 346 FROM %s AS foo 347 RUN touch abc 348 FROM %s 349 COPY --from=foo /abc / 350 `, repoName, repoName) 351 352 ctx := fakecontext.New(c, "", 353 fakecontext.WithDockerfile(dockerfile), 354 ) 355 defer ctx.Close() 356 357 res, body, err := request.Post( 358 "/build?pull=1", 359 request.RawContent(ctx.AsTarReader(c)), 360 request.ContentType("application/x-tar")) 361 require.NoError(c, err) 362 assert.Equal(c, http.StatusOK, res.StatusCode) 363 364 out, err := request.ReadBody(body) 365 require.NoError(c, err) 366 assert.Contains(c, string(out), "Successfully built") 367 } 368 369 func (s *DockerSuite) TestBuildAddRemoteNoDecompress(c *check.C) { 370 buffer := new(bytes.Buffer) 371 tw := tar.NewWriter(buffer) 372 dt := []byte("contents") 373 err := tw.WriteHeader(&tar.Header{ 374 Name: "foo", 375 Size: int64(len(dt)), 376 Mode: 0600, 377 Typeflag: tar.TypeReg, 378 }) 379 require.NoError(c, err) 380 _, err = tw.Write(dt) 381 require.NoError(c, err) 382 err = tw.Close() 383 require.NoError(c, err) 384 385 server := fakestorage.New(c, "", fakecontext.WithBinaryFiles(map[string]*bytes.Buffer{ 386 "test.tar": buffer, 387 })) 388 defer server.Close() 389 390 dockerfile := fmt.Sprintf(` 391 FROM busybox 392 ADD %s/test.tar / 393 RUN [ -f test.tar ] 394 `, server.URL()) 395 396 ctx := fakecontext.New(c, "", 397 fakecontext.WithDockerfile(dockerfile), 398 ) 399 defer ctx.Close() 400 401 res, body, err := request.Post( 402 "/build", 403 request.RawContent(ctx.AsTarReader(c)), 404 request.ContentType("application/x-tar")) 405 require.NoError(c, err) 406 assert.Equal(c, http.StatusOK, res.StatusCode) 407 408 out, err := request.ReadBody(body) 409 require.NoError(c, err) 410 assert.Contains(c, string(out), "Successfully built") 411 } 412 413 func (s *DockerSuite) TestBuildChownOnCopy(c *check.C) { 414 testRequires(c, DaemonIsLinux) 415 dockerfile := `FROM busybox 416 RUN echo 'test1:x:1001:1001::/bin:/bin/false' >> /etc/passwd 417 RUN echo 'test1:x:1001:' >> /etc/group 418 RUN echo 'test2:x:1002:' >> /etc/group 419 COPY --chown=test1:1002 . /new_dir 420 RUN ls -l / 421 RUN [ $(ls -l / | grep new_dir | awk '{print $3":"$4}') = 'test1:test2' ] 422 RUN [ $(ls -nl / | grep new_dir | awk '{print $3":"$4}') = '1001:1002' ] 423 ` 424 ctx := fakecontext.New(c, "", 425 fakecontext.WithDockerfile(dockerfile), 426 fakecontext.WithFile("test_file1", "some test content"), 427 ) 428 defer ctx.Close() 429 430 res, body, err := request.Post( 431 "/build", 432 request.RawContent(ctx.AsTarReader(c)), 433 request.ContentType("application/x-tar")) 434 c.Assert(err, checker.IsNil) 435 c.Assert(res.StatusCode, checker.Equals, http.StatusOK) 436 437 out, err := request.ReadBody(body) 438 require.NoError(c, err) 439 assert.Contains(c, string(out), "Successfully built") 440 } 441 442 func (s *DockerSuite) TestBuildCopyCacheOnFileChange(c *check.C) { 443 444 dockerfile := `FROM busybox 445 COPY file /file` 446 447 ctx1 := fakecontext.New(c, "", 448 fakecontext.WithDockerfile(dockerfile), 449 fakecontext.WithFile("file", "foo")) 450 ctx2 := fakecontext.New(c, "", 451 fakecontext.WithDockerfile(dockerfile), 452 fakecontext.WithFile("file", "bar")) 453 454 var build = func(ctx *fakecontext.Fake) string { 455 res, body, err := request.Post("/build", 456 request.RawContent(ctx.AsTarReader(c)), 457 request.ContentType("application/x-tar")) 458 459 require.NoError(c, err) 460 assert.Equal(c, http.StatusOK, res.StatusCode) 461 462 out, err := request.ReadBody(body) 463 464 ids := getImageIDsFromBuild(c, out) 465 return ids[len(ids)-1] 466 } 467 468 id1 := build(ctx1) 469 id2 := build(ctx1) 470 id3 := build(ctx2) 471 472 if id1 != id2 { 473 c.Fatal("didn't use the cache") 474 } 475 if id1 == id3 { 476 c.Fatal("COPY With different source file should not share same cache") 477 } 478 } 479 480 func (s *DockerSuite) TestBuildAddCacheOnFileChange(c *check.C) { 481 482 dockerfile := `FROM busybox 483 ADD file /file` 484 485 ctx1 := fakecontext.New(c, "", 486 fakecontext.WithDockerfile(dockerfile), 487 fakecontext.WithFile("file", "foo")) 488 ctx2 := fakecontext.New(c, "", 489 fakecontext.WithDockerfile(dockerfile), 490 fakecontext.WithFile("file", "bar")) 491 492 var build = func(ctx *fakecontext.Fake) string { 493 res, body, err := request.Post("/build", 494 request.RawContent(ctx.AsTarReader(c)), 495 request.ContentType("application/x-tar")) 496 497 require.NoError(c, err) 498 assert.Equal(c, http.StatusOK, res.StatusCode) 499 500 out, err := request.ReadBody(body) 501 502 ids := getImageIDsFromBuild(c, out) 503 return ids[len(ids)-1] 504 } 505 506 id1 := build(ctx1) 507 id2 := build(ctx1) 508 id3 := build(ctx2) 509 510 if id1 != id2 { 511 c.Fatal("didn't use the cache") 512 } 513 if id1 == id3 { 514 c.Fatal("COPY With different source file should not share same cache") 515 } 516 } 517 518 func (s *DockerSuite) TestBuildWithSession(c *check.C) { 519 testRequires(c, ExperimentalDaemon) 520 521 dockerfile := ` 522 FROM busybox 523 COPY file / 524 RUN cat /file 525 ` 526 527 fctx := fakecontext.New(c, "", 528 fakecontext.WithFile("file", "some content"), 529 ) 530 defer fctx.Close() 531 532 out := testBuildWithSession(c, fctx.Dir, dockerfile) 533 assert.Contains(c, out, "some content") 534 535 fctx.Add("second", "contentcontent") 536 537 dockerfile += ` 538 COPY second / 539 RUN cat /second 540 ` 541 542 out = testBuildWithSession(c, fctx.Dir, dockerfile) 543 assert.Equal(c, strings.Count(out, "Using cache"), 2) 544 assert.Contains(c, out, "contentcontent") 545 546 client, err := request.NewClient() 547 require.NoError(c, err) 548 549 du, err := client.DiskUsage(context.TODO()) 550 assert.Nil(c, err) 551 assert.True(c, du.BuilderSize > 10) 552 553 out = testBuildWithSession(c, fctx.Dir, dockerfile) 554 assert.Equal(c, strings.Count(out, "Using cache"), 4) 555 556 du2, err := client.DiskUsage(context.TODO()) 557 assert.Nil(c, err) 558 assert.Equal(c, du.BuilderSize, du2.BuilderSize) 559 560 // rebuild with regular tar, confirm cache still applies 561 fctx.Add("Dockerfile", dockerfile) 562 res, body, err := request.Post( 563 "/build", 564 request.RawContent(fctx.AsTarReader(c)), 565 request.ContentType("application/x-tar")) 566 require.NoError(c, err) 567 assert.Equal(c, http.StatusOK, res.StatusCode) 568 569 outBytes, err := request.ReadBody(body) 570 require.NoError(c, err) 571 assert.Contains(c, string(outBytes), "Successfully built") 572 assert.Equal(c, strings.Count(string(outBytes), "Using cache"), 4) 573 574 _, err = client.BuildCachePrune(context.TODO()) 575 assert.Nil(c, err) 576 577 du, err = client.DiskUsage(context.TODO()) 578 assert.Nil(c, err) 579 assert.Equal(c, du.BuilderSize, int64(0)) 580 } 581 582 func testBuildWithSession(c *check.C, dir, dockerfile string) (outStr string) { 583 client, err := request.NewClient() 584 require.NoError(c, err) 585 586 sess, err := session.NewSession("foo1", "foo") 587 assert.Nil(c, err) 588 589 fsProvider := filesync.NewFSSyncProvider([]filesync.SyncedDir{ 590 {Dir: dir}, 591 }) 592 sess.Allow(fsProvider) 593 594 g, ctx := errgroup.WithContext(context.Background()) 595 596 g.Go(func() error { 597 return sess.Run(ctx, client.DialSession) 598 }) 599 600 g.Go(func() error { 601 res, body, err := request.Post("/build?remote=client-session&session="+sess.ID(), func(req *http.Request) error { 602 req.Body = ioutil.NopCloser(strings.NewReader(dockerfile)) 603 return nil 604 }) 605 if err != nil { 606 return err 607 } 608 assert.Equal(c, res.StatusCode, http.StatusOK) 609 out, err := request.ReadBody(body) 610 require.NoError(c, err) 611 assert.Contains(c, string(out), "Successfully built") 612 sess.Close() 613 outStr = string(out) 614 return nil 615 }) 616 617 err = g.Wait() 618 assert.Nil(c, err) 619 return 620 } 621 622 func (s *DockerSuite) TestBuildScratchCopy(c *check.C) { 623 testRequires(c, DaemonIsLinux) 624 dockerfile := `FROM scratch 625 ADD Dockerfile / 626 ENV foo bar` 627 ctx := fakecontext.New(c, "", 628 fakecontext.WithDockerfile(dockerfile), 629 ) 630 defer ctx.Close() 631 632 res, body, err := request.Post( 633 "/build", 634 request.RawContent(ctx.AsTarReader(c)), 635 request.ContentType("application/x-tar")) 636 c.Assert(err, checker.IsNil) 637 c.Assert(res.StatusCode, checker.Equals, http.StatusOK) 638 639 out, err := request.ReadBody(body) 640 require.NoError(c, err) 641 assert.Contains(c, string(out), "Successfully built") 642 } 643 644 type buildLine struct { 645 Stream string 646 Aux struct { 647 ID string 648 } 649 } 650 651 func getImageIDsFromBuild(c *check.C, output []byte) []string { 652 ids := []string{} 653 for _, line := range bytes.Split(output, []byte("\n")) { 654 if len(line) == 0 { 655 continue 656 } 657 entry := buildLine{} 658 require.NoError(c, json.Unmarshal(line, &entry)) 659 if entry.Aux.ID != "" { 660 ids = append(ids, entry.Aux.ID) 661 } 662 } 663 return ids 664 }