github.com/moby/docker@v26.1.3+incompatible/layer/layer_test.go (about) 1 package layer // import "github.com/docker/docker/layer" 2 3 import ( 4 "bytes" 5 "errors" 6 "io" 7 "os" 8 "path/filepath" 9 "runtime" 10 "strings" 11 "testing" 12 13 "github.com/containerd/continuity/driver" 14 "github.com/docker/docker/daemon/graphdriver" 15 "github.com/docker/docker/daemon/graphdriver/vfs" 16 "github.com/docker/docker/pkg/archive" 17 "github.com/docker/docker/pkg/idtools" 18 "github.com/docker/docker/pkg/stringid" 19 "github.com/opencontainers/go-digest" 20 ) 21 22 func init() { 23 graphdriver.ApplyUncompressedLayer = archive.UnpackLayer 24 defaultArchiver := archive.NewDefaultArchiver() 25 vfs.CopyDir = defaultArchiver.CopyWithTar 26 } 27 28 func newVFSGraphDriver(td string) (graphdriver.Driver, error) { 29 uidMap := []idtools.IDMap{ 30 { 31 ContainerID: 0, 32 HostID: os.Getuid(), 33 Size: 1, 34 }, 35 } 36 gidMap := []idtools.IDMap{ 37 { 38 ContainerID: 0, 39 HostID: os.Getgid(), 40 Size: 1, 41 }, 42 } 43 44 options := graphdriver.Options{Root: td, IDMap: idtools.IdentityMapping{UIDMaps: uidMap, GIDMaps: gidMap}} 45 return graphdriver.GetDriver("vfs", nil, options) 46 } 47 48 func newTestGraphDriver(t *testing.T) (graphdriver.Driver, func()) { 49 td, err := os.MkdirTemp("", "graph-") 50 if err != nil { 51 t.Fatal(err) 52 } 53 54 driver, err := newVFSGraphDriver(td) 55 if err != nil { 56 t.Fatal(err) 57 } 58 59 return driver, func() { 60 os.RemoveAll(td) 61 } 62 } 63 64 func newTestStore(t *testing.T) (Store, string, func()) { 65 td, err := os.MkdirTemp("", "layerstore-") 66 if err != nil { 67 t.Fatal(err) 68 } 69 70 graph, graphcleanup := newTestGraphDriver(t) 71 72 ls, err := newStoreFromGraphDriver(td, graph) 73 if err != nil { 74 t.Fatal(err) 75 } 76 77 return ls, td, func() { 78 graphcleanup() 79 os.RemoveAll(td) 80 } 81 } 82 83 type layerInit func(root string) error 84 85 func createLayer(ls Store, parent ChainID, layerFunc layerInit) (Layer, error) { 86 containerID := stringid.GenerateRandomID() 87 mount, err := ls.CreateRWLayer(containerID, parent, nil) 88 if err != nil { 89 return nil, err 90 } 91 92 pathFS, err := mount.Mount("") 93 if err != nil { 94 return nil, err 95 } 96 97 if err := layerFunc(pathFS); err != nil { 98 return nil, err 99 } 100 101 ts, err := mount.TarStream() 102 if err != nil { 103 return nil, err 104 } 105 defer ts.Close() 106 107 layer, err := ls.Register(ts, parent) 108 if err != nil { 109 return nil, err 110 } 111 112 if err := mount.Unmount(); err != nil { 113 return nil, err 114 } 115 116 if _, err := ls.ReleaseRWLayer(mount); err != nil { 117 return nil, err 118 } 119 120 return layer, nil 121 } 122 123 type FileApplier interface { 124 ApplyFile(root string) error 125 } 126 127 type testFile struct { 128 name string 129 content []byte 130 permission os.FileMode 131 } 132 133 func newTestFile(name string, content []byte, perm os.FileMode) FileApplier { 134 return &testFile{ 135 name: name, 136 content: content, 137 permission: perm, 138 } 139 } 140 141 func (tf *testFile) ApplyFile(root string) error { 142 fullPath := filepath.Join(root, tf.name) 143 if err := os.MkdirAll(filepath.Dir(fullPath), 0o755); err != nil { 144 return err 145 } 146 // Check if already exists 147 if stat, err := os.Stat(fullPath); err == nil && stat.Mode().Perm() != tf.permission { 148 if err := driver.LocalDriver.Lchmod(fullPath, tf.permission); err != nil { 149 return err 150 } 151 } 152 return os.WriteFile(fullPath, tf.content, tf.permission) 153 } 154 155 func initWithFiles(files ...FileApplier) layerInit { 156 return func(root string) error { 157 for _, f := range files { 158 if err := f.ApplyFile(root); err != nil { 159 return err 160 } 161 } 162 return nil 163 } 164 } 165 166 func getCachedLayer(l Layer) *roLayer { 167 if rl, ok := l.(*referencedCacheLayer); ok { 168 return rl.roLayer 169 } 170 return l.(*roLayer) 171 } 172 173 func createMetadata(layers ...Layer) []Metadata { 174 metadata := make([]Metadata, len(layers)) 175 for i := range layers { 176 metadata[i].ChainID = layers[i].ChainID() 177 metadata[i].DiffID = layers[i].DiffID() 178 metadata[i].Size = layers[i].Size() 179 metadata[i].DiffSize = getCachedLayer(layers[i]).size 180 } 181 182 return metadata 183 } 184 185 func assertMetadata(t *testing.T, metadata, expectedMetadata []Metadata) { 186 if len(metadata) != len(expectedMetadata) { 187 t.Fatalf("Unexpected number of deletes %d, expected %d", len(metadata), len(expectedMetadata)) 188 } 189 190 for i := range metadata { 191 if metadata[i] != expectedMetadata[i] { 192 t.Errorf("Unexpected metadata\n\tExpected: %#v\n\tActual: %#v", expectedMetadata[i], metadata[i]) 193 } 194 } 195 if t.Failed() { 196 t.FailNow() 197 } 198 } 199 200 func releaseAndCheckDeleted(t *testing.T, ls Store, layer Layer, removed ...Layer) { 201 layerCount := len(ls.(*layerStore).layerMap) 202 expectedMetadata := createMetadata(removed...) 203 metadata, err := ls.Release(layer) 204 if err != nil { 205 t.Fatal(err) 206 } 207 208 assertMetadata(t, metadata, expectedMetadata) 209 210 if expected := layerCount - len(removed); len(ls.(*layerStore).layerMap) != expected { 211 t.Fatalf("Unexpected number of layers %d, expected %d", len(ls.(*layerStore).layerMap), expected) 212 } 213 } 214 215 func cacheID(l Layer) string { 216 return getCachedLayer(l).cacheID 217 } 218 219 func assertLayerEqual(t *testing.T, l1, l2 Layer) { 220 if l1.ChainID() != l2.ChainID() { 221 t.Fatalf("Mismatched ChainID: %s vs %s", l1.ChainID(), l2.ChainID()) 222 } 223 if l1.DiffID() != l2.DiffID() { 224 t.Fatalf("Mismatched DiffID: %s vs %s", l1.DiffID(), l2.DiffID()) 225 } 226 227 size1 := l1.Size() 228 size2 := l2.Size() 229 230 if size1 != size2 { 231 t.Fatalf("Mismatched size: %d vs %d", size1, size2) 232 } 233 234 if cacheID(l1) != cacheID(l2) { 235 t.Fatalf("Mismatched cache id: %s vs %s", cacheID(l1), cacheID(l2)) 236 } 237 238 p1 := l1.Parent() 239 p2 := l2.Parent() 240 if p1 != nil && p2 != nil { 241 assertLayerEqual(t, p1, p2) 242 } else if p1 != nil || p2 != nil { 243 t.Fatalf("Mismatched parents: %v vs %v", p1, p2) 244 } 245 } 246 247 func TestMountAndRegister(t *testing.T) { 248 ls, _, cleanup := newTestStore(t) 249 defer cleanup() 250 251 li := initWithFiles(newTestFile("testfile.txt", []byte("some test data"), 0o644)) 252 layer, err := createLayer(ls, "", li) 253 if err != nil { 254 t.Fatal(err) 255 } 256 257 size := layer.Size() 258 t.Logf("Layer size: %d", size) 259 260 mount2, err := ls.CreateRWLayer("new-test-mount", layer.ChainID(), nil) 261 if err != nil { 262 t.Fatal(err) 263 } 264 265 path2, err := mount2.Mount("") 266 if err != nil { 267 t.Fatal(err) 268 } 269 270 b, err := os.ReadFile(filepath.Join(path2, "testfile.txt")) 271 if err != nil { 272 t.Fatal(err) 273 } 274 275 if expected := "some test data"; string(b) != expected { 276 t.Fatalf("Wrong file data, expected %q, got %q", expected, string(b)) 277 } 278 279 if err := mount2.Unmount(); err != nil { 280 t.Fatal(err) 281 } 282 283 if _, err := ls.ReleaseRWLayer(mount2); err != nil { 284 t.Fatal(err) 285 } 286 } 287 288 func TestLayerRelease(t *testing.T) { 289 // TODO Windows: Figure out why this is failing 290 if runtime.GOOS == "windows" { 291 t.Skip("Failing on Windows") 292 } 293 ls, _, cleanup := newTestStore(t) 294 defer cleanup() 295 296 layer1, err := createLayer(ls, "", initWithFiles(newTestFile("layer1.txt", []byte("layer 1 file"), 0o644))) 297 if err != nil { 298 t.Fatal(err) 299 } 300 301 layer2, err := createLayer(ls, layer1.ChainID(), initWithFiles(newTestFile("layer2.txt", []byte("layer 2 file"), 0o644))) 302 if err != nil { 303 t.Fatal(err) 304 } 305 306 if _, err := ls.Release(layer1); err != nil { 307 t.Fatal(err) 308 } 309 310 layer3a, err := createLayer(ls, layer2.ChainID(), initWithFiles(newTestFile("layer3.txt", []byte("layer 3a file"), 0o644))) 311 if err != nil { 312 t.Fatal(err) 313 } 314 315 layer3b, err := createLayer(ls, layer2.ChainID(), initWithFiles(newTestFile("layer3.txt", []byte("layer 3b file"), 0o644))) 316 if err != nil { 317 t.Fatal(err) 318 } 319 320 if _, err := ls.Release(layer2); err != nil { 321 t.Fatal(err) 322 } 323 324 t.Logf("Layer1: %s", layer1.ChainID()) 325 t.Logf("Layer2: %s", layer2.ChainID()) 326 t.Logf("Layer3a: %s", layer3a.ChainID()) 327 t.Logf("Layer3b: %s", layer3b.ChainID()) 328 329 if expected := 4; len(ls.(*layerStore).layerMap) != expected { 330 t.Fatalf("Unexpected number of layers %d, expected %d", len(ls.(*layerStore).layerMap), expected) 331 } 332 333 releaseAndCheckDeleted(t, ls, layer3b, layer3b) 334 releaseAndCheckDeleted(t, ls, layer3a, layer3a, layer2, layer1) 335 } 336 337 func TestStoreRestore(t *testing.T) { 338 // TODO Windows: Figure out why this is failing 339 if runtime.GOOS == "windows" { 340 t.Skip("Failing on Windows") 341 } 342 ls, _, cleanup := newTestStore(t) 343 defer cleanup() 344 345 layer1, err := createLayer(ls, "", initWithFiles(newTestFile("layer1.txt", []byte("layer 1 file"), 0o644))) 346 if err != nil { 347 t.Fatal(err) 348 } 349 350 layer2, err := createLayer(ls, layer1.ChainID(), initWithFiles(newTestFile("layer2.txt", []byte("layer 2 file"), 0o644))) 351 if err != nil { 352 t.Fatal(err) 353 } 354 355 if _, err := ls.Release(layer1); err != nil { 356 t.Fatal(err) 357 } 358 359 layer3, err := createLayer(ls, layer2.ChainID(), initWithFiles(newTestFile("layer3.txt", []byte("layer 3 file"), 0o644))) 360 if err != nil { 361 t.Fatal(err) 362 } 363 364 if _, err := ls.Release(layer2); err != nil { 365 t.Fatal(err) 366 } 367 368 m, err := ls.CreateRWLayer("some-mount_name", layer3.ChainID(), nil) 369 if err != nil { 370 t.Fatal(err) 371 } 372 373 pathFS, err := m.Mount("") 374 if err != nil { 375 t.Fatal(err) 376 } 377 378 if err := os.WriteFile(filepath.Join(pathFS, "testfile.txt"), []byte("nothing here"), 0o644); err != nil { 379 t.Fatal(err) 380 } 381 382 if err := m.Unmount(); err != nil { 383 t.Fatal(err) 384 } 385 386 ls2, err := newStoreFromGraphDriver(ls.(*layerStore).store.root, ls.(*layerStore).driver) 387 if err != nil { 388 t.Fatal(err) 389 } 390 391 layer3b, err := ls2.Get(layer3.ChainID()) 392 if err != nil { 393 t.Fatal(err) 394 } 395 396 assertLayerEqual(t, layer3b, layer3) 397 398 // Create again with same name, should return error 399 if _, err := ls2.CreateRWLayer("some-mount_name", layer3b.ChainID(), nil); err == nil { 400 t.Fatal("Expected error creating mount with same name") 401 } else if !errors.Is(err, ErrMountNameConflict) { 402 t.Fatal(err) 403 } 404 405 m2, err := ls2.GetRWLayer("some-mount_name") 406 if err != nil { 407 t.Fatal(err) 408 } 409 410 if mountPath, err := m2.Mount(""); err != nil { 411 t.Fatal(err) 412 } else if pathFS != mountPath { 413 t.Fatalf("Unexpected path %s, expected %s", mountPath, pathFS) 414 } 415 416 if mountPath, err := m2.Mount(""); err != nil { 417 t.Fatal(err) 418 } else if pathFS != mountPath { 419 t.Fatalf("Unexpected path %s, expected %s", mountPath, pathFS) 420 } 421 if err := m2.Unmount(); err != nil { 422 t.Fatal(err) 423 } 424 425 b, err := os.ReadFile(filepath.Join(pathFS, "testfile.txt")) 426 if err != nil { 427 t.Fatal(err) 428 } 429 if expected := "nothing here"; string(b) != expected { 430 t.Fatalf("Unexpected content %q, expected %q", string(b), expected) 431 } 432 433 if err := m2.Unmount(); err != nil { 434 t.Fatal(err) 435 } 436 437 if metadata, err := ls2.ReleaseRWLayer(m2); err != nil { 438 t.Fatal(err) 439 } else if len(metadata) != 0 { 440 t.Fatalf("Unexpectedly deleted layers: %#v", metadata) 441 } 442 443 if metadata, err := ls2.ReleaseRWLayer(m2); err != nil { 444 t.Fatal(err) 445 } else if len(metadata) != 0 { 446 t.Fatalf("Unexpectedly deleted layers: %#v", metadata) 447 } 448 449 releaseAndCheckDeleted(t, ls2, layer3b, layer3, layer2, layer1) 450 } 451 452 func TestTarStreamStability(t *testing.T) { 453 // TODO Windows: Figure out why this is failing 454 if runtime.GOOS == "windows" { 455 t.Skip("Failing on Windows") 456 } 457 ls, _, cleanup := newTestStore(t) 458 defer cleanup() 459 460 files1 := []FileApplier{ 461 newTestFile("/etc/hosts", []byte("mydomain 10.0.0.1"), 0o644), 462 newTestFile("/etc/profile", []byte("PATH=/usr/bin"), 0o644), 463 } 464 addedFile := newTestFile("/etc/shadow", []byte("root:::::::"), 0o644) 465 files2 := []FileApplier{ 466 newTestFile("/etc/hosts", []byte("mydomain 10.0.0.2"), 0o644), 467 newTestFile("/etc/profile", []byte("PATH=/usr/bin"), 0o664), 468 newTestFile("/root/.bashrc", []byte("PATH=/usr/sbin:/usr/bin"), 0o644), 469 } 470 471 tar1, err := tarFromFiles(files1...) 472 if err != nil { 473 t.Fatal(err) 474 } 475 476 tar2, err := tarFromFiles(files2...) 477 if err != nil { 478 t.Fatal(err) 479 } 480 481 layer1, err := ls.Register(bytes.NewReader(tar1), "") 482 if err != nil { 483 t.Fatal(err) 484 } 485 486 // hack layer to add file 487 p, err := ls.(*layerStore).driver.Get(layer1.(*referencedCacheLayer).cacheID, "") 488 if err != nil { 489 t.Fatal(err) 490 } 491 492 if err := addedFile.ApplyFile(p); err != nil { 493 t.Fatal(err) 494 } 495 496 if err := ls.(*layerStore).driver.Put(layer1.(*referencedCacheLayer).cacheID); err != nil { 497 t.Fatal(err) 498 } 499 500 layer2, err := ls.Register(bytes.NewReader(tar2), layer1.ChainID()) 501 if err != nil { 502 t.Fatal(err) 503 } 504 505 id1 := layer1.ChainID() 506 t.Logf("Layer 1: %s", layer1.ChainID()) 507 t.Logf("Layer 2: %s", layer2.ChainID()) 508 509 if _, err := ls.Release(layer1); err != nil { 510 t.Fatal(err) 511 } 512 513 assertLayerDiff(t, tar2, layer2) 514 515 layer1b, err := ls.Get(id1) 516 if err != nil { 517 t.Logf("Content of layer map: %#v", ls.(*layerStore).layerMap) 518 t.Fatal(err) 519 } 520 521 if _, err := ls.Release(layer2); err != nil { 522 t.Fatal(err) 523 } 524 525 assertLayerDiff(t, tar1, layer1b) 526 527 if _, err := ls.Release(layer1b); err != nil { 528 t.Fatal(err) 529 } 530 } 531 532 func assertLayerDiff(t *testing.T, expected []byte, layer Layer) { 533 expectedDigest := digest.FromBytes(expected) 534 535 if digest.Digest(layer.DiffID()) != expectedDigest { 536 t.Fatalf("Mismatched diff id for %s, got %s, expected %s", layer.ChainID(), layer.DiffID(), expected) 537 } 538 539 ts, err := layer.TarStream() 540 if err != nil { 541 t.Fatal(err) 542 } 543 defer ts.Close() 544 545 actual, err := io.ReadAll(ts) 546 if err != nil { 547 t.Fatal(err) 548 } 549 550 if len(actual) != len(expected) { 551 logByteDiff(t, actual, expected) 552 t.Fatalf("Mismatched tar stream size for %s, got %d, expected %d", layer.ChainID(), len(actual), len(expected)) 553 } 554 555 actualDigest := digest.FromBytes(actual) 556 557 if actualDigest != expectedDigest { 558 logByteDiff(t, actual, expected) 559 t.Fatalf("Wrong digest of tar stream, got %s, expected %s", actualDigest, expectedDigest) 560 } 561 } 562 563 const maxByteLog = 4 * 1024 564 565 func logByteDiff(t *testing.T, actual, expected []byte) { 566 d1, d2 := byteDiff(actual, expected) 567 if len(d1) == 0 && len(d2) == 0 { 568 return 569 } 570 571 prefix := len(actual) - len(d1) 572 if len(d1) > maxByteLog || len(d2) > maxByteLog { 573 t.Logf("Byte diff after %d matching bytes", prefix) 574 } else { 575 t.Logf("Byte diff after %d matching bytes\nActual bytes after prefix:\n%x\nExpected bytes after prefix:\n%x", prefix, d1, d2) 576 } 577 } 578 579 // byteDiff returns the differing bytes after the matching prefix 580 func byteDiff(b1, b2 []byte) ([]byte, []byte) { 581 i := 0 582 for i < len(b1) && i < len(b2) { 583 if b1[i] != b2[i] { 584 break 585 } 586 i++ 587 } 588 589 return b1[i:], b2[i:] 590 } 591 592 func tarFromFiles(files ...FileApplier) ([]byte, error) { 593 td, err := os.MkdirTemp("", "tar-") 594 if err != nil { 595 return nil, err 596 } 597 defer os.RemoveAll(td) 598 599 for _, f := range files { 600 if err := f.ApplyFile(td); err != nil { 601 return nil, err 602 } 603 } 604 605 r, err := archive.Tar(td, archive.Uncompressed) 606 if err != nil { 607 return nil, err 608 } 609 610 buf := bytes.NewBuffer(nil) 611 if _, err := io.Copy(buf, r); err != nil { 612 return nil, err 613 } 614 615 return buf.Bytes(), nil 616 } 617 618 // assertReferences asserts that all the references are to the same 619 // image and represent the full set of references to that image. 620 func assertReferences(t *testing.T, references ...Layer) { 621 if len(references) == 0 { 622 return 623 } 624 base := references[0].(*referencedCacheLayer).roLayer 625 seenReferences := map[Layer]struct{}{ 626 references[0]: {}, 627 } 628 for i := 1; i < len(references); i++ { 629 other := references[i].(*referencedCacheLayer).roLayer 630 if base != other { 631 t.Fatalf("Unexpected referenced cache layer %s, expecting %s", other.ChainID(), base.ChainID()) 632 } 633 if _, ok := base.references[references[i]]; !ok { 634 t.Fatalf("Reference not part of reference list: %v", references[i]) 635 } 636 if _, ok := seenReferences[references[i]]; ok { 637 t.Fatalf("Duplicated reference %v", references[i]) 638 } 639 } 640 if rc := len(base.references); rc != len(references) { 641 t.Fatalf("Unexpected number of references %d, expecting %d", rc, len(references)) 642 } 643 } 644 645 func TestRegisterExistingLayer(t *testing.T) { 646 ls, _, cleanup := newTestStore(t) 647 defer cleanup() 648 649 baseFiles := []FileApplier{ 650 newTestFile("/etc/profile", []byte("# Base configuration"), 0o644), 651 } 652 653 layerFiles := []FileApplier{ 654 newTestFile("/root/.bashrc", []byte("# Root configuration"), 0o644), 655 } 656 657 li := initWithFiles(baseFiles...) 658 layer1, err := createLayer(ls, "", li) 659 if err != nil { 660 t.Fatal(err) 661 } 662 663 tar1, err := tarFromFiles(layerFiles...) 664 if err != nil { 665 t.Fatal(err) 666 } 667 668 layer2a, err := ls.Register(bytes.NewReader(tar1), layer1.ChainID()) 669 if err != nil { 670 t.Fatal(err) 671 } 672 673 layer2b, err := ls.Register(bytes.NewReader(tar1), layer1.ChainID()) 674 if err != nil { 675 t.Fatal(err) 676 } 677 678 assertReferences(t, layer2a, layer2b) 679 } 680 681 func TestTarStreamVerification(t *testing.T) { 682 // TODO Windows: Figure out why this is failing 683 if runtime.GOOS == "windows" { 684 t.Skip("Failing on Windows") 685 } 686 ls, tmpdir, cleanup := newTestStore(t) 687 defer cleanup() 688 689 files1 := []FileApplier{ 690 newTestFile("/foo", []byte("abc"), 0o644), 691 newTestFile("/bar", []byte("def"), 0o644), 692 } 693 files2 := []FileApplier{ 694 newTestFile("/foo", []byte("abc"), 0o644), 695 newTestFile("/bar", []byte("def"), 0o600), // different perm 696 } 697 698 tar1, err := tarFromFiles(files1...) 699 if err != nil { 700 t.Fatal(err) 701 } 702 703 tar2, err := tarFromFiles(files2...) 704 if err != nil { 705 t.Fatal(err) 706 } 707 708 layer1, err := ls.Register(bytes.NewReader(tar1), "") 709 if err != nil { 710 t.Fatal(err) 711 } 712 713 layer2, err := ls.Register(bytes.NewReader(tar2), "") 714 if err != nil { 715 t.Fatal(err) 716 } 717 id1 := digest.Digest(layer1.ChainID()) 718 id2 := digest.Digest(layer2.ChainID()) 719 720 // Replace tar data files 721 src, err := os.Open(filepath.Join(tmpdir, id1.Algorithm().String(), id1.Encoded(), "tar-split.json.gz")) 722 if err != nil { 723 t.Fatal(err) 724 } 725 defer src.Close() 726 727 dst, err := os.Create(filepath.Join(tmpdir, id2.Algorithm().String(), id2.Encoded(), "tar-split.json.gz")) 728 if err != nil { 729 t.Fatal(err) 730 } 731 defer dst.Close() 732 733 if _, err := io.Copy(dst, src); err != nil { 734 t.Fatal(err) 735 } 736 737 src.Sync() 738 dst.Sync() 739 740 ts, err := layer2.TarStream() 741 if err != nil { 742 t.Fatal(err) 743 } 744 _, err = io.Copy(io.Discard, ts) 745 if err == nil { 746 t.Fatal("expected data verification to fail") 747 } 748 if !strings.Contains(err.Error(), "could not verify layer data") { 749 t.Fatalf("wrong error returned from tarstream: %q", err) 750 } 751 }