github.com/vincentwoo/docker@v0.7.3-0.20160116130405-82401a4b13c0/layer/layer_test.go (about) 1 package layer 2 3 import ( 4 "bytes" 5 "io" 6 "io/ioutil" 7 "os" 8 "path/filepath" 9 "testing" 10 11 "github.com/docker/distribution/digest" 12 "github.com/docker/docker/daemon/graphdriver" 13 "github.com/docker/docker/daemon/graphdriver/vfs" 14 "github.com/docker/docker/pkg/archive" 15 "github.com/docker/docker/pkg/idtools" 16 "github.com/docker/docker/pkg/stringid" 17 ) 18 19 func init() { 20 graphdriver.ApplyUncompressedLayer = archive.UnpackLayer 21 vfs.CopyWithTar = archive.CopyWithTar 22 } 23 24 func newVFSGraphDriver(td string) (graphdriver.Driver, error) { 25 uidMap := []idtools.IDMap{ 26 { 27 ContainerID: 0, 28 HostID: os.Getuid(), 29 Size: 1, 30 }, 31 } 32 gidMap := []idtools.IDMap{ 33 { 34 ContainerID: 0, 35 HostID: os.Getgid(), 36 Size: 1, 37 }, 38 } 39 40 return graphdriver.GetDriver("vfs", td, nil, uidMap, gidMap) 41 } 42 43 func newTestGraphDriver(t *testing.T) (graphdriver.Driver, func()) { 44 td, err := ioutil.TempDir("", "graph-") 45 if err != nil { 46 t.Fatal(err) 47 } 48 49 driver, err := newVFSGraphDriver(td) 50 if err != nil { 51 t.Fatal(err) 52 } 53 54 return driver, func() { 55 os.RemoveAll(td) 56 } 57 } 58 59 func newTestStore(t *testing.T) (Store, func()) { 60 td, err := ioutil.TempDir("", "layerstore-") 61 if err != nil { 62 t.Fatal(err) 63 } 64 65 graph, graphcleanup := newTestGraphDriver(t) 66 fms, err := NewFSMetadataStore(td) 67 if err != nil { 68 t.Fatal(err) 69 } 70 ls, err := NewStoreFromGraphDriver(fms, graph) 71 if err != nil { 72 t.Fatal(err) 73 } 74 75 return ls, func() { 76 graphcleanup() 77 os.RemoveAll(td) 78 } 79 } 80 81 type layerInit func(root string) error 82 83 func createLayer(ls Store, parent ChainID, layerFunc layerInit) (Layer, error) { 84 containerID := stringid.GenerateRandomID() 85 mount, err := ls.CreateRWLayer(containerID, parent, "", nil) 86 if err != nil { 87 return nil, err 88 } 89 90 path, err := mount.Mount("") 91 if err != nil { 92 return nil, err 93 } 94 95 if err := layerFunc(path); err != nil { 96 return nil, err 97 } 98 99 ts, err := mount.TarStream() 100 if err != nil { 101 return nil, err 102 } 103 defer ts.Close() 104 105 layer, err := ls.Register(ts, parent) 106 if err != nil { 107 return nil, err 108 } 109 110 if err := mount.Unmount(); err != nil { 111 return nil, err 112 } 113 114 if _, err := ls.ReleaseRWLayer(mount); err != nil { 115 return nil, err 116 } 117 118 return layer, nil 119 } 120 121 type FileApplier interface { 122 ApplyFile(root string) error 123 } 124 125 type testFile struct { 126 name string 127 content []byte 128 permission os.FileMode 129 } 130 131 func newTestFile(name string, content []byte, perm os.FileMode) FileApplier { 132 return &testFile{ 133 name: name, 134 content: content, 135 permission: perm, 136 } 137 } 138 139 func (tf *testFile) ApplyFile(root string) error { 140 fullPath := filepath.Join(root, tf.name) 141 if err := os.MkdirAll(filepath.Dir(fullPath), 0755); err != nil { 142 return err 143 } 144 // Check if already exists 145 if stat, err := os.Stat(fullPath); err == nil && stat.Mode().Perm() != tf.permission { 146 if err := os.Chmod(fullPath, tf.permission); err != nil { 147 return err 148 } 149 } 150 if err := ioutil.WriteFile(fullPath, tf.content, tf.permission); err != nil { 151 return err 152 } 153 return nil 154 } 155 156 func initWithFiles(files ...FileApplier) layerInit { 157 return func(root string) error { 158 for _, f := range files { 159 if err := f.ApplyFile(root); err != nil { 160 return err 161 } 162 } 163 return nil 164 } 165 } 166 167 func getCachedLayer(l Layer) *roLayer { 168 if rl, ok := l.(*referencedCacheLayer); ok { 169 return rl.roLayer 170 } 171 return l.(*roLayer) 172 } 173 174 func getMountLayer(l RWLayer) *mountedLayer { 175 if rl, ok := l.(*referencedRWLayer); ok { 176 return rl.mountedLayer 177 } 178 return l.(*mountedLayer) 179 } 180 181 func createMetadata(layers ...Layer) []Metadata { 182 metadata := make([]Metadata, len(layers)) 183 for i := range layers { 184 size, err := layers[i].Size() 185 if err != nil { 186 panic(err) 187 } 188 189 metadata[i].ChainID = layers[i].ChainID() 190 metadata[i].DiffID = layers[i].DiffID() 191 metadata[i].Size = size 192 metadata[i].DiffSize = getCachedLayer(layers[i]).size 193 } 194 195 return metadata 196 } 197 198 func assertMetadata(t *testing.T, metadata, expectedMetadata []Metadata) { 199 if len(metadata) != len(expectedMetadata) { 200 t.Fatalf("Unexpected number of deletes %d, expected %d", len(metadata), len(expectedMetadata)) 201 } 202 203 for i := range metadata { 204 if metadata[i] != expectedMetadata[i] { 205 t.Errorf("Unexpected metadata\n\tExpected: %#v\n\tActual: %#v", expectedMetadata[i], metadata[i]) 206 } 207 } 208 if t.Failed() { 209 t.FailNow() 210 } 211 } 212 213 func releaseAndCheckDeleted(t *testing.T, ls Store, layer Layer, removed ...Layer) { 214 layerCount := len(ls.(*layerStore).layerMap) 215 expectedMetadata := createMetadata(removed...) 216 metadata, err := ls.Release(layer) 217 if err != nil { 218 t.Fatal(err) 219 } 220 221 assertMetadata(t, metadata, expectedMetadata) 222 223 if expected := layerCount - len(removed); len(ls.(*layerStore).layerMap) != expected { 224 t.Fatalf("Unexpected number of layers %d, expected %d", len(ls.(*layerStore).layerMap), expected) 225 } 226 } 227 228 func cacheID(l Layer) string { 229 return getCachedLayer(l).cacheID 230 } 231 232 func assertLayerEqual(t *testing.T, l1, l2 Layer) { 233 if l1.ChainID() != l2.ChainID() { 234 t.Fatalf("Mismatched ID: %s vs %s", l1.ChainID(), l2.ChainID()) 235 } 236 if l1.DiffID() != l2.DiffID() { 237 t.Fatalf("Mismatched DiffID: %s vs %s", l1.DiffID(), l2.DiffID()) 238 } 239 240 size1, err := l1.Size() 241 if err != nil { 242 t.Fatal(err) 243 } 244 245 size2, err := l2.Size() 246 if err != nil { 247 t.Fatal(err) 248 } 249 250 if size1 != size2 { 251 t.Fatalf("Mismatched size: %d vs %d", size1, size2) 252 } 253 254 if cacheID(l1) != cacheID(l2) { 255 t.Fatalf("Mismatched cache id: %s vs %s", cacheID(l1), cacheID(l2)) 256 } 257 258 p1 := l1.Parent() 259 p2 := l2.Parent() 260 if p1 != nil && p2 != nil { 261 assertLayerEqual(t, p1, p2) 262 } else if p1 != nil || p2 != nil { 263 t.Fatalf("Mismatched parents: %v vs %v", p1, p2) 264 } 265 } 266 267 func TestMountAndRegister(t *testing.T) { 268 ls, cleanup := newTestStore(t) 269 defer cleanup() 270 271 li := initWithFiles(newTestFile("testfile.txt", []byte("some test data"), 0644)) 272 layer, err := createLayer(ls, "", li) 273 if err != nil { 274 t.Fatal(err) 275 } 276 277 size, _ := layer.Size() 278 t.Logf("Layer size: %d", size) 279 280 mount2, err := ls.CreateRWLayer("new-test-mount", layer.ChainID(), "", nil) 281 if err != nil { 282 t.Fatal(err) 283 } 284 285 path2, err := mount2.Mount("") 286 if err != nil { 287 t.Fatal(err) 288 } 289 290 b, err := ioutil.ReadFile(filepath.Join(path2, "testfile.txt")) 291 if err != nil { 292 t.Fatal(err) 293 } 294 295 if expected := "some test data"; string(b) != expected { 296 t.Fatalf("Wrong file data, expected %q, got %q", expected, string(b)) 297 } 298 299 if err := mount2.Unmount(); err != nil { 300 t.Fatal(err) 301 } 302 303 if _, err := ls.ReleaseRWLayer(mount2); err != nil { 304 t.Fatal(err) 305 } 306 } 307 308 func TestLayerRelease(t *testing.T) { 309 ls, cleanup := newTestStore(t) 310 defer cleanup() 311 312 layer1, err := createLayer(ls, "", initWithFiles(newTestFile("layer1.txt", []byte("layer 1 file"), 0644))) 313 if err != nil { 314 t.Fatal(err) 315 } 316 317 layer2, err := createLayer(ls, layer1.ChainID(), initWithFiles(newTestFile("layer2.txt", []byte("layer 2 file"), 0644))) 318 if err != nil { 319 t.Fatal(err) 320 } 321 322 if _, err := ls.Release(layer1); err != nil { 323 t.Fatal(err) 324 } 325 326 layer3a, err := createLayer(ls, layer2.ChainID(), initWithFiles(newTestFile("layer3.txt", []byte("layer 3a file"), 0644))) 327 if err != nil { 328 t.Fatal(err) 329 } 330 331 layer3b, err := createLayer(ls, layer2.ChainID(), initWithFiles(newTestFile("layer3.txt", []byte("layer 3b file"), 0644))) 332 if err != nil { 333 t.Fatal(err) 334 } 335 336 if _, err := ls.Release(layer2); err != nil { 337 t.Fatal(err) 338 } 339 340 t.Logf("Layer1: %s", layer1.ChainID()) 341 t.Logf("Layer2: %s", layer2.ChainID()) 342 t.Logf("Layer3a: %s", layer3a.ChainID()) 343 t.Logf("Layer3b: %s", layer3b.ChainID()) 344 345 if expected := 4; len(ls.(*layerStore).layerMap) != expected { 346 t.Fatalf("Unexpected number of layers %d, expected %d", len(ls.(*layerStore).layerMap), expected) 347 } 348 349 releaseAndCheckDeleted(t, ls, layer3b, layer3b) 350 releaseAndCheckDeleted(t, ls, layer3a, layer3a, layer2, layer1) 351 } 352 353 func TestStoreRestore(t *testing.T) { 354 ls, cleanup := newTestStore(t) 355 defer cleanup() 356 357 layer1, err := createLayer(ls, "", initWithFiles(newTestFile("layer1.txt", []byte("layer 1 file"), 0644))) 358 if err != nil { 359 t.Fatal(err) 360 } 361 362 layer2, err := createLayer(ls, layer1.ChainID(), initWithFiles(newTestFile("layer2.txt", []byte("layer 2 file"), 0644))) 363 if err != nil { 364 t.Fatal(err) 365 } 366 367 if _, err := ls.Release(layer1); err != nil { 368 t.Fatal(err) 369 } 370 371 layer3, err := createLayer(ls, layer2.ChainID(), initWithFiles(newTestFile("layer3.txt", []byte("layer 3 file"), 0644))) 372 if err != nil { 373 t.Fatal(err) 374 } 375 376 if _, err := ls.Release(layer2); err != nil { 377 t.Fatal(err) 378 } 379 380 m, err := ls.CreateRWLayer("some-mount_name", layer3.ChainID(), "", nil) 381 if err != nil { 382 t.Fatal(err) 383 } 384 385 path, err := m.Mount("") 386 if err != nil { 387 t.Fatal(err) 388 } 389 390 if err := ioutil.WriteFile(filepath.Join(path, "testfile.txt"), []byte("nothing here"), 0644); err != nil { 391 t.Fatal(err) 392 } 393 assertActivityCount(t, m, 1) 394 395 if err := m.Unmount(); err != nil { 396 t.Fatal(err) 397 } 398 399 assertActivityCount(t, m, 0) 400 401 ls2, err := NewStoreFromGraphDriver(ls.(*layerStore).store, ls.(*layerStore).driver) 402 if err != nil { 403 t.Fatal(err) 404 } 405 406 layer3b, err := ls2.Get(layer3.ChainID()) 407 if err != nil { 408 t.Fatal(err) 409 } 410 411 assertLayerEqual(t, layer3b, layer3) 412 413 // Create again with same name, should return error 414 if _, err := ls2.CreateRWLayer("some-mount_name", layer3b.ChainID(), "", nil); err == nil { 415 t.Fatal("Expected error creating mount with same name") 416 } else if err != ErrMountNameConflict { 417 t.Fatal(err) 418 } 419 420 m2, err := ls2.GetRWLayer("some-mount_name") 421 if err != nil { 422 t.Fatal(err) 423 } 424 425 if mountPath, err := m2.Mount(""); err != nil { 426 t.Fatal(err) 427 } else if path != mountPath { 428 t.Fatalf("Unexpected path %s, expected %s", mountPath, path) 429 } 430 431 assertActivityCount(t, m2, 1) 432 433 if mountPath, err := m2.Mount(""); err != nil { 434 t.Fatal(err) 435 } else if path != mountPath { 436 t.Fatalf("Unexpected path %s, expected %s", mountPath, path) 437 } 438 assertActivityCount(t, m2, 2) 439 if err := m2.Unmount(); err != nil { 440 t.Fatal(err) 441 } 442 443 assertActivityCount(t, m2, 1) 444 445 b, err := ioutil.ReadFile(filepath.Join(path, "testfile.txt")) 446 if err != nil { 447 t.Fatal(err) 448 } 449 if expected := "nothing here"; string(b) != expected { 450 t.Fatalf("Unexpected content %q, expected %q", string(b), expected) 451 } 452 453 if err := m2.Unmount(); err != nil { 454 t.Fatal(err) 455 } 456 457 assertActivityCount(t, m2, 0) 458 459 if metadata, err := ls2.ReleaseRWLayer(m2); err != nil { 460 t.Fatal(err) 461 } else if len(metadata) != 0 { 462 t.Fatalf("Unexpectedly deleted layers: %#v", metadata) 463 } 464 465 if metadata, err := ls2.ReleaseRWLayer(m2); err != nil { 466 t.Fatal(err) 467 } else if len(metadata) != 0 { 468 t.Fatalf("Unexpectedly deleted layers: %#v", metadata) 469 } 470 471 releaseAndCheckDeleted(t, ls2, layer3b, layer3, layer2, layer1) 472 } 473 474 func TestTarStreamStability(t *testing.T) { 475 ls, cleanup := newTestStore(t) 476 defer cleanup() 477 478 files1 := []FileApplier{ 479 newTestFile("/etc/hosts", []byte("mydomain 10.0.0.1"), 0644), 480 newTestFile("/etc/profile", []byte("PATH=/usr/bin"), 0644), 481 } 482 addedFile := newTestFile("/etc/shadow", []byte("root:::::::"), 0644) 483 files2 := []FileApplier{ 484 newTestFile("/etc/hosts", []byte("mydomain 10.0.0.2"), 0644), 485 newTestFile("/etc/profile", []byte("PATH=/usr/bin"), 0664), 486 newTestFile("/root/.bashrc", []byte("PATH=/usr/sbin:/usr/bin"), 0644), 487 } 488 489 tar1, err := tarFromFiles(files1...) 490 if err != nil { 491 t.Fatal(err) 492 } 493 494 tar2, err := tarFromFiles(files2...) 495 if err != nil { 496 t.Fatal(err) 497 } 498 499 layer1, err := ls.Register(bytes.NewReader(tar1), "") 500 if err != nil { 501 t.Fatal(err) 502 } 503 504 // hack layer to add file 505 p, err := ls.(*layerStore).driver.Get(layer1.(*referencedCacheLayer).cacheID, "") 506 if err != nil { 507 t.Fatal(err) 508 } 509 510 if err := addedFile.ApplyFile(p); err != nil { 511 t.Fatal(err) 512 } 513 514 if err := ls.(*layerStore).driver.Put(layer1.(*referencedCacheLayer).cacheID); err != nil { 515 t.Fatal(err) 516 } 517 518 layer2, err := ls.Register(bytes.NewReader(tar2), layer1.ChainID()) 519 if err != nil { 520 t.Fatal(err) 521 } 522 523 id1 := layer1.ChainID() 524 t.Logf("Layer 1: %s", layer1.ChainID()) 525 t.Logf("Layer 2: %s", layer2.ChainID()) 526 527 if _, err := ls.Release(layer1); err != nil { 528 t.Fatal(err) 529 } 530 531 assertLayerDiff(t, tar2, layer2) 532 533 layer1b, err := ls.Get(id1) 534 if err != nil { 535 t.Logf("Content of layer map: %#v", ls.(*layerStore).layerMap) 536 t.Fatal(err) 537 } 538 539 if _, err := ls.Release(layer2); err != nil { 540 t.Fatal(err) 541 } 542 543 assertLayerDiff(t, tar1, layer1b) 544 545 if _, err := ls.Release(layer1b); err != nil { 546 t.Fatal(err) 547 } 548 } 549 550 func assertLayerDiff(t *testing.T, expected []byte, layer Layer) { 551 expectedDigest := digest.FromBytes(expected) 552 553 if digest.Digest(layer.DiffID()) != expectedDigest { 554 t.Fatalf("Mismatched diff id for %s, got %s, expected %s", layer.ChainID(), layer.DiffID(), expected) 555 } 556 557 ts, err := layer.TarStream() 558 if err != nil { 559 t.Fatal(err) 560 } 561 defer ts.Close() 562 563 actual, err := ioutil.ReadAll(ts) 564 if err != nil { 565 t.Fatal(err) 566 } 567 568 if len(actual) != len(expected) { 569 logByteDiff(t, actual, expected) 570 t.Fatalf("Mismatched tar stream size for %s, got %d, expected %d", layer.ChainID(), len(actual), len(expected)) 571 } 572 573 actualDigest := digest.FromBytes(actual) 574 575 if actualDigest != expectedDigest { 576 logByteDiff(t, actual, expected) 577 t.Fatalf("Wrong digest of tar stream, got %s, expected %s", actualDigest, expectedDigest) 578 } 579 } 580 581 const maxByteLog = 4 * 1024 582 583 func logByteDiff(t *testing.T, actual, expected []byte) { 584 d1, d2 := byteDiff(actual, expected) 585 if len(d1) == 0 && len(d2) == 0 { 586 return 587 } 588 589 prefix := len(actual) - len(d1) 590 if len(d1) > maxByteLog || len(d2) > maxByteLog { 591 t.Logf("Byte diff after %d matching bytes", prefix) 592 } else { 593 t.Logf("Byte diff after %d matching bytes\nActual bytes after prefix:\n%x\nExpected bytes after prefix:\n%x", prefix, d1, d2) 594 } 595 } 596 597 // byteDiff returns the differing bytes after the matching prefix 598 func byteDiff(b1, b2 []byte) ([]byte, []byte) { 599 i := 0 600 for i < len(b1) && i < len(b2) { 601 if b1[i] != b2[i] { 602 break 603 } 604 i++ 605 } 606 607 return b1[i:], b2[i:] 608 } 609 610 func tarFromFiles(files ...FileApplier) ([]byte, error) { 611 td, err := ioutil.TempDir("", "tar-") 612 if err != nil { 613 return nil, err 614 } 615 defer os.RemoveAll(td) 616 617 for _, f := range files { 618 if err := f.ApplyFile(td); err != nil { 619 return nil, err 620 } 621 } 622 623 r, err := archive.Tar(td, archive.Uncompressed) 624 if err != nil { 625 return nil, err 626 } 627 628 buf := bytes.NewBuffer(nil) 629 if _, err := io.Copy(buf, r); err != nil { 630 return nil, err 631 } 632 633 return buf.Bytes(), nil 634 } 635 636 // assertReferences asserts that all the references are to the same 637 // image and represent the full set of references to that image. 638 func assertReferences(t *testing.T, references ...Layer) { 639 if len(references) == 0 { 640 return 641 } 642 base := references[0].(*referencedCacheLayer).roLayer 643 seenReferences := map[Layer]struct{}{ 644 references[0]: {}, 645 } 646 for i := 1; i < len(references); i++ { 647 other := references[i].(*referencedCacheLayer).roLayer 648 if base != other { 649 t.Fatalf("Unexpected referenced cache layer %s, expecting %s", other.ChainID(), base.ChainID()) 650 } 651 if _, ok := base.references[references[i]]; !ok { 652 t.Fatalf("Reference not part of reference list: %v", references[i]) 653 } 654 if _, ok := seenReferences[references[i]]; ok { 655 t.Fatalf("Duplicated reference %v", references[i]) 656 } 657 } 658 if rc := len(base.references); rc != len(references) { 659 t.Fatalf("Unexpected number of references %d, expecting %d", rc, len(references)) 660 } 661 } 662 663 func assertActivityCount(t *testing.T, l RWLayer, expected int) { 664 rl := l.(*referencedRWLayer) 665 if rl.activityCount != expected { 666 t.Fatalf("Unexpected activity count %d, expected %d", rl.activityCount, expected) 667 } 668 } 669 670 func TestRegisterExistingLayer(t *testing.T) { 671 ls, cleanup := newTestStore(t) 672 defer cleanup() 673 674 baseFiles := []FileApplier{ 675 newTestFile("/etc/profile", []byte("# Base configuration"), 0644), 676 } 677 678 layerFiles := []FileApplier{ 679 newTestFile("/root/.bashrc", []byte("# Root configuration"), 0644), 680 } 681 682 li := initWithFiles(baseFiles...) 683 layer1, err := createLayer(ls, "", li) 684 if err != nil { 685 t.Fatal(err) 686 } 687 688 tar1, err := tarFromFiles(layerFiles...) 689 if err != nil { 690 t.Fatal(err) 691 } 692 693 layer2a, err := ls.Register(bytes.NewReader(tar1), layer1.ChainID()) 694 if err != nil { 695 t.Fatal(err) 696 } 697 698 layer2b, err := ls.Register(bytes.NewReader(tar1), layer1.ChainID()) 699 if err != nil { 700 t.Fatal(err) 701 } 702 703 assertReferences(t, layer2a, layer2b) 704 } 705 706 func graphDiffSize(ls Store, l Layer) (int64, error) { 707 cl := getCachedLayer(l) 708 var parent string 709 if cl.parent != nil { 710 parent = cl.parent.cacheID 711 } 712 return ls.(*layerStore).driver.DiffSize(cl.cacheID, parent) 713 } 714 715 func TestLayerSize(t *testing.T) { 716 ls, cleanup := newTestStore(t) 717 defer cleanup() 718 719 content1 := []byte("Base contents") 720 content2 := []byte("Added contents") 721 722 layer1, err := createLayer(ls, "", initWithFiles(newTestFile("file1", content1, 0644))) 723 if err != nil { 724 t.Fatal(err) 725 } 726 727 layer2, err := createLayer(ls, layer1.ChainID(), initWithFiles(newTestFile("file2", content2, 0644))) 728 if err != nil { 729 t.Fatal(err) 730 } 731 732 layer1DiffSize, err := graphDiffSize(ls, layer1) 733 if err != nil { 734 t.Fatal(err) 735 } 736 737 if int(layer1DiffSize) != len(content1) { 738 t.Fatalf("Unexpected diff size %d, expected %d", layer1DiffSize, len(content1)) 739 } 740 741 layer1Size, err := layer1.Size() 742 if err != nil { 743 t.Fatal(err) 744 } 745 746 if expected := len(content1); int(layer1Size) != expected { 747 t.Fatalf("Unexpected size %d, expected %d", layer1Size, expected) 748 } 749 750 layer2DiffSize, err := graphDiffSize(ls, layer2) 751 if err != nil { 752 t.Fatal(err) 753 } 754 755 if int(layer2DiffSize) != len(content2) { 756 t.Fatalf("Unexpected diff size %d, expected %d", layer2DiffSize, len(content2)) 757 } 758 759 layer2Size, err := layer2.Size() 760 if err != nil { 761 t.Fatal(err) 762 } 763 764 if expected := len(content1) + len(content2); int(layer2Size) != expected { 765 t.Fatalf("Unexpected size %d, expected %d", layer2Size, expected) 766 } 767 }