github.com/demonoid81/moby@v0.0.0-20200517203328-62dd8e17c460/pkg/archive/changes_test.go (about) 1 package archive // import "github.com/demonoid81/moby/pkg/archive" 2 3 import ( 4 "io/ioutil" 5 "os" 6 "os/exec" 7 "path" 8 "path/filepath" 9 "runtime" 10 "sort" 11 "syscall" 12 "testing" 13 "time" 14 15 "github.com/demonoid81/moby/pkg/system" 16 "gotest.tools/v3/assert" 17 "gotest.tools/v3/skip" 18 ) 19 20 func max(x, y int) int { 21 if x >= y { 22 return x 23 } 24 return y 25 } 26 27 func copyDir(src, dst string) error { 28 if runtime.GOOS != "windows" { 29 return exec.Command("cp", "-a", src, dst).Run() 30 } 31 32 // Could have used xcopy src dst /E /I /H /Y /B. However, xcopy has the 33 // unfortunate side effect of not preserving timestamps of newly created 34 // directories in the target directory, so we don't get accurate changes. 35 // Use robocopy instead. Note this isn't available in microsoft/nanoserver. 36 // But it has gotchas. See https://weblogs.sqlteam.com/robv/archive/2010/02/17/61106.aspx 37 err := exec.Command("robocopy", filepath.FromSlash(src), filepath.FromSlash(dst), "/SL", "/COPYALL", "/MIR").Run() 38 if exiterr, ok := err.(*exec.ExitError); ok { 39 if status, ok := exiterr.Sys().(syscall.WaitStatus); ok { 40 if status.ExitStatus()&24 == 0 { 41 return nil 42 } 43 } 44 } 45 return err 46 } 47 48 type FileType uint32 49 50 const ( 51 Regular FileType = iota 52 Dir 53 Symlink 54 ) 55 56 type FileData struct { 57 filetype FileType 58 path string 59 contents string 60 permissions os.FileMode 61 } 62 63 func createSampleDir(t *testing.T, root string) { 64 files := []FileData{ 65 {filetype: Regular, path: "file1", contents: "file1\n", permissions: 0600}, 66 {filetype: Regular, path: "file2", contents: "file2\n", permissions: 0666}, 67 {filetype: Regular, path: "file3", contents: "file3\n", permissions: 0404}, 68 {filetype: Regular, path: "file4", contents: "file4\n", permissions: 0600}, 69 {filetype: Regular, path: "file5", contents: "file5\n", permissions: 0600}, 70 {filetype: Regular, path: "file6", contents: "file6\n", permissions: 0600}, 71 {filetype: Regular, path: "file7", contents: "file7\n", permissions: 0600}, 72 {filetype: Dir, path: "dir1", contents: "", permissions: 0740}, 73 {filetype: Regular, path: "dir1/file1-1", contents: "file1-1\n", permissions: 01444}, 74 {filetype: Regular, path: "dir1/file1-2", contents: "file1-2\n", permissions: 0666}, 75 {filetype: Dir, path: "dir2", contents: "", permissions: 0700}, 76 {filetype: Regular, path: "dir2/file2-1", contents: "file2-1\n", permissions: 0666}, 77 {filetype: Regular, path: "dir2/file2-2", contents: "file2-2\n", permissions: 0666}, 78 {filetype: Dir, path: "dir3", contents: "", permissions: 0700}, 79 {filetype: Regular, path: "dir3/file3-1", contents: "file3-1\n", permissions: 0666}, 80 {filetype: Regular, path: "dir3/file3-2", contents: "file3-2\n", permissions: 0666}, 81 {filetype: Dir, path: "dir4", contents: "", permissions: 0700}, 82 {filetype: Regular, path: "dir4/file3-1", contents: "file4-1\n", permissions: 0666}, 83 {filetype: Regular, path: "dir4/file3-2", contents: "file4-2\n", permissions: 0666}, 84 {filetype: Symlink, path: "symlink1", contents: "target1", permissions: 0666}, 85 {filetype: Symlink, path: "symlink2", contents: "target2", permissions: 0666}, 86 {filetype: Symlink, path: "symlink3", contents: root + "/file1", permissions: 0666}, 87 {filetype: Symlink, path: "symlink4", contents: root + "/symlink3", permissions: 0666}, 88 {filetype: Symlink, path: "dirSymlink", contents: root + "/dir1", permissions: 0740}, 89 } 90 provisionSampleDir(t, root, files) 91 } 92 93 func provisionSampleDir(t *testing.T, root string, files []FileData) { 94 now := time.Now() 95 for _, info := range files { 96 p := path.Join(root, info.path) 97 if info.filetype == Dir { 98 err := os.MkdirAll(p, info.permissions) 99 assert.NilError(t, err) 100 } else if info.filetype == Regular { 101 err := ioutil.WriteFile(p, []byte(info.contents), info.permissions) 102 assert.NilError(t, err) 103 } else if info.filetype == Symlink { 104 err := os.Symlink(info.contents, p) 105 assert.NilError(t, err) 106 } 107 108 if info.filetype != Symlink { 109 // Set a consistent ctime, atime for all files and dirs 110 err := system.Chtimes(p, now, now) 111 assert.NilError(t, err) 112 } 113 } 114 } 115 116 func TestChangeString(t *testing.T) { 117 modifyChange := Change{"change", ChangeModify} 118 toString := modifyChange.String() 119 if toString != "C change" { 120 t.Fatalf("String() of a change with ChangeModify Kind should have been %s but was %s", "C change", toString) 121 } 122 addChange := Change{"change", ChangeAdd} 123 toString = addChange.String() 124 if toString != "A change" { 125 t.Fatalf("String() of a change with ChangeAdd Kind should have been %s but was %s", "A change", toString) 126 } 127 deleteChange := Change{"change", ChangeDelete} 128 toString = deleteChange.String() 129 if toString != "D change" { 130 t.Fatalf("String() of a change with ChangeDelete Kind should have been %s but was %s", "D change", toString) 131 } 132 } 133 134 func TestChangesWithNoChanges(t *testing.T) { 135 rwLayer, err := ioutil.TempDir("", "docker-changes-test") 136 assert.NilError(t, err) 137 defer os.RemoveAll(rwLayer) 138 layer, err := ioutil.TempDir("", "docker-changes-test-layer") 139 assert.NilError(t, err) 140 defer os.RemoveAll(layer) 141 createSampleDir(t, layer) 142 changes, err := Changes([]string{layer}, rwLayer) 143 assert.NilError(t, err) 144 if len(changes) != 0 { 145 t.Fatalf("Changes with no difference should have detect no changes, but detected %d", len(changes)) 146 } 147 } 148 149 func TestChangesWithChanges(t *testing.T) { 150 // Mock the readonly layer 151 layer, err := ioutil.TempDir("", "docker-changes-test-layer") 152 assert.NilError(t, err) 153 defer os.RemoveAll(layer) 154 createSampleDir(t, layer) 155 os.MkdirAll(path.Join(layer, "dir1/subfolder"), 0740) 156 157 // Mock the RW layer 158 rwLayer, err := ioutil.TempDir("", "docker-changes-test") 159 assert.NilError(t, err) 160 defer os.RemoveAll(rwLayer) 161 162 // Create a folder in RW layer 163 dir1 := path.Join(rwLayer, "dir1") 164 os.MkdirAll(dir1, 0740) 165 deletedFile := path.Join(dir1, ".wh.file1-2") 166 ioutil.WriteFile(deletedFile, []byte{}, 0600) 167 modifiedFile := path.Join(dir1, "file1-1") 168 ioutil.WriteFile(modifiedFile, []byte{0x00}, 01444) 169 // Let's add a subfolder for a newFile 170 subfolder := path.Join(dir1, "subfolder") 171 os.MkdirAll(subfolder, 0740) 172 newFile := path.Join(subfolder, "newFile") 173 ioutil.WriteFile(newFile, []byte{}, 0740) 174 175 changes, err := Changes([]string{layer}, rwLayer) 176 assert.NilError(t, err) 177 178 expectedChanges := []Change{ 179 {filepath.FromSlash("/dir1"), ChangeModify}, 180 {filepath.FromSlash("/dir1/file1-1"), ChangeModify}, 181 {filepath.FromSlash("/dir1/file1-2"), ChangeDelete}, 182 {filepath.FromSlash("/dir1/subfolder"), ChangeModify}, 183 {filepath.FromSlash("/dir1/subfolder/newFile"), ChangeAdd}, 184 } 185 checkChanges(expectedChanges, changes, t) 186 } 187 188 // See https://github.com/demonoid81/moby/pull/13590 189 func TestChangesWithChangesGH13590(t *testing.T) { 190 // TODO Windows. Needs further investigation to identify the failure 191 if runtime.GOOS == "windows" { 192 t.Skip("needs more investigation") 193 } 194 baseLayer, err := ioutil.TempDir("", "docker-changes-test.") 195 assert.NilError(t, err) 196 defer os.RemoveAll(baseLayer) 197 198 dir3 := path.Join(baseLayer, "dir1/dir2/dir3") 199 os.MkdirAll(dir3, 07400) 200 201 file := path.Join(dir3, "file.txt") 202 ioutil.WriteFile(file, []byte("hello"), 0666) 203 204 layer, err := ioutil.TempDir("", "docker-changes-test2.") 205 assert.NilError(t, err) 206 defer os.RemoveAll(layer) 207 208 // Test creating a new file 209 if err := copyDir(baseLayer+"/dir1", layer+"/"); err != nil { 210 t.Fatalf("Cmd failed: %q", err) 211 } 212 213 os.Remove(path.Join(layer, "dir1/dir2/dir3/file.txt")) 214 file = path.Join(layer, "dir1/dir2/dir3/file1.txt") 215 ioutil.WriteFile(file, []byte("bye"), 0666) 216 217 changes, err := Changes([]string{baseLayer}, layer) 218 assert.NilError(t, err) 219 220 expectedChanges := []Change{ 221 {"/dir1/dir2/dir3", ChangeModify}, 222 {"/dir1/dir2/dir3/file1.txt", ChangeAdd}, 223 } 224 checkChanges(expectedChanges, changes, t) 225 226 // Now test changing a file 227 layer, err = ioutil.TempDir("", "docker-changes-test3.") 228 assert.NilError(t, err) 229 defer os.RemoveAll(layer) 230 231 if err := copyDir(baseLayer+"/dir1", layer+"/"); err != nil { 232 t.Fatalf("Cmd failed: %q", err) 233 } 234 235 file = path.Join(layer, "dir1/dir2/dir3/file.txt") 236 ioutil.WriteFile(file, []byte("bye"), 0666) 237 238 changes, err = Changes([]string{baseLayer}, layer) 239 assert.NilError(t, err) 240 241 expectedChanges = []Change{ 242 {"/dir1/dir2/dir3/file.txt", ChangeModify}, 243 } 244 checkChanges(expectedChanges, changes, t) 245 } 246 247 // Create a directory, copy it, make sure we report no changes between the two 248 func TestChangesDirsEmpty(t *testing.T) { 249 src, err := ioutil.TempDir("", "docker-changes-test") 250 assert.NilError(t, err) 251 defer os.RemoveAll(src) 252 createSampleDir(t, src) 253 dst := src + "-copy" 254 err = copyDir(src, dst) 255 assert.NilError(t, err) 256 defer os.RemoveAll(dst) 257 changes, err := ChangesDirs(dst, src) 258 assert.NilError(t, err) 259 260 if len(changes) != 0 { 261 t.Fatalf("Reported changes for identical dirs: %v", changes) 262 } 263 os.RemoveAll(src) 264 os.RemoveAll(dst) 265 } 266 267 func mutateSampleDir(t *testing.T, root string) { 268 // Remove a regular file 269 err := os.RemoveAll(path.Join(root, "file1")) 270 assert.NilError(t, err) 271 272 // Remove a directory 273 err = os.RemoveAll(path.Join(root, "dir1")) 274 assert.NilError(t, err) 275 276 // Remove a symlink 277 err = os.RemoveAll(path.Join(root, "symlink1")) 278 assert.NilError(t, err) 279 280 // Rewrite a file 281 err = ioutil.WriteFile(path.Join(root, "file2"), []byte("fileNN\n"), 0777) 282 assert.NilError(t, err) 283 284 // Replace a file 285 err = os.RemoveAll(path.Join(root, "file3")) 286 assert.NilError(t, err) 287 err = ioutil.WriteFile(path.Join(root, "file3"), []byte("fileMM\n"), 0404) 288 assert.NilError(t, err) 289 290 // Touch file 291 err = system.Chtimes(path.Join(root, "file4"), time.Now().Add(time.Second), time.Now().Add(time.Second)) 292 assert.NilError(t, err) 293 294 // Replace file with dir 295 err = os.RemoveAll(path.Join(root, "file5")) 296 assert.NilError(t, err) 297 err = os.MkdirAll(path.Join(root, "file5"), 0666) 298 assert.NilError(t, err) 299 300 // Create new file 301 err = ioutil.WriteFile(path.Join(root, "filenew"), []byte("filenew\n"), 0777) 302 assert.NilError(t, err) 303 304 // Create new dir 305 err = os.MkdirAll(path.Join(root, "dirnew"), 0766) 306 assert.NilError(t, err) 307 308 // Create a new symlink 309 err = os.Symlink("targetnew", path.Join(root, "symlinknew")) 310 assert.NilError(t, err) 311 312 // Change a symlink 313 err = os.RemoveAll(path.Join(root, "symlink2")) 314 assert.NilError(t, err) 315 316 err = os.Symlink("target2change", path.Join(root, "symlink2")) 317 assert.NilError(t, err) 318 319 // Replace dir with file 320 err = os.RemoveAll(path.Join(root, "dir2")) 321 assert.NilError(t, err) 322 err = ioutil.WriteFile(path.Join(root, "dir2"), []byte("dir2\n"), 0777) 323 assert.NilError(t, err) 324 325 // Touch dir 326 err = system.Chtimes(path.Join(root, "dir3"), time.Now().Add(time.Second), time.Now().Add(time.Second)) 327 assert.NilError(t, err) 328 } 329 330 func TestChangesDirsMutated(t *testing.T) { 331 src, err := ioutil.TempDir("", "docker-changes-test") 332 assert.NilError(t, err) 333 createSampleDir(t, src) 334 dst := src + "-copy" 335 err = copyDir(src, dst) 336 assert.NilError(t, err) 337 defer os.RemoveAll(src) 338 defer os.RemoveAll(dst) 339 340 mutateSampleDir(t, dst) 341 342 changes, err := ChangesDirs(dst, src) 343 assert.NilError(t, err) 344 345 sort.Sort(changesByPath(changes)) 346 347 expectedChanges := []Change{ 348 {filepath.FromSlash("/dir1"), ChangeDelete}, 349 {filepath.FromSlash("/dir2"), ChangeModify}, 350 } 351 352 // Note there is slight difference between the Linux and Windows 353 // implementations here. Due to https://github.com/moby/moby/issues/9874, 354 // and the fix at https://github.com/moby/moby/pull/11422, Linux does not 355 // consider a change to the directory time as a change. Windows on NTFS 356 // does. See https://github.com/moby/moby/pull/37982 for more information. 357 // 358 // Note also: https://github.com/moby/moby/pull/37982#discussion_r223523114 359 // that differences are ordered in the way the test is currently written, hence 360 // this is in the middle of the list of changes rather than at the start or 361 // end. Potentially can be addressed later. 362 if runtime.GOOS == "windows" { 363 expectedChanges = append(expectedChanges, Change{filepath.FromSlash("/dir3"), ChangeModify}) 364 } 365 366 expectedChanges = append(expectedChanges, []Change{ 367 {filepath.FromSlash("/dirnew"), ChangeAdd}, 368 {filepath.FromSlash("/file1"), ChangeDelete}, 369 {filepath.FromSlash("/file2"), ChangeModify}, 370 {filepath.FromSlash("/file3"), ChangeModify}, 371 {filepath.FromSlash("/file4"), ChangeModify}, 372 {filepath.FromSlash("/file5"), ChangeModify}, 373 {filepath.FromSlash("/filenew"), ChangeAdd}, 374 {filepath.FromSlash("/symlink1"), ChangeDelete}, 375 {filepath.FromSlash("/symlink2"), ChangeModify}, 376 {filepath.FromSlash("/symlinknew"), ChangeAdd}, 377 }...) 378 379 for i := 0; i < max(len(changes), len(expectedChanges)); i++ { 380 if i >= len(expectedChanges) { 381 t.Fatalf("unexpected change %s\n", changes[i].String()) 382 } 383 if i >= len(changes) { 384 t.Fatalf("no change for expected change %s\n", expectedChanges[i].String()) 385 } 386 if changes[i].Path == expectedChanges[i].Path { 387 if changes[i] != expectedChanges[i] { 388 t.Fatalf("Wrong change for %s, expected %s, got %s\n", changes[i].Path, changes[i].String(), expectedChanges[i].String()) 389 } 390 } else if changes[i].Path < expectedChanges[i].Path { 391 t.Fatalf("unexpected change %q %q\n", changes[i].String(), expectedChanges[i].Path) 392 } else { 393 t.Fatalf("no change for expected change %s != %s\n", expectedChanges[i].String(), changes[i].String()) 394 } 395 } 396 } 397 398 func TestApplyLayer(t *testing.T) { 399 // TODO Windows. This is very close to working, but it fails with changes 400 // to \symlinknew and \symlink2. The destination has an updated 401 // Access/Modify/Change/Birth date to the source (~3/100th sec different). 402 // Needs further investigation as to why, but I currently believe this is 403 // just the way NTFS works. I don't think it's a bug in this test or archive. 404 if runtime.GOOS == "windows" { 405 t.Skip("needs further investigation") 406 } 407 src, err := ioutil.TempDir("", "docker-changes-test") 408 assert.NilError(t, err) 409 createSampleDir(t, src) 410 defer os.RemoveAll(src) 411 dst := src + "-copy" 412 err = copyDir(src, dst) 413 assert.NilError(t, err) 414 mutateSampleDir(t, dst) 415 defer os.RemoveAll(dst) 416 417 changes, err := ChangesDirs(dst, src) 418 assert.NilError(t, err) 419 420 layer, err := ExportChanges(dst, changes, nil, nil) 421 assert.NilError(t, err) 422 423 layerCopy, err := NewTempArchive(layer, "") 424 assert.NilError(t, err) 425 426 _, err = ApplyLayer(src, layerCopy) 427 assert.NilError(t, err) 428 429 changes2, err := ChangesDirs(src, dst) 430 assert.NilError(t, err) 431 432 if len(changes2) != 0 { 433 t.Fatalf("Unexpected differences after reapplying mutation: %v", changes2) 434 } 435 } 436 437 func TestChangesSizeWithHardlinks(t *testing.T) { 438 // TODO Windows. Needs further investigation. Likely in ChangeSizes not 439 // coping correctly with hardlinks on Windows. 440 if runtime.GOOS == "windows" { 441 t.Skip("needs further investigation") 442 } 443 srcDir, err := ioutil.TempDir("", "docker-test-srcDir") 444 assert.NilError(t, err) 445 defer os.RemoveAll(srcDir) 446 447 destDir, err := ioutil.TempDir("", "docker-test-destDir") 448 assert.NilError(t, err) 449 defer os.RemoveAll(destDir) 450 451 creationSize, err := prepareUntarSourceDirectory(100, destDir, true) 452 assert.NilError(t, err) 453 454 changes, err := ChangesDirs(destDir, srcDir) 455 assert.NilError(t, err) 456 457 got := ChangesSize(destDir, changes) 458 if got != int64(creationSize) { 459 t.Errorf("Expected %d bytes of changes, got %d", creationSize, got) 460 } 461 } 462 463 func TestChangesSizeWithNoChanges(t *testing.T) { 464 size := ChangesSize("/tmp", nil) 465 if size != 0 { 466 t.Fatalf("ChangesSizes with no changes should be 0, was %d", size) 467 } 468 } 469 470 func TestChangesSizeWithOnlyDeleteChanges(t *testing.T) { 471 changes := []Change{ 472 {Path: "deletedPath", Kind: ChangeDelete}, 473 } 474 size := ChangesSize("/tmp", changes) 475 if size != 0 { 476 t.Fatalf("ChangesSizes with only delete changes should be 0, was %d", size) 477 } 478 } 479 480 func TestChangesSize(t *testing.T) { 481 parentPath, err := ioutil.TempDir("", "docker-changes-test") 482 assert.NilError(t, err) 483 defer os.RemoveAll(parentPath) 484 addition := path.Join(parentPath, "addition") 485 err = ioutil.WriteFile(addition, []byte{0x01, 0x01, 0x01}, 0744) 486 assert.NilError(t, err) 487 modification := path.Join(parentPath, "modification") 488 err = ioutil.WriteFile(modification, []byte{0x01, 0x01, 0x01}, 0744) 489 assert.NilError(t, err) 490 491 changes := []Change{ 492 {Path: "addition", Kind: ChangeAdd}, 493 {Path: "modification", Kind: ChangeModify}, 494 } 495 size := ChangesSize(parentPath, changes) 496 if size != 6 { 497 t.Fatalf("Expected 6 bytes of changes, got %d", size) 498 } 499 } 500 501 func checkChanges(expectedChanges, changes []Change, t *testing.T) { 502 skip.If(t, runtime.GOOS != "windows" && os.Getuid() != 0, "skipping test that requires root") 503 sort.Sort(changesByPath(expectedChanges)) 504 sort.Sort(changesByPath(changes)) 505 for i := 0; i < max(len(changes), len(expectedChanges)); i++ { 506 if i >= len(expectedChanges) { 507 t.Fatalf("unexpected change %s\n", changes[i].String()) 508 } 509 if i >= len(changes) { 510 t.Fatalf("no change for expected change %s\n", expectedChanges[i].String()) 511 } 512 if changes[i].Path == expectedChanges[i].Path { 513 if changes[i] != expectedChanges[i] { 514 t.Fatalf("Wrong change for %s, expected %s, got %s\n", changes[i].Path, changes[i].String(), expectedChanges[i].String()) 515 } 516 } else if changes[i].Path < expectedChanges[i].Path { 517 t.Fatalf("unexpected change %s\n", changes[i].String()) 518 } else { 519 t.Fatalf("no change for expected change %s != %s\n", expectedChanges[i].String(), changes[i].String()) 520 } 521 } 522 }