github.com/demonoid81/moby@v0.0.0-20200517203328-62dd8e17c460/pkg/archive/changes_linux.go (about) 1 package archive // import "github.com/demonoid81/moby/pkg/archive" 2 3 import ( 4 "bytes" 5 "fmt" 6 "os" 7 "path/filepath" 8 "sort" 9 "syscall" 10 "unsafe" 11 12 "github.com/demonoid81/moby/pkg/system" 13 "golang.org/x/sys/unix" 14 ) 15 16 // walker is used to implement collectFileInfoForChanges on linux. Where this 17 // method in general returns the entire contents of two directory trees, we 18 // optimize some FS calls out on linux. In particular, we take advantage of the 19 // fact that getdents(2) returns the inode of each file in the directory being 20 // walked, which, when walking two trees in parallel to generate a list of 21 // changes, can be used to prune subtrees without ever having to lstat(2) them 22 // directly. Eliminating stat calls in this way can save up to seconds on large 23 // images. 24 type walker struct { 25 dir1 string 26 dir2 string 27 root1 *FileInfo 28 root2 *FileInfo 29 } 30 31 // collectFileInfoForChanges returns a complete representation of the trees 32 // rooted at dir1 and dir2, with one important exception: any subtree or 33 // leaf where the inode and device numbers are an exact match between dir1 34 // and dir2 will be pruned from the results. This method is *only* to be used 35 // to generating a list of changes between the two directories, as it does not 36 // reflect the full contents. 37 func collectFileInfoForChanges(dir1, dir2 string) (*FileInfo, *FileInfo, error) { 38 w := &walker{ 39 dir1: dir1, 40 dir2: dir2, 41 root1: newRootFileInfo(), 42 root2: newRootFileInfo(), 43 } 44 45 i1, err := os.Lstat(w.dir1) 46 if err != nil { 47 return nil, nil, err 48 } 49 i2, err := os.Lstat(w.dir2) 50 if err != nil { 51 return nil, nil, err 52 } 53 54 if err := w.walk("/", i1, i2); err != nil { 55 return nil, nil, err 56 } 57 58 return w.root1, w.root2, nil 59 } 60 61 // Given a FileInfo, its path info, and a reference to the root of the tree 62 // being constructed, register this file with the tree. 63 func walkchunk(path string, fi os.FileInfo, dir string, root *FileInfo) error { 64 if fi == nil { 65 return nil 66 } 67 parent := root.LookUp(filepath.Dir(path)) 68 if parent == nil { 69 return fmt.Errorf("walkchunk: Unexpectedly no parent for %s", path) 70 } 71 info := &FileInfo{ 72 name: filepath.Base(path), 73 children: make(map[string]*FileInfo), 74 parent: parent, 75 } 76 cpath := filepath.Join(dir, path) 77 stat, err := system.FromStatT(fi.Sys().(*syscall.Stat_t)) 78 if err != nil { 79 return err 80 } 81 info.stat = stat 82 info.capability, _ = system.Lgetxattr(cpath, "security.capability") // lgetxattr(2): fs access 83 parent.children[info.name] = info 84 return nil 85 } 86 87 // Walk a subtree rooted at the same path in both trees being iterated. For 88 // example, /docker/overlay/1234/a/b/c/d and /docker/overlay/8888/a/b/c/d 89 func (w *walker) walk(path string, i1, i2 os.FileInfo) (err error) { 90 // Register these nodes with the return trees, unless we're still at the 91 // (already-created) roots: 92 if path != "/" { 93 if err := walkchunk(path, i1, w.dir1, w.root1); err != nil { 94 return err 95 } 96 if err := walkchunk(path, i2, w.dir2, w.root2); err != nil { 97 return err 98 } 99 } 100 101 is1Dir := i1 != nil && i1.IsDir() 102 is2Dir := i2 != nil && i2.IsDir() 103 104 sameDevice := false 105 if i1 != nil && i2 != nil { 106 si1 := i1.Sys().(*syscall.Stat_t) 107 si2 := i2.Sys().(*syscall.Stat_t) 108 if si1.Dev == si2.Dev { 109 sameDevice = true 110 } 111 } 112 113 // If these files are both non-existent, or leaves (non-dirs), we are done. 114 if !is1Dir && !is2Dir { 115 return nil 116 } 117 118 // Fetch the names of all the files contained in both directories being walked: 119 var names1, names2 []nameIno 120 if is1Dir { 121 names1, err = readdirnames(filepath.Join(w.dir1, path)) // getdents(2): fs access 122 if err != nil { 123 return err 124 } 125 } 126 if is2Dir { 127 names2, err = readdirnames(filepath.Join(w.dir2, path)) // getdents(2): fs access 128 if err != nil { 129 return err 130 } 131 } 132 133 // We have lists of the files contained in both parallel directories, sorted 134 // in the same order. Walk them in parallel, generating a unique merged list 135 // of all items present in either or both directories. 136 var names []string 137 ix1 := 0 138 ix2 := 0 139 140 for { 141 if ix1 >= len(names1) { 142 break 143 } 144 if ix2 >= len(names2) { 145 break 146 } 147 148 ni1 := names1[ix1] 149 ni2 := names2[ix2] 150 151 switch bytes.Compare([]byte(ni1.name), []byte(ni2.name)) { 152 case -1: // ni1 < ni2 -- advance ni1 153 // we will not encounter ni1 in names2 154 names = append(names, ni1.name) 155 ix1++ 156 case 0: // ni1 == ni2 157 if ni1.ino != ni2.ino || !sameDevice { 158 names = append(names, ni1.name) 159 } 160 ix1++ 161 ix2++ 162 case 1: // ni1 > ni2 -- advance ni2 163 // we will not encounter ni2 in names1 164 names = append(names, ni2.name) 165 ix2++ 166 } 167 } 168 for ix1 < len(names1) { 169 names = append(names, names1[ix1].name) 170 ix1++ 171 } 172 for ix2 < len(names2) { 173 names = append(names, names2[ix2].name) 174 ix2++ 175 } 176 177 // For each of the names present in either or both of the directories being 178 // iterated, stat the name under each root, and recurse the pair of them: 179 for _, name := range names { 180 fname := filepath.Join(path, name) 181 var cInfo1, cInfo2 os.FileInfo 182 if is1Dir { 183 cInfo1, err = os.Lstat(filepath.Join(w.dir1, fname)) // lstat(2): fs access 184 if err != nil && !os.IsNotExist(err) { 185 return err 186 } 187 } 188 if is2Dir { 189 cInfo2, err = os.Lstat(filepath.Join(w.dir2, fname)) // lstat(2): fs access 190 if err != nil && !os.IsNotExist(err) { 191 return err 192 } 193 } 194 if err = w.walk(fname, cInfo1, cInfo2); err != nil { 195 return err 196 } 197 } 198 return nil 199 } 200 201 // {name,inode} pairs used to support the early-pruning logic of the walker type 202 type nameIno struct { 203 name string 204 ino uint64 205 } 206 207 type nameInoSlice []nameIno 208 209 func (s nameInoSlice) Len() int { return len(s) } 210 func (s nameInoSlice) Swap(i, j int) { s[i], s[j] = s[j], s[i] } 211 func (s nameInoSlice) Less(i, j int) bool { return s[i].name < s[j].name } 212 213 // readdirnames is a hacked-apart version of the Go stdlib code, exposing inode 214 // numbers further up the stack when reading directory contents. Unlike 215 // os.Readdirnames, which returns a list of filenames, this function returns a 216 // list of {filename,inode} pairs. 217 func readdirnames(dirname string) (names []nameIno, err error) { 218 var ( 219 size = 100 220 buf = make([]byte, 4096) 221 nbuf int 222 bufp int 223 nb int 224 ) 225 226 f, err := os.Open(dirname) 227 if err != nil { 228 return nil, err 229 } 230 defer f.Close() 231 232 names = make([]nameIno, 0, size) // Empty with room to grow. 233 for { 234 // Refill the buffer if necessary 235 if bufp >= nbuf { 236 bufp = 0 237 nbuf, err = unix.ReadDirent(int(f.Fd()), buf) // getdents on linux 238 if nbuf < 0 { 239 nbuf = 0 240 } 241 if err != nil { 242 return nil, os.NewSyscallError("readdirent", err) 243 } 244 if nbuf <= 0 { 245 break // EOF 246 } 247 } 248 249 // Drain the buffer 250 nb, names = parseDirent(buf[bufp:nbuf], names) 251 bufp += nb 252 } 253 254 sl := nameInoSlice(names) 255 sort.Sort(sl) 256 return sl, nil 257 } 258 259 // parseDirent is a minor modification of unix.ParseDirent (linux version) 260 // which returns {name,inode} pairs instead of just names. 261 func parseDirent(buf []byte, names []nameIno) (consumed int, newnames []nameIno) { 262 origlen := len(buf) 263 for len(buf) > 0 { 264 dirent := (*unix.Dirent)(unsafe.Pointer(&buf[0])) 265 buf = buf[dirent.Reclen:] 266 if dirent.Ino == 0 { // File absent in directory. 267 continue 268 } 269 bytes := (*[10000]byte)(unsafe.Pointer(&dirent.Name[0])) 270 var name = string(bytes[0:clen(bytes[:])]) 271 if name == "." || name == ".." { // Useless names 272 continue 273 } 274 names = append(names, nameIno{name, dirent.Ino}) 275 } 276 return origlen - len(buf), names 277 } 278 279 func clen(n []byte) int { 280 for i := 0; i < len(n); i++ { 281 if n[i] == 0 { 282 return i 283 } 284 } 285 return len(n) 286 }