github.com/hustcat/docker@v1.3.3-0.20160314103604-901c67a8eeab/pkg/archive/changes.go (about)

     1  package archive
     2  
     3  import (
     4  	"archive/tar"
     5  	"bytes"
     6  	"fmt"
     7  	"io"
     8  	"io/ioutil"
     9  	"os"
    10  	"path/filepath"
    11  	"sort"
    12  	"strings"
    13  	"syscall"
    14  	"time"
    15  
    16  	"github.com/Sirupsen/logrus"
    17  	"github.com/docker/docker/pkg/idtools"
    18  	"github.com/docker/docker/pkg/pools"
    19  	"github.com/docker/docker/pkg/system"
    20  )
    21  
    22  // ChangeType represents the change type.
    23  type ChangeType int
    24  
    25  const (
    26  	// ChangeModify represents the modify operation.
    27  	ChangeModify = iota
    28  	// ChangeAdd represents the add operation.
    29  	ChangeAdd
    30  	// ChangeDelete represents the delete operation.
    31  	ChangeDelete
    32  )
    33  
    34  func (c ChangeType) String() string {
    35  	switch c {
    36  	case ChangeModify:
    37  		return "C"
    38  	case ChangeAdd:
    39  		return "A"
    40  	case ChangeDelete:
    41  		return "D"
    42  	}
    43  	return ""
    44  }
    45  
    46  // Change represents a change, it wraps the change type and path.
    47  // It describes changes of the files in the path respect to the
    48  // parent layers. The change could be modify, add, delete.
    49  // This is used for layer diff.
    50  type Change struct {
    51  	Path string
    52  	Kind ChangeType
    53  }
    54  
    55  func (change *Change) String() string {
    56  	return fmt.Sprintf("%s %s", change.Kind, change.Path)
    57  }
    58  
    59  // for sort.Sort
    60  type changesByPath []Change
    61  
    62  func (c changesByPath) Less(i, j int) bool { return c[i].Path < c[j].Path }
    63  func (c changesByPath) Len() int           { return len(c) }
    64  func (c changesByPath) Swap(i, j int)      { c[j], c[i] = c[i], c[j] }
    65  
    66  // Gnu tar and the go tar writer don't have sub-second mtime
    67  // precision, which is problematic when we apply changes via tar
    68  // files, we handle this by comparing for exact times, *or* same
    69  // second count and either a or b having exactly 0 nanoseconds
    70  func sameFsTime(a, b time.Time) bool {
    71  	return a == b ||
    72  		(a.Unix() == b.Unix() &&
    73  			(a.Nanosecond() == 0 || b.Nanosecond() == 0))
    74  }
    75  
    76  func sameFsTimeSpec(a, b syscall.Timespec) bool {
    77  	return a.Sec == b.Sec &&
    78  		(a.Nsec == b.Nsec || a.Nsec == 0 || b.Nsec == 0)
    79  }
    80  
    81  // Changes walks the path rw and determines changes for the files in the path,
    82  // with respect to the parent layers
    83  func Changes(layers []string, rw string) ([]Change, error) {
    84  	var (
    85  		changes     []Change
    86  		changedDirs = make(map[string]struct{})
    87  	)
    88  
    89  	err := filepath.Walk(rw, func(path string, f os.FileInfo, err error) error {
    90  		if err != nil {
    91  			return err
    92  		}
    93  
    94  		// Rebase path
    95  		path, err = filepath.Rel(rw, path)
    96  		if err != nil {
    97  			return err
    98  		}
    99  
   100  		// As this runs on the daemon side, file paths are OS specific.
   101  		path = filepath.Join(string(os.PathSeparator), path)
   102  
   103  		// Skip root
   104  		if path == string(os.PathSeparator) {
   105  			return nil
   106  		}
   107  
   108  		// Skip AUFS metadata
   109  		if matched, err := filepath.Match(string(os.PathSeparator)+WhiteoutMetaPrefix+"*", path); err != nil || matched {
   110  			return err
   111  		}
   112  
   113  		change := Change{
   114  			Path: path,
   115  		}
   116  
   117  		// Find out what kind of modification happened
   118  		file := filepath.Base(path)
   119  		// If there is a whiteout, then the file was removed
   120  		if strings.HasPrefix(file, WhiteoutPrefix) {
   121  			originalFile := file[len(WhiteoutPrefix):]
   122  			change.Path = filepath.Join(filepath.Dir(path), originalFile)
   123  			change.Kind = ChangeDelete
   124  		} else {
   125  			// Otherwise, the file was added
   126  			change.Kind = ChangeAdd
   127  
   128  			// ...Unless it already existed in a top layer, in which case, it's a modification
   129  			for _, layer := range layers {
   130  				stat, err := os.Stat(filepath.Join(layer, path))
   131  				if err != nil && !os.IsNotExist(err) {
   132  					return err
   133  				}
   134  				if err == nil {
   135  					// The file existed in the top layer, so that's a modification
   136  
   137  					// However, if it's a directory, maybe it wasn't actually modified.
   138  					// If you modify /foo/bar/baz, then /foo will be part of the changed files only because it's the parent of bar
   139  					if stat.IsDir() && f.IsDir() {
   140  						if f.Size() == stat.Size() && f.Mode() == stat.Mode() && sameFsTime(f.ModTime(), stat.ModTime()) {
   141  							// Both directories are the same, don't record the change
   142  							return nil
   143  						}
   144  					}
   145  					change.Kind = ChangeModify
   146  					break
   147  				}
   148  			}
   149  		}
   150  
   151  		// If /foo/bar/file.txt is modified, then /foo/bar must be part of the changed files.
   152  		// This block is here to ensure the change is recorded even if the
   153  		// modify time, mode and size of the parent directory in the rw and ro layers are all equal.
   154  		// Check https://github.com/docker/docker/pull/13590 for details.
   155  		if f.IsDir() {
   156  			changedDirs[path] = struct{}{}
   157  		}
   158  		if change.Kind == ChangeAdd || change.Kind == ChangeDelete {
   159  			parent := filepath.Dir(path)
   160  			if _, ok := changedDirs[parent]; !ok && parent != "/" {
   161  				changes = append(changes, Change{Path: parent, Kind: ChangeModify})
   162  				changedDirs[parent] = struct{}{}
   163  			}
   164  		}
   165  
   166  		// Record change
   167  		changes = append(changes, change)
   168  		return nil
   169  	})
   170  	if err != nil && !os.IsNotExist(err) {
   171  		return nil, err
   172  	}
   173  	return changes, nil
   174  }
   175  
   176  // FileInfo describes the information of a file.
   177  type FileInfo struct {
   178  	parent     *FileInfo
   179  	name       string
   180  	stat       *system.StatT
   181  	children   map[string]*FileInfo
   182  	capability []byte
   183  	added      bool
   184  }
   185  
   186  // LookUp looks up the file information of a file.
   187  func (info *FileInfo) LookUp(path string) *FileInfo {
   188  	// As this runs on the daemon side, file paths are OS specific.
   189  	parent := info
   190  	if path == string(os.PathSeparator) {
   191  		return info
   192  	}
   193  
   194  	pathElements := strings.Split(path, string(os.PathSeparator))
   195  	for _, elem := range pathElements {
   196  		if elem != "" {
   197  			child := parent.children[elem]
   198  			if child == nil {
   199  				return nil
   200  			}
   201  			parent = child
   202  		}
   203  	}
   204  	return parent
   205  }
   206  
   207  func (info *FileInfo) path() string {
   208  	if info.parent == nil {
   209  		// As this runs on the daemon side, file paths are OS specific.
   210  		return string(os.PathSeparator)
   211  	}
   212  	return filepath.Join(info.parent.path(), info.name)
   213  }
   214  
   215  func (info *FileInfo) addChanges(oldInfo *FileInfo, changes *[]Change) {
   216  
   217  	sizeAtEntry := len(*changes)
   218  
   219  	if oldInfo == nil {
   220  		// add
   221  		change := Change{
   222  			Path: info.path(),
   223  			Kind: ChangeAdd,
   224  		}
   225  		*changes = append(*changes, change)
   226  		info.added = true
   227  	}
   228  
   229  	// We make a copy so we can modify it to detect additions
   230  	// also, we only recurse on the old dir if the new info is a directory
   231  	// otherwise any previous delete/change is considered recursive
   232  	oldChildren := make(map[string]*FileInfo)
   233  	if oldInfo != nil && info.isDir() {
   234  		for k, v := range oldInfo.children {
   235  			oldChildren[k] = v
   236  		}
   237  	}
   238  
   239  	for name, newChild := range info.children {
   240  		oldChild, _ := oldChildren[name]
   241  		if oldChild != nil {
   242  			// change?
   243  			oldStat := oldChild.stat
   244  			newStat := newChild.stat
   245  			// Note: We can't compare inode or ctime or blocksize here, because these change
   246  			// when copying a file into a container. However, that is not generally a problem
   247  			// because any content change will change mtime, and any status change should
   248  			// be visible when actually comparing the stat fields. The only time this
   249  			// breaks down is if some code intentionally hides a change by setting
   250  			// back mtime
   251  			if statDifferent(oldStat, newStat) ||
   252  				bytes.Compare(oldChild.capability, newChild.capability) != 0 {
   253  				change := Change{
   254  					Path: newChild.path(),
   255  					Kind: ChangeModify,
   256  				}
   257  				*changes = append(*changes, change)
   258  				newChild.added = true
   259  			}
   260  
   261  			// Remove from copy so we can detect deletions
   262  			delete(oldChildren, name)
   263  		}
   264  
   265  		newChild.addChanges(oldChild, changes)
   266  	}
   267  	for _, oldChild := range oldChildren {
   268  		// delete
   269  		change := Change{
   270  			Path: oldChild.path(),
   271  			Kind: ChangeDelete,
   272  		}
   273  		*changes = append(*changes, change)
   274  	}
   275  
   276  	// If there were changes inside this directory, we need to add it, even if the directory
   277  	// itself wasn't changed. This is needed to properly save and restore filesystem permissions.
   278  	// As this runs on the daemon side, file paths are OS specific.
   279  	if len(*changes) > sizeAtEntry && info.isDir() && !info.added && info.path() != string(os.PathSeparator) {
   280  		change := Change{
   281  			Path: info.path(),
   282  			Kind: ChangeModify,
   283  		}
   284  		// Let's insert the directory entry before the recently added entries located inside this dir
   285  		*changes = append(*changes, change) // just to resize the slice, will be overwritten
   286  		copy((*changes)[sizeAtEntry+1:], (*changes)[sizeAtEntry:])
   287  		(*changes)[sizeAtEntry] = change
   288  	}
   289  
   290  }
   291  
   292  // Changes add changes to file information.
   293  func (info *FileInfo) Changes(oldInfo *FileInfo) []Change {
   294  	var changes []Change
   295  
   296  	info.addChanges(oldInfo, &changes)
   297  
   298  	return changes
   299  }
   300  
   301  func newRootFileInfo() *FileInfo {
   302  	// As this runs on the daemon side, file paths are OS specific.
   303  	root := &FileInfo{
   304  		name:     string(os.PathSeparator),
   305  		children: make(map[string]*FileInfo),
   306  	}
   307  	return root
   308  }
   309  
   310  // ChangesDirs compares two directories and generates an array of Change objects describing the changes.
   311  // If oldDir is "", then all files in newDir will be Add-Changes.
   312  func ChangesDirs(newDir, oldDir string) ([]Change, error) {
   313  	var (
   314  		oldRoot, newRoot *FileInfo
   315  	)
   316  	if oldDir == "" {
   317  		emptyDir, err := ioutil.TempDir("", "empty")
   318  		if err != nil {
   319  			return nil, err
   320  		}
   321  		defer os.Remove(emptyDir)
   322  		oldDir = emptyDir
   323  	}
   324  	oldRoot, newRoot, err := collectFileInfoForChanges(oldDir, newDir)
   325  	if err != nil {
   326  		return nil, err
   327  	}
   328  
   329  	return newRoot.Changes(oldRoot), nil
   330  }
   331  
   332  // ChangesSize calculates the size in bytes of the provided changes, based on newDir.
   333  func ChangesSize(newDir string, changes []Change) int64 {
   334  	var (
   335  		size int64
   336  		sf   = make(map[uint64]struct{})
   337  	)
   338  	for _, change := range changes {
   339  		if change.Kind == ChangeModify || change.Kind == ChangeAdd {
   340  			file := filepath.Join(newDir, change.Path)
   341  			fileInfo, err := os.Lstat(file)
   342  			if err != nil {
   343  				logrus.Errorf("Can not stat %q: %s", file, err)
   344  				continue
   345  			}
   346  
   347  			if fileInfo != nil && !fileInfo.IsDir() {
   348  				if hasHardlinks(fileInfo) {
   349  					inode := getIno(fileInfo)
   350  					if _, ok := sf[inode]; !ok {
   351  						size += fileInfo.Size()
   352  						sf[inode] = struct{}{}
   353  					}
   354  				} else {
   355  					size += fileInfo.Size()
   356  				}
   357  			}
   358  		}
   359  	}
   360  	return size
   361  }
   362  
   363  // ExportChanges produces an Archive from the provided changes, relative to dir.
   364  func ExportChanges(dir string, changes []Change, uidMaps, gidMaps []idtools.IDMap) (Archive, error) {
   365  	reader, writer := io.Pipe()
   366  	go func() {
   367  		ta := &tarAppender{
   368  			TarWriter: tar.NewWriter(writer),
   369  			Buffer:    pools.BufioWriter32KPool.Get(nil),
   370  			SeenFiles: make(map[uint64]string),
   371  			UIDMaps:   uidMaps,
   372  			GIDMaps:   gidMaps,
   373  		}
   374  		// this buffer is needed for the duration of this piped stream
   375  		defer pools.BufioWriter32KPool.Put(ta.Buffer)
   376  
   377  		sort.Sort(changesByPath(changes))
   378  
   379  		// In general we log errors here but ignore them because
   380  		// during e.g. a diff operation the container can continue
   381  		// mutating the filesystem and we can see transient errors
   382  		// from this
   383  		for _, change := range changes {
   384  			if change.Kind == ChangeDelete {
   385  				whiteOutDir := filepath.Dir(change.Path)
   386  				whiteOutBase := filepath.Base(change.Path)
   387  				whiteOut := filepath.Join(whiteOutDir, WhiteoutPrefix+whiteOutBase)
   388  				timestamp := time.Now()
   389  				hdr := &tar.Header{
   390  					Name:       whiteOut[1:],
   391  					Size:       0,
   392  					ModTime:    timestamp,
   393  					AccessTime: timestamp,
   394  					ChangeTime: timestamp,
   395  				}
   396  				if err := ta.TarWriter.WriteHeader(hdr); err != nil {
   397  					logrus.Debugf("Can't write whiteout header: %s", err)
   398  				}
   399  			} else {
   400  				path := filepath.Join(dir, change.Path)
   401  				if err := ta.addTarFile(path, change.Path[1:]); err != nil {
   402  					logrus.Debugf("Can't add file %s to tar: %s", path, err)
   403  				}
   404  			}
   405  		}
   406  
   407  		// Make sure to check the error on Close.
   408  		if err := ta.TarWriter.Close(); err != nil {
   409  			logrus.Debugf("Can't close layer: %s", err)
   410  		}
   411  		if err := writer.Close(); err != nil {
   412  			logrus.Debugf("failed close Changes writer: %s", err)
   413  		}
   414  	}()
   415  	return reader, nil
   416  }