github.com/mgoltzsche/ctnr@v0.7.1-alpha/pkg/fs/tree/fsnode.go (about)

     1  package tree
     2  
     3  import (
     4  	"os"
     5  	"path/filepath"
     6  	"strings"
     7  
     8  	"github.com/mgoltzsche/ctnr/pkg/fs"
     9  	"github.com/mgoltzsche/ctnr/pkg/fs/source"
    10  	"github.com/mgoltzsche/ctnr/pkg/fs/writer"
    11  	"github.com/opencontainers/go-digest"
    12  	"github.com/pkg/errors"
    13  )
    14  
    15  const (
    16  	lookup lookupWalker = "path lookup"
    17  )
    18  
    19  var (
    20  	dirAttrs             = fs.FileAttrs{Mode: os.ModeDir | 0755}
    21  	srcOverlayPseudoRoot = sourceNoop("noop source")
    22  	srcParentDir         = sourceParentDir("parent dir source")
    23  	srcWhiteout          = source.NewSourceWhiteout()
    24  	emptyContent         = fs.NewReadableBytes([]byte{})
    25  )
    26  
    27  type sourceParentDir string
    28  
    29  func (s sourceParentDir) Attrs() fs.NodeInfo {
    30  	return fs.NodeInfo{fs.TypeDir, fs.FileAttrs{Mode: os.ModeDir | 0755}}
    31  }
    32  func (s sourceParentDir) DeriveAttrs() (fs.DerivedAttrs, error) {
    33  	return fs.DerivedAttrs{}, nil
    34  }
    35  func (s sourceParentDir) Write(dest, name string, w fs.Writer, written map[fs.Source]string) error {
    36  	return w.Mkdir(dest)
    37  }
    38  func (s sourceParentDir) Equal(other fs.Source) (bool, error) {
    39  	return s == other, nil
    40  }
    41  
    42  type sourceNoop string
    43  
    44  func (s sourceNoop) Attrs() fs.NodeInfo { return fs.NodeInfo{NodeType: fs.TypeDir} }
    45  func (s sourceNoop) DeriveAttrs() (fs.DerivedAttrs, error) {
    46  	return fs.DerivedAttrs{}, nil
    47  }
    48  func (s sourceNoop) Write(dest, name string, w fs.Writer, written map[fs.Source]string) error {
    49  	return nil
    50  }
    51  func (s sourceNoop) Equal(other fs.Source) (bool, error) {
    52  	return s == other, nil
    53  }
    54  
    55  type FsNode struct {
    56  	name string
    57  	fs.NodeInfo
    58  	source   fs.Source
    59  	parent   *FsNode
    60  	pathNode *FsNode
    61  	child    *FsNode
    62  	next     *FsNode
    63  	overlay  bool
    64  }
    65  
    66  func NewFS() fs.FsNode {
    67  	return newFS()
    68  }
    69  
    70  func newFS() *FsNode {
    71  	fs := &FsNode{name: "."}
    72  	fs.pathNode = fs
    73  	fs.SetSource(srcParentDir)
    74  	return fs
    75  }
    76  
    77  func (f *FsNode) String() (p string) {
    78  	return f.Path() + " " + f.NodeInfo.AttrString(fs.AttrsAll)
    79  }
    80  
    81  func (f *FsNode) Name() string {
    82  	return f.name
    83  }
    84  
    85  func (f *FsNode) Path() (p string) {
    86  	if f.parent == nil {
    87  		return string(filepath.Separator)
    88  	} else {
    89  		p = filepath.Join(f.parent.Path(), f.name)
    90  	}
    91  	return
    92  }
    93  
    94  func (f *FsNode) Empty() bool {
    95  	return f.source == srcParentDir
    96  }
    97  
    98  // Generates the file system's hash including the provided attribute set.
    99  func (f *FsNode) Hash(attrs fs.AttrSet) (digest.Digest, error) {
   100  	d := digest.SHA256.Digester()
   101  	err := f.WriteTo(d.Hash(), attrs)
   102  	return d.Digest(), err
   103  }
   104  
   105  // Walks through the tree writing all nodes into the provided Writer.
   106  func (f *FsNode) Write(w fs.Writer) (err error) {
   107  	return f.write(w, map[fs.Source]string{})
   108  }
   109  
   110  // Walks through the tree writing all nodes into the provided Writer.
   111  // The 'written' map is used to allow Source implementations to detect and write
   112  // a hardlink.
   113  func (f *FsNode) write(w fs.Writer, written map[fs.Source]string) (err error) {
   114  	if err = f.source.Write(f.Path(), f.name, w, written); err != nil {
   115  		return
   116  	}
   117  	if f.child != nil {
   118  		err = f.child.write(w, written)
   119  	}
   120  	if f.next != nil && err == nil {
   121  		err = f.next.write(w, written)
   122  	}
   123  	return
   124  }
   125  
   126  // Returns a new normalized file system tree
   127  func (f *FsNode) Normalized() (fs.FsNode, error) {
   128  	normalized := NewFS()
   129  	normalizer := writer.NewFsNodeWriter(normalized, fs.HashingNilWriter())
   130  	expander := fs.ExpandingWriter{normalizer}
   131  	if err := f.Write(&expander); err != nil {
   132  		return nil, errors.Wrap(err, "normalize fs spec")
   133  	}
   134  	normalized.(*FsNode).RemoveWhiteouts()
   135  	return normalized, nil
   136  }
   137  
   138  // Returns a file system tree containing only new and changed nodes.
   139  // Nodes that do not exist in the provided file system are returned as whiteouts.
   140  // Hardlinks from new to unchanged (upper to lower) nodes are supported by
   141  // adding hardlinked files to diff as well (to preserve hardlinks and stay
   142  // compatible with external tar tools when writing diff into tar file).
   143  func (f *FsNode) Diff(o fs.FsNode) (r fs.FsNode, err error) {
   144  	node := newFS()
   145  	from := map[string]*FsNode{}
   146  	to := map[string]*FsNode{}
   147  	unchangedSourceMap := map[fs.Source][]string{}
   148  	addedSources := []fs.Source{}
   149  	if err = o.(*FsNode).toMap(to); err == nil {
   150  		err = f.toMap(from)
   151  	}
   152  	if err != nil {
   153  		return nil, errors.WithMessage(err, "diff")
   154  	}
   155  	// Add new or changed nodes to tree
   156  	for k, v := range to {
   157  		old := from[k]
   158  		if old == nil {
   159  			added, err := node.addUpper(k, v.source)
   160  			if err != nil {
   161  				return nil, errors.WithMessage(err, "diff")
   162  			}
   163  			added.parent.applyParents(v.parent)
   164  		} else {
   165  			eq, err := v.Equal(old)
   166  			if err != nil {
   167  				return nil, errors.WithMessage(err, "diff")
   168  			}
   169  			if !eq {
   170  				// changed node - add new source to be written to layer
   171  				added, err := node.addUpper(k, v.source)
   172  				if err != nil {
   173  					return nil, errors.WithMessage(err, "diff")
   174  				}
   175  				added.parent.applyParents(v.parent)
   176  			} else {
   177  				// unchanged node - only map source to support hardlink from upper node
   178  				// Decision regarding hardlink preservation when upper layer contains hardlink to lower layer:
   179  				//  * support hardlinks to lower nodes (pointing to file in different archive) -> does not work with external tar tools, may not work in other engines
   180  				//  * or add hardlinked files to child archive again -> more compatible: works with external tar tools and other storage engines but takes more disk space
   181  				//  => Map source:paths to use add it if linked
   182  				paths := unchangedSourceMap[v.source]
   183  				if paths == nil {
   184  					paths = []string{k}
   185  				} else {
   186  					paths = append(paths, k)
   187  				}
   188  				unchangedSourceMap[v.source] = paths
   189  				continue
   190  			}
   191  		}
   192  		addedSources = append(addedSources, v.source)
   193  	}
   194  	// Add whiteout nodes for those that don't exist in dest tree
   195  	for k := range from {
   196  		p := to[filepath.Dir(k)]
   197  		if to[k] == nil && p != nil {
   198  			added, err := node.AddWhiteoutNode(k)
   199  			if err != nil {
   200  				return nil, errors.WithMessage(err, "diff")
   201  			}
   202  			added.parent.applyParents(p)
   203  		}
   204  	}
   205  	// Add hardlinked nodes (to preserve fs state while remaining compatible with other tools)
   206  	for _, added := range addedSources {
   207  		if paths := unchangedSourceMap[added]; paths != nil {
   208  			for _, path := range paths {
   209  				if _, err = node.AddUpper(path, added); err != nil {
   210  					return nil, errors.WithMessage(err, "diff")
   211  				}
   212  			}
   213  		}
   214  	}
   215  	return node, err
   216  }
   217  
   218  func (f *FsNode) Equal(o *FsNode) (bool, error) {
   219  	if !f.source.Attrs().Equal(o.source.Attrs()) {
   220  		return false, nil
   221  	}
   222  	fa, err := f.source.DeriveAttrs()
   223  	if err != nil {
   224  		return false, errors.Wrap(err, "equal")
   225  	}
   226  	oa, err := o.source.DeriveAttrs()
   227  	if err != nil {
   228  		return false, errors.Wrap(err, "equal")
   229  	}
   230  	return fa.Hash == oa.Hash, nil
   231  }
   232  
   233  func (f *FsNode) applyParents(o *FsNode) {
   234  	if f != nil && f.source != o.source {
   235  		f.SetSource(o.source)
   236  		f.parent.applyParents(o.parent)
   237  	}
   238  }
   239  
   240  func (f *FsNode) toMap(m map[string]*FsNode) (err error) {
   241  	if f.overlay {
   242  		return errors.Errorf("cannot map overlay %s - needs to be normalized first", f.Path())
   243  	}
   244  	m[f.Path()] = f
   245  	if f.child != nil {
   246  		err = f.child.toMap(m)
   247  	}
   248  	if f.next != nil && err == nil {
   249  		err = f.next.toMap(m)
   250  	}
   251  	return
   252  }
   253  
   254  func (f *FsNode) findChild(name string) *FsNode {
   255  	if name == "." || name == "/" {
   256  		return f
   257  	}
   258  	if f.overlay {
   259  		return nil
   260  	}
   261  	c := f.child
   262  	for c != nil {
   263  		if c.name == name {
   264  			return c
   265  		}
   266  		if c.name > name {
   267  			return nil
   268  		}
   269  		c = c.next
   270  	}
   271  	return c
   272  }
   273  
   274  func (f *FsNode) SetSource(src fs.Source) {
   275  	f.source = src
   276  	f.NodeInfo = src.Attrs()
   277  	if f.NodeType != fs.TypeDir {
   278  		f.child = nil
   279  	}
   280  }
   281  
   282  func (f *FsNode) AddUpper(path string, src fs.Source) (r fs.FsNode, err error) {
   283  	return f.addUpper(path, src)
   284  }
   285  
   286  func (f *FsNode) addUpper(path string, src fs.Source) (r *FsNode, err error) {
   287  	r, err = f.add(path, src, f.mkdirsUpper)
   288  	return r, errors.Wrap(err, "add upper fsnode")
   289  }
   290  
   291  func (f *FsNode) isLowerNode() bool {
   292  	_, ok := f.source.(*fs.NodeAttrs)
   293  	return ok
   294  }
   295  
   296  type mkdirsFn func(path string) (r *FsNode, err error)
   297  
   298  func (f *FsNode) AddLower(path string, src fs.Source) (r fs.FsNode, err error) {
   299  	r, err = f.add(path, src, f.mkdirsLower)
   300  	return r, errors.Wrap(err, "add lower fsnode")
   301  }
   302  func (f *FsNode) add(path string, src fs.Source, mkdirs mkdirsFn) (r *FsNode, err error) {
   303  	path = filepath.Clean(path)
   304  	dir := filepath.Dir(path)
   305  	var p *FsNode
   306  	if dir == "." {
   307  		p = f
   308  	} else {
   309  		if p, err = mkdirs(dir); err != nil {
   310  			return
   311  		}
   312  	}
   313  	name := filepath.Base(path)
   314  	r = p.findChild(name)
   315  	if r == nil {
   316  		r = p.newEntry(name)
   317  	}
   318  	t := src.Attrs().NodeType
   319  	if t == fs.TypeOverlay {
   320  		r = r.newOverlayEntry()
   321  	} else if r.overlay {
   322  		r = r.overlayRoot()
   323  	} else if t != fs.TypeDir {
   324  		r.child = nil
   325  	}
   326  	r.SetSource(src)
   327  	return
   328  }
   329  
   330  func (f *FsNode) AddWhiteout(path string) (fs.FsNode, error) {
   331  	return f.AddWhiteoutNode(path)
   332  }
   333  
   334  func (f *FsNode) AddWhiteoutNode(path string) (r *FsNode, err error) {
   335  	if r, err = f.addUpper(path, srcWhiteout); err != nil {
   336  		return nil, errors.WithMessage(err, "add whiteout")
   337  	}
   338  	return
   339  }
   340  
   341  // Removes whiteout nodes recursively in all children
   342  func (f *FsNode) RemoveWhiteouts() {
   343  	var (
   344  		c    = f.child
   345  		last *FsNode
   346  	)
   347  	for c != nil {
   348  		if c.NodeType == fs.TypeWhiteout {
   349  			if last == nil {
   350  				f.child = c.next
   351  			} else {
   352  				last.next = c.next
   353  			}
   354  		} else {
   355  			c.RemoveWhiteouts()
   356  			last = c
   357  		}
   358  		c = c.next
   359  	}
   360  }
   361  
   362  // Removes whiteout nodes recursively in all children
   363  func (f *FsNode) MockDevices() {
   364  	var (
   365  		c = f.child
   366  	)
   367  	for c != nil {
   368  		if c.NodeType == fs.TypeDevice {
   369  			c.SetSource(source.NewSourceFile(emptyContent, c.source.Attrs().FileAttrs))
   370  		} else {
   371  			c.MockDevices()
   372  		}
   373  		c = c.next
   374  	}
   375  }
   376  
   377  // Removes this node
   378  func (f *FsNode) Remove() {
   379  	p := f.Parent()
   380  	if p == nil {
   381  		panic("cannot remove detached node") // should not happen
   382  	}
   383  	c := p.child
   384  	var last *FsNode
   385  	for c != nil {
   386  		if c == f {
   387  			if last == nil {
   388  				p.child = c.next
   389  			} else {
   390  				last.next = c.next
   391  			}
   392  			c.parent = nil
   393  			c.next = nil
   394  			c.child = nil
   395  			break
   396  		}
   397  		last = c
   398  		c = c.next
   399  	}
   400  }
   401  
   402  func (f *FsNode) Link(path, target string) (linked fs.FsNode, dest fs.FsNode, err error) {
   403  	if !filepath.IsAbs(target) {
   404  		target = filepath.Join(filepath.Dir(path), target)
   405  	}
   406  	targetNode, err := f.node(target)
   407  	if err != nil {
   408  		err = errors.WithMessage(err, "link target")
   409  		return
   410  	}
   411  	linkedNode, err := targetNode.link(path)
   412  	return linkedNode, targetNode, errors.WithMessage(err, "link")
   413  }
   414  
   415  func (f *FsNode) link(dest string) (r *FsNode, err error) {
   416  	if f.NodeType == fs.TypeDir || f.NodeType == fs.TypeOverlay {
   417  		return nil, errors.Errorf("cannot link node %s to %s since it is of type %q", f.Path(), dest, f.NodeType)
   418  	}
   419  	p := f.Parent()
   420  	if p == nil {
   421  		return nil, errors.Errorf("link %s: cannot link file system root", dest)
   422  	}
   423  	src := f.source
   424  	if srcLink, ok := src.(*source.SourceUpperLink); ok {
   425  		src = srcLink.Source
   426  	}
   427  	r, err = p.addUpper(dest, source.NewSourceUpperLink(src))
   428  	if err != nil {
   429  		return nil, errors.Wrap(err, "link")
   430  	}
   431  	return
   432  }
   433  
   434  func (f *FsNode) Root() (r *FsNode) {
   435  	r = f
   436  	for r.parent != nil {
   437  		r = r.parent
   438  	}
   439  	return
   440  }
   441  
   442  func (f *FsNode) Parent() (p *FsNode) {
   443  	return f.pathNode.parent
   444  }
   445  
   446  func (f *FsNode) Node(path string) (r fs.FsNode, err error) {
   447  	return f.node(path)
   448  }
   449  
   450  func (f *FsNode) node(path string) (r *FsNode, err error) {
   451  	return f.walkPath(path, lookup)
   452  }
   453  
   454  func (f *FsNode) walkPath(path string, handler pathWalker) (r *FsNode, err error) {
   455  	path = filepath.Clean(path)
   456  
   457  	// if abs path delegate to root node
   458  	if filepath.IsAbs(path) {
   459  		return f.Root().walkPath(path[1:], handler)
   460  	}
   461  
   462  	// Resolve symlinks recursively
   463  	if f.NodeType == fs.TypeSymlink {
   464  		// Resolve link
   465  		if f, err = f.Parent().walkPath(f.Symlink, handler); err != nil {
   466  			return nil, err
   467  		}
   468  	}
   469  
   470  	handler.Visit(f)
   471  
   472  	if path == "." {
   473  		return f, nil // found
   474  	}
   475  
   476  	// Resolve relative path segments recursively
   477  	spos := strings.Index(path, string(filepath.Separator))
   478  	name := path
   479  	if spos != -1 {
   480  		if spos > 0 {
   481  			name = path[0:spos]
   482  		} else {
   483  			name = "."
   484  		}
   485  		path = path[spos+1:]
   486  		if path == "" {
   487  			path = "."
   488  		}
   489  	} else {
   490  		path = "."
   491  	}
   492  	if name == ".." {
   493  		r = f.Parent()
   494  		if r == nil {
   495  			return nil, errors.Errorf("path outside file system root: /%s", filepath.Join(name, path))
   496  		}
   497  	} else {
   498  		r = f.findChild(name)
   499  	}
   500  	if r == nil {
   501  		if r, err = handler.NotFound(f, name); err != nil {
   502  			return
   503  		}
   504  	}
   505  	return r.walkPath(path, handler)
   506  }
   507  
   508  type pathWalker interface {
   509  	Visit(*FsNode)
   510  	// Returns an error or creates a new node
   511  	NotFound(parent *FsNode, child string) (*FsNode, error)
   512  }
   513  
   514  type mkdirsWalker struct {
   515  	overlay bool
   516  }
   517  
   518  func (_ *mkdirsWalker) Visit(f *FsNode) {
   519  	if f.NodeType != fs.TypeDir {
   520  		f.SetSource(source.NewSourceDir(dirAttrs))
   521  	}
   522  }
   523  
   524  func (w *mkdirsWalker) NotFound(p *FsNode, name string) (r *FsNode, err error) {
   525  	if p.overlay {
   526  		w.overlay = true
   527  		p = p.overlayRoot()
   528  	}
   529  	r = p.newEntry(name)
   530  	if w.overlay {
   531  		r.SetSource(srcParentDir)
   532  	} else {
   533  		r.SetSource(source.NewSourceDir(dirAttrs))
   534  	}
   535  	return
   536  }
   537  
   538  type mkdirsUpperWalker struct {
   539  	mkdirsWalker
   540  }
   541  
   542  func (_ *mkdirsUpperWalker) Visit(f *FsNode) {
   543  	if f.NodeType != fs.TypeDir {
   544  		f.SetSource(source.NewSourceDir(dirAttrs))
   545  	} else if f.isLowerNode() || f.parent == nil {
   546  		f.SetSource(source.NewSourceDir(f.FileAttrs))
   547  	}
   548  }
   549  
   550  type lookupWalker string
   551  
   552  func (_ lookupWalker) Visit(f *FsNode) {}
   553  
   554  func (_ lookupWalker) NotFound(p *FsNode, name string) (*FsNode, error) {
   555  	return nil, errors.Errorf("path not found: %s", filepath.Join(p.Path(), name))
   556  }
   557  
   558  func (f *FsNode) Mkdirs(path string) (r fs.FsNode, err error) {
   559  	return f.mkdirsUpper(path)
   560  }
   561  
   562  func (f *FsNode) mkdirsUpper(path string) (r *FsNode, err error) {
   563  	if r, err = f.walkPath(path, &mkdirsUpperWalker{}); err != nil {
   564  		err = errors.WithMessage(err, "Mkdirs")
   565  	}
   566  	return
   567  }
   568  
   569  func (f *FsNode) mkdirsLower(path string) (r *FsNode, err error) {
   570  	if r, err = f.walkPath(path, &mkdirsWalker{}); err != nil {
   571  		err = errors.WithMessage(err, "mkdirs")
   572  	}
   573  	return
   574  }
   575  
   576  func (f *FsNode) newEntry(name string) (r *FsNode) {
   577  	r = f.newNode(name)
   578  	var last *FsNode
   579  	c := f.child
   580  	for c != nil {
   581  		if c.name > name {
   582  			// Insert before
   583  			r.next = c
   584  			break
   585  		}
   586  		last = c
   587  		c = c.next
   588  	}
   589  
   590  	// Append
   591  	if last == nil {
   592  		f.child = r
   593  	} else {
   594  		last.next = r
   595  	}
   596  
   597  	return
   598  }
   599  
   600  func (f *FsNode) newNode(name string) (r *FsNode) {
   601  	r = &FsNode{
   602  		name:   name,
   603  		parent: f.pathNode,
   604  	}
   605  	r.pathNode = r
   606  	return
   607  }
   608  
   609  func (f *FsNode) overlayRoot() (r *FsNode) {
   610  	c := f.child
   611  	for c.next != nil {
   612  		c = c.next
   613  	}
   614  	if c.NodeType != fs.TypeOverlay {
   615  		// Return last non-overlay entry (must be . entry)
   616  		return c
   617  	}
   618  	r = f.newNode(".")
   619  	r.SetSource(srcOverlayPseudoRoot)
   620  	r.parent = f.pathNode
   621  	r.pathNode = f
   622  	c.next = r
   623  	return
   624  }
   625  
   626  func (f *FsNode) newOverlayEntry() (r *FsNode) {
   627  	if !f.overlay && f.child != nil {
   628  		// Move existing children into overlay item to preserve insertion order
   629  		c := f.child
   630  		f.child = nil
   631  		f.newEntry(".")
   632  		f.child.SetSource(srcOverlayPseudoRoot)
   633  		f.child.pathNode = f
   634  		f.child.child = c
   635  	}
   636  	f.overlay = true
   637  	if f.NodeType != fs.TypeDir {
   638  		f.SetSource(source.NewSourceDir(dirAttrs))
   639  	}
   640  	r = f.newEntry(".")
   641  	r.pathNode = f
   642  	return
   643  }