github.com/mgoltzsche/ctnr@v0.7.1-alpha/pkg/fs/tree/fsnode.go (about) 1 package tree 2 3 import ( 4 "os" 5 "path/filepath" 6 "strings" 7 8 "github.com/mgoltzsche/ctnr/pkg/fs" 9 "github.com/mgoltzsche/ctnr/pkg/fs/source" 10 "github.com/mgoltzsche/ctnr/pkg/fs/writer" 11 "github.com/opencontainers/go-digest" 12 "github.com/pkg/errors" 13 ) 14 15 const ( 16 lookup lookupWalker = "path lookup" 17 ) 18 19 var ( 20 dirAttrs = fs.FileAttrs{Mode: os.ModeDir | 0755} 21 srcOverlayPseudoRoot = sourceNoop("noop source") 22 srcParentDir = sourceParentDir("parent dir source") 23 srcWhiteout = source.NewSourceWhiteout() 24 emptyContent = fs.NewReadableBytes([]byte{}) 25 ) 26 27 type sourceParentDir string 28 29 func (s sourceParentDir) Attrs() fs.NodeInfo { 30 return fs.NodeInfo{fs.TypeDir, fs.FileAttrs{Mode: os.ModeDir | 0755}} 31 } 32 func (s sourceParentDir) DeriveAttrs() (fs.DerivedAttrs, error) { 33 return fs.DerivedAttrs{}, nil 34 } 35 func (s sourceParentDir) Write(dest, name string, w fs.Writer, written map[fs.Source]string) error { 36 return w.Mkdir(dest) 37 } 38 func (s sourceParentDir) Equal(other fs.Source) (bool, error) { 39 return s == other, nil 40 } 41 42 type sourceNoop string 43 44 func (s sourceNoop) Attrs() fs.NodeInfo { return fs.NodeInfo{NodeType: fs.TypeDir} } 45 func (s sourceNoop) DeriveAttrs() (fs.DerivedAttrs, error) { 46 return fs.DerivedAttrs{}, nil 47 } 48 func (s sourceNoop) Write(dest, name string, w fs.Writer, written map[fs.Source]string) error { 49 return nil 50 } 51 func (s sourceNoop) Equal(other fs.Source) (bool, error) { 52 return s == other, nil 53 } 54 55 type FsNode struct { 56 name string 57 fs.NodeInfo 58 source fs.Source 59 parent *FsNode 60 pathNode *FsNode 61 child *FsNode 62 next *FsNode 63 overlay bool 64 } 65 66 func NewFS() fs.FsNode { 67 return newFS() 68 } 69 70 func newFS() *FsNode { 71 fs := &FsNode{name: "."} 72 fs.pathNode = fs 73 fs.SetSource(srcParentDir) 74 return fs 75 } 76 77 func (f *FsNode) String() (p string) { 78 return f.Path() + " " + f.NodeInfo.AttrString(fs.AttrsAll) 79 } 80 81 func (f *FsNode) Name() string { 82 return f.name 83 } 84 85 func (f *FsNode) Path() (p string) { 86 if f.parent == nil { 87 return string(filepath.Separator) 88 } else { 89 p = filepath.Join(f.parent.Path(), f.name) 90 } 91 return 92 } 93 94 func (f *FsNode) Empty() bool { 95 return f.source == srcParentDir 96 } 97 98 // Generates the file system's hash including the provided attribute set. 99 func (f *FsNode) Hash(attrs fs.AttrSet) (digest.Digest, error) { 100 d := digest.SHA256.Digester() 101 err := f.WriteTo(d.Hash(), attrs) 102 return d.Digest(), err 103 } 104 105 // Walks through the tree writing all nodes into the provided Writer. 106 func (f *FsNode) Write(w fs.Writer) (err error) { 107 return f.write(w, map[fs.Source]string{}) 108 } 109 110 // Walks through the tree writing all nodes into the provided Writer. 111 // The 'written' map is used to allow Source implementations to detect and write 112 // a hardlink. 113 func (f *FsNode) write(w fs.Writer, written map[fs.Source]string) (err error) { 114 if err = f.source.Write(f.Path(), f.name, w, written); err != nil { 115 return 116 } 117 if f.child != nil { 118 err = f.child.write(w, written) 119 } 120 if f.next != nil && err == nil { 121 err = f.next.write(w, written) 122 } 123 return 124 } 125 126 // Returns a new normalized file system tree 127 func (f *FsNode) Normalized() (fs.FsNode, error) { 128 normalized := NewFS() 129 normalizer := writer.NewFsNodeWriter(normalized, fs.HashingNilWriter()) 130 expander := fs.ExpandingWriter{normalizer} 131 if err := f.Write(&expander); err != nil { 132 return nil, errors.Wrap(err, "normalize fs spec") 133 } 134 normalized.(*FsNode).RemoveWhiteouts() 135 return normalized, nil 136 } 137 138 // Returns a file system tree containing only new and changed nodes. 139 // Nodes that do not exist in the provided file system are returned as whiteouts. 140 // Hardlinks from new to unchanged (upper to lower) nodes are supported by 141 // adding hardlinked files to diff as well (to preserve hardlinks and stay 142 // compatible with external tar tools when writing diff into tar file). 143 func (f *FsNode) Diff(o fs.FsNode) (r fs.FsNode, err error) { 144 node := newFS() 145 from := map[string]*FsNode{} 146 to := map[string]*FsNode{} 147 unchangedSourceMap := map[fs.Source][]string{} 148 addedSources := []fs.Source{} 149 if err = o.(*FsNode).toMap(to); err == nil { 150 err = f.toMap(from) 151 } 152 if err != nil { 153 return nil, errors.WithMessage(err, "diff") 154 } 155 // Add new or changed nodes to tree 156 for k, v := range to { 157 old := from[k] 158 if old == nil { 159 added, err := node.addUpper(k, v.source) 160 if err != nil { 161 return nil, errors.WithMessage(err, "diff") 162 } 163 added.parent.applyParents(v.parent) 164 } else { 165 eq, err := v.Equal(old) 166 if err != nil { 167 return nil, errors.WithMessage(err, "diff") 168 } 169 if !eq { 170 // changed node - add new source to be written to layer 171 added, err := node.addUpper(k, v.source) 172 if err != nil { 173 return nil, errors.WithMessage(err, "diff") 174 } 175 added.parent.applyParents(v.parent) 176 } else { 177 // unchanged node - only map source to support hardlink from upper node 178 // Decision regarding hardlink preservation when upper layer contains hardlink to lower layer: 179 // * support hardlinks to lower nodes (pointing to file in different archive) -> does not work with external tar tools, may not work in other engines 180 // * or add hardlinked files to child archive again -> more compatible: works with external tar tools and other storage engines but takes more disk space 181 // => Map source:paths to use add it if linked 182 paths := unchangedSourceMap[v.source] 183 if paths == nil { 184 paths = []string{k} 185 } else { 186 paths = append(paths, k) 187 } 188 unchangedSourceMap[v.source] = paths 189 continue 190 } 191 } 192 addedSources = append(addedSources, v.source) 193 } 194 // Add whiteout nodes for those that don't exist in dest tree 195 for k := range from { 196 p := to[filepath.Dir(k)] 197 if to[k] == nil && p != nil { 198 added, err := node.AddWhiteoutNode(k) 199 if err != nil { 200 return nil, errors.WithMessage(err, "diff") 201 } 202 added.parent.applyParents(p) 203 } 204 } 205 // Add hardlinked nodes (to preserve fs state while remaining compatible with other tools) 206 for _, added := range addedSources { 207 if paths := unchangedSourceMap[added]; paths != nil { 208 for _, path := range paths { 209 if _, err = node.AddUpper(path, added); err != nil { 210 return nil, errors.WithMessage(err, "diff") 211 } 212 } 213 } 214 } 215 return node, err 216 } 217 218 func (f *FsNode) Equal(o *FsNode) (bool, error) { 219 if !f.source.Attrs().Equal(o.source.Attrs()) { 220 return false, nil 221 } 222 fa, err := f.source.DeriveAttrs() 223 if err != nil { 224 return false, errors.Wrap(err, "equal") 225 } 226 oa, err := o.source.DeriveAttrs() 227 if err != nil { 228 return false, errors.Wrap(err, "equal") 229 } 230 return fa.Hash == oa.Hash, nil 231 } 232 233 func (f *FsNode) applyParents(o *FsNode) { 234 if f != nil && f.source != o.source { 235 f.SetSource(o.source) 236 f.parent.applyParents(o.parent) 237 } 238 } 239 240 func (f *FsNode) toMap(m map[string]*FsNode) (err error) { 241 if f.overlay { 242 return errors.Errorf("cannot map overlay %s - needs to be normalized first", f.Path()) 243 } 244 m[f.Path()] = f 245 if f.child != nil { 246 err = f.child.toMap(m) 247 } 248 if f.next != nil && err == nil { 249 err = f.next.toMap(m) 250 } 251 return 252 } 253 254 func (f *FsNode) findChild(name string) *FsNode { 255 if name == "." || name == "/" { 256 return f 257 } 258 if f.overlay { 259 return nil 260 } 261 c := f.child 262 for c != nil { 263 if c.name == name { 264 return c 265 } 266 if c.name > name { 267 return nil 268 } 269 c = c.next 270 } 271 return c 272 } 273 274 func (f *FsNode) SetSource(src fs.Source) { 275 f.source = src 276 f.NodeInfo = src.Attrs() 277 if f.NodeType != fs.TypeDir { 278 f.child = nil 279 } 280 } 281 282 func (f *FsNode) AddUpper(path string, src fs.Source) (r fs.FsNode, err error) { 283 return f.addUpper(path, src) 284 } 285 286 func (f *FsNode) addUpper(path string, src fs.Source) (r *FsNode, err error) { 287 r, err = f.add(path, src, f.mkdirsUpper) 288 return r, errors.Wrap(err, "add upper fsnode") 289 } 290 291 func (f *FsNode) isLowerNode() bool { 292 _, ok := f.source.(*fs.NodeAttrs) 293 return ok 294 } 295 296 type mkdirsFn func(path string) (r *FsNode, err error) 297 298 func (f *FsNode) AddLower(path string, src fs.Source) (r fs.FsNode, err error) { 299 r, err = f.add(path, src, f.mkdirsLower) 300 return r, errors.Wrap(err, "add lower fsnode") 301 } 302 func (f *FsNode) add(path string, src fs.Source, mkdirs mkdirsFn) (r *FsNode, err error) { 303 path = filepath.Clean(path) 304 dir := filepath.Dir(path) 305 var p *FsNode 306 if dir == "." { 307 p = f 308 } else { 309 if p, err = mkdirs(dir); err != nil { 310 return 311 } 312 } 313 name := filepath.Base(path) 314 r = p.findChild(name) 315 if r == nil { 316 r = p.newEntry(name) 317 } 318 t := src.Attrs().NodeType 319 if t == fs.TypeOverlay { 320 r = r.newOverlayEntry() 321 } else if r.overlay { 322 r = r.overlayRoot() 323 } else if t != fs.TypeDir { 324 r.child = nil 325 } 326 r.SetSource(src) 327 return 328 } 329 330 func (f *FsNode) AddWhiteout(path string) (fs.FsNode, error) { 331 return f.AddWhiteoutNode(path) 332 } 333 334 func (f *FsNode) AddWhiteoutNode(path string) (r *FsNode, err error) { 335 if r, err = f.addUpper(path, srcWhiteout); err != nil { 336 return nil, errors.WithMessage(err, "add whiteout") 337 } 338 return 339 } 340 341 // Removes whiteout nodes recursively in all children 342 func (f *FsNode) RemoveWhiteouts() { 343 var ( 344 c = f.child 345 last *FsNode 346 ) 347 for c != nil { 348 if c.NodeType == fs.TypeWhiteout { 349 if last == nil { 350 f.child = c.next 351 } else { 352 last.next = c.next 353 } 354 } else { 355 c.RemoveWhiteouts() 356 last = c 357 } 358 c = c.next 359 } 360 } 361 362 // Removes whiteout nodes recursively in all children 363 func (f *FsNode) MockDevices() { 364 var ( 365 c = f.child 366 ) 367 for c != nil { 368 if c.NodeType == fs.TypeDevice { 369 c.SetSource(source.NewSourceFile(emptyContent, c.source.Attrs().FileAttrs)) 370 } else { 371 c.MockDevices() 372 } 373 c = c.next 374 } 375 } 376 377 // Removes this node 378 func (f *FsNode) Remove() { 379 p := f.Parent() 380 if p == nil { 381 panic("cannot remove detached node") // should not happen 382 } 383 c := p.child 384 var last *FsNode 385 for c != nil { 386 if c == f { 387 if last == nil { 388 p.child = c.next 389 } else { 390 last.next = c.next 391 } 392 c.parent = nil 393 c.next = nil 394 c.child = nil 395 break 396 } 397 last = c 398 c = c.next 399 } 400 } 401 402 func (f *FsNode) Link(path, target string) (linked fs.FsNode, dest fs.FsNode, err error) { 403 if !filepath.IsAbs(target) { 404 target = filepath.Join(filepath.Dir(path), target) 405 } 406 targetNode, err := f.node(target) 407 if err != nil { 408 err = errors.WithMessage(err, "link target") 409 return 410 } 411 linkedNode, err := targetNode.link(path) 412 return linkedNode, targetNode, errors.WithMessage(err, "link") 413 } 414 415 func (f *FsNode) link(dest string) (r *FsNode, err error) { 416 if f.NodeType == fs.TypeDir || f.NodeType == fs.TypeOverlay { 417 return nil, errors.Errorf("cannot link node %s to %s since it is of type %q", f.Path(), dest, f.NodeType) 418 } 419 p := f.Parent() 420 if p == nil { 421 return nil, errors.Errorf("link %s: cannot link file system root", dest) 422 } 423 src := f.source 424 if srcLink, ok := src.(*source.SourceUpperLink); ok { 425 src = srcLink.Source 426 } 427 r, err = p.addUpper(dest, source.NewSourceUpperLink(src)) 428 if err != nil { 429 return nil, errors.Wrap(err, "link") 430 } 431 return 432 } 433 434 func (f *FsNode) Root() (r *FsNode) { 435 r = f 436 for r.parent != nil { 437 r = r.parent 438 } 439 return 440 } 441 442 func (f *FsNode) Parent() (p *FsNode) { 443 return f.pathNode.parent 444 } 445 446 func (f *FsNode) Node(path string) (r fs.FsNode, err error) { 447 return f.node(path) 448 } 449 450 func (f *FsNode) node(path string) (r *FsNode, err error) { 451 return f.walkPath(path, lookup) 452 } 453 454 func (f *FsNode) walkPath(path string, handler pathWalker) (r *FsNode, err error) { 455 path = filepath.Clean(path) 456 457 // if abs path delegate to root node 458 if filepath.IsAbs(path) { 459 return f.Root().walkPath(path[1:], handler) 460 } 461 462 // Resolve symlinks recursively 463 if f.NodeType == fs.TypeSymlink { 464 // Resolve link 465 if f, err = f.Parent().walkPath(f.Symlink, handler); err != nil { 466 return nil, err 467 } 468 } 469 470 handler.Visit(f) 471 472 if path == "." { 473 return f, nil // found 474 } 475 476 // Resolve relative path segments recursively 477 spos := strings.Index(path, string(filepath.Separator)) 478 name := path 479 if spos != -1 { 480 if spos > 0 { 481 name = path[0:spos] 482 } else { 483 name = "." 484 } 485 path = path[spos+1:] 486 if path == "" { 487 path = "." 488 } 489 } else { 490 path = "." 491 } 492 if name == ".." { 493 r = f.Parent() 494 if r == nil { 495 return nil, errors.Errorf("path outside file system root: /%s", filepath.Join(name, path)) 496 } 497 } else { 498 r = f.findChild(name) 499 } 500 if r == nil { 501 if r, err = handler.NotFound(f, name); err != nil { 502 return 503 } 504 } 505 return r.walkPath(path, handler) 506 } 507 508 type pathWalker interface { 509 Visit(*FsNode) 510 // Returns an error or creates a new node 511 NotFound(parent *FsNode, child string) (*FsNode, error) 512 } 513 514 type mkdirsWalker struct { 515 overlay bool 516 } 517 518 func (_ *mkdirsWalker) Visit(f *FsNode) { 519 if f.NodeType != fs.TypeDir { 520 f.SetSource(source.NewSourceDir(dirAttrs)) 521 } 522 } 523 524 func (w *mkdirsWalker) NotFound(p *FsNode, name string) (r *FsNode, err error) { 525 if p.overlay { 526 w.overlay = true 527 p = p.overlayRoot() 528 } 529 r = p.newEntry(name) 530 if w.overlay { 531 r.SetSource(srcParentDir) 532 } else { 533 r.SetSource(source.NewSourceDir(dirAttrs)) 534 } 535 return 536 } 537 538 type mkdirsUpperWalker struct { 539 mkdirsWalker 540 } 541 542 func (_ *mkdirsUpperWalker) Visit(f *FsNode) { 543 if f.NodeType != fs.TypeDir { 544 f.SetSource(source.NewSourceDir(dirAttrs)) 545 } else if f.isLowerNode() || f.parent == nil { 546 f.SetSource(source.NewSourceDir(f.FileAttrs)) 547 } 548 } 549 550 type lookupWalker string 551 552 func (_ lookupWalker) Visit(f *FsNode) {} 553 554 func (_ lookupWalker) NotFound(p *FsNode, name string) (*FsNode, error) { 555 return nil, errors.Errorf("path not found: %s", filepath.Join(p.Path(), name)) 556 } 557 558 func (f *FsNode) Mkdirs(path string) (r fs.FsNode, err error) { 559 return f.mkdirsUpper(path) 560 } 561 562 func (f *FsNode) mkdirsUpper(path string) (r *FsNode, err error) { 563 if r, err = f.walkPath(path, &mkdirsUpperWalker{}); err != nil { 564 err = errors.WithMessage(err, "Mkdirs") 565 } 566 return 567 } 568 569 func (f *FsNode) mkdirsLower(path string) (r *FsNode, err error) { 570 if r, err = f.walkPath(path, &mkdirsWalker{}); err != nil { 571 err = errors.WithMessage(err, "mkdirs") 572 } 573 return 574 } 575 576 func (f *FsNode) newEntry(name string) (r *FsNode) { 577 r = f.newNode(name) 578 var last *FsNode 579 c := f.child 580 for c != nil { 581 if c.name > name { 582 // Insert before 583 r.next = c 584 break 585 } 586 last = c 587 c = c.next 588 } 589 590 // Append 591 if last == nil { 592 f.child = r 593 } else { 594 last.next = r 595 } 596 597 return 598 } 599 600 func (f *FsNode) newNode(name string) (r *FsNode) { 601 r = &FsNode{ 602 name: name, 603 parent: f.pathNode, 604 } 605 r.pathNode = r 606 return 607 } 608 609 func (f *FsNode) overlayRoot() (r *FsNode) { 610 c := f.child 611 for c.next != nil { 612 c = c.next 613 } 614 if c.NodeType != fs.TypeOverlay { 615 // Return last non-overlay entry (must be . entry) 616 return c 617 } 618 r = f.newNode(".") 619 r.SetSource(srcOverlayPseudoRoot) 620 r.parent = f.pathNode 621 r.pathNode = f 622 c.next = r 623 return 624 } 625 626 func (f *FsNode) newOverlayEntry() (r *FsNode) { 627 if !f.overlay && f.child != nil { 628 // Move existing children into overlay item to preserve insertion order 629 c := f.child 630 f.child = nil 631 f.newEntry(".") 632 f.child.SetSource(srcOverlayPseudoRoot) 633 f.child.pathNode = f 634 f.child.child = c 635 } 636 f.overlay = true 637 if f.NodeType != fs.TypeDir { 638 f.SetSource(source.NewSourceDir(dirAttrs)) 639 } 640 r = f.newEntry(".") 641 r.pathNode = f 642 return 643 }