github.com/mckael/restic@v0.8.3/internal/restic/node.go (about) 1 package restic 2 3 import ( 4 "context" 5 "encoding/json" 6 "fmt" 7 "os" 8 "os/user" 9 "strconv" 10 "sync" 11 "syscall" 12 "time" 13 14 "github.com/restic/restic/internal/errors" 15 16 "bytes" 17 "runtime" 18 19 "github.com/restic/restic/internal/debug" 20 "github.com/restic/restic/internal/fs" 21 ) 22 23 // ExtendedAttribute is a tuple storing the xattr name and value. 24 type ExtendedAttribute struct { 25 Name string `json:"name"` 26 Value []byte `json:"value"` 27 } 28 29 // Node is a file, directory or other item in a backup. 30 type Node struct { 31 Name string `json:"name"` 32 Type string `json:"type"` 33 Mode os.FileMode `json:"mode,omitempty"` 34 ModTime time.Time `json:"mtime,omitempty"` 35 AccessTime time.Time `json:"atime,omitempty"` 36 ChangeTime time.Time `json:"ctime,omitempty"` 37 UID uint32 `json:"uid"` 38 GID uint32 `json:"gid"` 39 User string `json:"user,omitempty"` 40 Group string `json:"group,omitempty"` 41 Inode uint64 `json:"inode,omitempty"` 42 DeviceID uint64 `json:"device_id,omitempty"` // device id of the file, stat.st_dev 43 Size uint64 `json:"size,omitempty"` 44 Links uint64 `json:"links,omitempty"` 45 LinkTarget string `json:"linktarget,omitempty"` 46 ExtendedAttributes []ExtendedAttribute `json:"extended_attributes,omitempty"` 47 Device uint64 `json:"device,omitempty"` // in case of Type == "dev", stat.st_rdev 48 Content IDs `json:"content"` 49 Subtree *ID `json:"subtree,omitempty"` 50 51 Error string `json:"error,omitempty"` 52 53 Path string `json:"-"` 54 } 55 56 // Nodes is a slice of nodes that can be sorted. 57 type Nodes []*Node 58 59 func (n Nodes) Len() int { return len(n) } 60 func (n Nodes) Less(i, j int) bool { return n[i].Name < n[j].Name } 61 func (n Nodes) Swap(i, j int) { n[i], n[j] = n[j], n[i] } 62 63 func (node Node) String() string { 64 var mode os.FileMode 65 switch node.Type { 66 case "file": 67 mode = 0 68 case "dir": 69 mode = os.ModeDir 70 case "symlink": 71 mode = os.ModeSymlink 72 case "dev": 73 mode = os.ModeDevice 74 case "chardev": 75 mode = os.ModeDevice | os.ModeCharDevice 76 case "fifo": 77 mode = os.ModeNamedPipe 78 case "socket": 79 mode = os.ModeSocket 80 } 81 82 return fmt.Sprintf("%s %5d %5d %6d %s %s", 83 mode|node.Mode, node.UID, node.GID, node.Size, node.ModTime, node.Name) 84 } 85 86 // NodeFromFileInfo returns a new node from the given path and FileInfo. It 87 // returns the first error that is encountered, together with a node. 88 func NodeFromFileInfo(path string, fi os.FileInfo) (*Node, error) { 89 mask := os.ModePerm | os.ModeType | os.ModeSetuid | os.ModeSetgid | os.ModeSticky 90 node := &Node{ 91 Path: path, 92 Name: fi.Name(), 93 Mode: fi.Mode() & mask, 94 ModTime: fi.ModTime(), 95 } 96 97 node.Type = nodeTypeFromFileInfo(fi) 98 if node.Type == "file" { 99 node.Size = uint64(fi.Size()) 100 } 101 102 err := node.fillExtra(path, fi) 103 return node, err 104 } 105 106 func nodeTypeFromFileInfo(fi os.FileInfo) string { 107 switch fi.Mode() & (os.ModeType | os.ModeCharDevice) { 108 case 0: 109 return "file" 110 case os.ModeDir: 111 return "dir" 112 case os.ModeSymlink: 113 return "symlink" 114 case os.ModeDevice | os.ModeCharDevice: 115 return "chardev" 116 case os.ModeDevice: 117 return "dev" 118 case os.ModeNamedPipe: 119 return "fifo" 120 case os.ModeSocket: 121 return "socket" 122 } 123 124 return "" 125 } 126 127 // GetExtendedAttribute gets the extended attribute. 128 func (node Node) GetExtendedAttribute(a string) []byte { 129 for _, attr := range node.ExtendedAttributes { 130 if attr.Name == a { 131 return attr.Value 132 } 133 } 134 return nil 135 } 136 137 // CreateAt creates the node at the given path and restores all the meta data. 138 func (node *Node) CreateAt(ctx context.Context, path string, repo Repository, idx *HardlinkIndex) error { 139 debug.Log("create node %v at %v", node.Name, path) 140 141 switch node.Type { 142 case "dir": 143 if err := node.createDirAt(path); err != nil { 144 return err 145 } 146 case "file": 147 if err := node.createFileAt(ctx, path, repo, idx); err != nil { 148 return err 149 } 150 case "symlink": 151 if err := node.createSymlinkAt(path); err != nil { 152 return err 153 } 154 case "dev": 155 if err := node.createDevAt(path); err != nil { 156 return err 157 } 158 case "chardev": 159 if err := node.createCharDevAt(path); err != nil { 160 return err 161 } 162 case "fifo": 163 if err := node.createFifoAt(path); err != nil { 164 return err 165 } 166 case "socket": 167 return nil 168 default: 169 return errors.Errorf("filetype %q not implemented!\n", node.Type) 170 } 171 172 err := node.restoreMetadata(path) 173 if err != nil { 174 debug.Log("restoreMetadata(%s) error %v", path, err) 175 } 176 177 return err 178 } 179 180 func (node Node) restoreMetadata(path string) error { 181 var firsterr error 182 183 if err := lchown(path, int(node.UID), int(node.GID)); err != nil { 184 firsterr = errors.Wrap(err, "Lchown") 185 } 186 187 if node.Type != "symlink" { 188 if err := fs.Chmod(path, node.Mode); err != nil { 189 if firsterr != nil { 190 firsterr = errors.Wrap(err, "Chmod") 191 } 192 } 193 } 194 195 if node.Type != "dir" { 196 if err := node.RestoreTimestamps(path); err != nil { 197 debug.Log("error restoring timestamps for dir %v: %v", path, err) 198 if firsterr != nil { 199 firsterr = err 200 } 201 } 202 } 203 204 if err := node.restoreExtendedAttributes(path); err != nil { 205 debug.Log("error restoring extended attributes for %v: %v", path, err) 206 if firsterr != nil { 207 firsterr = err 208 } 209 } 210 211 return firsterr 212 } 213 214 func (node Node) restoreExtendedAttributes(path string) error { 215 for _, attr := range node.ExtendedAttributes { 216 err := Setxattr(path, attr.Name, attr.Value) 217 if err != nil { 218 return err 219 } 220 } 221 return nil 222 } 223 224 func (node Node) RestoreTimestamps(path string) error { 225 var utimes = [...]syscall.Timespec{ 226 syscall.NsecToTimespec(node.AccessTime.UnixNano()), 227 syscall.NsecToTimespec(node.ModTime.UnixNano()), 228 } 229 230 if node.Type == "symlink" { 231 return node.restoreSymlinkTimestamps(path, utimes) 232 } 233 234 if err := syscall.UtimesNano(path, utimes[:]); err != nil { 235 return errors.Wrap(err, "UtimesNano") 236 } 237 238 return nil 239 } 240 241 func (node Node) createDirAt(path string) error { 242 err := fs.Mkdir(path, node.Mode) 243 if err != nil && !os.IsExist(err) { 244 return errors.Wrap(err, "Mkdir") 245 } 246 247 return nil 248 } 249 250 func (node Node) createFileAt(ctx context.Context, path string, repo Repository, idx *HardlinkIndex) error { 251 if node.Links > 1 && idx.Has(node.Inode, node.DeviceID) { 252 if err := fs.Remove(path); !os.IsNotExist(err) { 253 return errors.Wrap(err, "RemoveCreateHardlink") 254 } 255 err := fs.Link(idx.GetFilename(node.Inode, node.DeviceID), path) 256 if err != nil { 257 return errors.Wrap(err, "CreateHardlink") 258 } 259 return nil 260 } 261 262 f, err := fs.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600) 263 if err != nil { 264 return errors.Wrap(err, "OpenFile") 265 } 266 267 err = node.writeNodeContent(ctx, repo, f) 268 closeErr := f.Close() 269 270 if err != nil { 271 return err 272 } 273 274 if closeErr != nil { 275 return errors.Wrap(closeErr, "Close") 276 } 277 278 if node.Links > 1 { 279 idx.Add(node.Inode, node.DeviceID, path) 280 } 281 282 return nil 283 } 284 285 func (node Node) writeNodeContent(ctx context.Context, repo Repository, f *os.File) error { 286 var buf []byte 287 for _, id := range node.Content { 288 size, found := repo.LookupBlobSize(id, DataBlob) 289 if !found { 290 return errors.Errorf("id %v not found in repository", id) 291 } 292 293 buf = buf[:cap(buf)] 294 if len(buf) < CiphertextLength(int(size)) { 295 buf = NewBlobBuffer(int(size)) 296 } 297 298 n, err := repo.LoadBlob(ctx, DataBlob, id, buf) 299 if err != nil { 300 return err 301 } 302 buf = buf[:n] 303 304 _, err = f.Write(buf) 305 if err != nil { 306 return errors.Wrap(err, "Write") 307 } 308 } 309 310 return nil 311 } 312 313 func (node Node) createSymlinkAt(path string) error { 314 // Windows does not allow non-admins to create soft links. 315 if runtime.GOOS == "windows" { 316 return nil 317 } 318 err := fs.Symlink(node.LinkTarget, path) 319 if err != nil { 320 return errors.Wrap(err, "Symlink") 321 } 322 323 return nil 324 } 325 326 func (node *Node) createDevAt(path string) error { 327 return mknod(path, syscall.S_IFBLK|0600, int(node.Device)) 328 } 329 330 func (node *Node) createCharDevAt(path string) error { 331 return mknod(path, syscall.S_IFCHR|0600, int(node.Device)) 332 } 333 334 func (node *Node) createFifoAt(path string) error { 335 return mkfifo(path, 0600) 336 } 337 338 func (node Node) MarshalJSON() ([]byte, error) { 339 if node.ModTime.Year() < 0 || node.ModTime.Year() > 9999 { 340 err := errors.Errorf("node %v has invalid ModTime year %d: %v", 341 node.Path, node.ModTime.Year(), node.ModTime) 342 return nil, err 343 } 344 345 if node.ChangeTime.Year() < 0 || node.ChangeTime.Year() > 9999 { 346 err := errors.Errorf("node %v has invalid ChangeTime year %d: %v", 347 node.Path, node.ChangeTime.Year(), node.ChangeTime) 348 return nil, err 349 } 350 351 if node.AccessTime.Year() < 0 || node.AccessTime.Year() > 9999 { 352 err := errors.Errorf("node %v has invalid AccessTime year %d: %v", 353 node.Path, node.AccessTime.Year(), node.AccessTime) 354 return nil, err 355 } 356 357 type nodeJSON Node 358 nj := nodeJSON(node) 359 name := strconv.Quote(node.Name) 360 nj.Name = name[1 : len(name)-1] 361 362 return json.Marshal(nj) 363 } 364 365 func (node *Node) UnmarshalJSON(data []byte) error { 366 type nodeJSON Node 367 nj := (*nodeJSON)(node) 368 369 err := json.Unmarshal(data, nj) 370 if err != nil { 371 return errors.Wrap(err, "Unmarshal") 372 } 373 374 nj.Name, err = strconv.Unquote(`"` + nj.Name + `"`) 375 return errors.Wrap(err, "Unquote") 376 } 377 378 func (node Node) Equals(other Node) bool { 379 if node.Name != other.Name { 380 return false 381 } 382 if node.Type != other.Type { 383 return false 384 } 385 if node.Mode != other.Mode { 386 return false 387 } 388 if !node.ModTime.Equal(other.ModTime) { 389 return false 390 } 391 if !node.AccessTime.Equal(other.AccessTime) { 392 return false 393 } 394 if !node.ChangeTime.Equal(other.ChangeTime) { 395 return false 396 } 397 if node.UID != other.UID { 398 return false 399 } 400 if node.GID != other.GID { 401 return false 402 } 403 if node.User != other.User { 404 return false 405 } 406 if node.Group != other.Group { 407 return false 408 } 409 if node.Inode != other.Inode { 410 return false 411 } 412 if node.DeviceID != other.DeviceID { 413 return false 414 } 415 if node.Size != other.Size { 416 return false 417 } 418 if node.Links != other.Links { 419 return false 420 } 421 if node.LinkTarget != other.LinkTarget { 422 return false 423 } 424 if node.Device != other.Device { 425 return false 426 } 427 if !node.sameContent(other) { 428 return false 429 } 430 if !node.sameExtendedAttributes(other) { 431 return false 432 } 433 if node.Subtree != nil { 434 if other.Subtree == nil { 435 return false 436 } 437 438 if !node.Subtree.Equal(*other.Subtree) { 439 return false 440 } 441 } else { 442 if other.Subtree != nil { 443 return false 444 } 445 } 446 if node.Error != other.Error { 447 return false 448 } 449 450 return true 451 } 452 453 func (node Node) sameContent(other Node) bool { 454 if node.Content == nil { 455 return other.Content == nil 456 } 457 458 if other.Content == nil { 459 return false 460 } 461 462 if len(node.Content) != len(other.Content) { 463 return false 464 } 465 466 for i := 0; i < len(node.Content); i++ { 467 if !node.Content[i].Equal(other.Content[i]) { 468 return false 469 } 470 } 471 return true 472 } 473 474 func (node Node) sameExtendedAttributes(other Node) bool { 475 if len(node.ExtendedAttributes) != len(other.ExtendedAttributes) { 476 return false 477 } 478 479 // build a set of all attributes that node has 480 type mapvalue struct { 481 value []byte 482 present bool 483 } 484 attributes := make(map[string]mapvalue) 485 for _, attr := range node.ExtendedAttributes { 486 attributes[attr.Name] = mapvalue{value: attr.Value} 487 } 488 489 for _, attr := range other.ExtendedAttributes { 490 v, ok := attributes[attr.Name] 491 if !ok { 492 // extended attribute is not set for node 493 debug.Log("other node has attribute %v, which is not present in node", attr.Name) 494 return false 495 496 } 497 498 if !bytes.Equal(v.value, attr.Value) { 499 // attribute has different value 500 debug.Log("attribute %v has different value", attr.Name) 501 return false 502 } 503 504 // remember that this attribute is present in other. 505 v.present = true 506 attributes[attr.Name] = v 507 } 508 509 // check for attributes that are not present in other 510 for name, v := range attributes { 511 if !v.present { 512 debug.Log("attribute %v not present in other node", name) 513 return false 514 } 515 } 516 517 return true 518 } 519 520 // IsNewer returns true of the file has been updated since the last Stat(). 521 func (node *Node) IsNewer(path string, fi os.FileInfo) bool { 522 if node.Type != "file" { 523 debug.Log("node %v is newer: not file", path) 524 return true 525 } 526 527 tpe := nodeTypeFromFileInfo(fi) 528 if node.Name != fi.Name() || node.Type != tpe { 529 debug.Log("node %v is newer: name or type changed", path) 530 return true 531 } 532 533 size := uint64(fi.Size()) 534 535 extendedStat, ok := toStatT(fi.Sys()) 536 if !ok { 537 if !node.ModTime.Equal(fi.ModTime()) || 538 node.Size != size { 539 debug.Log("node %v is newer: timestamp or size changed", path) 540 return true 541 } 542 return false 543 } 544 545 inode := extendedStat.ino() 546 547 if !node.ModTime.Equal(fi.ModTime()) || 548 !node.ChangeTime.Equal(changeTime(extendedStat)) || 549 node.Inode != uint64(inode) || 550 node.Size != size { 551 debug.Log("node %v is newer: timestamp, size or inode changed", path) 552 return true 553 } 554 555 debug.Log("node %v is not newer", path) 556 return false 557 } 558 559 func (node *Node) fillUser(stat statT) error { 560 node.UID = stat.uid() 561 node.GID = stat.gid() 562 563 username, err := lookupUsername(strconv.Itoa(int(stat.uid()))) 564 if err != nil { 565 return err 566 } 567 568 group, err := lookupGroup(strconv.Itoa(int(stat.gid()))) 569 if err != nil { 570 return err 571 } 572 573 node.User = username 574 node.Group = group 575 576 return nil 577 } 578 579 var ( 580 uidLookupCache = make(map[string]string) 581 uidLookupCacheMutex = sync.RWMutex{} 582 ) 583 584 func lookupUsername(uid string) (string, error) { 585 uidLookupCacheMutex.RLock() 586 value, ok := uidLookupCache[uid] 587 uidLookupCacheMutex.RUnlock() 588 589 if ok { 590 return value, nil 591 } 592 593 username := "" 594 595 u, err := user.LookupId(uid) 596 if err == nil { 597 username = u.Username 598 } 599 600 uidLookupCacheMutex.Lock() 601 uidLookupCache[uid] = username 602 uidLookupCacheMutex.Unlock() 603 604 return username, nil 605 } 606 607 var ( 608 gidLookupCache = make(map[string]string) 609 gidLookupCacheMutex = sync.RWMutex{} 610 ) 611 612 func lookupGroup(gid string) (string, error) { 613 gidLookupCacheMutex.RLock() 614 value, ok := gidLookupCache[gid] 615 gidLookupCacheMutex.RUnlock() 616 617 if ok { 618 return value, nil 619 } 620 621 group := "" 622 623 g, err := user.LookupGroupId(gid) 624 if err == nil { 625 group = g.Name 626 } 627 628 gidLookupCacheMutex.Lock() 629 gidLookupCache[gid] = group 630 gidLookupCacheMutex.Unlock() 631 632 return group, nil 633 } 634 635 func (node *Node) fillExtra(path string, fi os.FileInfo) error { 636 stat, ok := toStatT(fi.Sys()) 637 if !ok { 638 return nil 639 } 640 641 node.Inode = uint64(stat.ino()) 642 node.DeviceID = uint64(stat.dev()) 643 644 node.fillTimes(stat) 645 646 var err error 647 648 if err = node.fillUser(stat); err != nil { 649 return err 650 } 651 652 switch node.Type { 653 case "file": 654 node.Size = uint64(stat.size()) 655 node.Links = uint64(stat.nlink()) 656 case "dir": 657 case "symlink": 658 node.LinkTarget, err = fs.Readlink(path) 659 node.Links = uint64(stat.nlink()) 660 if err != nil { 661 return errors.Wrap(err, "Readlink") 662 } 663 case "dev": 664 node.Device = uint64(stat.rdev()) 665 node.Links = uint64(stat.nlink()) 666 case "chardev": 667 node.Device = uint64(stat.rdev()) 668 node.Links = uint64(stat.nlink()) 669 case "fifo": 670 case "socket": 671 default: 672 return errors.Errorf("invalid node type %q", node.Type) 673 } 674 675 if err = node.fillExtendedAttributes(path); err != nil { 676 return err 677 } 678 679 return nil 680 } 681 682 func (node *Node) fillExtendedAttributes(path string) error { 683 if node.Type == "symlink" { 684 return nil 685 } 686 687 xattrs, err := Listxattr(path) 688 debug.Log("fillExtendedAttributes(%v) %v %v", path, xattrs, err) 689 if err != nil { 690 return err 691 } 692 693 node.ExtendedAttributes = make([]ExtendedAttribute, 0, len(xattrs)) 694 for _, attr := range xattrs { 695 attrVal, err := Getxattr(path, attr) 696 if err != nil { 697 fmt.Fprintf(os.Stderr, "can not obtain extended attribute %v for %v:\n", attr, path) 698 continue 699 } 700 attr := ExtendedAttribute{ 701 Name: attr, 702 Value: attrVal, 703 } 704 705 node.ExtendedAttributes = append(node.ExtendedAttributes, attr) 706 } 707 708 return nil 709 } 710 711 type statT interface { 712 dev() uint64 713 ino() uint64 714 nlink() uint64 715 uid() uint32 716 gid() uint32 717 rdev() uint64 718 size() int64 719 atim() syscall.Timespec 720 mtim() syscall.Timespec 721 ctim() syscall.Timespec 722 } 723 724 func mkfifo(path string, mode uint32) (err error) { 725 return mknod(path, mode|syscall.S_IFIFO, 0) 726 } 727 728 func (node *Node) fillTimes(stat statT) { 729 ctim := stat.ctim() 730 atim := stat.atim() 731 node.ChangeTime = time.Unix(ctim.Unix()) 732 node.AccessTime = time.Unix(atim.Unix()) 733 } 734 735 func changeTime(stat statT) time.Time { 736 ctim := stat.ctim() 737 return time.Unix(ctim.Unix()) 738 }