github.com/cloud-foundations/dominator@v0.0.0-20221004181915-6e4fee580046/lib/filesystem/util/unpack.go (about)

     1  package util
     2  
     3  import (
     4  	"errors"
     5  	"fmt"
     6  	"io"
     7  	"os"
     8  	"path"
     9  	"syscall"
    10  	"time"
    11  
    12  	"github.com/Cloud-Foundations/Dominator/lib/filesystem"
    13  	"github.com/Cloud-Foundations/Dominator/lib/format"
    14  	"github.com/Cloud-Foundations/Dominator/lib/hash"
    15  	"github.com/Cloud-Foundations/Dominator/lib/log"
    16  	"github.com/Cloud-Foundations/Dominator/lib/objectserver"
    17  )
    18  
    19  const (
    20  	dirPerms  = syscall.S_IRWXU
    21  	filePerms = syscall.S_IRUSR | syscall.S_IWUSR | syscall.S_IRGRP
    22  )
    23  
    24  func createEmptyFile(filename string) error {
    25  	if file, err := os.Create(filename); err != nil {
    26  		return err
    27  	} else {
    28  		// Don't wait for finaliser to close, otherwise we can have too many
    29  		// open files.
    30  		file.Close()
    31  		return nil
    32  	}
    33  }
    34  
    35  func unpack(fs *filesystem.FileSystem, objectsGetter objectserver.ObjectsGetter,
    36  	dirname string, logger log.Logger) error {
    37  	for _, entry := range fs.EntryList {
    38  		if entry.Name == ".inodes" {
    39  			return errors.New("cannot unpack a file-system with /.inodes")
    40  		}
    41  	}
    42  	os.Mkdir(dirname, dirPerms)
    43  	inodesDir := path.Join(dirname, ".inodes")
    44  	if err := os.Mkdir(inodesDir, dirPerms); err != nil {
    45  		return err
    46  	}
    47  	defer os.RemoveAll(inodesDir)
    48  	var statfs syscall.Statfs_t
    49  	if err := syscall.Statfs(inodesDir, &statfs); err != nil {
    50  		return errors.New(fmt.Sprintf("Unable to Statfs: %s %s\n",
    51  			inodesDir, err))
    52  	}
    53  	if fs.TotalDataBytes > uint64(statfs.Bsize)*statfs.Bfree {
    54  		return errors.New("image will not fit on file-system")
    55  	}
    56  	hashes, inums, lengths := getHashes(fs)
    57  	err := writeObjects(objectsGetter, hashes, inums, lengths, inodesDir,
    58  		logger)
    59  	if err != nil {
    60  		return err
    61  	}
    62  	startWriteTime := time.Now()
    63  	if err := writeInodes(fs.InodeTable, inodesDir); err != nil {
    64  		return err
    65  	}
    66  	if err = fs.DirectoryInode.Write(dirname); err != nil {
    67  		return err
    68  	}
    69  	startBuildTime := time.Now()
    70  	writeDuration := startBuildTime.Sub(startWriteTime)
    71  	err = buildTree(&fs.DirectoryInode, dirname, "", inodesDir)
    72  	if err != nil {
    73  		return err
    74  	}
    75  	buildDuration := time.Since(startBuildTime)
    76  	logger.Printf("Unpacked file-system: made inodes in %s, built tree in %s\n",
    77  		format.Duration(writeDuration), format.Duration(buildDuration))
    78  	return nil
    79  }
    80  
    81  func getHashes(fs *filesystem.FileSystem) ([]hash.Hash, []uint64, []uint64) {
    82  	hashes := make([]hash.Hash, 0, fs.NumRegularInodes)
    83  	inums := make([]uint64, 0, fs.NumRegularInodes)
    84  	lengths := make([]uint64, 0, fs.NumRegularInodes)
    85  	for inum, inode := range fs.InodeTable {
    86  		if inode, ok := inode.(*filesystem.RegularInode); ok {
    87  			if inode.Size > 0 {
    88  				hashes = append(hashes, inode.Hash)
    89  				inums = append(inums, inum)
    90  				lengths = append(lengths, inode.Size)
    91  			}
    92  		}
    93  	}
    94  	return hashes, inums, lengths
    95  }
    96  
    97  func writeObjects(objectsGetter objectserver.ObjectsGetter, hashes []hash.Hash,
    98  	inums []uint64, lengths []uint64, inodesDir string,
    99  	logger log.Logger) error {
   100  	startTime := time.Now()
   101  	objectsReader, err := objectsGetter.GetObjects(hashes)
   102  	if err != nil {
   103  		return fmt.Errorf("error getting object reader: %s\n", err)
   104  	}
   105  	defer objectsReader.Close()
   106  	var totalLength uint64
   107  	buffer := make([]byte, 32<<10)
   108  	for index := range hashes {
   109  		err = writeObject(objectsReader, inums[index], lengths[index],
   110  			inodesDir, buffer)
   111  		if err != nil {
   112  			return err
   113  		}
   114  		totalLength += lengths[index]
   115  	}
   116  	duration := time.Since(startTime)
   117  	speed := uint64(float64(totalLength) / duration.Seconds())
   118  	logger.Printf("Copied %d objects (%s) in %s (%s/s)\n",
   119  		len(hashes), format.FormatBytes(totalLength), format.Duration(duration),
   120  		format.FormatBytes(speed))
   121  	return nil
   122  }
   123  
   124  func writeObject(objectsReader objectserver.ObjectsReader, inodeNumber uint64,
   125  	length uint64, inodesDir string, buffer []byte) error {
   126  	rlength, reader, err := objectsReader.NextObject()
   127  	if err != nil {
   128  		return err
   129  	}
   130  	defer reader.Close()
   131  	if rlength != length {
   132  		return errors.New("mismatched lengths")
   133  	}
   134  	filename := path.Join(inodesDir, fmt.Sprintf("%d", inodeNumber))
   135  	destFile, err := os.OpenFile(filename,
   136  		os.O_CREATE|os.O_TRUNC|os.O_WRONLY, filePerms)
   137  	if err != nil {
   138  		return err
   139  	}
   140  	doClose := true
   141  	defer func() {
   142  		if doClose {
   143  			destFile.Close()
   144  		}
   145  	}()
   146  	iLength := int64(length)
   147  	nCopied, err := io.CopyBuffer(destFile, io.LimitReader(reader, iLength),
   148  		buffer)
   149  	if err != nil {
   150  		return fmt.Errorf("error copying: %s", err)
   151  	}
   152  	if nCopied != iLength {
   153  		return fmt.Errorf("expected length: %d, got: %d for: %s\n",
   154  			length, nCopied, filename)
   155  	}
   156  	doClose = false
   157  	return destFile.Close()
   158  }
   159  
   160  func writeInode(inode filesystem.GenericInode, filename string) error {
   161  	switch inode := inode.(type) {
   162  	case *filesystem.RegularInode:
   163  		if inode.Size < 1 {
   164  			if err := createEmptyFile(filename); err != nil {
   165  				return err
   166  			}
   167  		}
   168  		if err := inode.WriteMetadata(filename); err != nil {
   169  			return err
   170  		}
   171  	case *filesystem.ComputedRegularInode:
   172  		if err := createEmptyFile(filename); err != nil {
   173  			return err
   174  		}
   175  		tmpInode := &filesystem.RegularInode{
   176  			Mode: inode.Mode,
   177  			Uid:  inode.Uid,
   178  			Gid:  inode.Gid,
   179  		}
   180  		if err := tmpInode.WriteMetadata(filename); err != nil {
   181  			return err
   182  		}
   183  	case *filesystem.SymlinkInode:
   184  		if err := inode.Write(filename); err != nil {
   185  			return err
   186  		}
   187  	case *filesystem.SpecialInode:
   188  		if err := inode.Write(filename); err != nil {
   189  			return err
   190  		}
   191  	case *filesystem.DirectoryInode:
   192  		if err := inode.Write(filename); err != nil {
   193  			return err
   194  		}
   195  	default:
   196  		return errors.New("unsupported inode type")
   197  	}
   198  	return nil
   199  }
   200  
   201  func writeInodes(inodeTable filesystem.InodeTable, inodesDir string) error {
   202  	for inodeNumber, inode := range inodeTable {
   203  		filename := path.Join(inodesDir, fmt.Sprintf("%d", inodeNumber))
   204  		if err := writeInode(inode, filename); err != nil {
   205  			return err
   206  		}
   207  	}
   208  	return nil
   209  }
   210  
   211  func buildTree(directory *filesystem.DirectoryInode,
   212  	rootDir, mySubPathName, inodesDir string) error {
   213  	for _, dirent := range directory.EntryList {
   214  		oldPath := path.Join(inodesDir, fmt.Sprintf("%d", dirent.InodeNumber))
   215  		newSubPath := path.Join(mySubPathName, dirent.Name)
   216  		newFullPath := path.Join(rootDir, newSubPath)
   217  		if inode := dirent.Inode(); inode == nil {
   218  			panic("no inode pointer for: " + newSubPath)
   219  		} else if dinode, ok := inode.(*filesystem.DirectoryInode); ok {
   220  			if err := renameDir(oldPath, newFullPath, dinode); err != nil {
   221  				return err
   222  			}
   223  			err := buildTree(dinode, rootDir, newSubPath, inodesDir)
   224  			if err != nil {
   225  				return err
   226  			}
   227  		} else {
   228  			if err := link(oldPath, newFullPath, inode); err != nil {
   229  				if !os.IsNotExist(err) {
   230  					return err
   231  				}
   232  			}
   233  		}
   234  	}
   235  	return nil
   236  }
   237  
   238  func link(oldname, newname string, inode filesystem.GenericInode) error {
   239  	if err := os.Link(oldname, newname); err == nil {
   240  		return nil
   241  	}
   242  	if finode, ok := inode.(*filesystem.RegularInode); ok {
   243  		if finode.Size > 0 {
   244  			reader, err := os.Open(oldname)
   245  			if err != nil {
   246  				return err
   247  			}
   248  			defer reader.Close()
   249  			writer, err := os.Create(newname)
   250  			if err != nil {
   251  				return err
   252  			}
   253  			defer writer.Close()
   254  			if _, err := io.Copy(writer, reader); err != nil {
   255  				return err
   256  			}
   257  		}
   258  	}
   259  	if err := writeInode(inode, newname); err != nil {
   260  		return err
   261  	}
   262  	return nil
   263  }
   264  
   265  func renameDir(oldpath, newpath string,
   266  	inode *filesystem.DirectoryInode) error {
   267  	if err := os.Rename(oldpath, newpath); err == nil {
   268  		return nil
   269  	}
   270  	if oldFi, err := os.Lstat(oldpath); err != nil {
   271  		return err
   272  	} else {
   273  		if !oldFi.IsDir() {
   274  			return fmt.Errorf("%s is not a directory", oldpath)
   275  		}
   276  	}
   277  	if newFi, err := os.Lstat(newpath); err != nil {
   278  		if !os.IsNotExist(err) {
   279  			return err
   280  		}
   281  		if err := inode.Write(newpath); err != nil {
   282  			return err
   283  		}
   284  	} else {
   285  		if !newFi.IsDir() {
   286  			return fmt.Errorf("%s is not a directory", newpath)
   287  		}
   288  		if err := inode.WriteMetadata(newpath); err != nil {
   289  			return err
   290  		}
   291  	}
   292  	if err := os.Remove(oldpath); err != nil {
   293  		return err
   294  	}
   295  	return nil
   296  }