github.com/cloud-foundations/dominator@v0.0.0-20221004181915-6e4fee580046/lib/filesystem/util/unpack.go (about) 1 package util 2 3 import ( 4 "errors" 5 "fmt" 6 "io" 7 "os" 8 "path" 9 "syscall" 10 "time" 11 12 "github.com/Cloud-Foundations/Dominator/lib/filesystem" 13 "github.com/Cloud-Foundations/Dominator/lib/format" 14 "github.com/Cloud-Foundations/Dominator/lib/hash" 15 "github.com/Cloud-Foundations/Dominator/lib/log" 16 "github.com/Cloud-Foundations/Dominator/lib/objectserver" 17 ) 18 19 const ( 20 dirPerms = syscall.S_IRWXU 21 filePerms = syscall.S_IRUSR | syscall.S_IWUSR | syscall.S_IRGRP 22 ) 23 24 func createEmptyFile(filename string) error { 25 if file, err := os.Create(filename); err != nil { 26 return err 27 } else { 28 // Don't wait for finaliser to close, otherwise we can have too many 29 // open files. 30 file.Close() 31 return nil 32 } 33 } 34 35 func unpack(fs *filesystem.FileSystem, objectsGetter objectserver.ObjectsGetter, 36 dirname string, logger log.Logger) error { 37 for _, entry := range fs.EntryList { 38 if entry.Name == ".inodes" { 39 return errors.New("cannot unpack a file-system with /.inodes") 40 } 41 } 42 os.Mkdir(dirname, dirPerms) 43 inodesDir := path.Join(dirname, ".inodes") 44 if err := os.Mkdir(inodesDir, dirPerms); err != nil { 45 return err 46 } 47 defer os.RemoveAll(inodesDir) 48 var statfs syscall.Statfs_t 49 if err := syscall.Statfs(inodesDir, &statfs); err != nil { 50 return errors.New(fmt.Sprintf("Unable to Statfs: %s %s\n", 51 inodesDir, err)) 52 } 53 if fs.TotalDataBytes > uint64(statfs.Bsize)*statfs.Bfree { 54 return errors.New("image will not fit on file-system") 55 } 56 hashes, inums, lengths := getHashes(fs) 57 err := writeObjects(objectsGetter, hashes, inums, lengths, inodesDir, 58 logger) 59 if err != nil { 60 return err 61 } 62 startWriteTime := time.Now() 63 if err := writeInodes(fs.InodeTable, inodesDir); err != nil { 64 return err 65 } 66 if err = fs.DirectoryInode.Write(dirname); err != nil { 67 return err 68 } 69 startBuildTime := time.Now() 70 writeDuration := startBuildTime.Sub(startWriteTime) 71 err = buildTree(&fs.DirectoryInode, dirname, "", inodesDir) 72 if err != nil { 73 return err 74 } 75 buildDuration := time.Since(startBuildTime) 76 logger.Printf("Unpacked file-system: made inodes in %s, built tree in %s\n", 77 format.Duration(writeDuration), format.Duration(buildDuration)) 78 return nil 79 } 80 81 func getHashes(fs *filesystem.FileSystem) ([]hash.Hash, []uint64, []uint64) { 82 hashes := make([]hash.Hash, 0, fs.NumRegularInodes) 83 inums := make([]uint64, 0, fs.NumRegularInodes) 84 lengths := make([]uint64, 0, fs.NumRegularInodes) 85 for inum, inode := range fs.InodeTable { 86 if inode, ok := inode.(*filesystem.RegularInode); ok { 87 if inode.Size > 0 { 88 hashes = append(hashes, inode.Hash) 89 inums = append(inums, inum) 90 lengths = append(lengths, inode.Size) 91 } 92 } 93 } 94 return hashes, inums, lengths 95 } 96 97 func writeObjects(objectsGetter objectserver.ObjectsGetter, hashes []hash.Hash, 98 inums []uint64, lengths []uint64, inodesDir string, 99 logger log.Logger) error { 100 startTime := time.Now() 101 objectsReader, err := objectsGetter.GetObjects(hashes) 102 if err != nil { 103 return fmt.Errorf("error getting object reader: %s\n", err) 104 } 105 defer objectsReader.Close() 106 var totalLength uint64 107 buffer := make([]byte, 32<<10) 108 for index := range hashes { 109 err = writeObject(objectsReader, inums[index], lengths[index], 110 inodesDir, buffer) 111 if err != nil { 112 return err 113 } 114 totalLength += lengths[index] 115 } 116 duration := time.Since(startTime) 117 speed := uint64(float64(totalLength) / duration.Seconds()) 118 logger.Printf("Copied %d objects (%s) in %s (%s/s)\n", 119 len(hashes), format.FormatBytes(totalLength), format.Duration(duration), 120 format.FormatBytes(speed)) 121 return nil 122 } 123 124 func writeObject(objectsReader objectserver.ObjectsReader, inodeNumber uint64, 125 length uint64, inodesDir string, buffer []byte) error { 126 rlength, reader, err := objectsReader.NextObject() 127 if err != nil { 128 return err 129 } 130 defer reader.Close() 131 if rlength != length { 132 return errors.New("mismatched lengths") 133 } 134 filename := path.Join(inodesDir, fmt.Sprintf("%d", inodeNumber)) 135 destFile, err := os.OpenFile(filename, 136 os.O_CREATE|os.O_TRUNC|os.O_WRONLY, filePerms) 137 if err != nil { 138 return err 139 } 140 doClose := true 141 defer func() { 142 if doClose { 143 destFile.Close() 144 } 145 }() 146 iLength := int64(length) 147 nCopied, err := io.CopyBuffer(destFile, io.LimitReader(reader, iLength), 148 buffer) 149 if err != nil { 150 return fmt.Errorf("error copying: %s", err) 151 } 152 if nCopied != iLength { 153 return fmt.Errorf("expected length: %d, got: %d for: %s\n", 154 length, nCopied, filename) 155 } 156 doClose = false 157 return destFile.Close() 158 } 159 160 func writeInode(inode filesystem.GenericInode, filename string) error { 161 switch inode := inode.(type) { 162 case *filesystem.RegularInode: 163 if inode.Size < 1 { 164 if err := createEmptyFile(filename); err != nil { 165 return err 166 } 167 } 168 if err := inode.WriteMetadata(filename); err != nil { 169 return err 170 } 171 case *filesystem.ComputedRegularInode: 172 if err := createEmptyFile(filename); err != nil { 173 return err 174 } 175 tmpInode := &filesystem.RegularInode{ 176 Mode: inode.Mode, 177 Uid: inode.Uid, 178 Gid: inode.Gid, 179 } 180 if err := tmpInode.WriteMetadata(filename); err != nil { 181 return err 182 } 183 case *filesystem.SymlinkInode: 184 if err := inode.Write(filename); err != nil { 185 return err 186 } 187 case *filesystem.SpecialInode: 188 if err := inode.Write(filename); err != nil { 189 return err 190 } 191 case *filesystem.DirectoryInode: 192 if err := inode.Write(filename); err != nil { 193 return err 194 } 195 default: 196 return errors.New("unsupported inode type") 197 } 198 return nil 199 } 200 201 func writeInodes(inodeTable filesystem.InodeTable, inodesDir string) error { 202 for inodeNumber, inode := range inodeTable { 203 filename := path.Join(inodesDir, fmt.Sprintf("%d", inodeNumber)) 204 if err := writeInode(inode, filename); err != nil { 205 return err 206 } 207 } 208 return nil 209 } 210 211 func buildTree(directory *filesystem.DirectoryInode, 212 rootDir, mySubPathName, inodesDir string) error { 213 for _, dirent := range directory.EntryList { 214 oldPath := path.Join(inodesDir, fmt.Sprintf("%d", dirent.InodeNumber)) 215 newSubPath := path.Join(mySubPathName, dirent.Name) 216 newFullPath := path.Join(rootDir, newSubPath) 217 if inode := dirent.Inode(); inode == nil { 218 panic("no inode pointer for: " + newSubPath) 219 } else if dinode, ok := inode.(*filesystem.DirectoryInode); ok { 220 if err := renameDir(oldPath, newFullPath, dinode); err != nil { 221 return err 222 } 223 err := buildTree(dinode, rootDir, newSubPath, inodesDir) 224 if err != nil { 225 return err 226 } 227 } else { 228 if err := link(oldPath, newFullPath, inode); err != nil { 229 if !os.IsNotExist(err) { 230 return err 231 } 232 } 233 } 234 } 235 return nil 236 } 237 238 func link(oldname, newname string, inode filesystem.GenericInode) error { 239 if err := os.Link(oldname, newname); err == nil { 240 return nil 241 } 242 if finode, ok := inode.(*filesystem.RegularInode); ok { 243 if finode.Size > 0 { 244 reader, err := os.Open(oldname) 245 if err != nil { 246 return err 247 } 248 defer reader.Close() 249 writer, err := os.Create(newname) 250 if err != nil { 251 return err 252 } 253 defer writer.Close() 254 if _, err := io.Copy(writer, reader); err != nil { 255 return err 256 } 257 } 258 } 259 if err := writeInode(inode, newname); err != nil { 260 return err 261 } 262 return nil 263 } 264 265 func renameDir(oldpath, newpath string, 266 inode *filesystem.DirectoryInode) error { 267 if err := os.Rename(oldpath, newpath); err == nil { 268 return nil 269 } 270 if oldFi, err := os.Lstat(oldpath); err != nil { 271 return err 272 } else { 273 if !oldFi.IsDir() { 274 return fmt.Errorf("%s is not a directory", oldpath) 275 } 276 } 277 if newFi, err := os.Lstat(newpath); err != nil { 278 if !os.IsNotExist(err) { 279 return err 280 } 281 if err := inode.Write(newpath); err != nil { 282 return err 283 } 284 } else { 285 if !newFi.IsDir() { 286 return fmt.Errorf("%s is not a directory", newpath) 287 } 288 if err := inode.WriteMetadata(newpath); err != nil { 289 return err 290 } 291 } 292 if err := os.Remove(oldpath); err != nil { 293 return err 294 } 295 return nil 296 }