github.com/stackdocker/rkt@v0.10.1-0.20151109095037-1aa827478248/store/tree.go (about) 1 // Copyright 2015 The rkt Authors 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package store 16 17 import ( 18 "archive/tar" 19 "crypto/sha512" 20 "encoding/json" 21 "fmt" 22 "io" 23 "io/ioutil" 24 "os" 25 "path/filepath" 26 "sort" 27 "syscall" 28 29 specaci "github.com/coreos/rkt/Godeps/_workspace/src/github.com/appc/spec/aci" 30 "github.com/coreos/rkt/Godeps/_workspace/src/github.com/appc/spec/pkg/tarheader" 31 "github.com/coreos/rkt/Godeps/_workspace/src/github.com/appc/spec/schema/types" 32 "github.com/coreos/rkt/pkg/aci" 33 "github.com/coreos/rkt/pkg/sys" 34 "github.com/coreos/rkt/pkg/uid" 35 ) 36 37 const ( 38 hashfilename = "hash" 39 renderedfilename = "rendered" 40 ) 41 42 // TreeStore represents a store of rendered ACIs 43 type TreeStore struct { 44 path string 45 } 46 47 // Write renders the ACI with the provided key in the treestore. id references 48 // that specific tree store rendered image. 49 // Write, to avoid having a rendered ACI with old stale files, requires that 50 // the destination directory doesn't exist (usually Remove should be called 51 // before Write) 52 func (ts *TreeStore) Write(id string, key string, s *Store) error { 53 treepath := ts.GetPath(id) 54 fi, _ := os.Stat(treepath) 55 if fi != nil { 56 return fmt.Errorf("treestore: path %s already exists", treepath) 57 } 58 imageID, err := types.NewHash(key) 59 if err != nil { 60 return fmt.Errorf("treestore: cannot convert key to imageID: %v", err) 61 } 62 if err := os.MkdirAll(treepath, 0755); err != nil { 63 return fmt.Errorf("treestore: cannot create treestore directory %s: %v", treepath, err) 64 } 65 err = aci.RenderACIWithImageID(*imageID, treepath, s, uid.NewBlankUidRange()) 66 if err != nil { 67 return fmt.Errorf("treestore: cannot render aci: %v", err) 68 } 69 hash, err := ts.Hash(id) 70 if err != nil { 71 return fmt.Errorf("treestore: cannot calculate tree hash: %v", err) 72 } 73 err = ioutil.WriteFile(filepath.Join(treepath, hashfilename), []byte(hash), 0644) 74 if err != nil { 75 return fmt.Errorf("treestore: cannot write hash file: %v", err) 76 } 77 // before creating the "rendered" flag file we need to ensure that all data is fsynced 78 dfd, err := syscall.Open(treepath, syscall.O_RDONLY, 0) 79 if err != nil { 80 return err 81 } 82 defer syscall.Close(dfd) 83 if err := sys.Syncfs(dfd); err != nil { 84 return fmt.Errorf("treestore: failed to sync data: %v", err) 85 } 86 // Create rendered file 87 f, err := os.Create(filepath.Join(treepath, renderedfilename)) 88 if err != nil { 89 return fmt.Errorf("treestore: failed to write rendered file: %v", err) 90 } 91 f.Close() 92 93 if err := syscall.Fsync(dfd); err != nil { 94 return fmt.Errorf("treestore: failed to sync tree store directory: %v", err) 95 } 96 return nil 97 } 98 99 // Remove cleans the directory for the provided id 100 func (ts *TreeStore) Remove(id string) error { 101 treepath := ts.GetPath(id) 102 // If tree path doesn't exist we're done 103 _, err := os.Stat(treepath) 104 if err != nil && os.IsNotExist(err) { 105 return nil 106 } 107 if err != nil { 108 return fmt.Errorf("treestore: failed to open tree store directory: %v", err) 109 } 110 111 renderedFilePath := filepath.Join(treepath, renderedfilename) 112 // The "rendered" flag file should be the firstly removed file. So if 113 // the removal ends with some error leaving some stale files IsRendered() 114 // will return false. 115 _, err = os.Stat(renderedFilePath) 116 if err != nil && !os.IsNotExist(err) { 117 return err 118 } 119 if !os.IsNotExist(err) { 120 err := os.Remove(renderedFilePath) 121 // Ensure that the treepath directory is fsynced after removing the 122 // "rendered" flag file 123 f, err := os.Open(treepath) 124 if err != nil { 125 return fmt.Errorf("treestore: failed to open tree store directory: %v", err) 126 } 127 defer f.Close() 128 err = f.Sync() 129 if err != nil { 130 return fmt.Errorf("treestore: failed to sync tree store directory: %v", err) 131 } 132 } 133 return os.RemoveAll(treepath) 134 } 135 136 // IsRendered checks if the tree store with the provided id is fully rendered 137 func (ts *TreeStore) IsRendered(id string) (bool, error) { 138 // if the "rendered" flag file exists, assume that the store is already 139 // fully rendered. 140 treepath := ts.GetPath(id) 141 _, err := os.Stat(filepath.Join(treepath, renderedfilename)) 142 if os.IsNotExist(err) { 143 return false, nil 144 } 145 if err != nil { 146 return false, err 147 } 148 return true, nil 149 } 150 151 // GetPath returns the absolute path of the treestore for the provided id. 152 // It doesn't ensure that the path exists and is fully rendered. This should 153 // be done calling IsRendered() 154 func (ts *TreeStore) GetPath(id string) string { 155 return filepath.Join(ts.path, id) 156 } 157 158 // GetRootFS returns the absolute path of the rootfs for the provided id. 159 // It doesn't ensure that the rootfs exists and is fully rendered. This should 160 // be done calling IsRendered() 161 func (ts *TreeStore) GetRootFS(id string) string { 162 return filepath.Join(ts.GetPath(id), "rootfs") 163 } 164 165 // Hash calculates an hash of the rendered ACI. It uses the same functions 166 // used to create a tar but instead of writing the full archive is just 167 // computes the sha512 sum of the file infos and contents. 168 func (ts *TreeStore) Hash(id string) (string, error) { 169 treepath := ts.GetPath(id) 170 171 hash := sha512.New() 172 iw := NewHashWriter(hash) 173 err := filepath.Walk(treepath, buildWalker(treepath, iw)) 174 if err != nil { 175 return "", fmt.Errorf("treestore: error walking rootfs: %v", err) 176 } 177 178 hashstring := hashToKey(hash) 179 180 return hashstring, nil 181 } 182 183 // Check calculates the actual rendered ACI's hash and verifies that it matches 184 // the saved value. 185 func (ts *TreeStore) Check(id string) error { 186 treepath := ts.GetPath(id) 187 hash, err := ioutil.ReadFile(filepath.Join(treepath, hashfilename)) 188 if err != nil { 189 return fmt.Errorf("treestore: cannot read hash file: %v", err) 190 } 191 curhash, err := ts.Hash(id) 192 if err != nil { 193 return fmt.Errorf("treestore: cannot calculate tree hash: %v", err) 194 } 195 if curhash != string(hash) { 196 return fmt.Errorf("treestore: wrong tree hash: %s, expected: %s", curhash, hash) 197 } 198 return nil 199 } 200 201 type xattr struct { 202 Name string 203 Value string 204 } 205 206 // Like tar Header but, to keep json output reproducible: 207 // * Xattrs as a slice 208 // * Skip Uname and Gname 209 // TODO. Should ModTime/AccessTime/ChangeTime be saved? For validation its 210 // probably enough to hash the file contents and the other infos and avoid 211 // problems due to them changing. 212 // TODO(sgotti) Is it possible that json output will change between go 213 // versions? Use another or our own Marshaller? 214 type fileInfo struct { 215 Name string // name of header file entry 216 Mode int64 // permission and mode bits 217 Uid int // user id of owner 218 Gid int // group id of owner 219 Size int64 // length in bytes 220 Typeflag byte // type of header entry 221 Linkname string // target name of link 222 Devmajor int64 // major number of character or block device 223 Devminor int64 // minor number of character or block device 224 Xattrs []xattr 225 } 226 227 func FileInfoFromHeader(hdr *tar.Header) *fileInfo { 228 fi := &fileInfo{ 229 Name: hdr.Name, 230 Mode: hdr.Mode, 231 Uid: hdr.Uid, 232 Gid: hdr.Gid, 233 Size: hdr.Size, 234 Typeflag: hdr.Typeflag, 235 Linkname: hdr.Linkname, 236 Devmajor: hdr.Devmajor, 237 Devminor: hdr.Devminor, 238 } 239 keys := make([]string, len(hdr.Xattrs)) 240 for k := range hdr.Xattrs { 241 keys = append(keys, k) 242 } 243 sort.Strings(keys) 244 245 xattrs := make([]xattr, 0) 246 for _, k := range keys { 247 xattrs = append(xattrs, xattr{Name: k, Value: hdr.Xattrs[k]}) 248 } 249 fi.Xattrs = xattrs 250 return fi 251 } 252 253 // TODO(sgotti) this func is copied from appcs/spec/aci/build.go but also 254 // removes the hashfile and the renderedfile. Find a way to reuse it. 255 func buildWalker(root string, aw specaci.ArchiveWriter) filepath.WalkFunc { 256 // cache of inode -> filepath, used to leverage hard links in the archive 257 inos := map[uint64]string{} 258 return func(path string, info os.FileInfo, err error) error { 259 if err != nil { 260 return err 261 } 262 relpath, err := filepath.Rel(root, path) 263 if err != nil { 264 return err 265 } 266 if relpath == "." { 267 return nil 268 } 269 if relpath == specaci.ManifestFile || relpath == hashfilename || relpath == renderedfilename { 270 // ignore; this will be written by the archive writer 271 // TODO(jonboulle): does this make sense? maybe just remove from archivewriter? 272 return nil 273 } 274 275 link := "" 276 var r io.Reader 277 switch info.Mode() & os.ModeType { 278 case os.ModeSocket: 279 return nil 280 case os.ModeNamedPipe: 281 case os.ModeCharDevice: 282 case os.ModeDevice: 283 case os.ModeDir: 284 case os.ModeSymlink: 285 target, err := os.Readlink(path) 286 if err != nil { 287 return err 288 } 289 link = target 290 default: 291 file, err := os.Open(path) 292 if err != nil { 293 return err 294 } 295 defer file.Close() 296 r = file 297 } 298 299 hdr, err := tar.FileInfoHeader(info, link) 300 if err != nil { 301 panic(err) 302 } 303 // Because os.FileInfo's Name method returns only the base 304 // name of the file it describes, it may be necessary to 305 // modify the Name field of the returned header to provide the 306 // full path name of the file. 307 hdr.Name = relpath 308 tarheader.Populate(hdr, info, inos) 309 // If the file is a hard link to a file we've already seen, we 310 // don't need the contents 311 if hdr.Typeflag == tar.TypeLink { 312 hdr.Size = 0 313 r = nil 314 } 315 316 if err := aw.AddFile(hdr, r); err != nil { 317 return err 318 } 319 return nil 320 } 321 } 322 323 type imageHashWriter struct { 324 io.Writer 325 } 326 327 func NewHashWriter(w io.Writer) specaci.ArchiveWriter { 328 return &imageHashWriter{w} 329 } 330 331 func (aw *imageHashWriter) AddFile(hdr *tar.Header, r io.Reader) error { 332 // Write the json encoding of the FileInfo struct 333 hdrj, err := json.Marshal(FileInfoFromHeader(hdr)) 334 if err != nil { 335 return err 336 } 337 _, err = aw.Writer.Write(hdrj) 338 if err != nil { 339 return err 340 } 341 342 if r != nil { 343 // Write the file data 344 _, err := io.Copy(aw.Writer, r) 345 if err != nil { 346 return err 347 } 348 } 349 350 return nil 351 } 352 353 func (aw *imageHashWriter) Close() error { 354 return nil 355 }