github.com/mheon/docker@v0.11.2-0.20150922122814-44f47903a831/graph/graph.go (about) 1 package graph 2 3 import ( 4 "compress/gzip" 5 "encoding/json" 6 "errors" 7 "fmt" 8 "io" 9 "io/ioutil" 10 "os" 11 "path/filepath" 12 "runtime" 13 "strconv" 14 "strings" 15 "sync" 16 "time" 17 18 "github.com/Sirupsen/logrus" 19 "github.com/docker/distribution/digest" 20 "github.com/docker/docker/autogen/dockerversion" 21 "github.com/docker/docker/daemon/graphdriver" 22 "github.com/docker/docker/image" 23 "github.com/docker/docker/pkg/archive" 24 "github.com/docker/docker/pkg/progressreader" 25 "github.com/docker/docker/pkg/streamformatter" 26 "github.com/docker/docker/pkg/stringid" 27 "github.com/docker/docker/pkg/system" 28 "github.com/docker/docker/pkg/truncindex" 29 "github.com/docker/docker/runconfig" 30 "github.com/vbatts/tar-split/tar/asm" 31 "github.com/vbatts/tar-split/tar/storage" 32 ) 33 34 // The type is used to protect pulling or building related image 35 // layers from deleteing when filtered by dangling=true 36 // The key of layers is the images ID which is pulling or building 37 // The value of layers is a slice which hold layer IDs referenced to 38 // pulling or building images 39 type retainedLayers struct { 40 layerHolders map[string]map[string]struct{} // map[layerID]map[sessionID] 41 sync.Mutex 42 } 43 44 func (r *retainedLayers) Add(sessionID string, layerIDs []string) { 45 r.Lock() 46 defer r.Unlock() 47 for _, layerID := range layerIDs { 48 if r.layerHolders[layerID] == nil { 49 r.layerHolders[layerID] = map[string]struct{}{} 50 } 51 r.layerHolders[layerID][sessionID] = struct{}{} 52 } 53 } 54 55 func (r *retainedLayers) Delete(sessionID string, layerIDs []string) { 56 r.Lock() 57 defer r.Unlock() 58 for _, layerID := range layerIDs { 59 holders, ok := r.layerHolders[layerID] 60 if !ok { 61 continue 62 } 63 delete(holders, sessionID) 64 if len(holders) == 0 { 65 delete(r.layerHolders, layerID) // Delete any empty reference set. 66 } 67 } 68 } 69 70 func (r *retainedLayers) Exists(layerID string) bool { 71 r.Lock() 72 _, exists := r.layerHolders[layerID] 73 r.Unlock() 74 return exists 75 } 76 77 // A Graph is a store for versioned filesystem images and the relationship between them. 78 type Graph struct { 79 root string 80 idIndex *truncindex.TruncIndex 81 driver graphdriver.Driver 82 imageMutex imageMutex // protect images in driver. 83 retained *retainedLayers 84 tarSplitDisabled bool 85 } 86 87 // file names for ./graph/<ID>/ 88 const ( 89 jsonFileName = "json" 90 layersizeFileName = "layersize" 91 digestFileName = "checksum" 92 tarDataFileName = "tar-data.json.gz" 93 ) 94 95 var ( 96 // ErrDigestNotSet is used when request the digest for a layer 97 // but the layer has no digest value or content to compute the 98 // the digest. 99 ErrDigestNotSet = errors.New("digest is not set for layer") 100 ) 101 102 // NewGraph instantiates a new graph at the given root path in the filesystem. 103 // `root` will be created if it doesn't exist. 104 func NewGraph(root string, driver graphdriver.Driver) (*Graph, error) { 105 abspath, err := filepath.Abs(root) 106 if err != nil { 107 return nil, err 108 } 109 // Create the root directory if it doesn't exists 110 if err := system.MkdirAll(root, 0700); err != nil { 111 return nil, err 112 } 113 114 graph := &Graph{ 115 root: abspath, 116 idIndex: truncindex.NewTruncIndex([]string{}), 117 driver: driver, 118 retained: &retainedLayers{layerHolders: make(map[string]map[string]struct{})}, 119 } 120 121 // Windows does not currently support tarsplit functionality. 122 if runtime.GOOS == "windows" { 123 graph.tarSplitDisabled = true 124 } 125 126 if err := graph.restore(); err != nil { 127 return nil, err 128 } 129 return graph, nil 130 } 131 132 // IsHeld returns whether the given layerID is being used by an ongoing pull or build. 133 func (graph *Graph) IsHeld(layerID string) bool { 134 return graph.retained.Exists(layerID) 135 } 136 137 func (graph *Graph) restore() error { 138 dir, err := ioutil.ReadDir(graph.root) 139 if err != nil { 140 return err 141 } 142 var ids = []string{} 143 for _, v := range dir { 144 id := v.Name() 145 if graph.driver.Exists(id) { 146 ids = append(ids, id) 147 } 148 } 149 150 graph.idIndex = truncindex.NewTruncIndex(ids) 151 logrus.Debugf("Restored %d elements", len(ids)) 152 return nil 153 } 154 155 // IsNotExist detects whether an image exists by parsing the incoming error 156 // message. 157 func (graph *Graph) IsNotExist(err error, id string) bool { 158 // FIXME: Implement error subclass instead of looking at the error text 159 // Note: This is the way golang implements os.IsNotExists on Plan9 160 return err != nil && (strings.Contains(strings.ToLower(err.Error()), "does not exist") || strings.Contains(strings.ToLower(err.Error()), "no such")) && strings.Contains(err.Error(), id) 161 } 162 163 // Exists returns true if an image is registered at the given id. 164 // If the image doesn't exist or if an error is encountered, false is returned. 165 func (graph *Graph) Exists(id string) bool { 166 if _, err := graph.Get(id); err != nil { 167 return false 168 } 169 return true 170 } 171 172 // Get returns the image with the given id, or an error if the image doesn't exist. 173 func (graph *Graph) Get(name string) (*image.Image, error) { 174 id, err := graph.idIndex.Get(name) 175 if err != nil { 176 return nil, fmt.Errorf("could not find image: %v", err) 177 } 178 img, err := graph.loadImage(id) 179 if err != nil { 180 return nil, err 181 } 182 if img.ID != id { 183 return nil, fmt.Errorf("Image stored at '%s' has wrong id '%s'", id, img.ID) 184 } 185 186 if img.Size < 0 { 187 size, err := graph.driver.DiffSize(img.ID, img.Parent) 188 if err != nil { 189 return nil, fmt.Errorf("unable to calculate size of image id %q: %s", img.ID, err) 190 } 191 192 img.Size = size 193 if err := graph.saveSize(graph.imageRoot(id), img.Size); err != nil { 194 return nil, err 195 } 196 } 197 return img, nil 198 } 199 200 // Create creates a new image and registers it in the graph. 201 func (graph *Graph) Create(layerData io.Reader, containerID, containerImage, comment, author string, containerConfig, config *runconfig.Config) (*image.Image, error) { 202 img := &image.Image{ 203 ID: stringid.GenerateRandomID(), 204 Comment: comment, 205 Created: time.Now().UTC(), 206 DockerVersion: dockerversion.VERSION, 207 Author: author, 208 Config: config, 209 Architecture: runtime.GOARCH, 210 OS: runtime.GOOS, 211 } 212 213 if containerID != "" { 214 img.Parent = containerImage 215 img.Container = containerID 216 img.ContainerConfig = *containerConfig 217 } 218 219 if err := graph.Register(img, layerData); err != nil { 220 return nil, err 221 } 222 return img, nil 223 } 224 225 // Register imports a pre-existing image into the graph. 226 // Returns nil if the image is already registered. 227 func (graph *Graph) Register(img *image.Image, layerData io.Reader) (err error) { 228 229 if err := image.ValidateID(img.ID); err != nil { 230 return err 231 } 232 233 // We need this entire operation to be atomic within the engine. Note that 234 // this doesn't mean Register is fully safe yet. 235 graph.imageMutex.Lock(img.ID) 236 defer graph.imageMutex.Unlock(img.ID) 237 238 // Skip register if image is already registered 239 if graph.Exists(img.ID) { 240 return nil 241 } 242 243 // The returned `error` must be named in this function's signature so that 244 // `err` is not shadowed in this deferred cleanup. 245 defer func() { 246 // If any error occurs, remove the new dir from the driver. 247 // Don't check for errors since the dir might not have been created. 248 if err != nil { 249 graph.driver.Remove(img.ID) 250 } 251 }() 252 253 // Ensure that the image root does not exist on the filesystem 254 // when it is not registered in the graph. 255 // This is common when you switch from one graph driver to another 256 if err := os.RemoveAll(graph.imageRoot(img.ID)); err != nil && !os.IsNotExist(err) { 257 return err 258 } 259 260 // If the driver has this ID but the graph doesn't, remove it from the driver to start fresh. 261 // (the graph is the source of truth). 262 // Ignore errors, since we don't know if the driver correctly returns ErrNotExist. 263 // (FIXME: make that mandatory for drivers). 264 graph.driver.Remove(img.ID) 265 266 tmp, err := graph.mktemp() 267 defer os.RemoveAll(tmp) 268 if err != nil { 269 return fmt.Errorf("mktemp failed: %s", err) 270 } 271 272 // Create root filesystem in the driver 273 if err := createRootFilesystemInDriver(graph, img); err != nil { 274 return err 275 } 276 277 // Apply the diff/layer 278 if err := graph.storeImage(img, layerData, tmp); err != nil { 279 return err 280 } 281 // Commit 282 if err := os.Rename(tmp, graph.imageRoot(img.ID)); err != nil { 283 return err 284 } 285 graph.idIndex.Add(img.ID) 286 return nil 287 } 288 289 func createRootFilesystemInDriver(graph *Graph, img *image.Image) error { 290 if err := graph.driver.Create(img.ID, img.Parent); err != nil { 291 return fmt.Errorf("Driver %s failed to create image rootfs %s: %s", graph.driver, img.ID, err) 292 } 293 return nil 294 } 295 296 // TempLayerArchive creates a temporary archive of the given image's filesystem layer. 297 // The archive is stored on disk and will be automatically deleted as soon as has been read. 298 // If output is not nil, a human-readable progress bar will be written to it. 299 func (graph *Graph) TempLayerArchive(id string, sf *streamformatter.StreamFormatter, output io.Writer) (*archive.TempArchive, error) { 300 image, err := graph.Get(id) 301 if err != nil { 302 return nil, err 303 } 304 tmp, err := graph.mktemp() 305 if err != nil { 306 return nil, err 307 } 308 a, err := graph.TarLayer(image) 309 if err != nil { 310 return nil, err 311 } 312 progressReader := progressreader.New(progressreader.Config{ 313 In: a, 314 Out: output, 315 Formatter: sf, 316 Size: 0, 317 NewLines: false, 318 ID: stringid.TruncateID(id), 319 Action: "Buffering to disk", 320 }) 321 defer progressReader.Close() 322 return archive.NewTempArchive(progressReader, tmp) 323 } 324 325 // mktemp creates a temporary sub-directory inside the graph's filesystem. 326 func (graph *Graph) mktemp() (string, error) { 327 dir := filepath.Join(graph.root, "_tmp", stringid.GenerateNonCryptoID()) 328 if err := system.MkdirAll(dir, 0700); err != nil { 329 return "", err 330 } 331 return dir, nil 332 } 333 334 func (graph *Graph) newTempFile() (*os.File, error) { 335 tmp, err := graph.mktemp() 336 if err != nil { 337 return nil, err 338 } 339 return ioutil.TempFile(tmp, "") 340 } 341 342 // Delete atomically removes an image from the graph. 343 func (graph *Graph) Delete(name string) error { 344 id, err := graph.idIndex.Get(name) 345 if err != nil { 346 return err 347 } 348 tmp, err := graph.mktemp() 349 graph.idIndex.Delete(id) 350 if err == nil { 351 if err := os.Rename(graph.imageRoot(id), tmp); err != nil { 352 // On err make tmp point to old dir and cleanup unused tmp dir 353 os.RemoveAll(tmp) 354 tmp = graph.imageRoot(id) 355 } 356 } else { 357 // On err make tmp point to old dir for cleanup 358 tmp = graph.imageRoot(id) 359 } 360 // Remove rootfs data from the driver 361 graph.driver.Remove(id) 362 // Remove the trashed image directory 363 return os.RemoveAll(tmp) 364 } 365 366 // Map returns a list of all images in the graph, addressable by ID. 367 func (graph *Graph) Map() map[string]*image.Image { 368 images := make(map[string]*image.Image) 369 graph.walkAll(func(image *image.Image) { 370 images[image.ID] = image 371 }) 372 return images 373 } 374 375 // walkAll iterates over each image in the graph, and passes it to a handler. 376 // The walking order is undetermined. 377 func (graph *Graph) walkAll(handler func(*image.Image)) { 378 graph.idIndex.Iterate(func(id string) { 379 if img, err := graph.Get(id); err != nil { 380 return 381 } else if handler != nil { 382 handler(img) 383 } 384 }) 385 } 386 387 // ByParent returns a lookup table of images by their parent. 388 // If an image of key ID has 3 children images, then the value for key ID 389 // will be a list of 3 images. 390 // If an image has no children, it will not have an entry in the table. 391 func (graph *Graph) ByParent() map[string][]*image.Image { 392 byParent := make(map[string][]*image.Image) 393 graph.walkAll(func(img *image.Image) { 394 parent, err := graph.Get(img.Parent) 395 if err != nil { 396 return 397 } 398 if children, exists := byParent[parent.ID]; exists { 399 byParent[parent.ID] = append(children, img) 400 } else { 401 byParent[parent.ID] = []*image.Image{img} 402 } 403 }) 404 return byParent 405 } 406 407 // HasChildren returns whether the given image has any child images. 408 func (graph *Graph) HasChildren(img *image.Image) bool { 409 return len(graph.ByParent()[img.ID]) > 0 410 } 411 412 // Retain keeps the images and layers that are in the pulling chain so that 413 // they are not deleted. If not retained, they may be deleted by rmi. 414 func (graph *Graph) Retain(sessionID string, layerIDs ...string) { 415 graph.retained.Add(sessionID, layerIDs) 416 } 417 418 // Release removes the referenced image ID from the provided set of layers. 419 func (graph *Graph) Release(sessionID string, layerIDs ...string) { 420 graph.retained.Delete(sessionID, layerIDs) 421 } 422 423 // Heads returns all heads in the graph, keyed by id. 424 // A head is an image which is not the parent of another image in the graph. 425 func (graph *Graph) Heads() map[string]*image.Image { 426 heads := make(map[string]*image.Image) 427 byParent := graph.ByParent() 428 graph.walkAll(func(image *image.Image) { 429 // If it's not in the byParent lookup table, then 430 // it's not a parent -> so it's a head! 431 if _, exists := byParent[image.ID]; !exists { 432 heads[image.ID] = image 433 } 434 }) 435 return heads 436 } 437 438 // TarLayer returns a tar archive of the image's filesystem layer. 439 func (graph *Graph) TarLayer(img *image.Image) (arch io.ReadCloser, err error) { 440 rdr, err := graph.assembleTarLayer(img) 441 if err != nil { 442 logrus.Debugf("[graph] TarLayer with traditional differ: %s", img.ID) 443 return graph.driver.Diff(img.ID, img.Parent) 444 } 445 return rdr, nil 446 } 447 448 func (graph *Graph) imageRoot(id string) string { 449 return filepath.Join(graph.root, id) 450 } 451 452 // loadImage fetches the image with the given id from the graph. 453 func (graph *Graph) loadImage(id string) (*image.Image, error) { 454 root := graph.imageRoot(id) 455 456 // Open the JSON file to decode by streaming 457 jsonSource, err := os.Open(jsonPath(root)) 458 if err != nil { 459 return nil, err 460 } 461 defer jsonSource.Close() 462 463 img := &image.Image{} 464 dec := json.NewDecoder(jsonSource) 465 466 // Decode the JSON data 467 if err := dec.Decode(img); err != nil { 468 return nil, err 469 } 470 if err := image.ValidateID(img.ID); err != nil { 471 return nil, err 472 } 473 474 if buf, err := ioutil.ReadFile(filepath.Join(root, layersizeFileName)); err != nil { 475 if !os.IsNotExist(err) { 476 return nil, err 477 } 478 // If the layersize file does not exist then set the size to a negative number 479 // because a layer size of 0 (zero) is valid 480 img.Size = -1 481 } else { 482 // Using Atoi here instead would temporarily convert the size to a machine 483 // dependent integer type, which causes images larger than 2^31 bytes to 484 // display negative sizes on 32-bit machines: 485 size, err := strconv.ParseInt(string(buf), 10, 64) 486 if err != nil { 487 return nil, err 488 } 489 img.Size = int64(size) 490 } 491 492 return img, nil 493 } 494 495 // saveSize stores the `size` in the provided graph `img` directory `root`. 496 func (graph *Graph) saveSize(root string, size int64) error { 497 if err := ioutil.WriteFile(filepath.Join(root, layersizeFileName), []byte(strconv.FormatInt(size, 10)), 0600); err != nil { 498 return fmt.Errorf("Error storing image size in %s/%s: %s", root, layersizeFileName, err) 499 } 500 return nil 501 } 502 503 // SetDigest sets the digest for the image layer to the provided value. 504 func (graph *Graph) SetDigest(id string, dgst digest.Digest) error { 505 graph.imageMutex.Lock(id) 506 defer graph.imageMutex.Unlock(id) 507 508 root := graph.imageRoot(id) 509 if err := ioutil.WriteFile(filepath.Join(root, digestFileName), []byte(dgst.String()), 0600); err != nil { 510 return fmt.Errorf("Error storing digest in %s/%s: %s", root, digestFileName, err) 511 } 512 return nil 513 } 514 515 // GetDigest gets the digest for the provide image layer id. 516 func (graph *Graph) GetDigest(id string) (digest.Digest, error) { 517 graph.imageMutex.Lock(id) 518 defer graph.imageMutex.Unlock(id) 519 520 root := graph.imageRoot(id) 521 cs, err := ioutil.ReadFile(filepath.Join(root, digestFileName)) 522 if err != nil { 523 if os.IsNotExist(err) { 524 return "", ErrDigestNotSet 525 } 526 return "", err 527 } 528 return digest.ParseDigest(string(cs)) 529 } 530 531 // RawJSON returns the JSON representation for an image as a byte array. 532 func (graph *Graph) RawJSON(id string) ([]byte, error) { 533 root := graph.imageRoot(id) 534 535 buf, err := ioutil.ReadFile(jsonPath(root)) 536 if err != nil { 537 return nil, fmt.Errorf("Failed to read json for image %s: %s", id, err) 538 } 539 540 return buf, nil 541 } 542 543 func jsonPath(root string) string { 544 return filepath.Join(root, jsonFileName) 545 } 546 547 // storeImage stores file system layer data for the given image to the 548 // graph's storage driver. Image metadata is stored in a file 549 // at the specified root directory. 550 func (graph *Graph) storeImage(img *image.Image, layerData io.Reader, root string) (err error) { 551 // Store the layer. If layerData is not nil, unpack it into the new layer 552 if layerData != nil { 553 if err := graph.disassembleAndApplyTarLayer(img, layerData, root); err != nil { 554 return err 555 } 556 } 557 558 if err := graph.saveSize(root, img.Size); err != nil { 559 return err 560 } 561 562 f, err := os.OpenFile(jsonPath(root), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(0600)) 563 if err != nil { 564 return err 565 } 566 567 defer f.Close() 568 569 return json.NewEncoder(f).Encode(img) 570 } 571 572 func (graph *Graph) disassembleAndApplyTarLayer(img *image.Image, layerData io.Reader, root string) (err error) { 573 var ar io.Reader 574 575 if graph.tarSplitDisabled { 576 ar = layerData 577 } else { 578 // this is saving the tar-split metadata 579 mf, err := os.OpenFile(filepath.Join(root, tarDataFileName), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(0600)) 580 if err != nil { 581 return err 582 } 583 584 mfz := gzip.NewWriter(mf) 585 metaPacker := storage.NewJSONPacker(mfz) 586 defer mf.Close() 587 defer mfz.Close() 588 589 inflatedLayerData, err := archive.DecompressStream(layerData) 590 if err != nil { 591 return err 592 } 593 594 // we're passing nil here for the file putter, because the ApplyDiff will 595 // handle the extraction of the archive 596 rdr, err := asm.NewInputTarStream(inflatedLayerData, metaPacker, nil) 597 if err != nil { 598 return err 599 } 600 601 ar = archive.Reader(rdr) 602 } 603 604 if img.Size, err = graph.driver.ApplyDiff(img.ID, img.Parent, ar); err != nil { 605 return err 606 } 607 608 return nil 609 } 610 611 func (graph *Graph) assembleTarLayer(img *image.Image) (io.ReadCloser, error) { 612 root := graph.imageRoot(img.ID) 613 mFileName := filepath.Join(root, tarDataFileName) 614 mf, err := os.Open(mFileName) 615 if err != nil { 616 if !os.IsNotExist(err) { 617 logrus.Errorf("failed to open %q: %s", mFileName, err) 618 } 619 return nil, err 620 } 621 pR, pW := io.Pipe() 622 // this will need to be in a goroutine, as we are returning the stream of a 623 // tar archive, but can not close the metadata reader early (when this 624 // function returns)... 625 go func() { 626 defer mf.Close() 627 // let's reassemble! 628 logrus.Debugf("[graph] TarLayer with reassembly: %s", img.ID) 629 mfz, err := gzip.NewReader(mf) 630 if err != nil { 631 pW.CloseWithError(fmt.Errorf("[graph] error with %s: %s", mFileName, err)) 632 return 633 } 634 defer mfz.Close() 635 636 // get our relative path to the container 637 fsLayer, err := graph.driver.Get(img.ID, "") 638 if err != nil { 639 pW.CloseWithError(err) 640 return 641 } 642 defer graph.driver.Put(img.ID) 643 644 metaUnpacker := storage.NewJSONUnpacker(mfz) 645 fileGetter := storage.NewPathFileGetter(fsLayer) 646 logrus.Debugf("[graph] %s is at %q", img.ID, fsLayer) 647 ots := asm.NewOutputTarStream(fileGetter, metaUnpacker) 648 defer ots.Close() 649 if _, err := io.Copy(pW, ots); err != nil { 650 pW.CloseWithError(err) 651 return 652 } 653 pW.Close() 654 }() 655 return pR, nil 656 }