github.com/tompao/docker@v1.9.1/daemon/graphdriver/devmapper/deviceset.go (about) 1 // +build linux 2 3 package devmapper 4 5 import ( 6 "bufio" 7 "encoding/json" 8 "errors" 9 "fmt" 10 "io" 11 "io/ioutil" 12 "os" 13 "os/exec" 14 "path" 15 "path/filepath" 16 "strconv" 17 "strings" 18 "sync" 19 "syscall" 20 "time" 21 22 "github.com/Sirupsen/logrus" 23 24 "github.com/docker/docker/daemon/graphdriver" 25 "github.com/docker/docker/pkg/devicemapper" 26 "github.com/docker/docker/pkg/idtools" 27 "github.com/docker/docker/pkg/mount" 28 "github.com/docker/docker/pkg/parsers" 29 "github.com/docker/docker/pkg/units" 30 31 "github.com/opencontainers/runc/libcontainer/label" 32 ) 33 34 var ( 35 defaultDataLoopbackSize int64 = 100 * 1024 * 1024 * 1024 36 defaultMetaDataLoopbackSize int64 = 2 * 1024 * 1024 * 1024 37 defaultBaseFsSize uint64 = 100 * 1024 * 1024 * 1024 38 defaultThinpBlockSize uint32 = 128 // 64K = 128 512b sectors 39 defaultUdevSyncOverride = false 40 maxDeviceID = 0xffffff // 24 bit, pool limit 41 deviceIDMapSz = (maxDeviceID + 1) / 8 42 // We retry device removal so many a times that even error messages 43 // will fill up console during normal operation. So only log Fatal 44 // messages by default. 45 logLevel = devicemapper.LogLevelFatal 46 driverDeferredRemovalSupport = false 47 enableDeferredRemoval = false 48 enableDeferredDeletion = false 49 ) 50 51 const deviceSetMetaFile string = "deviceset-metadata" 52 const transactionMetaFile string = "transaction-metadata" 53 54 type transaction struct { 55 OpenTransactionID uint64 `json:"open_transaction_id"` 56 DeviceIDHash string `json:"device_hash"` 57 DeviceID int `json:"device_id"` 58 } 59 60 type devInfo struct { 61 Hash string `json:"-"` 62 DeviceID int `json:"device_id"` 63 Size uint64 `json:"size"` 64 TransactionID uint64 `json:"transaction_id"` 65 Initialized bool `json:"initialized"` 66 Deleted bool `json:"deleted"` 67 devices *DeviceSet 68 69 mountCount int 70 mountPath string 71 72 // The global DeviceSet lock guarantees that we serialize all 73 // the calls to libdevmapper (which is not threadsafe), but we 74 // sometimes release that lock while sleeping. In that case 75 // this per-device lock is still held, protecting against 76 // other accesses to the device that we're doing the wait on. 77 // 78 // WARNING: In order to avoid AB-BA deadlocks when releasing 79 // the global lock while holding the per-device locks all 80 // device locks must be acquired *before* the device lock, and 81 // multiple device locks should be acquired parent before child. 82 lock sync.Mutex 83 } 84 85 type metaData struct { 86 Devices map[string]*devInfo `json:"Devices"` 87 } 88 89 // DeviceSet holds information about list of devices 90 type DeviceSet struct { 91 metaData `json:"-"` 92 sync.Mutex `json:"-"` // Protects all fields of DeviceSet and serializes calls into libdevmapper 93 root string 94 devicePrefix string 95 TransactionID uint64 `json:"-"` 96 NextDeviceID int `json:"next_device_id"` 97 deviceIDMap []byte 98 99 // Options 100 dataLoopbackSize int64 101 metaDataLoopbackSize int64 102 baseFsSize uint64 103 filesystem string 104 mountOptions string 105 mkfsArgs []string 106 dataDevice string // block or loop dev 107 dataLoopFile string // loopback file, if used 108 metadataDevice string // block or loop dev 109 metadataLoopFile string // loopback file, if used 110 doBlkDiscard bool 111 thinpBlockSize uint32 112 thinPoolDevice string 113 transaction `json:"-"` 114 overrideUdevSyncCheck bool 115 deferredRemove bool // use deferred removal 116 deferredDelete bool // use deferred deletion 117 BaseDeviceUUID string //save UUID of base device 118 nrDeletedDevices uint //number of deleted devices 119 deletionWorkerTicker *time.Ticker 120 uidMaps []idtools.IDMap 121 gidMaps []idtools.IDMap 122 } 123 124 // DiskUsage contains information about disk usage and is used when reporting Status of a device. 125 type DiskUsage struct { 126 // Used bytes on the disk. 127 Used uint64 128 // Total bytes on the disk. 129 Total uint64 130 // Available bytes on the disk. 131 Available uint64 132 } 133 134 // Status returns the information about the device. 135 type Status struct { 136 // PoolName is the name of the data pool. 137 PoolName string 138 // DataFile is the actual block device for data. 139 DataFile string 140 // DataLoopback loopback file, if used. 141 DataLoopback string 142 // MetadataFile is the actual block device for metadata. 143 MetadataFile string 144 // MetadataLoopback is the loopback file, if used. 145 MetadataLoopback string 146 // Data is the disk used for data. 147 Data DiskUsage 148 // Metadata is the disk used for meta data. 149 Metadata DiskUsage 150 // BaseDeviceSize is base size of container and image 151 BaseDeviceSize uint64 152 // BaseDeviceFS is backing filesystem. 153 BaseDeviceFS string 154 // SectorSize size of the vector. 155 SectorSize uint64 156 // UdevSyncSupported is true if sync is supported. 157 UdevSyncSupported bool 158 // DeferredRemoveEnabled is true then the device is not unmounted. 159 DeferredRemoveEnabled bool 160 // True if deferred deletion is enabled. This is different from 161 // deferred removal. "removal" means that device mapper device is 162 // deactivated. Thin device is still in thin pool and can be activated 163 // again. But "deletion" means that thin device will be deleted from 164 // thin pool and it can't be activated again. 165 DeferredDeleteEnabled bool 166 DeferredDeletedDeviceCount uint 167 } 168 169 // Structure used to export image/container metadata in docker inspect. 170 type deviceMetadata struct { 171 deviceID int 172 deviceSize uint64 // size in bytes 173 deviceName string // Device name as used during activation 174 } 175 176 // DevStatus returns information about device mounted containing its id, size and sector information. 177 type DevStatus struct { 178 // DeviceID is the id of the device. 179 DeviceID int 180 // Size is the size of the filesystem. 181 Size uint64 182 // TransactionID is a unique integer per device set used to identify an operation on the file system, this number is incremental. 183 TransactionID uint64 184 // SizeInSectors indicates the size of the sectors allocated. 185 SizeInSectors uint64 186 // MappedSectors indicates number of mapped sectors. 187 MappedSectors uint64 188 // HighestMappedSector is the pointer to the highest mapped sector. 189 HighestMappedSector uint64 190 } 191 192 func getDevName(name string) string { 193 return "/dev/mapper/" + name 194 } 195 196 func (info *devInfo) Name() string { 197 hash := info.Hash 198 if hash == "" { 199 hash = "base" 200 } 201 return fmt.Sprintf("%s-%s", info.devices.devicePrefix, hash) 202 } 203 204 func (info *devInfo) DevName() string { 205 return getDevName(info.Name()) 206 } 207 208 func (devices *DeviceSet) loopbackDir() string { 209 return path.Join(devices.root, "devicemapper") 210 } 211 212 func (devices *DeviceSet) metadataDir() string { 213 return path.Join(devices.root, "metadata") 214 } 215 216 func (devices *DeviceSet) metadataFile(info *devInfo) string { 217 file := info.Hash 218 if file == "" { 219 file = "base" 220 } 221 return path.Join(devices.metadataDir(), file) 222 } 223 224 func (devices *DeviceSet) transactionMetaFile() string { 225 return path.Join(devices.metadataDir(), transactionMetaFile) 226 } 227 228 func (devices *DeviceSet) deviceSetMetaFile() string { 229 return path.Join(devices.metadataDir(), deviceSetMetaFile) 230 } 231 232 func (devices *DeviceSet) oldMetadataFile() string { 233 return path.Join(devices.loopbackDir(), "json") 234 } 235 236 func (devices *DeviceSet) getPoolName() string { 237 if devices.thinPoolDevice == "" { 238 return devices.devicePrefix + "-pool" 239 } 240 return devices.thinPoolDevice 241 } 242 243 func (devices *DeviceSet) getPoolDevName() string { 244 return getDevName(devices.getPoolName()) 245 } 246 247 func (devices *DeviceSet) hasImage(name string) bool { 248 dirname := devices.loopbackDir() 249 filename := path.Join(dirname, name) 250 251 _, err := os.Stat(filename) 252 return err == nil 253 } 254 255 // ensureImage creates a sparse file of <size> bytes at the path 256 // <root>/devicemapper/<name>. 257 // If the file already exists and new size is larger than its current size, it grows to the new size. 258 // Either way it returns the full path. 259 func (devices *DeviceSet) ensureImage(name string, size int64) (string, error) { 260 dirname := devices.loopbackDir() 261 filename := path.Join(dirname, name) 262 263 uid, gid, err := idtools.GetRootUIDGID(devices.uidMaps, devices.gidMaps) 264 if err != nil { 265 return "", err 266 } 267 if err := idtools.MkdirAllAs(dirname, 0700, uid, gid); err != nil && !os.IsExist(err) { 268 return "", err 269 } 270 271 if fi, err := os.Stat(filename); err != nil { 272 if !os.IsNotExist(err) { 273 return "", err 274 } 275 logrus.Debugf("Creating loopback file %s for device-manage use", filename) 276 file, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE, 0600) 277 if err != nil { 278 return "", err 279 } 280 defer file.Close() 281 282 if err := file.Truncate(size); err != nil { 283 return "", err 284 } 285 } else { 286 if fi.Size() < size { 287 file, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE, 0600) 288 if err != nil { 289 return "", err 290 } 291 defer file.Close() 292 if err := file.Truncate(size); err != nil { 293 return "", fmt.Errorf("Unable to grow loopback file %s: %v", filename, err) 294 } 295 } else if fi.Size() > size { 296 logrus.Warnf("Can't shrink loopback file %s", filename) 297 } 298 } 299 return filename, nil 300 } 301 302 func (devices *DeviceSet) allocateTransactionID() uint64 { 303 devices.OpenTransactionID = devices.TransactionID + 1 304 return devices.OpenTransactionID 305 } 306 307 func (devices *DeviceSet) updatePoolTransactionID() error { 308 if err := devicemapper.SetTransactionID(devices.getPoolDevName(), devices.TransactionID, devices.OpenTransactionID); err != nil { 309 return fmt.Errorf("Error setting devmapper transaction ID: %s", err) 310 } 311 devices.TransactionID = devices.OpenTransactionID 312 return nil 313 } 314 315 func (devices *DeviceSet) removeMetadata(info *devInfo) error { 316 if err := os.RemoveAll(devices.metadataFile(info)); err != nil { 317 return fmt.Errorf("Error removing metadata file %s: %s", devices.metadataFile(info), err) 318 } 319 return nil 320 } 321 322 // Given json data and file path, write it to disk 323 func (devices *DeviceSet) writeMetaFile(jsonData []byte, filePath string) error { 324 tmpFile, err := ioutil.TempFile(devices.metadataDir(), ".tmp") 325 if err != nil { 326 return fmt.Errorf("Error creating metadata file: %s", err) 327 } 328 329 n, err := tmpFile.Write(jsonData) 330 if err != nil { 331 return fmt.Errorf("Error writing metadata to %s: %s", tmpFile.Name(), err) 332 } 333 if n < len(jsonData) { 334 return io.ErrShortWrite 335 } 336 if err := tmpFile.Sync(); err != nil { 337 return fmt.Errorf("Error syncing metadata file %s: %s", tmpFile.Name(), err) 338 } 339 if err := tmpFile.Close(); err != nil { 340 return fmt.Errorf("Error closing metadata file %s: %s", tmpFile.Name(), err) 341 } 342 if err := os.Rename(tmpFile.Name(), filePath); err != nil { 343 return fmt.Errorf("Error committing metadata file %s: %s", tmpFile.Name(), err) 344 } 345 346 return nil 347 } 348 349 func (devices *DeviceSet) saveMetadata(info *devInfo) error { 350 jsonData, err := json.Marshal(info) 351 if err != nil { 352 return fmt.Errorf("Error encoding metadata to json: %s", err) 353 } 354 if err := devices.writeMetaFile(jsonData, devices.metadataFile(info)); err != nil { 355 return err 356 } 357 return nil 358 } 359 360 func (devices *DeviceSet) markDeviceIDUsed(deviceID int) { 361 var mask byte 362 i := deviceID % 8 363 mask = 1 << uint(i) 364 devices.deviceIDMap[deviceID/8] = devices.deviceIDMap[deviceID/8] | mask 365 } 366 367 func (devices *DeviceSet) markDeviceIDFree(deviceID int) { 368 var mask byte 369 i := deviceID % 8 370 mask = ^(1 << uint(i)) 371 devices.deviceIDMap[deviceID/8] = devices.deviceIDMap[deviceID/8] & mask 372 } 373 374 func (devices *DeviceSet) isDeviceIDFree(deviceID int) bool { 375 var mask byte 376 i := deviceID % 8 377 mask = (1 << uint(i)) 378 if (devices.deviceIDMap[deviceID/8] & mask) != 0 { 379 return false 380 } 381 return true 382 } 383 384 // Should be called with devices.Lock() held. 385 func (devices *DeviceSet) lookupDevice(hash string) (*devInfo, error) { 386 info := devices.Devices[hash] 387 if info == nil { 388 info = devices.loadMetadata(hash) 389 if info == nil { 390 return nil, fmt.Errorf("Unknown device %s", hash) 391 } 392 393 devices.Devices[hash] = info 394 } 395 return info, nil 396 } 397 398 func (devices *DeviceSet) lookupDeviceWithLock(hash string) (*devInfo, error) { 399 devices.Lock() 400 defer devices.Unlock() 401 info, err := devices.lookupDevice(hash) 402 return info, err 403 } 404 405 // This function relies on that device hash map has been loaded in advance. 406 // Should be called with devices.Lock() held. 407 func (devices *DeviceSet) constructDeviceIDMap() { 408 logrus.Debugf("[deviceset] constructDeviceIDMap()") 409 defer logrus.Debugf("[deviceset] constructDeviceIDMap() END") 410 411 for _, info := range devices.Devices { 412 devices.markDeviceIDUsed(info.DeviceID) 413 logrus.Debugf("Added deviceId=%d to DeviceIdMap", info.DeviceID) 414 } 415 } 416 417 func (devices *DeviceSet) deviceFileWalkFunction(path string, finfo os.FileInfo) error { 418 419 // Skip some of the meta files which are not device files. 420 if strings.HasSuffix(finfo.Name(), ".migrated") { 421 logrus.Debugf("Skipping file %s", path) 422 return nil 423 } 424 425 if strings.HasPrefix(finfo.Name(), ".") { 426 logrus.Debugf("Skipping file %s", path) 427 return nil 428 } 429 430 if finfo.Name() == deviceSetMetaFile { 431 logrus.Debugf("Skipping file %s", path) 432 return nil 433 } 434 435 if finfo.Name() == transactionMetaFile { 436 logrus.Debugf("Skipping file %s", path) 437 return nil 438 } 439 440 logrus.Debugf("Loading data for file %s", path) 441 442 hash := finfo.Name() 443 if hash == "base" { 444 hash = "" 445 } 446 447 // Include deleted devices also as cleanup delete device logic 448 // will go through it and see if there are any deleted devices. 449 if _, err := devices.lookupDevice(hash); err != nil { 450 return fmt.Errorf("Error looking up device %s:%v", hash, err) 451 } 452 453 return nil 454 } 455 456 func (devices *DeviceSet) loadDeviceFilesOnStart() error { 457 logrus.Debugf("[deviceset] loadDeviceFilesOnStart()") 458 defer logrus.Debugf("[deviceset] loadDeviceFilesOnStart() END") 459 460 var scan = func(path string, info os.FileInfo, err error) error { 461 if err != nil { 462 logrus.Debugf("Can't walk the file %s", path) 463 return nil 464 } 465 466 // Skip any directories 467 if info.IsDir() { 468 return nil 469 } 470 471 return devices.deviceFileWalkFunction(path, info) 472 } 473 474 return filepath.Walk(devices.metadataDir(), scan) 475 } 476 477 // Should be called with devices.Lock() held. 478 func (devices *DeviceSet) unregisterDevice(id int, hash string) error { 479 logrus.Debugf("unregisterDevice(%v, %v)", id, hash) 480 info := &devInfo{ 481 Hash: hash, 482 DeviceID: id, 483 } 484 485 delete(devices.Devices, hash) 486 487 if err := devices.removeMetadata(info); err != nil { 488 logrus.Debugf("Error removing metadata: %s", err) 489 return err 490 } 491 492 return nil 493 } 494 495 // Should be called with devices.Lock() held. 496 func (devices *DeviceSet) registerDevice(id int, hash string, size uint64, transactionID uint64) (*devInfo, error) { 497 logrus.Debugf("registerDevice(%v, %v)", id, hash) 498 info := &devInfo{ 499 Hash: hash, 500 DeviceID: id, 501 Size: size, 502 TransactionID: transactionID, 503 Initialized: false, 504 devices: devices, 505 } 506 507 devices.Devices[hash] = info 508 509 if err := devices.saveMetadata(info); err != nil { 510 // Try to remove unused device 511 delete(devices.Devices, hash) 512 return nil, err 513 } 514 515 return info, nil 516 } 517 518 func (devices *DeviceSet) activateDeviceIfNeeded(info *devInfo, ignoreDeleted bool) error { 519 logrus.Debugf("activateDeviceIfNeeded(%v)", info.Hash) 520 521 if info.Deleted && !ignoreDeleted { 522 return fmt.Errorf("devmapper: Can't activate device %v as it is marked for deletion", info.Hash) 523 } 524 525 // Make sure deferred removal on device is canceled, if one was 526 // scheduled. 527 if err := devices.cancelDeferredRemoval(info); err != nil { 528 return fmt.Errorf("Deivce Deferred Removal Cancellation Failed: %s", err) 529 } 530 531 if devinfo, _ := devicemapper.GetInfo(info.Name()); devinfo != nil && devinfo.Exists != 0 { 532 return nil 533 } 534 535 return devicemapper.ActivateDevice(devices.getPoolDevName(), info.Name(), info.DeviceID, info.Size) 536 } 537 538 // Return true only if kernel supports xfs and mkfs.xfs is available 539 func xfsSupported() bool { 540 // Make sure mkfs.xfs is available 541 if _, err := exec.LookPath("mkfs.xfs"); err != nil { 542 return false 543 } 544 545 // Check if kernel supports xfs filesystem or not. 546 exec.Command("modprobe", "xfs").Run() 547 548 f, err := os.Open("/proc/filesystems") 549 if err != nil { 550 logrus.Warnf("Could not check if xfs is supported: %v", err) 551 return false 552 } 553 defer f.Close() 554 555 s := bufio.NewScanner(f) 556 for s.Scan() { 557 if strings.HasSuffix(s.Text(), "\txfs") { 558 return true 559 } 560 } 561 562 if err := s.Err(); err != nil { 563 logrus.Warnf("Could not check if xfs is supported: %v", err) 564 } 565 return false 566 } 567 568 func determineDefaultFS() string { 569 if xfsSupported() { 570 return "xfs" 571 } 572 573 logrus.Warn("XFS is not supported in your system. Either the kernel doesnt support it or mkfs.xfs is not in your PATH. Defaulting to ext4 filesystem") 574 return "ext4" 575 } 576 577 func (devices *DeviceSet) createFilesystem(info *devInfo) error { 578 devname := info.DevName() 579 580 args := []string{} 581 for _, arg := range devices.mkfsArgs { 582 args = append(args, arg) 583 } 584 585 args = append(args, devname) 586 587 var err error 588 589 if devices.filesystem == "" { 590 devices.filesystem = determineDefaultFS() 591 } 592 593 switch devices.filesystem { 594 case "xfs": 595 err = exec.Command("mkfs.xfs", args...).Run() 596 case "ext4": 597 err = exec.Command("mkfs.ext4", append([]string{"-E", "nodiscard,lazy_itable_init=0,lazy_journal_init=0"}, args...)...).Run() 598 if err != nil { 599 err = exec.Command("mkfs.ext4", append([]string{"-E", "nodiscard,lazy_itable_init=0"}, args...)...).Run() 600 } 601 if err != nil { 602 return err 603 } 604 err = exec.Command("tune2fs", append([]string{"-c", "-1", "-i", "0"}, devname)...).Run() 605 default: 606 err = fmt.Errorf("Unsupported filesystem type %s", devices.filesystem) 607 } 608 if err != nil { 609 return err 610 } 611 612 return nil 613 } 614 615 func (devices *DeviceSet) migrateOldMetaData() error { 616 // Migrate old metadata file 617 jsonData, err := ioutil.ReadFile(devices.oldMetadataFile()) 618 if err != nil && !os.IsNotExist(err) { 619 return err 620 } 621 622 if jsonData != nil { 623 m := metaData{Devices: make(map[string]*devInfo)} 624 625 if err := json.Unmarshal(jsonData, &m); err != nil { 626 return err 627 } 628 629 for hash, info := range m.Devices { 630 info.Hash = hash 631 devices.saveMetadata(info) 632 } 633 if err := os.Rename(devices.oldMetadataFile(), devices.oldMetadataFile()+".migrated"); err != nil { 634 return err 635 } 636 637 } 638 639 return nil 640 } 641 642 // Cleanup deleted devices. It assumes that all the devices have been 643 // loaded in the hash table. 644 func (devices *DeviceSet) cleanupDeletedDevices() error { 645 devices.Lock() 646 647 // If there are no deleted devices, there is nothing to do. 648 if devices.nrDeletedDevices == 0 { 649 devices.Unlock() 650 return nil 651 } 652 653 var deletedDevices []*devInfo 654 655 for _, info := range devices.Devices { 656 if !info.Deleted { 657 continue 658 } 659 logrus.Debugf("devmapper: Found deleted device %s.", info.Hash) 660 deletedDevices = append(deletedDevices, info) 661 } 662 663 // Delete the deleted devices. DeleteDevice() first takes the info lock 664 // and then devices.Lock(). So drop it to avoid deadlock. 665 devices.Unlock() 666 667 for _, info := range deletedDevices { 668 // This will again try deferred deletion. 669 if err := devices.DeleteDevice(info.Hash, false); err != nil { 670 logrus.Warnf("devmapper: Deletion of device %s, device_id=%v failed:%v", info.Hash, info.DeviceID, err) 671 } 672 } 673 674 return nil 675 } 676 677 func (devices *DeviceSet) countDeletedDevices() { 678 for _, info := range devices.Devices { 679 if !info.Deleted { 680 continue 681 } 682 devices.nrDeletedDevices++ 683 } 684 } 685 686 func (devices *DeviceSet) startDeviceDeletionWorker() { 687 // Deferred deletion is not enabled. Don't do anything. 688 if !devices.deferredDelete { 689 return 690 } 691 692 logrus.Debugf("devmapper: Worker to cleanup deleted devices started") 693 for range devices.deletionWorkerTicker.C { 694 devices.cleanupDeletedDevices() 695 } 696 } 697 698 func (devices *DeviceSet) initMetaData() error { 699 devices.Lock() 700 defer devices.Unlock() 701 702 if err := devices.migrateOldMetaData(); err != nil { 703 return err 704 } 705 706 _, transactionID, _, _, _, _, err := devices.poolStatus() 707 if err != nil { 708 return err 709 } 710 711 devices.TransactionID = transactionID 712 713 if err := devices.loadDeviceFilesOnStart(); err != nil { 714 return fmt.Errorf("devmapper: Failed to load device files:%v", err) 715 } 716 717 devices.constructDeviceIDMap() 718 devices.countDeletedDevices() 719 720 if err := devices.processPendingTransaction(); err != nil { 721 return err 722 } 723 724 // Start a goroutine to cleanup Deleted Devices 725 go devices.startDeviceDeletionWorker() 726 return nil 727 } 728 729 func (devices *DeviceSet) incNextDeviceID() { 730 // IDs are 24bit, so wrap around 731 devices.NextDeviceID = (devices.NextDeviceID + 1) & maxDeviceID 732 } 733 734 func (devices *DeviceSet) getNextFreeDeviceID() (int, error) { 735 devices.incNextDeviceID() 736 for i := 0; i <= maxDeviceID; i++ { 737 if devices.isDeviceIDFree(devices.NextDeviceID) { 738 devices.markDeviceIDUsed(devices.NextDeviceID) 739 return devices.NextDeviceID, nil 740 } 741 devices.incNextDeviceID() 742 } 743 744 return 0, fmt.Errorf("Unable to find a free device ID") 745 } 746 747 func (devices *DeviceSet) createRegisterDevice(hash string) (*devInfo, error) { 748 devices.Lock() 749 defer devices.Unlock() 750 751 deviceID, err := devices.getNextFreeDeviceID() 752 if err != nil { 753 return nil, err 754 } 755 756 if err := devices.openTransaction(hash, deviceID); err != nil { 757 logrus.Debugf("Error opening transaction hash = %s deviceID = %d", hash, deviceID) 758 devices.markDeviceIDFree(deviceID) 759 return nil, err 760 } 761 762 for { 763 if err := devicemapper.CreateDevice(devices.getPoolDevName(), deviceID); err != nil { 764 if devicemapper.DeviceIDExists(err) { 765 // Device ID already exists. This should not 766 // happen. Now we have a mechianism to find 767 // a free device ID. So something is not right. 768 // Give a warning and continue. 769 logrus.Errorf("Device ID %d exists in pool but it is supposed to be unused", deviceID) 770 deviceID, err = devices.getNextFreeDeviceID() 771 if err != nil { 772 return nil, err 773 } 774 // Save new device id into transaction 775 devices.refreshTransaction(deviceID) 776 continue 777 } 778 logrus.Debugf("Error creating device: %s", err) 779 devices.markDeviceIDFree(deviceID) 780 return nil, err 781 } 782 break 783 } 784 785 logrus.Debugf("Registering device (id %v) with FS size %v", deviceID, devices.baseFsSize) 786 info, err := devices.registerDevice(deviceID, hash, devices.baseFsSize, devices.OpenTransactionID) 787 if err != nil { 788 _ = devicemapper.DeleteDevice(devices.getPoolDevName(), deviceID) 789 devices.markDeviceIDFree(deviceID) 790 return nil, err 791 } 792 793 if err := devices.closeTransaction(); err != nil { 794 devices.unregisterDevice(deviceID, hash) 795 devicemapper.DeleteDevice(devices.getPoolDevName(), deviceID) 796 devices.markDeviceIDFree(deviceID) 797 return nil, err 798 } 799 return info, nil 800 } 801 802 func (devices *DeviceSet) createRegisterSnapDevice(hash string, baseInfo *devInfo) error { 803 deviceID, err := devices.getNextFreeDeviceID() 804 if err != nil { 805 return err 806 } 807 808 if err := devices.openTransaction(hash, deviceID); err != nil { 809 logrus.Debugf("Error opening transaction hash = %s deviceID = %d", hash, deviceID) 810 devices.markDeviceIDFree(deviceID) 811 return err 812 } 813 814 for { 815 if err := devicemapper.CreateSnapDevice(devices.getPoolDevName(), deviceID, baseInfo.Name(), baseInfo.DeviceID); err != nil { 816 if devicemapper.DeviceIDExists(err) { 817 // Device ID already exists. This should not 818 // happen. Now we have a mechianism to find 819 // a free device ID. So something is not right. 820 // Give a warning and continue. 821 logrus.Errorf("Device ID %d exists in pool but it is supposed to be unused", deviceID) 822 deviceID, err = devices.getNextFreeDeviceID() 823 if err != nil { 824 return err 825 } 826 // Save new device id into transaction 827 devices.refreshTransaction(deviceID) 828 continue 829 } 830 logrus.Debugf("Error creating snap device: %s", err) 831 devices.markDeviceIDFree(deviceID) 832 return err 833 } 834 break 835 } 836 837 if _, err := devices.registerDevice(deviceID, hash, baseInfo.Size, devices.OpenTransactionID); err != nil { 838 devicemapper.DeleteDevice(devices.getPoolDevName(), deviceID) 839 devices.markDeviceIDFree(deviceID) 840 logrus.Debugf("Error registering device: %s", err) 841 return err 842 } 843 844 if err := devices.closeTransaction(); err != nil { 845 devices.unregisterDevice(deviceID, hash) 846 devicemapper.DeleteDevice(devices.getPoolDevName(), deviceID) 847 devices.markDeviceIDFree(deviceID) 848 return err 849 } 850 return nil 851 } 852 853 func (devices *DeviceSet) loadMetadata(hash string) *devInfo { 854 info := &devInfo{Hash: hash, devices: devices} 855 856 jsonData, err := ioutil.ReadFile(devices.metadataFile(info)) 857 if err != nil { 858 return nil 859 } 860 861 if err := json.Unmarshal(jsonData, &info); err != nil { 862 return nil 863 } 864 865 if info.DeviceID > maxDeviceID { 866 logrus.Errorf("Ignoring Invalid DeviceId=%d", info.DeviceID) 867 return nil 868 } 869 870 return info 871 } 872 873 func getDeviceUUID(device string) (string, error) { 874 out, err := exec.Command("blkid", "-s", "UUID", "-o", "value", device).Output() 875 if err != nil { 876 logrus.Debugf("Failed to find uuid for device %s:%v", device, err) 877 return "", err 878 } 879 880 uuid := strings.TrimSuffix(string(out), "\n") 881 uuid = strings.TrimSpace(uuid) 882 logrus.Debugf("UUID for device: %s is:%s", device, uuid) 883 return uuid, nil 884 } 885 886 func (devices *DeviceSet) getBaseDeviceSize() uint64 { 887 info, _ := devices.lookupDevice("") 888 if info == nil { 889 return 0 890 } 891 return info.Size 892 } 893 894 func (devices *DeviceSet) getBaseDeviceFS() string { 895 return devices.filesystem 896 } 897 898 func (devices *DeviceSet) verifyBaseDeviceUUIDFS(baseInfo *devInfo) error { 899 devices.Lock() 900 defer devices.Unlock() 901 902 if err := devices.activateDeviceIfNeeded(baseInfo, false); err != nil { 903 return err 904 } 905 906 defer devices.deactivateDevice(baseInfo) 907 908 uuid, err := getDeviceUUID(baseInfo.DevName()) 909 if err != nil { 910 return err 911 } 912 913 if devices.BaseDeviceUUID != uuid { 914 return fmt.Errorf("Current Base Device UUID:%s does not match with stored UUID:%s. Possibly using a different thin pool than last invocation", uuid, devices.BaseDeviceUUID) 915 } 916 917 // If user specified a filesystem using dm.fs option and current 918 // file system of base image is not same, warn user that dm.fs 919 // will be ignored. 920 if devices.filesystem != "" { 921 fs, err := ProbeFsType(baseInfo.DevName()) 922 if err != nil { 923 return err 924 } 925 926 if fs != devices.filesystem { 927 logrus.Warnf("Base device already exists and has filesystem %s on it. User specified filesystem %s will be ignored.", fs, devices.filesystem) 928 devices.filesystem = fs 929 } 930 } 931 return nil 932 } 933 934 func (devices *DeviceSet) saveBaseDeviceUUID(baseInfo *devInfo) error { 935 devices.Lock() 936 defer devices.Unlock() 937 938 if err := devices.activateDeviceIfNeeded(baseInfo, false); err != nil { 939 return err 940 } 941 942 defer devices.deactivateDevice(baseInfo) 943 944 uuid, err := getDeviceUUID(baseInfo.DevName()) 945 if err != nil { 946 return err 947 } 948 949 devices.BaseDeviceUUID = uuid 950 devices.saveDeviceSetMetaData() 951 return nil 952 } 953 954 func (devices *DeviceSet) createBaseImage() error { 955 logrus.Debugf("Initializing base device-mapper thin volume") 956 957 // Create initial device 958 info, err := devices.createRegisterDevice("") 959 if err != nil { 960 return err 961 } 962 963 logrus.Debugf("Creating filesystem on base device-mapper thin volume") 964 965 if err := devices.activateDeviceIfNeeded(info, false); err != nil { 966 return err 967 } 968 969 if err := devices.createFilesystem(info); err != nil { 970 return err 971 } 972 973 info.Initialized = true 974 if err := devices.saveMetadata(info); err != nil { 975 info.Initialized = false 976 return err 977 } 978 979 if err := devices.saveBaseDeviceUUID(info); err != nil { 980 return fmt.Errorf("Could not query and save base device UUID:%v", err) 981 } 982 983 return nil 984 } 985 986 // Returns if thin pool device exists or not. If device exists, also makes 987 // sure it is a thin pool device and not some other type of device. 988 func (devices *DeviceSet) thinPoolExists(thinPoolDevice string) (bool, error) { 989 logrus.Debugf("devmapper: Checking for existence of the pool %s", thinPoolDevice) 990 991 info, err := devicemapper.GetInfo(thinPoolDevice) 992 if err != nil { 993 return false, fmt.Errorf("devmapper: GetInfo() on device %s failed: %v", thinPoolDevice, err) 994 } 995 996 // Device does not exist. 997 if info.Exists == 0 { 998 return false, nil 999 } 1000 1001 _, _, deviceType, _, err := devicemapper.GetStatus(thinPoolDevice) 1002 if err != nil { 1003 return false, fmt.Errorf("devmapper: GetStatus() on device %s failed: %v", thinPoolDevice, err) 1004 } 1005 1006 if deviceType != "thin-pool" { 1007 return false, fmt.Errorf("devmapper: Device %s is not a thin pool", thinPoolDevice) 1008 } 1009 1010 return true, nil 1011 } 1012 1013 func (devices *DeviceSet) checkThinPool() error { 1014 _, transactionID, dataUsed, _, _, _, err := devices.poolStatus() 1015 if err != nil { 1016 return err 1017 } 1018 if dataUsed != 0 { 1019 return fmt.Errorf("Unable to take ownership of thin-pool (%s) that already has used data blocks", 1020 devices.thinPoolDevice) 1021 } 1022 if transactionID != 0 { 1023 return fmt.Errorf("Unable to take ownership of thin-pool (%s) with non-zero transaction ID", 1024 devices.thinPoolDevice) 1025 } 1026 return nil 1027 } 1028 1029 // Base image is initialized properly. Either save UUID for first time (for 1030 // upgrade case or verify UUID. 1031 func (devices *DeviceSet) setupVerifyBaseImageUUIDFS(baseInfo *devInfo) error { 1032 // If BaseDeviceUUID is nil (upgrade case), save it and return success. 1033 if devices.BaseDeviceUUID == "" { 1034 if err := devices.saveBaseDeviceUUID(baseInfo); err != nil { 1035 return fmt.Errorf("Could not query and save base device UUID:%v", err) 1036 } 1037 return nil 1038 } 1039 1040 if err := devices.verifyBaseDeviceUUIDFS(baseInfo); err != nil { 1041 return fmt.Errorf("Base Device UUID and Filesystem verification failed.%v", err) 1042 } 1043 1044 return nil 1045 } 1046 1047 func (devices *DeviceSet) setupBaseImage() error { 1048 oldInfo, _ := devices.lookupDeviceWithLock("") 1049 1050 // base image already exists. If it is initialized properly, do UUID 1051 // verification and return. Otherwise remove image and set it up 1052 // fresh. 1053 1054 if oldInfo != nil { 1055 if oldInfo.Initialized && !oldInfo.Deleted { 1056 if err := devices.setupVerifyBaseImageUUIDFS(oldInfo); err != nil { 1057 return err 1058 } 1059 1060 return nil 1061 } 1062 1063 logrus.Debugf("Removing uninitialized base image") 1064 // If previous base device is in deferred delete state, 1065 // that needs to be cleaned up first. So don't try 1066 // deferred deletion. 1067 if err := devices.DeleteDevice("", true); err != nil { 1068 return err 1069 } 1070 } 1071 1072 // If we are setting up base image for the first time, make sure 1073 // thin pool is empty. 1074 if devices.thinPoolDevice != "" && oldInfo == nil { 1075 if err := devices.checkThinPool(); err != nil { 1076 return err 1077 } 1078 } 1079 1080 // Create new base image device 1081 if err := devices.createBaseImage(); err != nil { 1082 return err 1083 } 1084 1085 return nil 1086 } 1087 1088 func setCloseOnExec(name string) { 1089 if fileInfos, _ := ioutil.ReadDir("/proc/self/fd"); fileInfos != nil { 1090 for _, i := range fileInfos { 1091 link, _ := os.Readlink(filepath.Join("/proc/self/fd", i.Name())) 1092 if link == name { 1093 fd, err := strconv.Atoi(i.Name()) 1094 if err == nil { 1095 syscall.CloseOnExec(fd) 1096 } 1097 } 1098 } 1099 } 1100 } 1101 1102 // DMLog implements logging using DevMapperLogger interface. 1103 func (devices *DeviceSet) DMLog(level int, file string, line int, dmError int, message string) { 1104 // By default libdm sends us all the messages including debug ones. 1105 // We need to filter out messages here and figure out which one 1106 // should be printed. 1107 if level > logLevel { 1108 return 1109 } 1110 1111 // FIXME(vbatts) push this back into ./pkg/devicemapper/ 1112 if level <= devicemapper.LogLevelErr { 1113 logrus.Errorf("libdevmapper(%d): %s:%d (%d) %s", level, file, line, dmError, message) 1114 } else if level <= devicemapper.LogLevelInfo { 1115 logrus.Infof("libdevmapper(%d): %s:%d (%d) %s", level, file, line, dmError, message) 1116 } else { 1117 // FIXME(vbatts) push this back into ./pkg/devicemapper/ 1118 logrus.Debugf("libdevmapper(%d): %s:%d (%d) %s", level, file, line, dmError, message) 1119 } 1120 } 1121 1122 func major(device uint64) uint64 { 1123 return (device >> 8) & 0xfff 1124 } 1125 1126 func minor(device uint64) uint64 { 1127 return (device & 0xff) | ((device >> 12) & 0xfff00) 1128 } 1129 1130 // ResizePool increases the size of the pool. 1131 func (devices *DeviceSet) ResizePool(size int64) error { 1132 dirname := devices.loopbackDir() 1133 datafilename := path.Join(dirname, "data") 1134 if len(devices.dataDevice) > 0 { 1135 datafilename = devices.dataDevice 1136 } 1137 metadatafilename := path.Join(dirname, "metadata") 1138 if len(devices.metadataDevice) > 0 { 1139 metadatafilename = devices.metadataDevice 1140 } 1141 1142 datafile, err := os.OpenFile(datafilename, os.O_RDWR, 0) 1143 if datafile == nil { 1144 return err 1145 } 1146 defer datafile.Close() 1147 1148 fi, err := datafile.Stat() 1149 if fi == nil { 1150 return err 1151 } 1152 1153 if fi.Size() > size { 1154 return fmt.Errorf("Can't shrink file") 1155 } 1156 1157 dataloopback := devicemapper.FindLoopDeviceFor(datafile) 1158 if dataloopback == nil { 1159 return fmt.Errorf("Unable to find loopback mount for: %s", datafilename) 1160 } 1161 defer dataloopback.Close() 1162 1163 metadatafile, err := os.OpenFile(metadatafilename, os.O_RDWR, 0) 1164 if metadatafile == nil { 1165 return err 1166 } 1167 defer metadatafile.Close() 1168 1169 metadataloopback := devicemapper.FindLoopDeviceFor(metadatafile) 1170 if metadataloopback == nil { 1171 return fmt.Errorf("Unable to find loopback mount for: %s", metadatafilename) 1172 } 1173 defer metadataloopback.Close() 1174 1175 // Grow loopback file 1176 if err := datafile.Truncate(size); err != nil { 1177 return fmt.Errorf("Unable to grow loopback file: %s", err) 1178 } 1179 1180 // Reload size for loopback device 1181 if err := devicemapper.LoopbackSetCapacity(dataloopback); err != nil { 1182 return fmt.Errorf("Unable to update loopback capacity: %s", err) 1183 } 1184 1185 // Suspend the pool 1186 if err := devicemapper.SuspendDevice(devices.getPoolName()); err != nil { 1187 return fmt.Errorf("Unable to suspend pool: %s", err) 1188 } 1189 1190 // Reload with the new block sizes 1191 if err := devicemapper.ReloadPool(devices.getPoolName(), dataloopback, metadataloopback, devices.thinpBlockSize); err != nil { 1192 return fmt.Errorf("Unable to reload pool: %s", err) 1193 } 1194 1195 // Resume the pool 1196 if err := devicemapper.ResumeDevice(devices.getPoolName()); err != nil { 1197 return fmt.Errorf("Unable to resume pool: %s", err) 1198 } 1199 1200 return nil 1201 } 1202 1203 func (devices *DeviceSet) loadTransactionMetaData() error { 1204 jsonData, err := ioutil.ReadFile(devices.transactionMetaFile()) 1205 if err != nil { 1206 // There is no active transaction. This will be the case 1207 // during upgrade. 1208 if os.IsNotExist(err) { 1209 devices.OpenTransactionID = devices.TransactionID 1210 return nil 1211 } 1212 return err 1213 } 1214 1215 json.Unmarshal(jsonData, &devices.transaction) 1216 return nil 1217 } 1218 1219 func (devices *DeviceSet) saveTransactionMetaData() error { 1220 jsonData, err := json.Marshal(&devices.transaction) 1221 if err != nil { 1222 return fmt.Errorf("Error encoding metadata to json: %s", err) 1223 } 1224 1225 return devices.writeMetaFile(jsonData, devices.transactionMetaFile()) 1226 } 1227 1228 func (devices *DeviceSet) removeTransactionMetaData() error { 1229 if err := os.RemoveAll(devices.transactionMetaFile()); err != nil { 1230 return err 1231 } 1232 return nil 1233 } 1234 1235 func (devices *DeviceSet) rollbackTransaction() error { 1236 logrus.Debugf("Rolling back open transaction: TransactionID=%d hash=%s device_id=%d", devices.OpenTransactionID, devices.DeviceIDHash, devices.DeviceID) 1237 1238 // A device id might have already been deleted before transaction 1239 // closed. In that case this call will fail. Just leave a message 1240 // in case of failure. 1241 if err := devicemapper.DeleteDevice(devices.getPoolDevName(), devices.DeviceID); err != nil { 1242 logrus.Errorf("Unable to delete device: %s", err) 1243 } 1244 1245 dinfo := &devInfo{Hash: devices.DeviceIDHash} 1246 if err := devices.removeMetadata(dinfo); err != nil { 1247 logrus.Errorf("Unable to remove metadata: %s", err) 1248 } else { 1249 devices.markDeviceIDFree(devices.DeviceID) 1250 } 1251 1252 if err := devices.removeTransactionMetaData(); err != nil { 1253 logrus.Errorf("Unable to remove transaction meta file %s: %s", devices.transactionMetaFile(), err) 1254 } 1255 1256 return nil 1257 } 1258 1259 func (devices *DeviceSet) processPendingTransaction() error { 1260 if err := devices.loadTransactionMetaData(); err != nil { 1261 return err 1262 } 1263 1264 // If there was open transaction but pool transaction ID is same 1265 // as open transaction ID, nothing to roll back. 1266 if devices.TransactionID == devices.OpenTransactionID { 1267 return nil 1268 } 1269 1270 // If open transaction ID is less than pool transaction ID, something 1271 // is wrong. Bail out. 1272 if devices.OpenTransactionID < devices.TransactionID { 1273 logrus.Errorf("Open Transaction id %d is less than pool transaction id %d", devices.OpenTransactionID, devices.TransactionID) 1274 return nil 1275 } 1276 1277 // Pool transaction ID is not same as open transaction. There is 1278 // a transaction which was not completed. 1279 if err := devices.rollbackTransaction(); err != nil { 1280 return fmt.Errorf("Rolling back open transaction failed: %s", err) 1281 } 1282 1283 devices.OpenTransactionID = devices.TransactionID 1284 return nil 1285 } 1286 1287 func (devices *DeviceSet) loadDeviceSetMetaData() error { 1288 jsonData, err := ioutil.ReadFile(devices.deviceSetMetaFile()) 1289 if err != nil { 1290 // For backward compatibility return success if file does 1291 // not exist. 1292 if os.IsNotExist(err) { 1293 return nil 1294 } 1295 return err 1296 } 1297 1298 return json.Unmarshal(jsonData, devices) 1299 } 1300 1301 func (devices *DeviceSet) saveDeviceSetMetaData() error { 1302 jsonData, err := json.Marshal(devices) 1303 if err != nil { 1304 return fmt.Errorf("Error encoding metadata to json: %s", err) 1305 } 1306 1307 return devices.writeMetaFile(jsonData, devices.deviceSetMetaFile()) 1308 } 1309 1310 func (devices *DeviceSet) openTransaction(hash string, DeviceID int) error { 1311 devices.allocateTransactionID() 1312 devices.DeviceIDHash = hash 1313 devices.DeviceID = DeviceID 1314 if err := devices.saveTransactionMetaData(); err != nil { 1315 return fmt.Errorf("Error saving transaction metadata: %s", err) 1316 } 1317 return nil 1318 } 1319 1320 func (devices *DeviceSet) refreshTransaction(DeviceID int) error { 1321 devices.DeviceID = DeviceID 1322 if err := devices.saveTransactionMetaData(); err != nil { 1323 return fmt.Errorf("Error saving transaction metadata: %s", err) 1324 } 1325 return nil 1326 } 1327 1328 func (devices *DeviceSet) closeTransaction() error { 1329 if err := devices.updatePoolTransactionID(); err != nil { 1330 logrus.Debugf("Failed to close Transaction") 1331 return err 1332 } 1333 return nil 1334 } 1335 1336 func determineDriverCapabilities(version string) error { 1337 /* 1338 * Driver version 4.27.0 and greater support deferred activation 1339 * feature. 1340 */ 1341 1342 logrus.Debugf("devicemapper: driver version is %s", version) 1343 1344 versionSplit := strings.Split(version, ".") 1345 major, err := strconv.Atoi(versionSplit[0]) 1346 if err != nil { 1347 return graphdriver.ErrNotSupported 1348 } 1349 1350 if major > 4 { 1351 driverDeferredRemovalSupport = true 1352 return nil 1353 } 1354 1355 if major < 4 { 1356 return nil 1357 } 1358 1359 minor, err := strconv.Atoi(versionSplit[1]) 1360 if err != nil { 1361 return graphdriver.ErrNotSupported 1362 } 1363 1364 /* 1365 * If major is 4 and minor is 27, then there is no need to 1366 * check for patch level as it can not be less than 0. 1367 */ 1368 if minor >= 27 { 1369 driverDeferredRemovalSupport = true 1370 return nil 1371 } 1372 1373 return nil 1374 } 1375 1376 // Determine the major and minor number of loopback device 1377 func getDeviceMajorMinor(file *os.File) (uint64, uint64, error) { 1378 stat, err := file.Stat() 1379 if err != nil { 1380 return 0, 0, err 1381 } 1382 1383 dev := stat.Sys().(*syscall.Stat_t).Rdev 1384 majorNum := major(dev) 1385 minorNum := minor(dev) 1386 1387 logrus.Debugf("[devmapper]: Major:Minor for device: %s is:%v:%v", file.Name(), majorNum, minorNum) 1388 return majorNum, minorNum, nil 1389 } 1390 1391 // Given a file which is backing file of a loop back device, find the 1392 // loopback device name and its major/minor number. 1393 func getLoopFileDeviceMajMin(filename string) (string, uint64, uint64, error) { 1394 file, err := os.Open(filename) 1395 if err != nil { 1396 logrus.Debugf("[devmapper]: Failed to open file %s", filename) 1397 return "", 0, 0, err 1398 } 1399 1400 defer file.Close() 1401 loopbackDevice := devicemapper.FindLoopDeviceFor(file) 1402 if loopbackDevice == nil { 1403 return "", 0, 0, fmt.Errorf("[devmapper]: Unable to find loopback mount for: %s", filename) 1404 } 1405 defer loopbackDevice.Close() 1406 1407 Major, Minor, err := getDeviceMajorMinor(loopbackDevice) 1408 if err != nil { 1409 return "", 0, 0, err 1410 } 1411 return loopbackDevice.Name(), Major, Minor, nil 1412 } 1413 1414 // Get the major/minor numbers of thin pool data and metadata devices 1415 func (devices *DeviceSet) getThinPoolDataMetaMajMin() (uint64, uint64, uint64, uint64, error) { 1416 var params, poolDataMajMin, poolMetadataMajMin string 1417 1418 _, _, _, params, err := devicemapper.GetTable(devices.getPoolName()) 1419 if err != nil { 1420 return 0, 0, 0, 0, err 1421 } 1422 1423 if _, err = fmt.Sscanf(params, "%s %s", &poolMetadataMajMin, &poolDataMajMin); err != nil { 1424 return 0, 0, 0, 0, err 1425 } 1426 1427 logrus.Debugf("[devmapper]: poolDataMajMin=%s poolMetaMajMin=%s\n", poolDataMajMin, poolMetadataMajMin) 1428 1429 poolDataMajMinorSplit := strings.Split(poolDataMajMin, ":") 1430 poolDataMajor, err := strconv.ParseUint(poolDataMajMinorSplit[0], 10, 32) 1431 if err != nil { 1432 return 0, 0, 0, 0, err 1433 } 1434 1435 poolDataMinor, err := strconv.ParseUint(poolDataMajMinorSplit[1], 10, 32) 1436 if err != nil { 1437 return 0, 0, 0, 0, err 1438 } 1439 1440 poolMetadataMajMinorSplit := strings.Split(poolMetadataMajMin, ":") 1441 poolMetadataMajor, err := strconv.ParseUint(poolMetadataMajMinorSplit[0], 10, 32) 1442 if err != nil { 1443 return 0, 0, 0, 0, err 1444 } 1445 1446 poolMetadataMinor, err := strconv.ParseUint(poolMetadataMajMinorSplit[1], 10, 32) 1447 if err != nil { 1448 return 0, 0, 0, 0, err 1449 } 1450 1451 return poolDataMajor, poolDataMinor, poolMetadataMajor, poolMetadataMinor, nil 1452 } 1453 1454 func (devices *DeviceSet) loadThinPoolLoopBackInfo() error { 1455 poolDataMajor, poolDataMinor, poolMetadataMajor, poolMetadataMinor, err := devices.getThinPoolDataMetaMajMin() 1456 if err != nil { 1457 return err 1458 } 1459 1460 dirname := devices.loopbackDir() 1461 1462 // data device has not been passed in. So there should be a data file 1463 // which is being mounted as loop device. 1464 if devices.dataDevice == "" { 1465 datafilename := path.Join(dirname, "data") 1466 dataLoopDevice, dataMajor, dataMinor, err := getLoopFileDeviceMajMin(datafilename) 1467 if err != nil { 1468 return err 1469 } 1470 1471 // Compare the two 1472 if poolDataMajor == dataMajor && poolDataMinor == dataMinor { 1473 devices.dataDevice = dataLoopDevice 1474 devices.dataLoopFile = datafilename 1475 } 1476 1477 } 1478 1479 // metadata device has not been passed in. So there should be a 1480 // metadata file which is being mounted as loop device. 1481 if devices.metadataDevice == "" { 1482 metadatafilename := path.Join(dirname, "metadata") 1483 metadataLoopDevice, metadataMajor, metadataMinor, err := getLoopFileDeviceMajMin(metadatafilename) 1484 if err != nil { 1485 return err 1486 } 1487 if poolMetadataMajor == metadataMajor && poolMetadataMinor == metadataMinor { 1488 devices.metadataDevice = metadataLoopDevice 1489 devices.metadataLoopFile = metadatafilename 1490 } 1491 } 1492 1493 return nil 1494 } 1495 1496 func (devices *DeviceSet) initDevmapper(doInit bool) error { 1497 // give ourselves to libdm as a log handler 1498 devicemapper.LogInit(devices) 1499 1500 version, err := devicemapper.GetDriverVersion() 1501 if err != nil { 1502 // Can't even get driver version, assume not supported 1503 return graphdriver.ErrNotSupported 1504 } 1505 1506 if err := determineDriverCapabilities(version); err != nil { 1507 return graphdriver.ErrNotSupported 1508 } 1509 1510 // If user asked for deferred removal then check both libdm library 1511 // and kernel driver support deferred removal otherwise error out. 1512 if enableDeferredRemoval { 1513 if !driverDeferredRemovalSupport { 1514 return fmt.Errorf("devmapper: Deferred removal can not be enabled as kernel does not support it") 1515 } 1516 if !devicemapper.LibraryDeferredRemovalSupport { 1517 return fmt.Errorf("devmapper: Deferred removal can not be enabled as libdm does not support it") 1518 } 1519 logrus.Debugf("devmapper: Deferred removal support enabled.") 1520 devices.deferredRemove = true 1521 } 1522 1523 if enableDeferredDeletion { 1524 if !devices.deferredRemove { 1525 return fmt.Errorf("devmapper: Deferred deletion can not be enabled as deferred removal is not enabled. Enable deferred removal using --storage-opt dm.use_deferred_removal=true parameter") 1526 } 1527 logrus.Debugf("devmapper: Deferred deletion support enabled.") 1528 devices.deferredDelete = true 1529 } 1530 1531 // https://github.com/docker/docker/issues/4036 1532 if supported := devicemapper.UdevSetSyncSupport(true); !supported { 1533 logrus.Warn("Udev sync is not supported. This will lead to unexpected behavior, data loss and errors. For more information, see https://docs.docker.com/reference/commandline/daemon/#daemon-storage-driver-option") 1534 } 1535 1536 //create the root dir of the devmapper driver ownership to match this 1537 //daemon's remapped root uid/gid so containers can start properly 1538 uid, gid, err := idtools.GetRootUIDGID(devices.uidMaps, devices.gidMaps) 1539 if err != nil { 1540 return err 1541 } 1542 if err := idtools.MkdirAs(devices.root, 0700, uid, gid); err != nil && !os.IsExist(err) { 1543 return err 1544 } 1545 if err := os.MkdirAll(devices.metadataDir(), 0700); err != nil && !os.IsExist(err) { 1546 return err 1547 } 1548 1549 // Set the device prefix from the device id and inode of the docker root dir 1550 1551 st, err := os.Stat(devices.root) 1552 if err != nil { 1553 return fmt.Errorf("Error looking up dir %s: %s", devices.root, err) 1554 } 1555 sysSt := st.Sys().(*syscall.Stat_t) 1556 // "reg-" stands for "regular file". 1557 // In the future we might use "dev-" for "device file", etc. 1558 // docker-maj,min[-inode] stands for: 1559 // - Managed by docker 1560 // - The target of this device is at major <maj> and minor <min> 1561 // - If <inode> is defined, use that file inside the device as a loopback image. Otherwise use the device itself. 1562 devices.devicePrefix = fmt.Sprintf("docker-%d:%d-%d", major(sysSt.Dev), minor(sysSt.Dev), sysSt.Ino) 1563 logrus.Debugf("Generated prefix: %s", devices.devicePrefix) 1564 1565 // Check for the existence of the thin-pool device 1566 poolExists, err := devices.thinPoolExists(devices.getPoolName()) 1567 if err != nil { 1568 return err 1569 } 1570 1571 // It seems libdevmapper opens this without O_CLOEXEC, and go exec will not close files 1572 // that are not Close-on-exec, and lxc-start will die if it inherits any unexpected files, 1573 // so we add this badhack to make sure it closes itself 1574 setCloseOnExec("/dev/mapper/control") 1575 1576 // Make sure the sparse images exist in <root>/devicemapper/data and 1577 // <root>/devicemapper/metadata 1578 1579 createdLoopback := false 1580 1581 // If the pool doesn't exist, create it 1582 if !poolExists && devices.thinPoolDevice == "" { 1583 logrus.Debugf("Pool doesn't exist. Creating it.") 1584 1585 var ( 1586 dataFile *os.File 1587 metadataFile *os.File 1588 ) 1589 1590 if devices.dataDevice == "" { 1591 // Make sure the sparse images exist in <root>/devicemapper/data 1592 1593 hasData := devices.hasImage("data") 1594 1595 if !doInit && !hasData { 1596 return errors.New("Loopback data file not found") 1597 } 1598 1599 if !hasData { 1600 createdLoopback = true 1601 } 1602 1603 data, err := devices.ensureImage("data", devices.dataLoopbackSize) 1604 if err != nil { 1605 logrus.Debugf("Error device ensureImage (data): %s", err) 1606 return err 1607 } 1608 1609 dataFile, err = devicemapper.AttachLoopDevice(data) 1610 if err != nil { 1611 return err 1612 } 1613 devices.dataLoopFile = data 1614 devices.dataDevice = dataFile.Name() 1615 } else { 1616 dataFile, err = os.OpenFile(devices.dataDevice, os.O_RDWR, 0600) 1617 if err != nil { 1618 return err 1619 } 1620 } 1621 defer dataFile.Close() 1622 1623 if devices.metadataDevice == "" { 1624 // Make sure the sparse images exist in <root>/devicemapper/metadata 1625 1626 hasMetadata := devices.hasImage("metadata") 1627 1628 if !doInit && !hasMetadata { 1629 return errors.New("Loopback metadata file not found") 1630 } 1631 1632 if !hasMetadata { 1633 createdLoopback = true 1634 } 1635 1636 metadata, err := devices.ensureImage("metadata", devices.metaDataLoopbackSize) 1637 if err != nil { 1638 logrus.Debugf("Error device ensureImage (metadata): %s", err) 1639 return err 1640 } 1641 1642 metadataFile, err = devicemapper.AttachLoopDevice(metadata) 1643 if err != nil { 1644 return err 1645 } 1646 devices.metadataLoopFile = metadata 1647 devices.metadataDevice = metadataFile.Name() 1648 } else { 1649 metadataFile, err = os.OpenFile(devices.metadataDevice, os.O_RDWR, 0600) 1650 if err != nil { 1651 return err 1652 } 1653 } 1654 defer metadataFile.Close() 1655 1656 if err := devicemapper.CreatePool(devices.getPoolName(), dataFile, metadataFile, devices.thinpBlockSize); err != nil { 1657 return err 1658 } 1659 } 1660 1661 // Pool already exists and caller did not pass us a pool. That means 1662 // we probably created pool earlier and could not remove it as some 1663 // containers were still using it. Detect some of the properties of 1664 // pool, like is it using loop devices. 1665 if poolExists && devices.thinPoolDevice == "" { 1666 if err := devices.loadThinPoolLoopBackInfo(); err != nil { 1667 logrus.Debugf("Failed to load thin pool loopback device information:%v", err) 1668 return err 1669 } 1670 } 1671 1672 // If we didn't just create the data or metadata image, we need to 1673 // load the transaction id and migrate old metadata 1674 if !createdLoopback { 1675 if err := devices.initMetaData(); err != nil { 1676 return err 1677 } 1678 } 1679 1680 if devices.thinPoolDevice == "" { 1681 if devices.metadataLoopFile != "" || devices.dataLoopFile != "" { 1682 logrus.Warnf("Usage of loopback devices is strongly discouraged for production use. Please use `--storage-opt dm.thinpooldev` or use `man docker` to refer to dm.thinpooldev section.") 1683 } 1684 } 1685 1686 // Right now this loads only NextDeviceID. If there is more metadata 1687 // down the line, we might have to move it earlier. 1688 if err := devices.loadDeviceSetMetaData(); err != nil { 1689 return err 1690 } 1691 1692 // Setup the base image 1693 if doInit { 1694 if err := devices.setupBaseImage(); err != nil { 1695 logrus.Debugf("Error device setupBaseImage: %s", err) 1696 return err 1697 } 1698 } 1699 1700 return nil 1701 } 1702 1703 // AddDevice adds a device and registers in the hash. 1704 func (devices *DeviceSet) AddDevice(hash, baseHash string) error { 1705 logrus.Debugf("[deviceset] AddDevice(hash=%s basehash=%s)", hash, baseHash) 1706 defer logrus.Debugf("[deviceset] AddDevice(hash=%s basehash=%s) END", hash, baseHash) 1707 1708 // If a deleted device exists, return error. 1709 baseInfo, err := devices.lookupDeviceWithLock(baseHash) 1710 if err != nil { 1711 return err 1712 } 1713 1714 if baseInfo.Deleted { 1715 return fmt.Errorf("devmapper: Base device %v has been marked for deferred deletion", baseInfo.Hash) 1716 } 1717 1718 baseInfo.lock.Lock() 1719 defer baseInfo.lock.Unlock() 1720 1721 devices.Lock() 1722 defer devices.Unlock() 1723 1724 // Also include deleted devices in case hash of new device is 1725 // same as one of the deleted devices. 1726 if info, _ := devices.lookupDevice(hash); info != nil { 1727 return fmt.Errorf("device %s already exists. Deleted=%v", hash, info.Deleted) 1728 } 1729 1730 if err := devices.createRegisterSnapDevice(hash, baseInfo); err != nil { 1731 return err 1732 } 1733 1734 return nil 1735 } 1736 1737 func (devices *DeviceSet) markForDeferredDeletion(info *devInfo) error { 1738 // If device is already in deleted state, there is nothing to be done. 1739 if info.Deleted { 1740 return nil 1741 } 1742 1743 logrus.Debugf("devmapper: Marking device %s for deferred deletion.", info.Hash) 1744 1745 info.Deleted = true 1746 1747 // save device metadata to refelect deleted state. 1748 if err := devices.saveMetadata(info); err != nil { 1749 info.Deleted = false 1750 return err 1751 } 1752 1753 devices.nrDeletedDevices++ 1754 return nil 1755 } 1756 1757 // Should be caled with devices.Lock() held. 1758 func (devices *DeviceSet) deleteTransaction(info *devInfo, syncDelete bool) error { 1759 if err := devices.openTransaction(info.Hash, info.DeviceID); err != nil { 1760 logrus.Debugf("Error opening transaction hash = %s deviceId = %d", "", info.DeviceID) 1761 return err 1762 } 1763 1764 defer devices.closeTransaction() 1765 1766 err := devicemapper.DeleteDevice(devices.getPoolDevName(), info.DeviceID) 1767 if err != nil { 1768 // If syncDelete is true, we want to return error. If deferred 1769 // deletion is not enabled, we return an error. If error is 1770 // something other then EBUSY, return an error. 1771 if syncDelete || !devices.deferredDelete || err != devicemapper.ErrBusy { 1772 logrus.Debugf("Error deleting device: %s", err) 1773 return err 1774 } 1775 } 1776 1777 if err == nil { 1778 if err := devices.unregisterDevice(info.DeviceID, info.Hash); err != nil { 1779 return err 1780 } 1781 // If device was already in deferred delete state that means 1782 // deletion was being tried again later. Reduce the deleted 1783 // device count. 1784 if info.Deleted { 1785 devices.nrDeletedDevices-- 1786 } 1787 } else { 1788 if err := devices.markForDeferredDeletion(info); err != nil { 1789 return err 1790 } 1791 } 1792 1793 return nil 1794 } 1795 1796 // Issue discard only if device open count is zero. 1797 func (devices *DeviceSet) issueDiscard(info *devInfo) error { 1798 logrus.Debugf("devmapper: issueDiscard(device: %s). START", info.Hash) 1799 defer logrus.Debugf("devmapper: issueDiscard(device: %s). END", info.Hash) 1800 // This is a workaround for the kernel not discarding block so 1801 // on the thin pool when we remove a thinp device, so we do it 1802 // manually. 1803 // Even if device is deferred deleted, activate it and isue 1804 // discards. 1805 if err := devices.activateDeviceIfNeeded(info, true); err != nil { 1806 return err 1807 } 1808 1809 devinfo, err := devicemapper.GetInfo(info.Name()) 1810 if err != nil { 1811 return err 1812 } 1813 1814 if devinfo.OpenCount != 0 { 1815 logrus.Debugf("devmapper: Device: %s is in use. OpenCount=%d. Not issuing discards.", info.Hash, devinfo.OpenCount) 1816 return nil 1817 } 1818 1819 if err := devicemapper.BlockDeviceDiscard(info.DevName()); err != nil { 1820 logrus.Debugf("Error discarding block on device: %s (ignoring)", err) 1821 } 1822 return nil 1823 } 1824 1825 // Should be called with devices.Lock() held. 1826 func (devices *DeviceSet) deleteDevice(info *devInfo, syncDelete bool) error { 1827 if devices.doBlkDiscard { 1828 devices.issueDiscard(info) 1829 } 1830 1831 // Try to deactivate device in case it is active. 1832 if err := devices.deactivateDevice(info); err != nil { 1833 logrus.Debugf("Error deactivating device: %s", err) 1834 return err 1835 } 1836 1837 if err := devices.deleteTransaction(info, syncDelete); err != nil { 1838 return err 1839 } 1840 1841 devices.markDeviceIDFree(info.DeviceID) 1842 1843 return nil 1844 } 1845 1846 // DeleteDevice will return success if device has been marked for deferred 1847 // removal. If one wants to override that and want DeleteDevice() to fail if 1848 // device was busy and could not be deleted, set syncDelete=true. 1849 func (devices *DeviceSet) DeleteDevice(hash string, syncDelete bool) error { 1850 logrus.Debugf("devmapper: DeleteDevice(hash=%v syncDelete=%v) START", hash, syncDelete) 1851 defer logrus.Debugf("devmapper: DeleteDevice(hash=%v syncDelete=%v) END", hash, syncDelete) 1852 info, err := devices.lookupDeviceWithLock(hash) 1853 if err != nil { 1854 return err 1855 } 1856 1857 info.lock.Lock() 1858 defer info.lock.Unlock() 1859 1860 devices.Lock() 1861 defer devices.Unlock() 1862 1863 // If mountcount is not zero, that means devices is still in use 1864 // or has not been Put() properly. Fail device deletion. 1865 1866 if info.mountCount != 0 { 1867 return fmt.Errorf("devmapper: Can't delete device %v as it is still mounted. mntCount=%v", info.Hash, info.mountCount) 1868 } 1869 1870 return devices.deleteDevice(info, syncDelete) 1871 } 1872 1873 func (devices *DeviceSet) deactivatePool() error { 1874 logrus.Debugf("[devmapper] deactivatePool()") 1875 defer logrus.Debugf("[devmapper] deactivatePool END") 1876 devname := devices.getPoolDevName() 1877 1878 devinfo, err := devicemapper.GetInfo(devname) 1879 if err != nil { 1880 return err 1881 } 1882 1883 if devinfo.Exists == 0 { 1884 return nil 1885 } 1886 if err := devicemapper.RemoveDevice(devname); err != nil { 1887 return err 1888 } 1889 1890 if d, err := devicemapper.GetDeps(devname); err == nil { 1891 logrus.Warnf("[devmapper] device %s still has %d active dependents", devname, d.Count) 1892 } 1893 1894 return nil 1895 } 1896 1897 func (devices *DeviceSet) deactivateDevice(info *devInfo) error { 1898 logrus.Debugf("[devmapper] deactivateDevice(%s)", info.Hash) 1899 defer logrus.Debugf("[devmapper] deactivateDevice END(%s)", info.Hash) 1900 1901 devinfo, err := devicemapper.GetInfo(info.Name()) 1902 if err != nil { 1903 return err 1904 } 1905 1906 if devinfo.Exists == 0 { 1907 return nil 1908 } 1909 1910 if devices.deferredRemove { 1911 if err := devicemapper.RemoveDeviceDeferred(info.Name()); err != nil { 1912 return err 1913 } 1914 } else { 1915 if err := devices.removeDevice(info.Name()); err != nil { 1916 return err 1917 } 1918 } 1919 return nil 1920 } 1921 1922 // Issues the underlying dm remove operation. 1923 func (devices *DeviceSet) removeDevice(devname string) error { 1924 var err error 1925 1926 logrus.Debugf("[devmapper] removeDevice START(%s)", devname) 1927 defer logrus.Debugf("[devmapper] removeDevice END(%s)", devname) 1928 1929 for i := 0; i < 200; i++ { 1930 err = devicemapper.RemoveDevice(devname) 1931 if err == nil { 1932 break 1933 } 1934 if err != devicemapper.ErrBusy { 1935 return err 1936 } 1937 1938 // If we see EBUSY it may be a transient error, 1939 // sleep a bit a retry a few times. 1940 devices.Unlock() 1941 time.Sleep(100 * time.Millisecond) 1942 devices.Lock() 1943 } 1944 1945 return err 1946 } 1947 1948 func (devices *DeviceSet) cancelDeferredRemoval(info *devInfo) error { 1949 if !devices.deferredRemove { 1950 return nil 1951 } 1952 1953 logrus.Debugf("[devmapper] cancelDeferredRemoval START(%s)", info.Name()) 1954 defer logrus.Debugf("[devmapper] cancelDeferredRemoval END(%s)", info.Name()) 1955 1956 devinfo, err := devicemapper.GetInfoWithDeferred(info.Name()) 1957 1958 if devinfo != nil && devinfo.DeferredRemove == 0 { 1959 return nil 1960 } 1961 1962 // Cancel deferred remove 1963 for i := 0; i < 100; i++ { 1964 err = devicemapper.CancelDeferredRemove(info.Name()) 1965 if err == nil { 1966 break 1967 } 1968 1969 if err == devicemapper.ErrEnxio { 1970 // Device is probably already gone. Return success. 1971 return nil 1972 } 1973 1974 if err != devicemapper.ErrBusy { 1975 return err 1976 } 1977 1978 // If we see EBUSY it may be a transient error, 1979 // sleep a bit a retry a few times. 1980 devices.Unlock() 1981 time.Sleep(100 * time.Millisecond) 1982 devices.Lock() 1983 } 1984 return err 1985 } 1986 1987 // Shutdown shuts down the device by unmounting the root. 1988 func (devices *DeviceSet) Shutdown() error { 1989 logrus.Debugf("[deviceset %s] Shutdown()", devices.devicePrefix) 1990 logrus.Debugf("[devmapper] Shutting down DeviceSet: %s", devices.root) 1991 defer logrus.Debugf("[deviceset %s] Shutdown() END", devices.devicePrefix) 1992 1993 var devs []*devInfo 1994 1995 // Stop deletion worker. This should start delivering new events to 1996 // ticker channel. That means no new instance of cleanupDeletedDevice() 1997 // will run after this call. If one instance is already running at 1998 // the time of the call, it must be holding devices.Lock() and 1999 // we will block on this lock till cleanup function exits. 2000 devices.deletionWorkerTicker.Stop() 2001 2002 devices.Lock() 2003 // Save DeviceSet Metadata first. Docker kills all threads if they 2004 // don't finish in certain time. It is possible that Shutdown() 2005 // routine does not finish in time as we loop trying to deactivate 2006 // some devices while these are busy. In that case shutdown() routine 2007 // will be killed and we will not get a chance to save deviceset 2008 // metadata. Hence save this early before trying to deactivate devices. 2009 devices.saveDeviceSetMetaData() 2010 2011 for _, info := range devices.Devices { 2012 devs = append(devs, info) 2013 } 2014 devices.Unlock() 2015 2016 for _, info := range devs { 2017 info.lock.Lock() 2018 if info.mountCount > 0 { 2019 // We use MNT_DETACH here in case it is still busy in some running 2020 // container. This means it'll go away from the global scope directly, 2021 // and the device will be released when that container dies. 2022 if err := syscall.Unmount(info.mountPath, syscall.MNT_DETACH); err != nil { 2023 logrus.Debugf("Shutdown unmounting %s, error: %s", info.mountPath, err) 2024 } 2025 2026 devices.Lock() 2027 if err := devices.deactivateDevice(info); err != nil { 2028 logrus.Debugf("Shutdown deactivate %s , error: %s", info.Hash, err) 2029 } 2030 devices.Unlock() 2031 } 2032 info.lock.Unlock() 2033 } 2034 2035 info, _ := devices.lookupDeviceWithLock("") 2036 if info != nil { 2037 info.lock.Lock() 2038 devices.Lock() 2039 if err := devices.deactivateDevice(info); err != nil { 2040 logrus.Debugf("Shutdown deactivate base , error: %s", err) 2041 } 2042 devices.Unlock() 2043 info.lock.Unlock() 2044 } 2045 2046 devices.Lock() 2047 if devices.thinPoolDevice == "" { 2048 if err := devices.deactivatePool(); err != nil { 2049 logrus.Debugf("Shutdown deactivate pool , error: %s", err) 2050 } 2051 } 2052 devices.Unlock() 2053 2054 return nil 2055 } 2056 2057 // MountDevice mounts the device if not already mounted. 2058 func (devices *DeviceSet) MountDevice(hash, path, mountLabel string) error { 2059 info, err := devices.lookupDeviceWithLock(hash) 2060 if err != nil { 2061 return err 2062 } 2063 2064 if info.Deleted { 2065 return fmt.Errorf("devmapper: Can't mount device %v as it has been marked for deferred deletion", info.Hash) 2066 } 2067 2068 info.lock.Lock() 2069 defer info.lock.Unlock() 2070 2071 devices.Lock() 2072 defer devices.Unlock() 2073 2074 if info.mountCount > 0 { 2075 if path != info.mountPath { 2076 return fmt.Errorf("Trying to mount devmapper device in multiple places (%s, %s)", info.mountPath, path) 2077 } 2078 2079 info.mountCount++ 2080 return nil 2081 } 2082 2083 if err := devices.activateDeviceIfNeeded(info, false); err != nil { 2084 return fmt.Errorf("Error activating devmapper device for '%s': %s", hash, err) 2085 } 2086 2087 fstype, err := ProbeFsType(info.DevName()) 2088 if err != nil { 2089 return err 2090 } 2091 2092 options := "" 2093 2094 if fstype == "xfs" { 2095 // XFS needs nouuid or it can't mount filesystems with the same fs 2096 options = joinMountOptions(options, "nouuid") 2097 } 2098 2099 options = joinMountOptions(options, devices.mountOptions) 2100 options = joinMountOptions(options, label.FormatMountLabel("", mountLabel)) 2101 2102 if err := mount.Mount(info.DevName(), path, fstype, options); err != nil { 2103 return fmt.Errorf("Error mounting '%s' on '%s': %s", info.DevName(), path, err) 2104 } 2105 2106 info.mountCount = 1 2107 info.mountPath = path 2108 2109 return nil 2110 } 2111 2112 // UnmountDevice unmounts the device and removes it from hash. 2113 func (devices *DeviceSet) UnmountDevice(hash string) error { 2114 logrus.Debugf("[devmapper] UnmountDevice(hash=%s)", hash) 2115 defer logrus.Debugf("[devmapper] UnmountDevice(hash=%s) END", hash) 2116 2117 info, err := devices.lookupDeviceWithLock(hash) 2118 if err != nil { 2119 return err 2120 } 2121 2122 info.lock.Lock() 2123 defer info.lock.Unlock() 2124 2125 devices.Lock() 2126 defer devices.Unlock() 2127 2128 if info.mountCount == 0 { 2129 return fmt.Errorf("UnmountDevice: device not-mounted id %s", hash) 2130 } 2131 2132 info.mountCount-- 2133 if info.mountCount > 0 { 2134 return nil 2135 } 2136 2137 logrus.Debugf("[devmapper] Unmount(%s)", info.mountPath) 2138 if err := syscall.Unmount(info.mountPath, syscall.MNT_DETACH); err != nil { 2139 return err 2140 } 2141 logrus.Debugf("[devmapper] Unmount done") 2142 2143 if err := devices.deactivateDevice(info); err != nil { 2144 return err 2145 } 2146 2147 info.mountPath = "" 2148 2149 return nil 2150 } 2151 2152 // HasDevice returns true if the device metadata exists. 2153 func (devices *DeviceSet) HasDevice(hash string) bool { 2154 info, _ := devices.lookupDeviceWithLock(hash) 2155 return info != nil 2156 } 2157 2158 // List returns a list of device ids. 2159 func (devices *DeviceSet) List() []string { 2160 devices.Lock() 2161 defer devices.Unlock() 2162 2163 ids := make([]string, len(devices.Devices)) 2164 i := 0 2165 for k := range devices.Devices { 2166 ids[i] = k 2167 i++ 2168 } 2169 return ids 2170 } 2171 2172 func (devices *DeviceSet) deviceStatus(devName string) (sizeInSectors, mappedSectors, highestMappedSector uint64, err error) { 2173 var params string 2174 _, sizeInSectors, _, params, err = devicemapper.GetStatus(devName) 2175 if err != nil { 2176 return 2177 } 2178 if _, err = fmt.Sscanf(params, "%d %d", &mappedSectors, &highestMappedSector); err == nil { 2179 return 2180 } 2181 return 2182 } 2183 2184 // GetDeviceStatus provides size, mapped sectors 2185 func (devices *DeviceSet) GetDeviceStatus(hash string) (*DevStatus, error) { 2186 info, err := devices.lookupDeviceWithLock(hash) 2187 if err != nil { 2188 return nil, err 2189 } 2190 2191 info.lock.Lock() 2192 defer info.lock.Unlock() 2193 2194 devices.Lock() 2195 defer devices.Unlock() 2196 2197 status := &DevStatus{ 2198 DeviceID: info.DeviceID, 2199 Size: info.Size, 2200 TransactionID: info.TransactionID, 2201 } 2202 2203 if err := devices.activateDeviceIfNeeded(info, false); err != nil { 2204 return nil, fmt.Errorf("Error activating devmapper device for '%s': %s", hash, err) 2205 } 2206 2207 sizeInSectors, mappedSectors, highestMappedSector, err := devices.deviceStatus(info.DevName()) 2208 2209 if err != nil { 2210 return nil, err 2211 } 2212 2213 status.SizeInSectors = sizeInSectors 2214 status.MappedSectors = mappedSectors 2215 status.HighestMappedSector = highestMappedSector 2216 2217 return status, nil 2218 } 2219 2220 func (devices *DeviceSet) poolStatus() (totalSizeInSectors, transactionID, dataUsed, dataTotal, metadataUsed, metadataTotal uint64, err error) { 2221 var params string 2222 if _, totalSizeInSectors, _, params, err = devicemapper.GetStatus(devices.getPoolName()); err == nil { 2223 _, err = fmt.Sscanf(params, "%d %d/%d %d/%d", &transactionID, &metadataUsed, &metadataTotal, &dataUsed, &dataTotal) 2224 } 2225 return 2226 } 2227 2228 // DataDevicePath returns the path to the data storage for this deviceset, 2229 // regardless of loopback or block device 2230 func (devices *DeviceSet) DataDevicePath() string { 2231 return devices.dataDevice 2232 } 2233 2234 // MetadataDevicePath returns the path to the metadata storage for this deviceset, 2235 // regardless of loopback or block device 2236 func (devices *DeviceSet) MetadataDevicePath() string { 2237 return devices.metadataDevice 2238 } 2239 2240 func (devices *DeviceSet) getUnderlyingAvailableSpace(loopFile string) (uint64, error) { 2241 buf := new(syscall.Statfs_t) 2242 if err := syscall.Statfs(loopFile, buf); err != nil { 2243 logrus.Warnf("Couldn't stat loopfile filesystem %v: %v", loopFile, err) 2244 return 0, err 2245 } 2246 return buf.Bfree * uint64(buf.Bsize), nil 2247 } 2248 2249 func (devices *DeviceSet) isRealFile(loopFile string) (bool, error) { 2250 if loopFile != "" { 2251 fi, err := os.Stat(loopFile) 2252 if err != nil { 2253 logrus.Warnf("Couldn't stat loopfile %v: %v", loopFile, err) 2254 return false, err 2255 } 2256 return fi.Mode().IsRegular(), nil 2257 } 2258 return false, nil 2259 } 2260 2261 // Status returns the current status of this deviceset 2262 func (devices *DeviceSet) Status() *Status { 2263 devices.Lock() 2264 defer devices.Unlock() 2265 2266 status := &Status{} 2267 2268 status.PoolName = devices.getPoolName() 2269 status.DataFile = devices.DataDevicePath() 2270 status.DataLoopback = devices.dataLoopFile 2271 status.MetadataFile = devices.MetadataDevicePath() 2272 status.MetadataLoopback = devices.metadataLoopFile 2273 status.UdevSyncSupported = devicemapper.UdevSyncSupported() 2274 status.DeferredRemoveEnabled = devices.deferredRemove 2275 status.DeferredDeleteEnabled = devices.deferredDelete 2276 status.DeferredDeletedDeviceCount = devices.nrDeletedDevices 2277 status.BaseDeviceSize = devices.getBaseDeviceSize() 2278 status.BaseDeviceFS = devices.getBaseDeviceFS() 2279 2280 totalSizeInSectors, _, dataUsed, dataTotal, metadataUsed, metadataTotal, err := devices.poolStatus() 2281 if err == nil { 2282 // Convert from blocks to bytes 2283 blockSizeInSectors := totalSizeInSectors / dataTotal 2284 2285 status.Data.Used = dataUsed * blockSizeInSectors * 512 2286 status.Data.Total = dataTotal * blockSizeInSectors * 512 2287 status.Data.Available = status.Data.Total - status.Data.Used 2288 2289 // metadata blocks are always 4k 2290 status.Metadata.Used = metadataUsed * 4096 2291 status.Metadata.Total = metadataTotal * 4096 2292 status.Metadata.Available = status.Metadata.Total - status.Metadata.Used 2293 2294 status.SectorSize = blockSizeInSectors * 512 2295 2296 if check, _ := devices.isRealFile(devices.dataLoopFile); check { 2297 actualSpace, err := devices.getUnderlyingAvailableSpace(devices.dataLoopFile) 2298 if err == nil && actualSpace < status.Data.Available { 2299 status.Data.Available = actualSpace 2300 } 2301 } 2302 2303 if check, _ := devices.isRealFile(devices.metadataLoopFile); check { 2304 actualSpace, err := devices.getUnderlyingAvailableSpace(devices.metadataLoopFile) 2305 if err == nil && actualSpace < status.Metadata.Available { 2306 status.Metadata.Available = actualSpace 2307 } 2308 } 2309 } 2310 2311 return status 2312 } 2313 2314 // Status returns the current status of this deviceset 2315 func (devices *DeviceSet) exportDeviceMetadata(hash string) (*deviceMetadata, error) { 2316 info, err := devices.lookupDeviceWithLock(hash) 2317 if err != nil { 2318 return nil, err 2319 } 2320 2321 info.lock.Lock() 2322 defer info.lock.Unlock() 2323 2324 metadata := &deviceMetadata{info.DeviceID, info.Size, info.Name()} 2325 return metadata, nil 2326 } 2327 2328 // NewDeviceSet creates the device set based on the options provided. 2329 func NewDeviceSet(root string, doInit bool, options []string, uidMaps, gidMaps []idtools.IDMap) (*DeviceSet, error) { 2330 devicemapper.SetDevDir("/dev") 2331 2332 devices := &DeviceSet{ 2333 root: root, 2334 metaData: metaData{Devices: make(map[string]*devInfo)}, 2335 dataLoopbackSize: defaultDataLoopbackSize, 2336 metaDataLoopbackSize: defaultMetaDataLoopbackSize, 2337 baseFsSize: defaultBaseFsSize, 2338 overrideUdevSyncCheck: defaultUdevSyncOverride, 2339 doBlkDiscard: true, 2340 thinpBlockSize: defaultThinpBlockSize, 2341 deviceIDMap: make([]byte, deviceIDMapSz), 2342 deletionWorkerTicker: time.NewTicker(time.Second * 30), 2343 uidMaps: uidMaps, 2344 gidMaps: gidMaps, 2345 } 2346 2347 foundBlkDiscard := false 2348 for _, option := range options { 2349 key, val, err := parsers.ParseKeyValueOpt(option) 2350 if err != nil { 2351 return nil, err 2352 } 2353 key = strings.ToLower(key) 2354 switch key { 2355 case "dm.basesize": 2356 size, err := units.RAMInBytes(val) 2357 if err != nil { 2358 return nil, err 2359 } 2360 devices.baseFsSize = uint64(size) 2361 case "dm.loopdatasize": 2362 size, err := units.RAMInBytes(val) 2363 if err != nil { 2364 return nil, err 2365 } 2366 devices.dataLoopbackSize = size 2367 case "dm.loopmetadatasize": 2368 size, err := units.RAMInBytes(val) 2369 if err != nil { 2370 return nil, err 2371 } 2372 devices.metaDataLoopbackSize = size 2373 case "dm.fs": 2374 if val != "ext4" && val != "xfs" { 2375 return nil, fmt.Errorf("Unsupported filesystem %s\n", val) 2376 } 2377 devices.filesystem = val 2378 case "dm.mkfsarg": 2379 devices.mkfsArgs = append(devices.mkfsArgs, val) 2380 case "dm.mountopt": 2381 devices.mountOptions = joinMountOptions(devices.mountOptions, val) 2382 case "dm.metadatadev": 2383 devices.metadataDevice = val 2384 case "dm.datadev": 2385 devices.dataDevice = val 2386 case "dm.thinpooldev": 2387 devices.thinPoolDevice = strings.TrimPrefix(val, "/dev/mapper/") 2388 case "dm.blkdiscard": 2389 foundBlkDiscard = true 2390 devices.doBlkDiscard, err = strconv.ParseBool(val) 2391 if err != nil { 2392 return nil, err 2393 } 2394 case "dm.blocksize": 2395 size, err := units.RAMInBytes(val) 2396 if err != nil { 2397 return nil, err 2398 } 2399 // convert to 512b sectors 2400 devices.thinpBlockSize = uint32(size) >> 9 2401 case "dm.override_udev_sync_check": 2402 devices.overrideUdevSyncCheck, err = strconv.ParseBool(val) 2403 if err != nil { 2404 return nil, err 2405 } 2406 2407 case "dm.use_deferred_removal": 2408 enableDeferredRemoval, err = strconv.ParseBool(val) 2409 if err != nil { 2410 return nil, err 2411 } 2412 2413 case "dm.use_deferred_deletion": 2414 enableDeferredDeletion, err = strconv.ParseBool(val) 2415 if err != nil { 2416 return nil, err 2417 } 2418 2419 default: 2420 return nil, fmt.Errorf("Unknown option %s\n", key) 2421 } 2422 } 2423 2424 // By default, don't do blk discard hack on raw devices, its rarely useful and is expensive 2425 if !foundBlkDiscard && (devices.dataDevice != "" || devices.thinPoolDevice != "") { 2426 devices.doBlkDiscard = false 2427 } 2428 2429 if err := devices.initDevmapper(doInit); err != nil { 2430 return nil, err 2431 } 2432 2433 return devices, nil 2434 }