github.com/gondor/docker@v1.9.0-rc1/daemon/graphdriver/devmapper/deviceset.go (about) 1 // +build linux 2 3 package devmapper 4 5 import ( 6 "encoding/json" 7 "errors" 8 "fmt" 9 "io" 10 "io/ioutil" 11 "os" 12 "os/exec" 13 "path" 14 "path/filepath" 15 "strconv" 16 "strings" 17 "sync" 18 "syscall" 19 "time" 20 21 "github.com/Sirupsen/logrus" 22 23 "github.com/docker/docker/daemon/graphdriver" 24 "github.com/docker/docker/pkg/devicemapper" 25 "github.com/docker/docker/pkg/idtools" 26 "github.com/docker/docker/pkg/mount" 27 "github.com/docker/docker/pkg/parsers" 28 "github.com/docker/docker/pkg/units" 29 30 "github.com/opencontainers/runc/libcontainer/label" 31 ) 32 33 var ( 34 defaultDataLoopbackSize int64 = 100 * 1024 * 1024 * 1024 35 defaultMetaDataLoopbackSize int64 = 2 * 1024 * 1024 * 1024 36 defaultBaseFsSize uint64 = 100 * 1024 * 1024 * 1024 37 defaultThinpBlockSize uint32 = 128 // 64K = 128 512b sectors 38 defaultUdevSyncOverride = false 39 maxDeviceID = 0xffffff // 24 bit, pool limit 40 deviceIDMapSz = (maxDeviceID + 1) / 8 41 // We retry device removal so many a times that even error messages 42 // will fill up console during normal operation. So only log Fatal 43 // messages by default. 44 logLevel = devicemapper.LogLevelFatal 45 driverDeferredRemovalSupport = false 46 enableDeferredRemoval = false 47 enableDeferredDeletion = false 48 ) 49 50 const deviceSetMetaFile string = "deviceset-metadata" 51 const transactionMetaFile string = "transaction-metadata" 52 53 type transaction struct { 54 OpenTransactionID uint64 `json:"open_transaction_id"` 55 DeviceIDHash string `json:"device_hash"` 56 DeviceID int `json:"device_id"` 57 } 58 59 type devInfo struct { 60 Hash string `json:"-"` 61 DeviceID int `json:"device_id"` 62 Size uint64 `json:"size"` 63 TransactionID uint64 `json:"transaction_id"` 64 Initialized bool `json:"initialized"` 65 Deleted bool `json:"deleted"` 66 devices *DeviceSet 67 68 mountCount int 69 mountPath string 70 71 // The global DeviceSet lock guarantees that we serialize all 72 // the calls to libdevmapper (which is not threadsafe), but we 73 // sometimes release that lock while sleeping. In that case 74 // this per-device lock is still held, protecting against 75 // other accesses to the device that we're doing the wait on. 76 // 77 // WARNING: In order to avoid AB-BA deadlocks when releasing 78 // the global lock while holding the per-device locks all 79 // device locks must be acquired *before* the device lock, and 80 // multiple device locks should be acquired parent before child. 81 lock sync.Mutex 82 } 83 84 type metaData struct { 85 Devices map[string]*devInfo `json:"Devices"` 86 } 87 88 // DeviceSet holds information about list of devices 89 type DeviceSet struct { 90 metaData `json:"-"` 91 sync.Mutex `json:"-"` // Protects all fields of DeviceSet and serializes calls into libdevmapper 92 root string 93 devicePrefix string 94 TransactionID uint64 `json:"-"` 95 NextDeviceID int `json:"next_device_id"` 96 deviceIDMap []byte 97 98 // Options 99 dataLoopbackSize int64 100 metaDataLoopbackSize int64 101 baseFsSize uint64 102 filesystem string 103 mountOptions string 104 mkfsArgs []string 105 dataDevice string // block or loop dev 106 dataLoopFile string // loopback file, if used 107 metadataDevice string // block or loop dev 108 metadataLoopFile string // loopback file, if used 109 doBlkDiscard bool 110 thinpBlockSize uint32 111 thinPoolDevice string 112 transaction `json:"-"` 113 overrideUdevSyncCheck bool 114 deferredRemove bool // use deferred removal 115 deferredDelete bool // use deferred deletion 116 BaseDeviceUUID string //save UUID of base device 117 nrDeletedDevices uint //number of deleted devices 118 deletionWorkerTicker *time.Ticker 119 uidMaps []idtools.IDMap 120 gidMaps []idtools.IDMap 121 } 122 123 // DiskUsage contains information about disk usage and is used when reporting Status of a device. 124 type DiskUsage struct { 125 // Used bytes on the disk. 126 Used uint64 127 // Total bytes on the disk. 128 Total uint64 129 // Available bytes on the disk. 130 Available uint64 131 } 132 133 // Status returns the information about the device. 134 type Status struct { 135 // PoolName is the name of the data pool. 136 PoolName string 137 // DataFile is the actual block device for data. 138 DataFile string 139 // DataLoopback loopback file, if used. 140 DataLoopback string 141 // MetadataFile is the actual block device for metadata. 142 MetadataFile string 143 // MetadataLoopback is the loopback file, if used. 144 MetadataLoopback string 145 // Data is the disk used for data. 146 Data DiskUsage 147 // Metadata is the disk used for meta data. 148 Metadata DiskUsage 149 // BaseDeviceSize is base size of container and image 150 BaseDeviceSize uint64 151 // SectorSize size of the vector. 152 SectorSize uint64 153 // UdevSyncSupported is true if sync is supported. 154 UdevSyncSupported bool 155 // DeferredRemoveEnabled is true then the device is not unmounted. 156 DeferredRemoveEnabled bool 157 // True if deferred deletion is enabled. This is different from 158 // deferred removal. "removal" means that device mapper device is 159 // deactivated. Thin device is still in thin pool and can be activated 160 // again. But "deletion" means that thin device will be deleted from 161 // thin pool and it can't be activated again. 162 DeferredDeleteEnabled bool 163 DeferredDeletedDeviceCount uint 164 } 165 166 // Structure used to export image/container metadata in docker inspect. 167 type deviceMetadata struct { 168 deviceID int 169 deviceSize uint64 // size in bytes 170 deviceName string // Device name as used during activation 171 } 172 173 // DevStatus returns information about device mounted containing its id, size and sector information. 174 type DevStatus struct { 175 // DeviceID is the id of the device. 176 DeviceID int 177 // Size is the size of the filesystem. 178 Size uint64 179 // TransactionID is a unique integer per device set used to identify an operation on the file system, this number is incremental. 180 TransactionID uint64 181 // SizeInSectors indicates the size of the sectors allocated. 182 SizeInSectors uint64 183 // MappedSectors indicates number of mapped sectors. 184 MappedSectors uint64 185 // HighestMappedSector is the pointer to the highest mapped sector. 186 HighestMappedSector uint64 187 } 188 189 func getDevName(name string) string { 190 return "/dev/mapper/" + name 191 } 192 193 func (info *devInfo) Name() string { 194 hash := info.Hash 195 if hash == "" { 196 hash = "base" 197 } 198 return fmt.Sprintf("%s-%s", info.devices.devicePrefix, hash) 199 } 200 201 func (info *devInfo) DevName() string { 202 return getDevName(info.Name()) 203 } 204 205 func (devices *DeviceSet) loopbackDir() string { 206 return path.Join(devices.root, "devicemapper") 207 } 208 209 func (devices *DeviceSet) metadataDir() string { 210 return path.Join(devices.root, "metadata") 211 } 212 213 func (devices *DeviceSet) metadataFile(info *devInfo) string { 214 file := info.Hash 215 if file == "" { 216 file = "base" 217 } 218 return path.Join(devices.metadataDir(), file) 219 } 220 221 func (devices *DeviceSet) transactionMetaFile() string { 222 return path.Join(devices.metadataDir(), transactionMetaFile) 223 } 224 225 func (devices *DeviceSet) deviceSetMetaFile() string { 226 return path.Join(devices.metadataDir(), deviceSetMetaFile) 227 } 228 229 func (devices *DeviceSet) oldMetadataFile() string { 230 return path.Join(devices.loopbackDir(), "json") 231 } 232 233 func (devices *DeviceSet) getPoolName() string { 234 if devices.thinPoolDevice == "" { 235 return devices.devicePrefix + "-pool" 236 } 237 return devices.thinPoolDevice 238 } 239 240 func (devices *DeviceSet) getPoolDevName() string { 241 return getDevName(devices.getPoolName()) 242 } 243 244 func (devices *DeviceSet) hasImage(name string) bool { 245 dirname := devices.loopbackDir() 246 filename := path.Join(dirname, name) 247 248 _, err := os.Stat(filename) 249 return err == nil 250 } 251 252 // ensureImage creates a sparse file of <size> bytes at the path 253 // <root>/devicemapper/<name>. 254 // If the file already exists and new size is larger than its current size, it grows to the new size. 255 // Either way it returns the full path. 256 func (devices *DeviceSet) ensureImage(name string, size int64) (string, error) { 257 dirname := devices.loopbackDir() 258 filename := path.Join(dirname, name) 259 260 uid, gid, err := idtools.GetRootUIDGID(devices.uidMaps, devices.gidMaps) 261 if err != nil { 262 return "", err 263 } 264 if err := idtools.MkdirAllAs(dirname, 0700, uid, gid); err != nil && !os.IsExist(err) { 265 return "", err 266 } 267 268 if fi, err := os.Stat(filename); err != nil { 269 if !os.IsNotExist(err) { 270 return "", err 271 } 272 logrus.Debugf("Creating loopback file %s for device-manage use", filename) 273 file, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE, 0600) 274 if err != nil { 275 return "", err 276 } 277 defer file.Close() 278 279 if err := file.Truncate(size); err != nil { 280 return "", err 281 } 282 } else { 283 if fi.Size() < size { 284 file, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE, 0600) 285 if err != nil { 286 return "", err 287 } 288 defer file.Close() 289 if err := file.Truncate(size); err != nil { 290 return "", fmt.Errorf("Unable to grow loopback file %s: %v", filename, err) 291 } 292 } else if fi.Size() > size { 293 logrus.Warnf("Can't shrink loopback file %s", filename) 294 } 295 } 296 return filename, nil 297 } 298 299 func (devices *DeviceSet) allocateTransactionID() uint64 { 300 devices.OpenTransactionID = devices.TransactionID + 1 301 return devices.OpenTransactionID 302 } 303 304 func (devices *DeviceSet) updatePoolTransactionID() error { 305 if err := devicemapper.SetTransactionID(devices.getPoolDevName(), devices.TransactionID, devices.OpenTransactionID); err != nil { 306 return fmt.Errorf("Error setting devmapper transaction ID: %s", err) 307 } 308 devices.TransactionID = devices.OpenTransactionID 309 return nil 310 } 311 312 func (devices *DeviceSet) removeMetadata(info *devInfo) error { 313 if err := os.RemoveAll(devices.metadataFile(info)); err != nil { 314 return fmt.Errorf("Error removing metadata file %s: %s", devices.metadataFile(info), err) 315 } 316 return nil 317 } 318 319 // Given json data and file path, write it to disk 320 func (devices *DeviceSet) writeMetaFile(jsonData []byte, filePath string) error { 321 tmpFile, err := ioutil.TempFile(devices.metadataDir(), ".tmp") 322 if err != nil { 323 return fmt.Errorf("Error creating metadata file: %s", err) 324 } 325 326 n, err := tmpFile.Write(jsonData) 327 if err != nil { 328 return fmt.Errorf("Error writing metadata to %s: %s", tmpFile.Name(), err) 329 } 330 if n < len(jsonData) { 331 return io.ErrShortWrite 332 } 333 if err := tmpFile.Sync(); err != nil { 334 return fmt.Errorf("Error syncing metadata file %s: %s", tmpFile.Name(), err) 335 } 336 if err := tmpFile.Close(); err != nil { 337 return fmt.Errorf("Error closing metadata file %s: %s", tmpFile.Name(), err) 338 } 339 if err := os.Rename(tmpFile.Name(), filePath); err != nil { 340 return fmt.Errorf("Error committing metadata file %s: %s", tmpFile.Name(), err) 341 } 342 343 return nil 344 } 345 346 func (devices *DeviceSet) saveMetadata(info *devInfo) error { 347 jsonData, err := json.Marshal(info) 348 if err != nil { 349 return fmt.Errorf("Error encoding metadata to json: %s", err) 350 } 351 if err := devices.writeMetaFile(jsonData, devices.metadataFile(info)); err != nil { 352 return err 353 } 354 return nil 355 } 356 357 func (devices *DeviceSet) markDeviceIDUsed(deviceID int) { 358 var mask byte 359 i := deviceID % 8 360 mask = 1 << uint(i) 361 devices.deviceIDMap[deviceID/8] = devices.deviceIDMap[deviceID/8] | mask 362 } 363 364 func (devices *DeviceSet) markDeviceIDFree(deviceID int) { 365 var mask byte 366 i := deviceID % 8 367 mask = ^(1 << uint(i)) 368 devices.deviceIDMap[deviceID/8] = devices.deviceIDMap[deviceID/8] & mask 369 } 370 371 func (devices *DeviceSet) isDeviceIDFree(deviceID int) bool { 372 var mask byte 373 i := deviceID % 8 374 mask = (1 << uint(i)) 375 if (devices.deviceIDMap[deviceID/8] & mask) != 0 { 376 return false 377 } 378 return true 379 } 380 381 // Should be called with devices.Lock() held. 382 func (devices *DeviceSet) lookupDevice(hash string) (*devInfo, error) { 383 info := devices.Devices[hash] 384 if info == nil { 385 info = devices.loadMetadata(hash) 386 if info == nil { 387 return nil, fmt.Errorf("Unknown device %s", hash) 388 } 389 390 devices.Devices[hash] = info 391 } 392 return info, nil 393 } 394 395 func (devices *DeviceSet) lookupDeviceWithLock(hash string) (*devInfo, error) { 396 devices.Lock() 397 defer devices.Unlock() 398 info, err := devices.lookupDevice(hash) 399 return info, err 400 } 401 402 // This function relies on that device hash map has been loaded in advance. 403 // Should be called with devices.Lock() held. 404 func (devices *DeviceSet) constructDeviceIDMap() { 405 logrus.Debugf("[deviceset] constructDeviceIDMap()") 406 defer logrus.Debugf("[deviceset] constructDeviceIDMap() END") 407 408 for _, info := range devices.Devices { 409 devices.markDeviceIDUsed(info.DeviceID) 410 logrus.Debugf("Added deviceId=%d to DeviceIdMap", info.DeviceID) 411 } 412 } 413 414 func (devices *DeviceSet) deviceFileWalkFunction(path string, finfo os.FileInfo) error { 415 416 // Skip some of the meta files which are not device files. 417 if strings.HasSuffix(finfo.Name(), ".migrated") { 418 logrus.Debugf("Skipping file %s", path) 419 return nil 420 } 421 422 if strings.HasPrefix(finfo.Name(), ".") { 423 logrus.Debugf("Skipping file %s", path) 424 return nil 425 } 426 427 if finfo.Name() == deviceSetMetaFile { 428 logrus.Debugf("Skipping file %s", path) 429 return nil 430 } 431 432 if finfo.Name() == transactionMetaFile { 433 logrus.Debugf("Skipping file %s", path) 434 return nil 435 } 436 437 logrus.Debugf("Loading data for file %s", path) 438 439 hash := finfo.Name() 440 if hash == "base" { 441 hash = "" 442 } 443 444 // Include deleted devices also as cleanup delete device logic 445 // will go through it and see if there are any deleted devices. 446 if _, err := devices.lookupDevice(hash); err != nil { 447 return fmt.Errorf("Error looking up device %s:%v", hash, err) 448 } 449 450 return nil 451 } 452 453 func (devices *DeviceSet) loadDeviceFilesOnStart() error { 454 logrus.Debugf("[deviceset] loadDeviceFilesOnStart()") 455 defer logrus.Debugf("[deviceset] loadDeviceFilesOnStart() END") 456 457 var scan = func(path string, info os.FileInfo, err error) error { 458 if err != nil { 459 logrus.Debugf("Can't walk the file %s", path) 460 return nil 461 } 462 463 // Skip any directories 464 if info.IsDir() { 465 return nil 466 } 467 468 return devices.deviceFileWalkFunction(path, info) 469 } 470 471 return filepath.Walk(devices.metadataDir(), scan) 472 } 473 474 // Should be called with devices.Lock() held. 475 func (devices *DeviceSet) unregisterDevice(id int, hash string) error { 476 logrus.Debugf("unregisterDevice(%v, %v)", id, hash) 477 info := &devInfo{ 478 Hash: hash, 479 DeviceID: id, 480 } 481 482 delete(devices.Devices, hash) 483 484 if err := devices.removeMetadata(info); err != nil { 485 logrus.Debugf("Error removing metadata: %s", err) 486 return err 487 } 488 489 return nil 490 } 491 492 // Should be called with devices.Lock() held. 493 func (devices *DeviceSet) registerDevice(id int, hash string, size uint64, transactionID uint64) (*devInfo, error) { 494 logrus.Debugf("registerDevice(%v, %v)", id, hash) 495 info := &devInfo{ 496 Hash: hash, 497 DeviceID: id, 498 Size: size, 499 TransactionID: transactionID, 500 Initialized: false, 501 devices: devices, 502 } 503 504 devices.Devices[hash] = info 505 506 if err := devices.saveMetadata(info); err != nil { 507 // Try to remove unused device 508 delete(devices.Devices, hash) 509 return nil, err 510 } 511 512 return info, nil 513 } 514 515 func (devices *DeviceSet) activateDeviceIfNeeded(info *devInfo, ignoreDeleted bool) error { 516 logrus.Debugf("activateDeviceIfNeeded(%v)", info.Hash) 517 518 if info.Deleted && !ignoreDeleted { 519 return fmt.Errorf("devmapper: Can't activate device %v as it is marked for deletion", info.Hash) 520 } 521 522 // Make sure deferred removal on device is canceled, if one was 523 // scheduled. 524 if err := devices.cancelDeferredRemoval(info); err != nil { 525 return fmt.Errorf("Deivce Deferred Removal Cancellation Failed: %s", err) 526 } 527 528 if devinfo, _ := devicemapper.GetInfo(info.Name()); devinfo != nil && devinfo.Exists != 0 { 529 return nil 530 } 531 532 return devicemapper.ActivateDevice(devices.getPoolDevName(), info.Name(), info.DeviceID, info.Size) 533 } 534 535 func (devices *DeviceSet) createFilesystem(info *devInfo) error { 536 devname := info.DevName() 537 538 args := []string{} 539 for _, arg := range devices.mkfsArgs { 540 args = append(args, arg) 541 } 542 543 args = append(args, devname) 544 545 var err error 546 switch devices.filesystem { 547 case "xfs": 548 err = exec.Command("mkfs.xfs", args...).Run() 549 case "ext4": 550 err = exec.Command("mkfs.ext4", append([]string{"-E", "nodiscard,lazy_itable_init=0,lazy_journal_init=0"}, args...)...).Run() 551 if err != nil { 552 err = exec.Command("mkfs.ext4", append([]string{"-E", "nodiscard,lazy_itable_init=0"}, args...)...).Run() 553 } 554 if err != nil { 555 return err 556 } 557 err = exec.Command("tune2fs", append([]string{"-c", "-1", "-i", "0"}, devname)...).Run() 558 default: 559 err = fmt.Errorf("Unsupported filesystem type %s", devices.filesystem) 560 } 561 if err != nil { 562 return err 563 } 564 565 return nil 566 } 567 568 func (devices *DeviceSet) migrateOldMetaData() error { 569 // Migrate old metadata file 570 jsonData, err := ioutil.ReadFile(devices.oldMetadataFile()) 571 if err != nil && !os.IsNotExist(err) { 572 return err 573 } 574 575 if jsonData != nil { 576 m := metaData{Devices: make(map[string]*devInfo)} 577 578 if err := json.Unmarshal(jsonData, &m); err != nil { 579 return err 580 } 581 582 for hash, info := range m.Devices { 583 info.Hash = hash 584 devices.saveMetadata(info) 585 } 586 if err := os.Rename(devices.oldMetadataFile(), devices.oldMetadataFile()+".migrated"); err != nil { 587 return err 588 } 589 590 } 591 592 return nil 593 } 594 595 // Cleanup deleted devices. It assumes that all the devices have been 596 // loaded in the hash table. 597 func (devices *DeviceSet) cleanupDeletedDevices() error { 598 devices.Lock() 599 600 // If there are no deleted devices, there is nothing to do. 601 if devices.nrDeletedDevices == 0 { 602 return nil 603 } 604 605 var deletedDevices []*devInfo 606 607 for _, info := range devices.Devices { 608 if !info.Deleted { 609 continue 610 } 611 logrus.Debugf("devmapper: Found deleted device %s.", info.Hash) 612 deletedDevices = append(deletedDevices, info) 613 } 614 615 // Delete the deleted devices. DeleteDevice() first takes the info lock 616 // and then devices.Lock(). So drop it to avoid deadlock. 617 devices.Unlock() 618 619 for _, info := range deletedDevices { 620 // This will again try deferred deletion. 621 if err := devices.DeleteDevice(info.Hash, false); err != nil { 622 logrus.Warnf("devmapper: Deletion of device %s, device_id=%v failed:%v", info.Hash, info.DeviceID, err) 623 } 624 } 625 626 return nil 627 } 628 629 func (devices *DeviceSet) countDeletedDevices() { 630 for _, info := range devices.Devices { 631 if !info.Deleted { 632 continue 633 } 634 devices.nrDeletedDevices++ 635 } 636 } 637 638 func (devices *DeviceSet) startDeviceDeletionWorker() { 639 // Deferred deletion is not enabled. Don't do anything. 640 if !devices.deferredDelete { 641 return 642 } 643 644 logrus.Debugf("devmapper: Worker to cleanup deleted devices started") 645 for range devices.deletionWorkerTicker.C { 646 devices.cleanupDeletedDevices() 647 } 648 } 649 650 func (devices *DeviceSet) initMetaData() error { 651 devices.Lock() 652 defer devices.Unlock() 653 654 if err := devices.migrateOldMetaData(); err != nil { 655 return err 656 } 657 658 _, transactionID, _, _, _, _, err := devices.poolStatus() 659 if err != nil { 660 return err 661 } 662 663 devices.TransactionID = transactionID 664 665 if err := devices.loadDeviceFilesOnStart(); err != nil { 666 return fmt.Errorf("devmapper: Failed to load device files:%v", err) 667 } 668 669 devices.constructDeviceIDMap() 670 devices.countDeletedDevices() 671 672 if err := devices.processPendingTransaction(); err != nil { 673 return err 674 } 675 676 // Start a goroutine to cleanup Deleted Devices 677 go devices.startDeviceDeletionWorker() 678 return nil 679 } 680 681 func (devices *DeviceSet) incNextDeviceID() { 682 // IDs are 24bit, so wrap around 683 devices.NextDeviceID = (devices.NextDeviceID + 1) & maxDeviceID 684 } 685 686 func (devices *DeviceSet) getNextFreeDeviceID() (int, error) { 687 devices.incNextDeviceID() 688 for i := 0; i <= maxDeviceID; i++ { 689 if devices.isDeviceIDFree(devices.NextDeviceID) { 690 devices.markDeviceIDUsed(devices.NextDeviceID) 691 return devices.NextDeviceID, nil 692 } 693 devices.incNextDeviceID() 694 } 695 696 return 0, fmt.Errorf("Unable to find a free device ID") 697 } 698 699 func (devices *DeviceSet) createRegisterDevice(hash string) (*devInfo, error) { 700 devices.Lock() 701 defer devices.Unlock() 702 703 deviceID, err := devices.getNextFreeDeviceID() 704 if err != nil { 705 return nil, err 706 } 707 708 if err := devices.openTransaction(hash, deviceID); err != nil { 709 logrus.Debugf("Error opening transaction hash = %s deviceID = %d", hash, deviceID) 710 devices.markDeviceIDFree(deviceID) 711 return nil, err 712 } 713 714 for { 715 if err := devicemapper.CreateDevice(devices.getPoolDevName(), deviceID); err != nil { 716 if devicemapper.DeviceIDExists(err) { 717 // Device ID already exists. This should not 718 // happen. Now we have a mechianism to find 719 // a free device ID. So something is not right. 720 // Give a warning and continue. 721 logrus.Errorf("Device ID %d exists in pool but it is supposed to be unused", deviceID) 722 deviceID, err = devices.getNextFreeDeviceID() 723 if err != nil { 724 return nil, err 725 } 726 // Save new device id into transaction 727 devices.refreshTransaction(deviceID) 728 continue 729 } 730 logrus.Debugf("Error creating device: %s", err) 731 devices.markDeviceIDFree(deviceID) 732 return nil, err 733 } 734 break 735 } 736 737 logrus.Debugf("Registering device (id %v) with FS size %v", deviceID, devices.baseFsSize) 738 info, err := devices.registerDevice(deviceID, hash, devices.baseFsSize, devices.OpenTransactionID) 739 if err != nil { 740 _ = devicemapper.DeleteDevice(devices.getPoolDevName(), deviceID) 741 devices.markDeviceIDFree(deviceID) 742 return nil, err 743 } 744 745 if err := devices.closeTransaction(); err != nil { 746 devices.unregisterDevice(deviceID, hash) 747 devicemapper.DeleteDevice(devices.getPoolDevName(), deviceID) 748 devices.markDeviceIDFree(deviceID) 749 return nil, err 750 } 751 return info, nil 752 } 753 754 func (devices *DeviceSet) createRegisterSnapDevice(hash string, baseInfo *devInfo) error { 755 deviceID, err := devices.getNextFreeDeviceID() 756 if err != nil { 757 return err 758 } 759 760 if err := devices.openTransaction(hash, deviceID); err != nil { 761 logrus.Debugf("Error opening transaction hash = %s deviceID = %d", hash, deviceID) 762 devices.markDeviceIDFree(deviceID) 763 return err 764 } 765 766 for { 767 if err := devicemapper.CreateSnapDevice(devices.getPoolDevName(), deviceID, baseInfo.Name(), baseInfo.DeviceID); err != nil { 768 if devicemapper.DeviceIDExists(err) { 769 // Device ID already exists. This should not 770 // happen. Now we have a mechianism to find 771 // a free device ID. So something is not right. 772 // Give a warning and continue. 773 logrus.Errorf("Device ID %d exists in pool but it is supposed to be unused", deviceID) 774 deviceID, err = devices.getNextFreeDeviceID() 775 if err != nil { 776 return err 777 } 778 // Save new device id into transaction 779 devices.refreshTransaction(deviceID) 780 continue 781 } 782 logrus.Debugf("Error creating snap device: %s", err) 783 devices.markDeviceIDFree(deviceID) 784 return err 785 } 786 break 787 } 788 789 if _, err := devices.registerDevice(deviceID, hash, baseInfo.Size, devices.OpenTransactionID); err != nil { 790 devicemapper.DeleteDevice(devices.getPoolDevName(), deviceID) 791 devices.markDeviceIDFree(deviceID) 792 logrus.Debugf("Error registering device: %s", err) 793 return err 794 } 795 796 if err := devices.closeTransaction(); err != nil { 797 devices.unregisterDevice(deviceID, hash) 798 devicemapper.DeleteDevice(devices.getPoolDevName(), deviceID) 799 devices.markDeviceIDFree(deviceID) 800 return err 801 } 802 return nil 803 } 804 805 func (devices *DeviceSet) loadMetadata(hash string) *devInfo { 806 info := &devInfo{Hash: hash, devices: devices} 807 808 jsonData, err := ioutil.ReadFile(devices.metadataFile(info)) 809 if err != nil { 810 return nil 811 } 812 813 if err := json.Unmarshal(jsonData, &info); err != nil { 814 return nil 815 } 816 817 if info.DeviceID > maxDeviceID { 818 logrus.Errorf("Ignoring Invalid DeviceId=%d", info.DeviceID) 819 return nil 820 } 821 822 return info 823 } 824 825 func getDeviceUUID(device string) (string, error) { 826 out, err := exec.Command("blkid", "-s", "UUID", "-o", "value", device).Output() 827 if err != nil { 828 logrus.Debugf("Failed to find uuid for device %s:%v", device, err) 829 return "", err 830 } 831 832 uuid := strings.TrimSuffix(string(out), "\n") 833 uuid = strings.TrimSpace(uuid) 834 logrus.Debugf("UUID for device: %s is:%s", device, uuid) 835 return uuid, nil 836 } 837 838 func (devices *DeviceSet) getBaseDeviceSize() uint64 { 839 info, _ := devices.lookupDevice("") 840 if info == nil { 841 return 0 842 } 843 return info.Size 844 } 845 846 func (devices *DeviceSet) verifyBaseDeviceUUID(baseInfo *devInfo) error { 847 devices.Lock() 848 defer devices.Unlock() 849 850 if err := devices.activateDeviceIfNeeded(baseInfo, false); err != nil { 851 return err 852 } 853 854 defer devices.deactivateDevice(baseInfo) 855 856 uuid, err := getDeviceUUID(baseInfo.DevName()) 857 if err != nil { 858 return err 859 } 860 861 if devices.BaseDeviceUUID != uuid { 862 return fmt.Errorf("Current Base Device UUID:%s does not match with stored UUID:%s", uuid, devices.BaseDeviceUUID) 863 } 864 865 return nil 866 } 867 868 func (devices *DeviceSet) saveBaseDeviceUUID(baseInfo *devInfo) error { 869 devices.Lock() 870 defer devices.Unlock() 871 872 if err := devices.activateDeviceIfNeeded(baseInfo, false); err != nil { 873 return err 874 } 875 876 defer devices.deactivateDevice(baseInfo) 877 878 uuid, err := getDeviceUUID(baseInfo.DevName()) 879 if err != nil { 880 return err 881 } 882 883 devices.BaseDeviceUUID = uuid 884 devices.saveDeviceSetMetaData() 885 return nil 886 } 887 888 func (devices *DeviceSet) createBaseImage() error { 889 logrus.Debugf("Initializing base device-mapper thin volume") 890 891 // Create initial device 892 info, err := devices.createRegisterDevice("") 893 if err != nil { 894 return err 895 } 896 897 logrus.Debugf("Creating filesystem on base device-mapper thin volume") 898 899 if err := devices.activateDeviceIfNeeded(info, false); err != nil { 900 return err 901 } 902 903 if err := devices.createFilesystem(info); err != nil { 904 return err 905 } 906 907 info.Initialized = true 908 if err := devices.saveMetadata(info); err != nil { 909 info.Initialized = false 910 return err 911 } 912 913 if err := devices.saveBaseDeviceUUID(info); err != nil { 914 return fmt.Errorf("Could not query and save base device UUID:%v", err) 915 } 916 917 return nil 918 } 919 920 // Returns if thin pool device exists or not. If device exists, also makes 921 // sure it is a thin pool device and not some other type of device. 922 func (devices *DeviceSet) thinPoolExists(thinPoolDevice string) (bool, error) { 923 logrus.Debugf("devmapper: Checking for existence of the pool %s", thinPoolDevice) 924 925 info, err := devicemapper.GetInfo(thinPoolDevice) 926 if err != nil { 927 return false, fmt.Errorf("devmapper: GetInfo() on device %s failed: %v", thinPoolDevice, err) 928 } 929 930 // Device does not exist. 931 if info.Exists == 0 { 932 return false, nil 933 } 934 935 _, _, deviceType, _, err := devicemapper.GetStatus(thinPoolDevice) 936 if err != nil { 937 return false, fmt.Errorf("devmapper: GetStatus() on device %s failed: %v", thinPoolDevice, err) 938 } 939 940 if deviceType != "thin-pool" { 941 return false, fmt.Errorf("devmapper: Device %s is not a thin pool", thinPoolDevice) 942 } 943 944 return true, nil 945 } 946 947 func (devices *DeviceSet) checkThinPool() error { 948 _, transactionID, dataUsed, _, _, _, err := devices.poolStatus() 949 if err != nil { 950 return err 951 } 952 if dataUsed != 0 { 953 return fmt.Errorf("Unable to take ownership of thin-pool (%s) that already has used data blocks", 954 devices.thinPoolDevice) 955 } 956 if transactionID != 0 { 957 return fmt.Errorf("Unable to take ownership of thin-pool (%s) with non-zero transaction ID", 958 devices.thinPoolDevice) 959 } 960 return nil 961 } 962 963 // Base image is initialized properly. Either save UUID for first time (for 964 // upgrade case or verify UUID. 965 func (devices *DeviceSet) setupVerifyBaseImageUUID(baseInfo *devInfo) error { 966 // If BaseDeviceUUID is nil (upgrade case), save it and return success. 967 if devices.BaseDeviceUUID == "" { 968 if err := devices.saveBaseDeviceUUID(baseInfo); err != nil { 969 return fmt.Errorf("Could not query and save base device UUID:%v", err) 970 } 971 return nil 972 } 973 974 if err := devices.verifyBaseDeviceUUID(baseInfo); err != nil { 975 return fmt.Errorf("Base Device UUID verification failed. Possibly using a different thin pool than last invocation:%v", err) 976 } 977 978 return nil 979 } 980 981 func (devices *DeviceSet) setupBaseImage() error { 982 oldInfo, _ := devices.lookupDeviceWithLock("") 983 984 // base image already exists. If it is initialized properly, do UUID 985 // verification and return. Otherwise remove image and set it up 986 // fresh. 987 988 if oldInfo != nil { 989 if oldInfo.Initialized && !oldInfo.Deleted { 990 if err := devices.setupVerifyBaseImageUUID(oldInfo); err != nil { 991 return err 992 } 993 994 return nil 995 } 996 997 logrus.Debugf("Removing uninitialized base image") 998 // If previous base device is in deferred delete state, 999 // that needs to be cleaned up first. So don't try 1000 // deferred deletion. 1001 if err := devices.DeleteDevice("", true); err != nil { 1002 return err 1003 } 1004 } 1005 1006 // If we are setting up base image for the first time, make sure 1007 // thin pool is empty. 1008 if devices.thinPoolDevice != "" && oldInfo == nil { 1009 if err := devices.checkThinPool(); err != nil { 1010 return err 1011 } 1012 } 1013 1014 // Create new base image device 1015 if err := devices.createBaseImage(); err != nil { 1016 return err 1017 } 1018 1019 return nil 1020 } 1021 1022 func setCloseOnExec(name string) { 1023 if fileInfos, _ := ioutil.ReadDir("/proc/self/fd"); fileInfos != nil { 1024 for _, i := range fileInfos { 1025 link, _ := os.Readlink(filepath.Join("/proc/self/fd", i.Name())) 1026 if link == name { 1027 fd, err := strconv.Atoi(i.Name()) 1028 if err == nil { 1029 syscall.CloseOnExec(fd) 1030 } 1031 } 1032 } 1033 } 1034 } 1035 1036 // DMLog implements logging using DevMapperLogger interface. 1037 func (devices *DeviceSet) DMLog(level int, file string, line int, dmError int, message string) { 1038 // By default libdm sends us all the messages including debug ones. 1039 // We need to filter out messages here and figure out which one 1040 // should be printed. 1041 if level > logLevel { 1042 return 1043 } 1044 1045 // FIXME(vbatts) push this back into ./pkg/devicemapper/ 1046 if level <= devicemapper.LogLevelErr { 1047 logrus.Errorf("libdevmapper(%d): %s:%d (%d) %s", level, file, line, dmError, message) 1048 } else if level <= devicemapper.LogLevelInfo { 1049 logrus.Infof("libdevmapper(%d): %s:%d (%d) %s", level, file, line, dmError, message) 1050 } else { 1051 // FIXME(vbatts) push this back into ./pkg/devicemapper/ 1052 logrus.Debugf("libdevmapper(%d): %s:%d (%d) %s", level, file, line, dmError, message) 1053 } 1054 } 1055 1056 func major(device uint64) uint64 { 1057 return (device >> 8) & 0xfff 1058 } 1059 1060 func minor(device uint64) uint64 { 1061 return (device & 0xff) | ((device >> 12) & 0xfff00) 1062 } 1063 1064 // ResizePool increases the size of the pool. 1065 func (devices *DeviceSet) ResizePool(size int64) error { 1066 dirname := devices.loopbackDir() 1067 datafilename := path.Join(dirname, "data") 1068 if len(devices.dataDevice) > 0 { 1069 datafilename = devices.dataDevice 1070 } 1071 metadatafilename := path.Join(dirname, "metadata") 1072 if len(devices.metadataDevice) > 0 { 1073 metadatafilename = devices.metadataDevice 1074 } 1075 1076 datafile, err := os.OpenFile(datafilename, os.O_RDWR, 0) 1077 if datafile == nil { 1078 return err 1079 } 1080 defer datafile.Close() 1081 1082 fi, err := datafile.Stat() 1083 if fi == nil { 1084 return err 1085 } 1086 1087 if fi.Size() > size { 1088 return fmt.Errorf("Can't shrink file") 1089 } 1090 1091 dataloopback := devicemapper.FindLoopDeviceFor(datafile) 1092 if dataloopback == nil { 1093 return fmt.Errorf("Unable to find loopback mount for: %s", datafilename) 1094 } 1095 defer dataloopback.Close() 1096 1097 metadatafile, err := os.OpenFile(metadatafilename, os.O_RDWR, 0) 1098 if metadatafile == nil { 1099 return err 1100 } 1101 defer metadatafile.Close() 1102 1103 metadataloopback := devicemapper.FindLoopDeviceFor(metadatafile) 1104 if metadataloopback == nil { 1105 return fmt.Errorf("Unable to find loopback mount for: %s", metadatafilename) 1106 } 1107 defer metadataloopback.Close() 1108 1109 // Grow loopback file 1110 if err := datafile.Truncate(size); err != nil { 1111 return fmt.Errorf("Unable to grow loopback file: %s", err) 1112 } 1113 1114 // Reload size for loopback device 1115 if err := devicemapper.LoopbackSetCapacity(dataloopback); err != nil { 1116 return fmt.Errorf("Unable to update loopback capacity: %s", err) 1117 } 1118 1119 // Suspend the pool 1120 if err := devicemapper.SuspendDevice(devices.getPoolName()); err != nil { 1121 return fmt.Errorf("Unable to suspend pool: %s", err) 1122 } 1123 1124 // Reload with the new block sizes 1125 if err := devicemapper.ReloadPool(devices.getPoolName(), dataloopback, metadataloopback, devices.thinpBlockSize); err != nil { 1126 return fmt.Errorf("Unable to reload pool: %s", err) 1127 } 1128 1129 // Resume the pool 1130 if err := devicemapper.ResumeDevice(devices.getPoolName()); err != nil { 1131 return fmt.Errorf("Unable to resume pool: %s", err) 1132 } 1133 1134 return nil 1135 } 1136 1137 func (devices *DeviceSet) loadTransactionMetaData() error { 1138 jsonData, err := ioutil.ReadFile(devices.transactionMetaFile()) 1139 if err != nil { 1140 // There is no active transaction. This will be the case 1141 // during upgrade. 1142 if os.IsNotExist(err) { 1143 devices.OpenTransactionID = devices.TransactionID 1144 return nil 1145 } 1146 return err 1147 } 1148 1149 json.Unmarshal(jsonData, &devices.transaction) 1150 return nil 1151 } 1152 1153 func (devices *DeviceSet) saveTransactionMetaData() error { 1154 jsonData, err := json.Marshal(&devices.transaction) 1155 if err != nil { 1156 return fmt.Errorf("Error encoding metadata to json: %s", err) 1157 } 1158 1159 return devices.writeMetaFile(jsonData, devices.transactionMetaFile()) 1160 } 1161 1162 func (devices *DeviceSet) removeTransactionMetaData() error { 1163 if err := os.RemoveAll(devices.transactionMetaFile()); err != nil { 1164 return err 1165 } 1166 return nil 1167 } 1168 1169 func (devices *DeviceSet) rollbackTransaction() error { 1170 logrus.Debugf("Rolling back open transaction: TransactionID=%d hash=%s device_id=%d", devices.OpenTransactionID, devices.DeviceIDHash, devices.DeviceID) 1171 1172 // A device id might have already been deleted before transaction 1173 // closed. In that case this call will fail. Just leave a message 1174 // in case of failure. 1175 if err := devicemapper.DeleteDevice(devices.getPoolDevName(), devices.DeviceID); err != nil { 1176 logrus.Errorf("Unable to delete device: %s", err) 1177 } 1178 1179 dinfo := &devInfo{Hash: devices.DeviceIDHash} 1180 if err := devices.removeMetadata(dinfo); err != nil { 1181 logrus.Errorf("Unable to remove metadata: %s", err) 1182 } else { 1183 devices.markDeviceIDFree(devices.DeviceID) 1184 } 1185 1186 if err := devices.removeTransactionMetaData(); err != nil { 1187 logrus.Errorf("Unable to remove transaction meta file %s: %s", devices.transactionMetaFile(), err) 1188 } 1189 1190 return nil 1191 } 1192 1193 func (devices *DeviceSet) processPendingTransaction() error { 1194 if err := devices.loadTransactionMetaData(); err != nil { 1195 return err 1196 } 1197 1198 // If there was open transaction but pool transaction ID is same 1199 // as open transaction ID, nothing to roll back. 1200 if devices.TransactionID == devices.OpenTransactionID { 1201 return nil 1202 } 1203 1204 // If open transaction ID is less than pool transaction ID, something 1205 // is wrong. Bail out. 1206 if devices.OpenTransactionID < devices.TransactionID { 1207 logrus.Errorf("Open Transaction id %d is less than pool transaction id %d", devices.OpenTransactionID, devices.TransactionID) 1208 return nil 1209 } 1210 1211 // Pool transaction ID is not same as open transaction. There is 1212 // a transaction which was not completed. 1213 if err := devices.rollbackTransaction(); err != nil { 1214 return fmt.Errorf("Rolling back open transaction failed: %s", err) 1215 } 1216 1217 devices.OpenTransactionID = devices.TransactionID 1218 return nil 1219 } 1220 1221 func (devices *DeviceSet) loadDeviceSetMetaData() error { 1222 jsonData, err := ioutil.ReadFile(devices.deviceSetMetaFile()) 1223 if err != nil { 1224 // For backward compatibility return success if file does 1225 // not exist. 1226 if os.IsNotExist(err) { 1227 return nil 1228 } 1229 return err 1230 } 1231 1232 return json.Unmarshal(jsonData, devices) 1233 } 1234 1235 func (devices *DeviceSet) saveDeviceSetMetaData() error { 1236 jsonData, err := json.Marshal(devices) 1237 if err != nil { 1238 return fmt.Errorf("Error encoding metadata to json: %s", err) 1239 } 1240 1241 return devices.writeMetaFile(jsonData, devices.deviceSetMetaFile()) 1242 } 1243 1244 func (devices *DeviceSet) openTransaction(hash string, DeviceID int) error { 1245 devices.allocateTransactionID() 1246 devices.DeviceIDHash = hash 1247 devices.DeviceID = DeviceID 1248 if err := devices.saveTransactionMetaData(); err != nil { 1249 return fmt.Errorf("Error saving transaction metadata: %s", err) 1250 } 1251 return nil 1252 } 1253 1254 func (devices *DeviceSet) refreshTransaction(DeviceID int) error { 1255 devices.DeviceID = DeviceID 1256 if err := devices.saveTransactionMetaData(); err != nil { 1257 return fmt.Errorf("Error saving transaction metadata: %s", err) 1258 } 1259 return nil 1260 } 1261 1262 func (devices *DeviceSet) closeTransaction() error { 1263 if err := devices.updatePoolTransactionID(); err != nil { 1264 logrus.Debugf("Failed to close Transaction") 1265 return err 1266 } 1267 return nil 1268 } 1269 1270 func determineDriverCapabilities(version string) error { 1271 /* 1272 * Driver version 4.27.0 and greater support deferred activation 1273 * feature. 1274 */ 1275 1276 logrus.Debugf("devicemapper: driver version is %s", version) 1277 1278 versionSplit := strings.Split(version, ".") 1279 major, err := strconv.Atoi(versionSplit[0]) 1280 if err != nil { 1281 return graphdriver.ErrNotSupported 1282 } 1283 1284 if major > 4 { 1285 driverDeferredRemovalSupport = true 1286 return nil 1287 } 1288 1289 if major < 4 { 1290 return nil 1291 } 1292 1293 minor, err := strconv.Atoi(versionSplit[1]) 1294 if err != nil { 1295 return graphdriver.ErrNotSupported 1296 } 1297 1298 /* 1299 * If major is 4 and minor is 27, then there is no need to 1300 * check for patch level as it can not be less than 0. 1301 */ 1302 if minor >= 27 { 1303 driverDeferredRemovalSupport = true 1304 return nil 1305 } 1306 1307 return nil 1308 } 1309 1310 // Determine the major and minor number of loopback device 1311 func getDeviceMajorMinor(file *os.File) (uint64, uint64, error) { 1312 stat, err := file.Stat() 1313 if err != nil { 1314 return 0, 0, err 1315 } 1316 1317 dev := stat.Sys().(*syscall.Stat_t).Rdev 1318 majorNum := major(dev) 1319 minorNum := minor(dev) 1320 1321 logrus.Debugf("[devmapper]: Major:Minor for device: %s is:%v:%v", file.Name(), majorNum, minorNum) 1322 return majorNum, minorNum, nil 1323 } 1324 1325 // Given a file which is backing file of a loop back device, find the 1326 // loopback device name and its major/minor number. 1327 func getLoopFileDeviceMajMin(filename string) (string, uint64, uint64, error) { 1328 file, err := os.Open(filename) 1329 if err != nil { 1330 logrus.Debugf("[devmapper]: Failed to open file %s", filename) 1331 return "", 0, 0, err 1332 } 1333 1334 defer file.Close() 1335 loopbackDevice := devicemapper.FindLoopDeviceFor(file) 1336 if loopbackDevice == nil { 1337 return "", 0, 0, fmt.Errorf("[devmapper]: Unable to find loopback mount for: %s", filename) 1338 } 1339 defer loopbackDevice.Close() 1340 1341 Major, Minor, err := getDeviceMajorMinor(loopbackDevice) 1342 if err != nil { 1343 return "", 0, 0, err 1344 } 1345 return loopbackDevice.Name(), Major, Minor, nil 1346 } 1347 1348 // Get the major/minor numbers of thin pool data and metadata devices 1349 func (devices *DeviceSet) getThinPoolDataMetaMajMin() (uint64, uint64, uint64, uint64, error) { 1350 var params, poolDataMajMin, poolMetadataMajMin string 1351 1352 _, _, _, params, err := devicemapper.GetTable(devices.getPoolName()) 1353 if err != nil { 1354 return 0, 0, 0, 0, err 1355 } 1356 1357 if _, err = fmt.Sscanf(params, "%s %s", &poolMetadataMajMin, &poolDataMajMin); err != nil { 1358 return 0, 0, 0, 0, err 1359 } 1360 1361 logrus.Debugf("[devmapper]: poolDataMajMin=%s poolMetaMajMin=%s\n", poolDataMajMin, poolMetadataMajMin) 1362 1363 poolDataMajMinorSplit := strings.Split(poolDataMajMin, ":") 1364 poolDataMajor, err := strconv.ParseUint(poolDataMajMinorSplit[0], 10, 32) 1365 if err != nil { 1366 return 0, 0, 0, 0, err 1367 } 1368 1369 poolDataMinor, err := strconv.ParseUint(poolDataMajMinorSplit[1], 10, 32) 1370 if err != nil { 1371 return 0, 0, 0, 0, err 1372 } 1373 1374 poolMetadataMajMinorSplit := strings.Split(poolMetadataMajMin, ":") 1375 poolMetadataMajor, err := strconv.ParseUint(poolMetadataMajMinorSplit[0], 10, 32) 1376 if err != nil { 1377 return 0, 0, 0, 0, err 1378 } 1379 1380 poolMetadataMinor, err := strconv.ParseUint(poolMetadataMajMinorSplit[1], 10, 32) 1381 if err != nil { 1382 return 0, 0, 0, 0, err 1383 } 1384 1385 return poolDataMajor, poolDataMinor, poolMetadataMajor, poolMetadataMinor, nil 1386 } 1387 1388 func (devices *DeviceSet) loadThinPoolLoopBackInfo() error { 1389 poolDataMajor, poolDataMinor, poolMetadataMajor, poolMetadataMinor, err := devices.getThinPoolDataMetaMajMin() 1390 if err != nil { 1391 return err 1392 } 1393 1394 dirname := devices.loopbackDir() 1395 1396 // data device has not been passed in. So there should be a data file 1397 // which is being mounted as loop device. 1398 if devices.dataDevice == "" { 1399 datafilename := path.Join(dirname, "data") 1400 dataLoopDevice, dataMajor, dataMinor, err := getLoopFileDeviceMajMin(datafilename) 1401 if err != nil { 1402 return err 1403 } 1404 1405 // Compare the two 1406 if poolDataMajor == dataMajor && poolDataMinor == dataMinor { 1407 devices.dataDevice = dataLoopDevice 1408 devices.dataLoopFile = datafilename 1409 } 1410 1411 } 1412 1413 // metadata device has not been passed in. So there should be a 1414 // metadata file which is being mounted as loop device. 1415 if devices.metadataDevice == "" { 1416 metadatafilename := path.Join(dirname, "metadata") 1417 metadataLoopDevice, metadataMajor, metadataMinor, err := getLoopFileDeviceMajMin(metadatafilename) 1418 if err != nil { 1419 return err 1420 } 1421 if poolMetadataMajor == metadataMajor && poolMetadataMinor == metadataMinor { 1422 devices.metadataDevice = metadataLoopDevice 1423 devices.metadataLoopFile = metadatafilename 1424 } 1425 } 1426 1427 return nil 1428 } 1429 1430 func (devices *DeviceSet) initDevmapper(doInit bool) error { 1431 // give ourselves to libdm as a log handler 1432 devicemapper.LogInit(devices) 1433 1434 version, err := devicemapper.GetDriverVersion() 1435 if err != nil { 1436 // Can't even get driver version, assume not supported 1437 return graphdriver.ErrNotSupported 1438 } 1439 1440 if err := determineDriverCapabilities(version); err != nil { 1441 return graphdriver.ErrNotSupported 1442 } 1443 1444 // If user asked for deferred removal then check both libdm library 1445 // and kernel driver support deferred removal otherwise error out. 1446 if enableDeferredRemoval { 1447 if !driverDeferredRemovalSupport { 1448 return fmt.Errorf("devmapper: Deferred removal can not be enabled as kernel does not support it") 1449 } 1450 if !devicemapper.LibraryDeferredRemovalSupport { 1451 return fmt.Errorf("devmapper: Deferred removal can not be enabled as libdm does not support it") 1452 } 1453 logrus.Debugf("devmapper: Deferred removal support enabled.") 1454 devices.deferredRemove = true 1455 } 1456 1457 if enableDeferredDeletion { 1458 if !devices.deferredRemove { 1459 return fmt.Errorf("devmapper: Deferred deletion can not be enabled as deferred removal is not enabled. Enable deferred removal using --storage-opt dm.use_deferred_removal=true parameter") 1460 } 1461 logrus.Debugf("devmapper: Deferred deletion support enabled.") 1462 devices.deferredDelete = true 1463 } 1464 1465 // https://github.com/docker/docker/issues/4036 1466 if supported := devicemapper.UdevSetSyncSupport(true); !supported { 1467 logrus.Warn("Udev sync is not supported. This will lead to unexpected behavior, data loss and errors. For more information, see https://docs.docker.com/reference/commandline/daemon/#daemon-storage-driver-option") 1468 } 1469 1470 //create the root dir of the devmapper driver ownership to match this 1471 //daemon's remapped root uid/gid so containers can start properly 1472 uid, gid, err := idtools.GetRootUIDGID(devices.uidMaps, devices.gidMaps) 1473 if err != nil { 1474 return err 1475 } 1476 if err := idtools.MkdirAs(devices.root, 0700, uid, gid); err != nil && !os.IsExist(err) { 1477 return err 1478 } 1479 if err := os.MkdirAll(devices.metadataDir(), 0700); err != nil && !os.IsExist(err) { 1480 return err 1481 } 1482 1483 // Set the device prefix from the device id and inode of the docker root dir 1484 1485 st, err := os.Stat(devices.root) 1486 if err != nil { 1487 return fmt.Errorf("Error looking up dir %s: %s", devices.root, err) 1488 } 1489 sysSt := st.Sys().(*syscall.Stat_t) 1490 // "reg-" stands for "regular file". 1491 // In the future we might use "dev-" for "device file", etc. 1492 // docker-maj,min[-inode] stands for: 1493 // - Managed by docker 1494 // - The target of this device is at major <maj> and minor <min> 1495 // - If <inode> is defined, use that file inside the device as a loopback image. Otherwise use the device itself. 1496 devices.devicePrefix = fmt.Sprintf("docker-%d:%d-%d", major(sysSt.Dev), minor(sysSt.Dev), sysSt.Ino) 1497 logrus.Debugf("Generated prefix: %s", devices.devicePrefix) 1498 1499 // Check for the existence of the thin-pool device 1500 poolExists, err := devices.thinPoolExists(devices.getPoolName()) 1501 if err != nil { 1502 return err 1503 } 1504 1505 // It seems libdevmapper opens this without O_CLOEXEC, and go exec will not close files 1506 // that are not Close-on-exec, and lxc-start will die if it inherits any unexpected files, 1507 // so we add this badhack to make sure it closes itself 1508 setCloseOnExec("/dev/mapper/control") 1509 1510 // Make sure the sparse images exist in <root>/devicemapper/data and 1511 // <root>/devicemapper/metadata 1512 1513 createdLoopback := false 1514 1515 // If the pool doesn't exist, create it 1516 if !poolExists && devices.thinPoolDevice == "" { 1517 logrus.Debugf("Pool doesn't exist. Creating it.") 1518 1519 var ( 1520 dataFile *os.File 1521 metadataFile *os.File 1522 ) 1523 1524 if devices.dataDevice == "" { 1525 // Make sure the sparse images exist in <root>/devicemapper/data 1526 1527 hasData := devices.hasImage("data") 1528 1529 if !doInit && !hasData { 1530 return errors.New("Loopback data file not found") 1531 } 1532 1533 if !hasData { 1534 createdLoopback = true 1535 } 1536 1537 data, err := devices.ensureImage("data", devices.dataLoopbackSize) 1538 if err != nil { 1539 logrus.Debugf("Error device ensureImage (data): %s", err) 1540 return err 1541 } 1542 1543 dataFile, err = devicemapper.AttachLoopDevice(data) 1544 if err != nil { 1545 return err 1546 } 1547 devices.dataLoopFile = data 1548 devices.dataDevice = dataFile.Name() 1549 } else { 1550 dataFile, err = os.OpenFile(devices.dataDevice, os.O_RDWR, 0600) 1551 if err != nil { 1552 return err 1553 } 1554 } 1555 defer dataFile.Close() 1556 1557 if devices.metadataDevice == "" { 1558 // Make sure the sparse images exist in <root>/devicemapper/metadata 1559 1560 hasMetadata := devices.hasImage("metadata") 1561 1562 if !doInit && !hasMetadata { 1563 return errors.New("Loopback metadata file not found") 1564 } 1565 1566 if !hasMetadata { 1567 createdLoopback = true 1568 } 1569 1570 metadata, err := devices.ensureImage("metadata", devices.metaDataLoopbackSize) 1571 if err != nil { 1572 logrus.Debugf("Error device ensureImage (metadata): %s", err) 1573 return err 1574 } 1575 1576 metadataFile, err = devicemapper.AttachLoopDevice(metadata) 1577 if err != nil { 1578 return err 1579 } 1580 devices.metadataLoopFile = metadata 1581 devices.metadataDevice = metadataFile.Name() 1582 } else { 1583 metadataFile, err = os.OpenFile(devices.metadataDevice, os.O_RDWR, 0600) 1584 if err != nil { 1585 return err 1586 } 1587 } 1588 defer metadataFile.Close() 1589 1590 if err := devicemapper.CreatePool(devices.getPoolName(), dataFile, metadataFile, devices.thinpBlockSize); err != nil { 1591 return err 1592 } 1593 } 1594 1595 // Pool already exists and caller did not pass us a pool. That means 1596 // we probably created pool earlier and could not remove it as some 1597 // containers were still using it. Detect some of the properties of 1598 // pool, like is it using loop devices. 1599 if poolExists && devices.thinPoolDevice == "" { 1600 if err := devices.loadThinPoolLoopBackInfo(); err != nil { 1601 logrus.Debugf("Failed to load thin pool loopback device information:%v", err) 1602 return err 1603 } 1604 } 1605 1606 // If we didn't just create the data or metadata image, we need to 1607 // load the transaction id and migrate old metadata 1608 if !createdLoopback { 1609 if err := devices.initMetaData(); err != nil { 1610 return err 1611 } 1612 } 1613 1614 if devices.thinPoolDevice == "" { 1615 if devices.metadataLoopFile != "" || devices.dataLoopFile != "" { 1616 logrus.Warnf("Usage of loopback devices is strongly discouraged for production use. Please use `--storage-opt dm.thinpooldev` or use `man docker` to refer to dm.thinpooldev section.") 1617 } 1618 } 1619 1620 // Right now this loads only NextDeviceID. If there is more metadata 1621 // down the line, we might have to move it earlier. 1622 if err := devices.loadDeviceSetMetaData(); err != nil { 1623 return err 1624 } 1625 1626 // Setup the base image 1627 if doInit { 1628 if err := devices.setupBaseImage(); err != nil { 1629 logrus.Debugf("Error device setupBaseImage: %s", err) 1630 return err 1631 } 1632 } 1633 1634 return nil 1635 } 1636 1637 // AddDevice adds a device and registers in the hash. 1638 func (devices *DeviceSet) AddDevice(hash, baseHash string) error { 1639 logrus.Debugf("[deviceset] AddDevice(hash=%s basehash=%s)", hash, baseHash) 1640 defer logrus.Debugf("[deviceset] AddDevice(hash=%s basehash=%s) END", hash, baseHash) 1641 1642 // If a deleted device exists, return error. 1643 baseInfo, err := devices.lookupDeviceWithLock(baseHash) 1644 if err != nil { 1645 return err 1646 } 1647 1648 if baseInfo.Deleted { 1649 return fmt.Errorf("devmapper: Base device %v has been marked for deferred deletion", baseInfo.Hash) 1650 } 1651 1652 baseInfo.lock.Lock() 1653 defer baseInfo.lock.Unlock() 1654 1655 devices.Lock() 1656 defer devices.Unlock() 1657 1658 // Also include deleted devices in case hash of new device is 1659 // same as one of the deleted devices. 1660 if info, _ := devices.lookupDevice(hash); info != nil { 1661 return fmt.Errorf("device %s already exists. Deleted=%v", hash, info.Deleted) 1662 } 1663 1664 if err := devices.createRegisterSnapDevice(hash, baseInfo); err != nil { 1665 return err 1666 } 1667 1668 return nil 1669 } 1670 1671 func (devices *DeviceSet) markForDeferredDeletion(info *devInfo) error { 1672 // If device is already in deleted state, there is nothing to be done. 1673 if info.Deleted { 1674 return nil 1675 } 1676 1677 logrus.Debugf("devmapper: Marking device %s for deferred deletion.", info.Hash) 1678 1679 info.Deleted = true 1680 1681 // save device metadata to refelect deleted state. 1682 if err := devices.saveMetadata(info); err != nil { 1683 info.Deleted = false 1684 return err 1685 } 1686 1687 devices.nrDeletedDevices++ 1688 return nil 1689 } 1690 1691 // Should be caled with devices.Lock() held. 1692 func (devices *DeviceSet) deleteTransaction(info *devInfo, syncDelete bool) error { 1693 if err := devices.openTransaction(info.Hash, info.DeviceID); err != nil { 1694 logrus.Debugf("Error opening transaction hash = %s deviceId = %d", "", info.DeviceID) 1695 return err 1696 } 1697 1698 defer devices.closeTransaction() 1699 1700 err := devicemapper.DeleteDevice(devices.getPoolDevName(), info.DeviceID) 1701 if err != nil { 1702 // If syncDelete is true, we want to return error. If deferred 1703 // deletion is not enabled, we return an error. If error is 1704 // something other then EBUSY, return an error. 1705 if syncDelete || !devices.deferredDelete || err != devicemapper.ErrBusy { 1706 logrus.Debugf("Error deleting device: %s", err) 1707 return err 1708 } 1709 } 1710 1711 if err == nil { 1712 if err := devices.unregisterDevice(info.DeviceID, info.Hash); err != nil { 1713 return err 1714 } 1715 // If device was already in deferred delete state that means 1716 // deletion was being tried again later. Reduce the deleted 1717 // device count. 1718 if info.Deleted { 1719 devices.nrDeletedDevices-- 1720 } 1721 } else { 1722 if err := devices.markForDeferredDeletion(info); err != nil { 1723 return err 1724 } 1725 } 1726 1727 return nil 1728 } 1729 1730 // Issue discard only if device open count is zero. 1731 func (devices *DeviceSet) issueDiscard(info *devInfo) error { 1732 logrus.Debugf("devmapper: issueDiscard(device: %s). START", info.Hash) 1733 defer logrus.Debugf("devmapper: issueDiscard(device: %s). END", info.Hash) 1734 // This is a workaround for the kernel not discarding block so 1735 // on the thin pool when we remove a thinp device, so we do it 1736 // manually. 1737 // Even if device is deferred deleted, activate it and isue 1738 // discards. 1739 if err := devices.activateDeviceIfNeeded(info, true); err != nil { 1740 return err 1741 } 1742 1743 devinfo, err := devicemapper.GetInfo(info.Name()) 1744 if err != nil { 1745 return err 1746 } 1747 1748 if devinfo.OpenCount != 0 { 1749 logrus.Debugf("devmapper: Device: %s is in use. OpenCount=%d. Not issuing discards.", info.Hash, devinfo.OpenCount) 1750 return nil 1751 } 1752 1753 if err := devicemapper.BlockDeviceDiscard(info.DevName()); err != nil { 1754 logrus.Debugf("Error discarding block on device: %s (ignoring)", err) 1755 } 1756 return nil 1757 } 1758 1759 // Should be called with devices.Lock() held. 1760 func (devices *DeviceSet) deleteDevice(info *devInfo, syncDelete bool) error { 1761 if devices.doBlkDiscard { 1762 devices.issueDiscard(info) 1763 } 1764 1765 // Try to deactivate device in case it is active. 1766 if err := devices.deactivateDevice(info); err != nil { 1767 logrus.Debugf("Error deactivating device: %s", err) 1768 return err 1769 } 1770 1771 if err := devices.deleteTransaction(info, syncDelete); err != nil { 1772 return err 1773 } 1774 1775 devices.markDeviceIDFree(info.DeviceID) 1776 1777 return nil 1778 } 1779 1780 // DeleteDevice will return success if device has been marked for deferred 1781 // removal. If one wants to override that and want DeleteDevice() to fail if 1782 // device was busy and could not be deleted, set syncDelete=true. 1783 func (devices *DeviceSet) DeleteDevice(hash string, syncDelete bool) error { 1784 logrus.Debugf("devmapper: DeleteDevice(hash=%v syncDelete=%v) START", hash, syncDelete) 1785 defer logrus.Debugf("devmapper: DeleteDevice(hash=%v syncDelete=%v) END", hash, syncDelete) 1786 info, err := devices.lookupDeviceWithLock(hash) 1787 if err != nil { 1788 return err 1789 } 1790 1791 info.lock.Lock() 1792 defer info.lock.Unlock() 1793 1794 devices.Lock() 1795 defer devices.Unlock() 1796 1797 // If mountcount is not zero, that means devices is still in use 1798 // or has not been Put() properly. Fail device deletion. 1799 1800 if info.mountCount != 0 { 1801 return fmt.Errorf("devmapper: Can't delete device %v as it is still mounted. mntCount=%v", info.Hash, info.mountCount) 1802 } 1803 1804 return devices.deleteDevice(info, syncDelete) 1805 } 1806 1807 func (devices *DeviceSet) deactivatePool() error { 1808 logrus.Debugf("[devmapper] deactivatePool()") 1809 defer logrus.Debugf("[devmapper] deactivatePool END") 1810 devname := devices.getPoolDevName() 1811 1812 devinfo, err := devicemapper.GetInfo(devname) 1813 if err != nil { 1814 return err 1815 } 1816 1817 if devinfo.Exists == 0 { 1818 return nil 1819 } 1820 if err := devicemapper.RemoveDevice(devname); err != nil { 1821 return err 1822 } 1823 1824 if d, err := devicemapper.GetDeps(devname); err == nil { 1825 logrus.Warnf("[devmapper] device %s still has %d active dependents", devname, d.Count) 1826 } 1827 1828 return nil 1829 } 1830 1831 func (devices *DeviceSet) deactivateDevice(info *devInfo) error { 1832 logrus.Debugf("[devmapper] deactivateDevice(%s)", info.Hash) 1833 defer logrus.Debugf("[devmapper] deactivateDevice END(%s)", info.Hash) 1834 1835 devinfo, err := devicemapper.GetInfo(info.Name()) 1836 if err != nil { 1837 return err 1838 } 1839 1840 if devinfo.Exists == 0 { 1841 return nil 1842 } 1843 1844 if devices.deferredRemove { 1845 if err := devicemapper.RemoveDeviceDeferred(info.Name()); err != nil { 1846 return err 1847 } 1848 } else { 1849 if err := devices.removeDevice(info.Name()); err != nil { 1850 return err 1851 } 1852 } 1853 return nil 1854 } 1855 1856 // Issues the underlying dm remove operation. 1857 func (devices *DeviceSet) removeDevice(devname string) error { 1858 var err error 1859 1860 logrus.Debugf("[devmapper] removeDevice START(%s)", devname) 1861 defer logrus.Debugf("[devmapper] removeDevice END(%s)", devname) 1862 1863 for i := 0; i < 200; i++ { 1864 err = devicemapper.RemoveDevice(devname) 1865 if err == nil { 1866 break 1867 } 1868 if err != devicemapper.ErrBusy { 1869 return err 1870 } 1871 1872 // If we see EBUSY it may be a transient error, 1873 // sleep a bit a retry a few times. 1874 devices.Unlock() 1875 time.Sleep(100 * time.Millisecond) 1876 devices.Lock() 1877 } 1878 1879 return err 1880 } 1881 1882 func (devices *DeviceSet) cancelDeferredRemoval(info *devInfo) error { 1883 if !devices.deferredRemove { 1884 return nil 1885 } 1886 1887 logrus.Debugf("[devmapper] cancelDeferredRemoval START(%s)", info.Name()) 1888 defer logrus.Debugf("[devmapper] cancelDeferredRemoval END(%s)", info.Name()) 1889 1890 devinfo, err := devicemapper.GetInfoWithDeferred(info.Name()) 1891 1892 if devinfo != nil && devinfo.DeferredRemove == 0 { 1893 return nil 1894 } 1895 1896 // Cancel deferred remove 1897 for i := 0; i < 100; i++ { 1898 err = devicemapper.CancelDeferredRemove(info.Name()) 1899 if err == nil { 1900 break 1901 } 1902 1903 if err == devicemapper.ErrEnxio { 1904 // Device is probably already gone. Return success. 1905 return nil 1906 } 1907 1908 if err != devicemapper.ErrBusy { 1909 return err 1910 } 1911 1912 // If we see EBUSY it may be a transient error, 1913 // sleep a bit a retry a few times. 1914 devices.Unlock() 1915 time.Sleep(100 * time.Millisecond) 1916 devices.Lock() 1917 } 1918 return err 1919 } 1920 1921 // Shutdown shuts down the device by unmounting the root. 1922 func (devices *DeviceSet) Shutdown() error { 1923 logrus.Debugf("[deviceset %s] Shutdown()", devices.devicePrefix) 1924 logrus.Debugf("[devmapper] Shutting down DeviceSet: %s", devices.root) 1925 defer logrus.Debugf("[deviceset %s] Shutdown() END", devices.devicePrefix) 1926 1927 var devs []*devInfo 1928 1929 // Stop deletion worker. This should start delivering new events to 1930 // ticker channel. That means no new instance of cleanupDeletedDevice() 1931 // will run after this call. If one instance is already running at 1932 // the time of the call, it must be holding devices.Lock() and 1933 // we will block on this lock till cleanup function exits. 1934 devices.deletionWorkerTicker.Stop() 1935 1936 devices.Lock() 1937 // Save DeviceSet Metadata first. Docker kills all threads if they 1938 // don't finish in certain time. It is possible that Shutdown() 1939 // routine does not finish in time as we loop trying to deactivate 1940 // some devices while these are busy. In that case shutdown() routine 1941 // will be killed and we will not get a chance to save deviceset 1942 // metadata. Hence save this early before trying to deactivate devices. 1943 devices.saveDeviceSetMetaData() 1944 1945 for _, info := range devices.Devices { 1946 devs = append(devs, info) 1947 } 1948 devices.Unlock() 1949 1950 for _, info := range devs { 1951 info.lock.Lock() 1952 if info.mountCount > 0 { 1953 // We use MNT_DETACH here in case it is still busy in some running 1954 // container. This means it'll go away from the global scope directly, 1955 // and the device will be released when that container dies. 1956 if err := syscall.Unmount(info.mountPath, syscall.MNT_DETACH); err != nil { 1957 logrus.Debugf("Shutdown unmounting %s, error: %s", info.mountPath, err) 1958 } 1959 1960 devices.Lock() 1961 if err := devices.deactivateDevice(info); err != nil { 1962 logrus.Debugf("Shutdown deactivate %s , error: %s", info.Hash, err) 1963 } 1964 devices.Unlock() 1965 } 1966 info.lock.Unlock() 1967 } 1968 1969 info, _ := devices.lookupDeviceWithLock("") 1970 if info != nil { 1971 info.lock.Lock() 1972 devices.Lock() 1973 if err := devices.deactivateDevice(info); err != nil { 1974 logrus.Debugf("Shutdown deactivate base , error: %s", err) 1975 } 1976 devices.Unlock() 1977 info.lock.Unlock() 1978 } 1979 1980 devices.Lock() 1981 if devices.thinPoolDevice == "" { 1982 if err := devices.deactivatePool(); err != nil { 1983 logrus.Debugf("Shutdown deactivate pool , error: %s", err) 1984 } 1985 } 1986 devices.Unlock() 1987 1988 return nil 1989 } 1990 1991 // MountDevice mounts the device if not already mounted. 1992 func (devices *DeviceSet) MountDevice(hash, path, mountLabel string) error { 1993 info, err := devices.lookupDeviceWithLock(hash) 1994 if err != nil { 1995 return err 1996 } 1997 1998 if info.Deleted { 1999 return fmt.Errorf("devmapper: Can't mount device %v as it has been marked for deferred deletion", info.Hash) 2000 } 2001 2002 info.lock.Lock() 2003 defer info.lock.Unlock() 2004 2005 devices.Lock() 2006 defer devices.Unlock() 2007 2008 if info.mountCount > 0 { 2009 if path != info.mountPath { 2010 return fmt.Errorf("Trying to mount devmapper device in multiple places (%s, %s)", info.mountPath, path) 2011 } 2012 2013 info.mountCount++ 2014 return nil 2015 } 2016 2017 if err := devices.activateDeviceIfNeeded(info, false); err != nil { 2018 return fmt.Errorf("Error activating devmapper device for '%s': %s", hash, err) 2019 } 2020 2021 fstype, err := ProbeFsType(info.DevName()) 2022 if err != nil { 2023 return err 2024 } 2025 2026 options := "" 2027 2028 if fstype == "xfs" { 2029 // XFS needs nouuid or it can't mount filesystems with the same fs 2030 options = joinMountOptions(options, "nouuid") 2031 } 2032 2033 options = joinMountOptions(options, devices.mountOptions) 2034 options = joinMountOptions(options, label.FormatMountLabel("", mountLabel)) 2035 2036 if err := mount.Mount(info.DevName(), path, fstype, options); err != nil { 2037 return fmt.Errorf("Error mounting '%s' on '%s': %s", info.DevName(), path, err) 2038 } 2039 2040 info.mountCount = 1 2041 info.mountPath = path 2042 2043 return nil 2044 } 2045 2046 // UnmountDevice unmounts the device and removes it from hash. 2047 func (devices *DeviceSet) UnmountDevice(hash string) error { 2048 logrus.Debugf("[devmapper] UnmountDevice(hash=%s)", hash) 2049 defer logrus.Debugf("[devmapper] UnmountDevice(hash=%s) END", hash) 2050 2051 info, err := devices.lookupDeviceWithLock(hash) 2052 if err != nil { 2053 return err 2054 } 2055 2056 info.lock.Lock() 2057 defer info.lock.Unlock() 2058 2059 devices.Lock() 2060 defer devices.Unlock() 2061 2062 if info.mountCount == 0 { 2063 return fmt.Errorf("UnmountDevice: device not-mounted id %s", hash) 2064 } 2065 2066 info.mountCount-- 2067 if info.mountCount > 0 { 2068 return nil 2069 } 2070 2071 logrus.Debugf("[devmapper] Unmount(%s)", info.mountPath) 2072 if err := syscall.Unmount(info.mountPath, syscall.MNT_DETACH); err != nil { 2073 return err 2074 } 2075 logrus.Debugf("[devmapper] Unmount done") 2076 2077 if err := devices.deactivateDevice(info); err != nil { 2078 return err 2079 } 2080 2081 info.mountPath = "" 2082 2083 return nil 2084 } 2085 2086 // HasDevice returns true if the device metadata exists. 2087 func (devices *DeviceSet) HasDevice(hash string) bool { 2088 info, _ := devices.lookupDeviceWithLock(hash) 2089 return info != nil 2090 } 2091 2092 // List returns a list of device ids. 2093 func (devices *DeviceSet) List() []string { 2094 devices.Lock() 2095 defer devices.Unlock() 2096 2097 ids := make([]string, len(devices.Devices)) 2098 i := 0 2099 for k := range devices.Devices { 2100 ids[i] = k 2101 i++ 2102 } 2103 return ids 2104 } 2105 2106 func (devices *DeviceSet) deviceStatus(devName string) (sizeInSectors, mappedSectors, highestMappedSector uint64, err error) { 2107 var params string 2108 _, sizeInSectors, _, params, err = devicemapper.GetStatus(devName) 2109 if err != nil { 2110 return 2111 } 2112 if _, err = fmt.Sscanf(params, "%d %d", &mappedSectors, &highestMappedSector); err == nil { 2113 return 2114 } 2115 return 2116 } 2117 2118 // GetDeviceStatus provides size, mapped sectors 2119 func (devices *DeviceSet) GetDeviceStatus(hash string) (*DevStatus, error) { 2120 info, err := devices.lookupDeviceWithLock(hash) 2121 if err != nil { 2122 return nil, err 2123 } 2124 2125 info.lock.Lock() 2126 defer info.lock.Unlock() 2127 2128 devices.Lock() 2129 defer devices.Unlock() 2130 2131 status := &DevStatus{ 2132 DeviceID: info.DeviceID, 2133 Size: info.Size, 2134 TransactionID: info.TransactionID, 2135 } 2136 2137 if err := devices.activateDeviceIfNeeded(info, false); err != nil { 2138 return nil, fmt.Errorf("Error activating devmapper device for '%s': %s", hash, err) 2139 } 2140 2141 sizeInSectors, mappedSectors, highestMappedSector, err := devices.deviceStatus(info.DevName()) 2142 2143 if err != nil { 2144 return nil, err 2145 } 2146 2147 status.SizeInSectors = sizeInSectors 2148 status.MappedSectors = mappedSectors 2149 status.HighestMappedSector = highestMappedSector 2150 2151 return status, nil 2152 } 2153 2154 func (devices *DeviceSet) poolStatus() (totalSizeInSectors, transactionID, dataUsed, dataTotal, metadataUsed, metadataTotal uint64, err error) { 2155 var params string 2156 if _, totalSizeInSectors, _, params, err = devicemapper.GetStatus(devices.getPoolName()); err == nil { 2157 _, err = fmt.Sscanf(params, "%d %d/%d %d/%d", &transactionID, &metadataUsed, &metadataTotal, &dataUsed, &dataTotal) 2158 } 2159 return 2160 } 2161 2162 // DataDevicePath returns the path to the data storage for this deviceset, 2163 // regardless of loopback or block device 2164 func (devices *DeviceSet) DataDevicePath() string { 2165 return devices.dataDevice 2166 } 2167 2168 // MetadataDevicePath returns the path to the metadata storage for this deviceset, 2169 // regardless of loopback or block device 2170 func (devices *DeviceSet) MetadataDevicePath() string { 2171 return devices.metadataDevice 2172 } 2173 2174 func (devices *DeviceSet) getUnderlyingAvailableSpace(loopFile string) (uint64, error) { 2175 buf := new(syscall.Statfs_t) 2176 if err := syscall.Statfs(loopFile, buf); err != nil { 2177 logrus.Warnf("Couldn't stat loopfile filesystem %v: %v", loopFile, err) 2178 return 0, err 2179 } 2180 return buf.Bfree * uint64(buf.Bsize), nil 2181 } 2182 2183 func (devices *DeviceSet) isRealFile(loopFile string) (bool, error) { 2184 if loopFile != "" { 2185 fi, err := os.Stat(loopFile) 2186 if err != nil { 2187 logrus.Warnf("Couldn't stat loopfile %v: %v", loopFile, err) 2188 return false, err 2189 } 2190 return fi.Mode().IsRegular(), nil 2191 } 2192 return false, nil 2193 } 2194 2195 // Status returns the current status of this deviceset 2196 func (devices *DeviceSet) Status() *Status { 2197 devices.Lock() 2198 defer devices.Unlock() 2199 2200 status := &Status{} 2201 2202 status.PoolName = devices.getPoolName() 2203 status.DataFile = devices.DataDevicePath() 2204 status.DataLoopback = devices.dataLoopFile 2205 status.MetadataFile = devices.MetadataDevicePath() 2206 status.MetadataLoopback = devices.metadataLoopFile 2207 status.UdevSyncSupported = devicemapper.UdevSyncSupported() 2208 status.DeferredRemoveEnabled = devices.deferredRemove 2209 status.DeferredDeleteEnabled = devices.deferredDelete 2210 status.DeferredDeletedDeviceCount = devices.nrDeletedDevices 2211 status.BaseDeviceSize = devices.getBaseDeviceSize() 2212 2213 totalSizeInSectors, _, dataUsed, dataTotal, metadataUsed, metadataTotal, err := devices.poolStatus() 2214 if err == nil { 2215 // Convert from blocks to bytes 2216 blockSizeInSectors := totalSizeInSectors / dataTotal 2217 2218 status.Data.Used = dataUsed * blockSizeInSectors * 512 2219 status.Data.Total = dataTotal * blockSizeInSectors * 512 2220 status.Data.Available = status.Data.Total - status.Data.Used 2221 2222 // metadata blocks are always 4k 2223 status.Metadata.Used = metadataUsed * 4096 2224 status.Metadata.Total = metadataTotal * 4096 2225 status.Metadata.Available = status.Metadata.Total - status.Metadata.Used 2226 2227 status.SectorSize = blockSizeInSectors * 512 2228 2229 if check, _ := devices.isRealFile(devices.dataLoopFile); check { 2230 actualSpace, err := devices.getUnderlyingAvailableSpace(devices.dataLoopFile) 2231 if err == nil && actualSpace < status.Data.Available { 2232 status.Data.Available = actualSpace 2233 } 2234 } 2235 2236 if check, _ := devices.isRealFile(devices.metadataLoopFile); check { 2237 actualSpace, err := devices.getUnderlyingAvailableSpace(devices.metadataLoopFile) 2238 if err == nil && actualSpace < status.Metadata.Available { 2239 status.Metadata.Available = actualSpace 2240 } 2241 } 2242 } 2243 2244 return status 2245 } 2246 2247 // Status returns the current status of this deviceset 2248 func (devices *DeviceSet) exportDeviceMetadata(hash string) (*deviceMetadata, error) { 2249 info, err := devices.lookupDeviceWithLock(hash) 2250 if err != nil { 2251 return nil, err 2252 } 2253 2254 info.lock.Lock() 2255 defer info.lock.Unlock() 2256 2257 metadata := &deviceMetadata{info.DeviceID, info.Size, info.Name()} 2258 return metadata, nil 2259 } 2260 2261 // NewDeviceSet creates the device set based on the options provided. 2262 func NewDeviceSet(root string, doInit bool, options []string, uidMaps, gidMaps []idtools.IDMap) (*DeviceSet, error) { 2263 devicemapper.SetDevDir("/dev") 2264 2265 devices := &DeviceSet{ 2266 root: root, 2267 metaData: metaData{Devices: make(map[string]*devInfo)}, 2268 dataLoopbackSize: defaultDataLoopbackSize, 2269 metaDataLoopbackSize: defaultMetaDataLoopbackSize, 2270 baseFsSize: defaultBaseFsSize, 2271 overrideUdevSyncCheck: defaultUdevSyncOverride, 2272 filesystem: "ext4", 2273 doBlkDiscard: true, 2274 thinpBlockSize: defaultThinpBlockSize, 2275 deviceIDMap: make([]byte, deviceIDMapSz), 2276 deletionWorkerTicker: time.NewTicker(time.Second * 30), 2277 uidMaps: uidMaps, 2278 gidMaps: gidMaps, 2279 } 2280 2281 foundBlkDiscard := false 2282 for _, option := range options { 2283 key, val, err := parsers.ParseKeyValueOpt(option) 2284 if err != nil { 2285 return nil, err 2286 } 2287 key = strings.ToLower(key) 2288 switch key { 2289 case "dm.basesize": 2290 size, err := units.RAMInBytes(val) 2291 if err != nil { 2292 return nil, err 2293 } 2294 devices.baseFsSize = uint64(size) 2295 case "dm.loopdatasize": 2296 size, err := units.RAMInBytes(val) 2297 if err != nil { 2298 return nil, err 2299 } 2300 devices.dataLoopbackSize = size 2301 case "dm.loopmetadatasize": 2302 size, err := units.RAMInBytes(val) 2303 if err != nil { 2304 return nil, err 2305 } 2306 devices.metaDataLoopbackSize = size 2307 case "dm.fs": 2308 if val != "ext4" && val != "xfs" { 2309 return nil, fmt.Errorf("Unsupported filesystem %s\n", val) 2310 } 2311 devices.filesystem = val 2312 case "dm.mkfsarg": 2313 devices.mkfsArgs = append(devices.mkfsArgs, val) 2314 case "dm.mountopt": 2315 devices.mountOptions = joinMountOptions(devices.mountOptions, val) 2316 case "dm.metadatadev": 2317 devices.metadataDevice = val 2318 case "dm.datadev": 2319 devices.dataDevice = val 2320 case "dm.thinpooldev": 2321 devices.thinPoolDevice = strings.TrimPrefix(val, "/dev/mapper/") 2322 case "dm.blkdiscard": 2323 foundBlkDiscard = true 2324 devices.doBlkDiscard, err = strconv.ParseBool(val) 2325 if err != nil { 2326 return nil, err 2327 } 2328 case "dm.blocksize": 2329 size, err := units.RAMInBytes(val) 2330 if err != nil { 2331 return nil, err 2332 } 2333 // convert to 512b sectors 2334 devices.thinpBlockSize = uint32(size) >> 9 2335 case "dm.override_udev_sync_check": 2336 devices.overrideUdevSyncCheck, err = strconv.ParseBool(val) 2337 if err != nil { 2338 return nil, err 2339 } 2340 2341 case "dm.use_deferred_removal": 2342 enableDeferredRemoval, err = strconv.ParseBool(val) 2343 if err != nil { 2344 return nil, err 2345 } 2346 2347 case "dm.use_deferred_deletion": 2348 enableDeferredDeletion, err = strconv.ParseBool(val) 2349 if err != nil { 2350 return nil, err 2351 } 2352 2353 default: 2354 return nil, fmt.Errorf("Unknown option %s\n", key) 2355 } 2356 } 2357 2358 // By default, don't do blk discard hack on raw devices, its rarely useful and is expensive 2359 if !foundBlkDiscard && (devices.dataDevice != "" || devices.thinPoolDevice != "") { 2360 devices.doBlkDiscard = false 2361 } 2362 2363 if err := devices.initDevmapper(doInit); err != nil { 2364 return nil, err 2365 } 2366 2367 return devices, nil 2368 }