github.com/psychoss/docker@v1.9.0/daemon/graphdriver/devmapper/deviceset.go (about) 1 // +build linux 2 3 package devmapper 4 5 import ( 6 "encoding/json" 7 "errors" 8 "fmt" 9 "io" 10 "io/ioutil" 11 "os" 12 "os/exec" 13 "path" 14 "path/filepath" 15 "strconv" 16 "strings" 17 "sync" 18 "syscall" 19 "time" 20 21 "github.com/Sirupsen/logrus" 22 23 "github.com/docker/docker/daemon/graphdriver" 24 "github.com/docker/docker/pkg/devicemapper" 25 "github.com/docker/docker/pkg/idtools" 26 "github.com/docker/docker/pkg/mount" 27 "github.com/docker/docker/pkg/parsers" 28 "github.com/docker/docker/pkg/units" 29 30 "github.com/opencontainers/runc/libcontainer/label" 31 ) 32 33 var ( 34 defaultDataLoopbackSize int64 = 100 * 1024 * 1024 * 1024 35 defaultMetaDataLoopbackSize int64 = 2 * 1024 * 1024 * 1024 36 defaultBaseFsSize uint64 = 100 * 1024 * 1024 * 1024 37 defaultThinpBlockSize uint32 = 128 // 64K = 128 512b sectors 38 defaultUdevSyncOverride = false 39 maxDeviceID = 0xffffff // 24 bit, pool limit 40 deviceIDMapSz = (maxDeviceID + 1) / 8 41 // We retry device removal so many a times that even error messages 42 // will fill up console during normal operation. So only log Fatal 43 // messages by default. 44 logLevel = devicemapper.LogLevelFatal 45 driverDeferredRemovalSupport = false 46 enableDeferredRemoval = false 47 enableDeferredDeletion = false 48 ) 49 50 const deviceSetMetaFile string = "deviceset-metadata" 51 const transactionMetaFile string = "transaction-metadata" 52 53 type transaction struct { 54 OpenTransactionID uint64 `json:"open_transaction_id"` 55 DeviceIDHash string `json:"device_hash"` 56 DeviceID int `json:"device_id"` 57 } 58 59 type devInfo struct { 60 Hash string `json:"-"` 61 DeviceID int `json:"device_id"` 62 Size uint64 `json:"size"` 63 TransactionID uint64 `json:"transaction_id"` 64 Initialized bool `json:"initialized"` 65 Deleted bool `json:"deleted"` 66 devices *DeviceSet 67 68 mountCount int 69 mountPath string 70 71 // The global DeviceSet lock guarantees that we serialize all 72 // the calls to libdevmapper (which is not threadsafe), but we 73 // sometimes release that lock while sleeping. In that case 74 // this per-device lock is still held, protecting against 75 // other accesses to the device that we're doing the wait on. 76 // 77 // WARNING: In order to avoid AB-BA deadlocks when releasing 78 // the global lock while holding the per-device locks all 79 // device locks must be acquired *before* the device lock, and 80 // multiple device locks should be acquired parent before child. 81 lock sync.Mutex 82 } 83 84 type metaData struct { 85 Devices map[string]*devInfo `json:"Devices"` 86 } 87 88 // DeviceSet holds information about list of devices 89 type DeviceSet struct { 90 metaData `json:"-"` 91 sync.Mutex `json:"-"` // Protects all fields of DeviceSet and serializes calls into libdevmapper 92 root string 93 devicePrefix string 94 TransactionID uint64 `json:"-"` 95 NextDeviceID int `json:"next_device_id"` 96 deviceIDMap []byte 97 98 // Options 99 dataLoopbackSize int64 100 metaDataLoopbackSize int64 101 baseFsSize uint64 102 filesystem string 103 mountOptions string 104 mkfsArgs []string 105 dataDevice string // block or loop dev 106 dataLoopFile string // loopback file, if used 107 metadataDevice string // block or loop dev 108 metadataLoopFile string // loopback file, if used 109 doBlkDiscard bool 110 thinpBlockSize uint32 111 thinPoolDevice string 112 transaction `json:"-"` 113 overrideUdevSyncCheck bool 114 deferredRemove bool // use deferred removal 115 deferredDelete bool // use deferred deletion 116 BaseDeviceUUID string //save UUID of base device 117 nrDeletedDevices uint //number of deleted devices 118 deletionWorkerTicker *time.Ticker 119 uidMaps []idtools.IDMap 120 gidMaps []idtools.IDMap 121 } 122 123 // DiskUsage contains information about disk usage and is used when reporting Status of a device. 124 type DiskUsage struct { 125 // Used bytes on the disk. 126 Used uint64 127 // Total bytes on the disk. 128 Total uint64 129 // Available bytes on the disk. 130 Available uint64 131 } 132 133 // Status returns the information about the device. 134 type Status struct { 135 // PoolName is the name of the data pool. 136 PoolName string 137 // DataFile is the actual block device for data. 138 DataFile string 139 // DataLoopback loopback file, if used. 140 DataLoopback string 141 // MetadataFile is the actual block device for metadata. 142 MetadataFile string 143 // MetadataLoopback is the loopback file, if used. 144 MetadataLoopback string 145 // Data is the disk used for data. 146 Data DiskUsage 147 // Metadata is the disk used for meta data. 148 Metadata DiskUsage 149 // BaseDeviceSize is base size of container and image 150 BaseDeviceSize uint64 151 // SectorSize size of the vector. 152 SectorSize uint64 153 // UdevSyncSupported is true if sync is supported. 154 UdevSyncSupported bool 155 // DeferredRemoveEnabled is true then the device is not unmounted. 156 DeferredRemoveEnabled bool 157 // True if deferred deletion is enabled. This is different from 158 // deferred removal. "removal" means that device mapper device is 159 // deactivated. Thin device is still in thin pool and can be activated 160 // again. But "deletion" means that thin device will be deleted from 161 // thin pool and it can't be activated again. 162 DeferredDeleteEnabled bool 163 DeferredDeletedDeviceCount uint 164 } 165 166 // Structure used to export image/container metadata in docker inspect. 167 type deviceMetadata struct { 168 deviceID int 169 deviceSize uint64 // size in bytes 170 deviceName string // Device name as used during activation 171 } 172 173 // DevStatus returns information about device mounted containing its id, size and sector information. 174 type DevStatus struct { 175 // DeviceID is the id of the device. 176 DeviceID int 177 // Size is the size of the filesystem. 178 Size uint64 179 // TransactionID is a unique integer per device set used to identify an operation on the file system, this number is incremental. 180 TransactionID uint64 181 // SizeInSectors indicates the size of the sectors allocated. 182 SizeInSectors uint64 183 // MappedSectors indicates number of mapped sectors. 184 MappedSectors uint64 185 // HighestMappedSector is the pointer to the highest mapped sector. 186 HighestMappedSector uint64 187 } 188 189 func getDevName(name string) string { 190 return "/dev/mapper/" + name 191 } 192 193 func (info *devInfo) Name() string { 194 hash := info.Hash 195 if hash == "" { 196 hash = "base" 197 } 198 return fmt.Sprintf("%s-%s", info.devices.devicePrefix, hash) 199 } 200 201 func (info *devInfo) DevName() string { 202 return getDevName(info.Name()) 203 } 204 205 func (devices *DeviceSet) loopbackDir() string { 206 return path.Join(devices.root, "devicemapper") 207 } 208 209 func (devices *DeviceSet) metadataDir() string { 210 return path.Join(devices.root, "metadata") 211 } 212 213 func (devices *DeviceSet) metadataFile(info *devInfo) string { 214 file := info.Hash 215 if file == "" { 216 file = "base" 217 } 218 return path.Join(devices.metadataDir(), file) 219 } 220 221 func (devices *DeviceSet) transactionMetaFile() string { 222 return path.Join(devices.metadataDir(), transactionMetaFile) 223 } 224 225 func (devices *DeviceSet) deviceSetMetaFile() string { 226 return path.Join(devices.metadataDir(), deviceSetMetaFile) 227 } 228 229 func (devices *DeviceSet) oldMetadataFile() string { 230 return path.Join(devices.loopbackDir(), "json") 231 } 232 233 func (devices *DeviceSet) getPoolName() string { 234 if devices.thinPoolDevice == "" { 235 return devices.devicePrefix + "-pool" 236 } 237 return devices.thinPoolDevice 238 } 239 240 func (devices *DeviceSet) getPoolDevName() string { 241 return getDevName(devices.getPoolName()) 242 } 243 244 func (devices *DeviceSet) hasImage(name string) bool { 245 dirname := devices.loopbackDir() 246 filename := path.Join(dirname, name) 247 248 _, err := os.Stat(filename) 249 return err == nil 250 } 251 252 // ensureImage creates a sparse file of <size> bytes at the path 253 // <root>/devicemapper/<name>. 254 // If the file already exists and new size is larger than its current size, it grows to the new size. 255 // Either way it returns the full path. 256 func (devices *DeviceSet) ensureImage(name string, size int64) (string, error) { 257 dirname := devices.loopbackDir() 258 filename := path.Join(dirname, name) 259 260 uid, gid, err := idtools.GetRootUIDGID(devices.uidMaps, devices.gidMaps) 261 if err != nil { 262 return "", err 263 } 264 if err := idtools.MkdirAllAs(dirname, 0700, uid, gid); err != nil && !os.IsExist(err) { 265 return "", err 266 } 267 268 if fi, err := os.Stat(filename); err != nil { 269 if !os.IsNotExist(err) { 270 return "", err 271 } 272 logrus.Debugf("Creating loopback file %s for device-manage use", filename) 273 file, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE, 0600) 274 if err != nil { 275 return "", err 276 } 277 defer file.Close() 278 279 if err := file.Truncate(size); err != nil { 280 return "", err 281 } 282 } else { 283 if fi.Size() < size { 284 file, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE, 0600) 285 if err != nil { 286 return "", err 287 } 288 defer file.Close() 289 if err := file.Truncate(size); err != nil { 290 return "", fmt.Errorf("Unable to grow loopback file %s: %v", filename, err) 291 } 292 } else if fi.Size() > size { 293 logrus.Warnf("Can't shrink loopback file %s", filename) 294 } 295 } 296 return filename, nil 297 } 298 299 func (devices *DeviceSet) allocateTransactionID() uint64 { 300 devices.OpenTransactionID = devices.TransactionID + 1 301 return devices.OpenTransactionID 302 } 303 304 func (devices *DeviceSet) updatePoolTransactionID() error { 305 if err := devicemapper.SetTransactionID(devices.getPoolDevName(), devices.TransactionID, devices.OpenTransactionID); err != nil { 306 return fmt.Errorf("Error setting devmapper transaction ID: %s", err) 307 } 308 devices.TransactionID = devices.OpenTransactionID 309 return nil 310 } 311 312 func (devices *DeviceSet) removeMetadata(info *devInfo) error { 313 if err := os.RemoveAll(devices.metadataFile(info)); err != nil { 314 return fmt.Errorf("Error removing metadata file %s: %s", devices.metadataFile(info), err) 315 } 316 return nil 317 } 318 319 // Given json data and file path, write it to disk 320 func (devices *DeviceSet) writeMetaFile(jsonData []byte, filePath string) error { 321 tmpFile, err := ioutil.TempFile(devices.metadataDir(), ".tmp") 322 if err != nil { 323 return fmt.Errorf("Error creating metadata file: %s", err) 324 } 325 326 n, err := tmpFile.Write(jsonData) 327 if err != nil { 328 return fmt.Errorf("Error writing metadata to %s: %s", tmpFile.Name(), err) 329 } 330 if n < len(jsonData) { 331 return io.ErrShortWrite 332 } 333 if err := tmpFile.Sync(); err != nil { 334 return fmt.Errorf("Error syncing metadata file %s: %s", tmpFile.Name(), err) 335 } 336 if err := tmpFile.Close(); err != nil { 337 return fmt.Errorf("Error closing metadata file %s: %s", tmpFile.Name(), err) 338 } 339 if err := os.Rename(tmpFile.Name(), filePath); err != nil { 340 return fmt.Errorf("Error committing metadata file %s: %s", tmpFile.Name(), err) 341 } 342 343 return nil 344 } 345 346 func (devices *DeviceSet) saveMetadata(info *devInfo) error { 347 jsonData, err := json.Marshal(info) 348 if err != nil { 349 return fmt.Errorf("Error encoding metadata to json: %s", err) 350 } 351 if err := devices.writeMetaFile(jsonData, devices.metadataFile(info)); err != nil { 352 return err 353 } 354 return nil 355 } 356 357 func (devices *DeviceSet) markDeviceIDUsed(deviceID int) { 358 var mask byte 359 i := deviceID % 8 360 mask = 1 << uint(i) 361 devices.deviceIDMap[deviceID/8] = devices.deviceIDMap[deviceID/8] | mask 362 } 363 364 func (devices *DeviceSet) markDeviceIDFree(deviceID int) { 365 var mask byte 366 i := deviceID % 8 367 mask = ^(1 << uint(i)) 368 devices.deviceIDMap[deviceID/8] = devices.deviceIDMap[deviceID/8] & mask 369 } 370 371 func (devices *DeviceSet) isDeviceIDFree(deviceID int) bool { 372 var mask byte 373 i := deviceID % 8 374 mask = (1 << uint(i)) 375 if (devices.deviceIDMap[deviceID/8] & mask) != 0 { 376 return false 377 } 378 return true 379 } 380 381 // Should be called with devices.Lock() held. 382 func (devices *DeviceSet) lookupDevice(hash string) (*devInfo, error) { 383 info := devices.Devices[hash] 384 if info == nil { 385 info = devices.loadMetadata(hash) 386 if info == nil { 387 return nil, fmt.Errorf("Unknown device %s", hash) 388 } 389 390 devices.Devices[hash] = info 391 } 392 return info, nil 393 } 394 395 func (devices *DeviceSet) lookupDeviceWithLock(hash string) (*devInfo, error) { 396 devices.Lock() 397 defer devices.Unlock() 398 info, err := devices.lookupDevice(hash) 399 return info, err 400 } 401 402 // This function relies on that device hash map has been loaded in advance. 403 // Should be called with devices.Lock() held. 404 func (devices *DeviceSet) constructDeviceIDMap() { 405 logrus.Debugf("[deviceset] constructDeviceIDMap()") 406 defer logrus.Debugf("[deviceset] constructDeviceIDMap() END") 407 408 for _, info := range devices.Devices { 409 devices.markDeviceIDUsed(info.DeviceID) 410 logrus.Debugf("Added deviceId=%d to DeviceIdMap", info.DeviceID) 411 } 412 } 413 414 func (devices *DeviceSet) deviceFileWalkFunction(path string, finfo os.FileInfo) error { 415 416 // Skip some of the meta files which are not device files. 417 if strings.HasSuffix(finfo.Name(), ".migrated") { 418 logrus.Debugf("Skipping file %s", path) 419 return nil 420 } 421 422 if strings.HasPrefix(finfo.Name(), ".") { 423 logrus.Debugf("Skipping file %s", path) 424 return nil 425 } 426 427 if finfo.Name() == deviceSetMetaFile { 428 logrus.Debugf("Skipping file %s", path) 429 return nil 430 } 431 432 if finfo.Name() == transactionMetaFile { 433 logrus.Debugf("Skipping file %s", path) 434 return nil 435 } 436 437 logrus.Debugf("Loading data for file %s", path) 438 439 hash := finfo.Name() 440 if hash == "base" { 441 hash = "" 442 } 443 444 // Include deleted devices also as cleanup delete device logic 445 // will go through it and see if there are any deleted devices. 446 if _, err := devices.lookupDevice(hash); err != nil { 447 return fmt.Errorf("Error looking up device %s:%v", hash, err) 448 } 449 450 return nil 451 } 452 453 func (devices *DeviceSet) loadDeviceFilesOnStart() error { 454 logrus.Debugf("[deviceset] loadDeviceFilesOnStart()") 455 defer logrus.Debugf("[deviceset] loadDeviceFilesOnStart() END") 456 457 var scan = func(path string, info os.FileInfo, err error) error { 458 if err != nil { 459 logrus.Debugf("Can't walk the file %s", path) 460 return nil 461 } 462 463 // Skip any directories 464 if info.IsDir() { 465 return nil 466 } 467 468 return devices.deviceFileWalkFunction(path, info) 469 } 470 471 return filepath.Walk(devices.metadataDir(), scan) 472 } 473 474 // Should be called with devices.Lock() held. 475 func (devices *DeviceSet) unregisterDevice(id int, hash string) error { 476 logrus.Debugf("unregisterDevice(%v, %v)", id, hash) 477 info := &devInfo{ 478 Hash: hash, 479 DeviceID: id, 480 } 481 482 delete(devices.Devices, hash) 483 484 if err := devices.removeMetadata(info); err != nil { 485 logrus.Debugf("Error removing metadata: %s", err) 486 return err 487 } 488 489 return nil 490 } 491 492 // Should be called with devices.Lock() held. 493 func (devices *DeviceSet) registerDevice(id int, hash string, size uint64, transactionID uint64) (*devInfo, error) { 494 logrus.Debugf("registerDevice(%v, %v)", id, hash) 495 info := &devInfo{ 496 Hash: hash, 497 DeviceID: id, 498 Size: size, 499 TransactionID: transactionID, 500 Initialized: false, 501 devices: devices, 502 } 503 504 devices.Devices[hash] = info 505 506 if err := devices.saveMetadata(info); err != nil { 507 // Try to remove unused device 508 delete(devices.Devices, hash) 509 return nil, err 510 } 511 512 return info, nil 513 } 514 515 func (devices *DeviceSet) activateDeviceIfNeeded(info *devInfo, ignoreDeleted bool) error { 516 logrus.Debugf("activateDeviceIfNeeded(%v)", info.Hash) 517 518 if info.Deleted && !ignoreDeleted { 519 return fmt.Errorf("devmapper: Can't activate device %v as it is marked for deletion", info.Hash) 520 } 521 522 // Make sure deferred removal on device is canceled, if one was 523 // scheduled. 524 if err := devices.cancelDeferredRemoval(info); err != nil { 525 return fmt.Errorf("Deivce Deferred Removal Cancellation Failed: %s", err) 526 } 527 528 if devinfo, _ := devicemapper.GetInfo(info.Name()); devinfo != nil && devinfo.Exists != 0 { 529 return nil 530 } 531 532 return devicemapper.ActivateDevice(devices.getPoolDevName(), info.Name(), info.DeviceID, info.Size) 533 } 534 535 func (devices *DeviceSet) createFilesystem(info *devInfo) error { 536 devname := info.DevName() 537 538 args := []string{} 539 for _, arg := range devices.mkfsArgs { 540 args = append(args, arg) 541 } 542 543 args = append(args, devname) 544 545 var err error 546 switch devices.filesystem { 547 case "xfs": 548 err = exec.Command("mkfs.xfs", args...).Run() 549 case "ext4": 550 err = exec.Command("mkfs.ext4", append([]string{"-E", "nodiscard,lazy_itable_init=0,lazy_journal_init=0"}, args...)...).Run() 551 if err != nil { 552 err = exec.Command("mkfs.ext4", append([]string{"-E", "nodiscard,lazy_itable_init=0"}, args...)...).Run() 553 } 554 if err != nil { 555 return err 556 } 557 err = exec.Command("tune2fs", append([]string{"-c", "-1", "-i", "0"}, devname)...).Run() 558 default: 559 err = fmt.Errorf("Unsupported filesystem type %s", devices.filesystem) 560 } 561 if err != nil { 562 return err 563 } 564 565 return nil 566 } 567 568 func (devices *DeviceSet) migrateOldMetaData() error { 569 // Migrate old metadata file 570 jsonData, err := ioutil.ReadFile(devices.oldMetadataFile()) 571 if err != nil && !os.IsNotExist(err) { 572 return err 573 } 574 575 if jsonData != nil { 576 m := metaData{Devices: make(map[string]*devInfo)} 577 578 if err := json.Unmarshal(jsonData, &m); err != nil { 579 return err 580 } 581 582 for hash, info := range m.Devices { 583 info.Hash = hash 584 devices.saveMetadata(info) 585 } 586 if err := os.Rename(devices.oldMetadataFile(), devices.oldMetadataFile()+".migrated"); err != nil { 587 return err 588 } 589 590 } 591 592 return nil 593 } 594 595 // Cleanup deleted devices. It assumes that all the devices have been 596 // loaded in the hash table. 597 func (devices *DeviceSet) cleanupDeletedDevices() error { 598 devices.Lock() 599 600 // If there are no deleted devices, there is nothing to do. 601 if devices.nrDeletedDevices == 0 { 602 devices.Unlock() 603 return nil 604 } 605 606 var deletedDevices []*devInfo 607 608 for _, info := range devices.Devices { 609 if !info.Deleted { 610 continue 611 } 612 logrus.Debugf("devmapper: Found deleted device %s.", info.Hash) 613 deletedDevices = append(deletedDevices, info) 614 } 615 616 // Delete the deleted devices. DeleteDevice() first takes the info lock 617 // and then devices.Lock(). So drop it to avoid deadlock. 618 devices.Unlock() 619 620 for _, info := range deletedDevices { 621 // This will again try deferred deletion. 622 if err := devices.DeleteDevice(info.Hash, false); err != nil { 623 logrus.Warnf("devmapper: Deletion of device %s, device_id=%v failed:%v", info.Hash, info.DeviceID, err) 624 } 625 } 626 627 return nil 628 } 629 630 func (devices *DeviceSet) countDeletedDevices() { 631 for _, info := range devices.Devices { 632 if !info.Deleted { 633 continue 634 } 635 devices.nrDeletedDevices++ 636 } 637 } 638 639 func (devices *DeviceSet) startDeviceDeletionWorker() { 640 // Deferred deletion is not enabled. Don't do anything. 641 if !devices.deferredDelete { 642 return 643 } 644 645 logrus.Debugf("devmapper: Worker to cleanup deleted devices started") 646 for range devices.deletionWorkerTicker.C { 647 devices.cleanupDeletedDevices() 648 } 649 } 650 651 func (devices *DeviceSet) initMetaData() error { 652 devices.Lock() 653 defer devices.Unlock() 654 655 if err := devices.migrateOldMetaData(); err != nil { 656 return err 657 } 658 659 _, transactionID, _, _, _, _, err := devices.poolStatus() 660 if err != nil { 661 return err 662 } 663 664 devices.TransactionID = transactionID 665 666 if err := devices.loadDeviceFilesOnStart(); err != nil { 667 return fmt.Errorf("devmapper: Failed to load device files:%v", err) 668 } 669 670 devices.constructDeviceIDMap() 671 devices.countDeletedDevices() 672 673 if err := devices.processPendingTransaction(); err != nil { 674 return err 675 } 676 677 // Start a goroutine to cleanup Deleted Devices 678 go devices.startDeviceDeletionWorker() 679 return nil 680 } 681 682 func (devices *DeviceSet) incNextDeviceID() { 683 // IDs are 24bit, so wrap around 684 devices.NextDeviceID = (devices.NextDeviceID + 1) & maxDeviceID 685 } 686 687 func (devices *DeviceSet) getNextFreeDeviceID() (int, error) { 688 devices.incNextDeviceID() 689 for i := 0; i <= maxDeviceID; i++ { 690 if devices.isDeviceIDFree(devices.NextDeviceID) { 691 devices.markDeviceIDUsed(devices.NextDeviceID) 692 return devices.NextDeviceID, nil 693 } 694 devices.incNextDeviceID() 695 } 696 697 return 0, fmt.Errorf("Unable to find a free device ID") 698 } 699 700 func (devices *DeviceSet) createRegisterDevice(hash string) (*devInfo, error) { 701 devices.Lock() 702 defer devices.Unlock() 703 704 deviceID, err := devices.getNextFreeDeviceID() 705 if err != nil { 706 return nil, err 707 } 708 709 if err := devices.openTransaction(hash, deviceID); err != nil { 710 logrus.Debugf("Error opening transaction hash = %s deviceID = %d", hash, deviceID) 711 devices.markDeviceIDFree(deviceID) 712 return nil, err 713 } 714 715 for { 716 if err := devicemapper.CreateDevice(devices.getPoolDevName(), deviceID); err != nil { 717 if devicemapper.DeviceIDExists(err) { 718 // Device ID already exists. This should not 719 // happen. Now we have a mechianism to find 720 // a free device ID. So something is not right. 721 // Give a warning and continue. 722 logrus.Errorf("Device ID %d exists in pool but it is supposed to be unused", deviceID) 723 deviceID, err = devices.getNextFreeDeviceID() 724 if err != nil { 725 return nil, err 726 } 727 // Save new device id into transaction 728 devices.refreshTransaction(deviceID) 729 continue 730 } 731 logrus.Debugf("Error creating device: %s", err) 732 devices.markDeviceIDFree(deviceID) 733 return nil, err 734 } 735 break 736 } 737 738 logrus.Debugf("Registering device (id %v) with FS size %v", deviceID, devices.baseFsSize) 739 info, err := devices.registerDevice(deviceID, hash, devices.baseFsSize, devices.OpenTransactionID) 740 if err != nil { 741 _ = devicemapper.DeleteDevice(devices.getPoolDevName(), deviceID) 742 devices.markDeviceIDFree(deviceID) 743 return nil, err 744 } 745 746 if err := devices.closeTransaction(); err != nil { 747 devices.unregisterDevice(deviceID, hash) 748 devicemapper.DeleteDevice(devices.getPoolDevName(), deviceID) 749 devices.markDeviceIDFree(deviceID) 750 return nil, err 751 } 752 return info, nil 753 } 754 755 func (devices *DeviceSet) createRegisterSnapDevice(hash string, baseInfo *devInfo) error { 756 deviceID, err := devices.getNextFreeDeviceID() 757 if err != nil { 758 return err 759 } 760 761 if err := devices.openTransaction(hash, deviceID); err != nil { 762 logrus.Debugf("Error opening transaction hash = %s deviceID = %d", hash, deviceID) 763 devices.markDeviceIDFree(deviceID) 764 return err 765 } 766 767 for { 768 if err := devicemapper.CreateSnapDevice(devices.getPoolDevName(), deviceID, baseInfo.Name(), baseInfo.DeviceID); err != nil { 769 if devicemapper.DeviceIDExists(err) { 770 // Device ID already exists. This should not 771 // happen. Now we have a mechianism to find 772 // a free device ID. So something is not right. 773 // Give a warning and continue. 774 logrus.Errorf("Device ID %d exists in pool but it is supposed to be unused", deviceID) 775 deviceID, err = devices.getNextFreeDeviceID() 776 if err != nil { 777 return err 778 } 779 // Save new device id into transaction 780 devices.refreshTransaction(deviceID) 781 continue 782 } 783 logrus.Debugf("Error creating snap device: %s", err) 784 devices.markDeviceIDFree(deviceID) 785 return err 786 } 787 break 788 } 789 790 if _, err := devices.registerDevice(deviceID, hash, baseInfo.Size, devices.OpenTransactionID); err != nil { 791 devicemapper.DeleteDevice(devices.getPoolDevName(), deviceID) 792 devices.markDeviceIDFree(deviceID) 793 logrus.Debugf("Error registering device: %s", err) 794 return err 795 } 796 797 if err := devices.closeTransaction(); err != nil { 798 devices.unregisterDevice(deviceID, hash) 799 devicemapper.DeleteDevice(devices.getPoolDevName(), deviceID) 800 devices.markDeviceIDFree(deviceID) 801 return err 802 } 803 return nil 804 } 805 806 func (devices *DeviceSet) loadMetadata(hash string) *devInfo { 807 info := &devInfo{Hash: hash, devices: devices} 808 809 jsonData, err := ioutil.ReadFile(devices.metadataFile(info)) 810 if err != nil { 811 return nil 812 } 813 814 if err := json.Unmarshal(jsonData, &info); err != nil { 815 return nil 816 } 817 818 if info.DeviceID > maxDeviceID { 819 logrus.Errorf("Ignoring Invalid DeviceId=%d", info.DeviceID) 820 return nil 821 } 822 823 return info 824 } 825 826 func getDeviceUUID(device string) (string, error) { 827 out, err := exec.Command("blkid", "-s", "UUID", "-o", "value", device).Output() 828 if err != nil { 829 logrus.Debugf("Failed to find uuid for device %s:%v", device, err) 830 return "", err 831 } 832 833 uuid := strings.TrimSuffix(string(out), "\n") 834 uuid = strings.TrimSpace(uuid) 835 logrus.Debugf("UUID for device: %s is:%s", device, uuid) 836 return uuid, nil 837 } 838 839 func (devices *DeviceSet) getBaseDeviceSize() uint64 { 840 info, _ := devices.lookupDevice("") 841 if info == nil { 842 return 0 843 } 844 return info.Size 845 } 846 847 func (devices *DeviceSet) verifyBaseDeviceUUID(baseInfo *devInfo) error { 848 devices.Lock() 849 defer devices.Unlock() 850 851 if err := devices.activateDeviceIfNeeded(baseInfo, false); err != nil { 852 return err 853 } 854 855 defer devices.deactivateDevice(baseInfo) 856 857 uuid, err := getDeviceUUID(baseInfo.DevName()) 858 if err != nil { 859 return err 860 } 861 862 if devices.BaseDeviceUUID != uuid { 863 return fmt.Errorf("Current Base Device UUID:%s does not match with stored UUID:%s", uuid, devices.BaseDeviceUUID) 864 } 865 866 return nil 867 } 868 869 func (devices *DeviceSet) saveBaseDeviceUUID(baseInfo *devInfo) error { 870 devices.Lock() 871 defer devices.Unlock() 872 873 if err := devices.activateDeviceIfNeeded(baseInfo, false); err != nil { 874 return err 875 } 876 877 defer devices.deactivateDevice(baseInfo) 878 879 uuid, err := getDeviceUUID(baseInfo.DevName()) 880 if err != nil { 881 return err 882 } 883 884 devices.BaseDeviceUUID = uuid 885 devices.saveDeviceSetMetaData() 886 return nil 887 } 888 889 func (devices *DeviceSet) createBaseImage() error { 890 logrus.Debugf("Initializing base device-mapper thin volume") 891 892 // Create initial device 893 info, err := devices.createRegisterDevice("") 894 if err != nil { 895 return err 896 } 897 898 logrus.Debugf("Creating filesystem on base device-mapper thin volume") 899 900 if err := devices.activateDeviceIfNeeded(info, false); err != nil { 901 return err 902 } 903 904 if err := devices.createFilesystem(info); err != nil { 905 return err 906 } 907 908 info.Initialized = true 909 if err := devices.saveMetadata(info); err != nil { 910 info.Initialized = false 911 return err 912 } 913 914 if err := devices.saveBaseDeviceUUID(info); err != nil { 915 return fmt.Errorf("Could not query and save base device UUID:%v", err) 916 } 917 918 return nil 919 } 920 921 // Returns if thin pool device exists or not. If device exists, also makes 922 // sure it is a thin pool device and not some other type of device. 923 func (devices *DeviceSet) thinPoolExists(thinPoolDevice string) (bool, error) { 924 logrus.Debugf("devmapper: Checking for existence of the pool %s", thinPoolDevice) 925 926 info, err := devicemapper.GetInfo(thinPoolDevice) 927 if err != nil { 928 return false, fmt.Errorf("devmapper: GetInfo() on device %s failed: %v", thinPoolDevice, err) 929 } 930 931 // Device does not exist. 932 if info.Exists == 0 { 933 return false, nil 934 } 935 936 _, _, deviceType, _, err := devicemapper.GetStatus(thinPoolDevice) 937 if err != nil { 938 return false, fmt.Errorf("devmapper: GetStatus() on device %s failed: %v", thinPoolDevice, err) 939 } 940 941 if deviceType != "thin-pool" { 942 return false, fmt.Errorf("devmapper: Device %s is not a thin pool", thinPoolDevice) 943 } 944 945 return true, nil 946 } 947 948 func (devices *DeviceSet) checkThinPool() error { 949 _, transactionID, dataUsed, _, _, _, err := devices.poolStatus() 950 if err != nil { 951 return err 952 } 953 if dataUsed != 0 { 954 return fmt.Errorf("Unable to take ownership of thin-pool (%s) that already has used data blocks", 955 devices.thinPoolDevice) 956 } 957 if transactionID != 0 { 958 return fmt.Errorf("Unable to take ownership of thin-pool (%s) with non-zero transaction ID", 959 devices.thinPoolDevice) 960 } 961 return nil 962 } 963 964 // Base image is initialized properly. Either save UUID for first time (for 965 // upgrade case or verify UUID. 966 func (devices *DeviceSet) setupVerifyBaseImageUUID(baseInfo *devInfo) error { 967 // If BaseDeviceUUID is nil (upgrade case), save it and return success. 968 if devices.BaseDeviceUUID == "" { 969 if err := devices.saveBaseDeviceUUID(baseInfo); err != nil { 970 return fmt.Errorf("Could not query and save base device UUID:%v", err) 971 } 972 return nil 973 } 974 975 if err := devices.verifyBaseDeviceUUID(baseInfo); err != nil { 976 return fmt.Errorf("Base Device UUID verification failed. Possibly using a different thin pool than last invocation:%v", err) 977 } 978 979 return nil 980 } 981 982 func (devices *DeviceSet) setupBaseImage() error { 983 oldInfo, _ := devices.lookupDeviceWithLock("") 984 985 // base image already exists. If it is initialized properly, do UUID 986 // verification and return. Otherwise remove image and set it up 987 // fresh. 988 989 if oldInfo != nil { 990 if oldInfo.Initialized && !oldInfo.Deleted { 991 if err := devices.setupVerifyBaseImageUUID(oldInfo); err != nil { 992 return err 993 } 994 995 return nil 996 } 997 998 logrus.Debugf("Removing uninitialized base image") 999 // If previous base device is in deferred delete state, 1000 // that needs to be cleaned up first. So don't try 1001 // deferred deletion. 1002 if err := devices.DeleteDevice("", true); err != nil { 1003 return err 1004 } 1005 } 1006 1007 // If we are setting up base image for the first time, make sure 1008 // thin pool is empty. 1009 if devices.thinPoolDevice != "" && oldInfo == nil { 1010 if err := devices.checkThinPool(); err != nil { 1011 return err 1012 } 1013 } 1014 1015 // Create new base image device 1016 if err := devices.createBaseImage(); err != nil { 1017 return err 1018 } 1019 1020 return nil 1021 } 1022 1023 func setCloseOnExec(name string) { 1024 if fileInfos, _ := ioutil.ReadDir("/proc/self/fd"); fileInfos != nil { 1025 for _, i := range fileInfos { 1026 link, _ := os.Readlink(filepath.Join("/proc/self/fd", i.Name())) 1027 if link == name { 1028 fd, err := strconv.Atoi(i.Name()) 1029 if err == nil { 1030 syscall.CloseOnExec(fd) 1031 } 1032 } 1033 } 1034 } 1035 } 1036 1037 // DMLog implements logging using DevMapperLogger interface. 1038 func (devices *DeviceSet) DMLog(level int, file string, line int, dmError int, message string) { 1039 // By default libdm sends us all the messages including debug ones. 1040 // We need to filter out messages here and figure out which one 1041 // should be printed. 1042 if level > logLevel { 1043 return 1044 } 1045 1046 // FIXME(vbatts) push this back into ./pkg/devicemapper/ 1047 if level <= devicemapper.LogLevelErr { 1048 logrus.Errorf("libdevmapper(%d): %s:%d (%d) %s", level, file, line, dmError, message) 1049 } else if level <= devicemapper.LogLevelInfo { 1050 logrus.Infof("libdevmapper(%d): %s:%d (%d) %s", level, file, line, dmError, message) 1051 } else { 1052 // FIXME(vbatts) push this back into ./pkg/devicemapper/ 1053 logrus.Debugf("libdevmapper(%d): %s:%d (%d) %s", level, file, line, dmError, message) 1054 } 1055 } 1056 1057 func major(device uint64) uint64 { 1058 return (device >> 8) & 0xfff 1059 } 1060 1061 func minor(device uint64) uint64 { 1062 return (device & 0xff) | ((device >> 12) & 0xfff00) 1063 } 1064 1065 // ResizePool increases the size of the pool. 1066 func (devices *DeviceSet) ResizePool(size int64) error { 1067 dirname := devices.loopbackDir() 1068 datafilename := path.Join(dirname, "data") 1069 if len(devices.dataDevice) > 0 { 1070 datafilename = devices.dataDevice 1071 } 1072 metadatafilename := path.Join(dirname, "metadata") 1073 if len(devices.metadataDevice) > 0 { 1074 metadatafilename = devices.metadataDevice 1075 } 1076 1077 datafile, err := os.OpenFile(datafilename, os.O_RDWR, 0) 1078 if datafile == nil { 1079 return err 1080 } 1081 defer datafile.Close() 1082 1083 fi, err := datafile.Stat() 1084 if fi == nil { 1085 return err 1086 } 1087 1088 if fi.Size() > size { 1089 return fmt.Errorf("Can't shrink file") 1090 } 1091 1092 dataloopback := devicemapper.FindLoopDeviceFor(datafile) 1093 if dataloopback == nil { 1094 return fmt.Errorf("Unable to find loopback mount for: %s", datafilename) 1095 } 1096 defer dataloopback.Close() 1097 1098 metadatafile, err := os.OpenFile(metadatafilename, os.O_RDWR, 0) 1099 if metadatafile == nil { 1100 return err 1101 } 1102 defer metadatafile.Close() 1103 1104 metadataloopback := devicemapper.FindLoopDeviceFor(metadatafile) 1105 if metadataloopback == nil { 1106 return fmt.Errorf("Unable to find loopback mount for: %s", metadatafilename) 1107 } 1108 defer metadataloopback.Close() 1109 1110 // Grow loopback file 1111 if err := datafile.Truncate(size); err != nil { 1112 return fmt.Errorf("Unable to grow loopback file: %s", err) 1113 } 1114 1115 // Reload size for loopback device 1116 if err := devicemapper.LoopbackSetCapacity(dataloopback); err != nil { 1117 return fmt.Errorf("Unable to update loopback capacity: %s", err) 1118 } 1119 1120 // Suspend the pool 1121 if err := devicemapper.SuspendDevice(devices.getPoolName()); err != nil { 1122 return fmt.Errorf("Unable to suspend pool: %s", err) 1123 } 1124 1125 // Reload with the new block sizes 1126 if err := devicemapper.ReloadPool(devices.getPoolName(), dataloopback, metadataloopback, devices.thinpBlockSize); err != nil { 1127 return fmt.Errorf("Unable to reload pool: %s", err) 1128 } 1129 1130 // Resume the pool 1131 if err := devicemapper.ResumeDevice(devices.getPoolName()); err != nil { 1132 return fmt.Errorf("Unable to resume pool: %s", err) 1133 } 1134 1135 return nil 1136 } 1137 1138 func (devices *DeviceSet) loadTransactionMetaData() error { 1139 jsonData, err := ioutil.ReadFile(devices.transactionMetaFile()) 1140 if err != nil { 1141 // There is no active transaction. This will be the case 1142 // during upgrade. 1143 if os.IsNotExist(err) { 1144 devices.OpenTransactionID = devices.TransactionID 1145 return nil 1146 } 1147 return err 1148 } 1149 1150 json.Unmarshal(jsonData, &devices.transaction) 1151 return nil 1152 } 1153 1154 func (devices *DeviceSet) saveTransactionMetaData() error { 1155 jsonData, err := json.Marshal(&devices.transaction) 1156 if err != nil { 1157 return fmt.Errorf("Error encoding metadata to json: %s", err) 1158 } 1159 1160 return devices.writeMetaFile(jsonData, devices.transactionMetaFile()) 1161 } 1162 1163 func (devices *DeviceSet) removeTransactionMetaData() error { 1164 if err := os.RemoveAll(devices.transactionMetaFile()); err != nil { 1165 return err 1166 } 1167 return nil 1168 } 1169 1170 func (devices *DeviceSet) rollbackTransaction() error { 1171 logrus.Debugf("Rolling back open transaction: TransactionID=%d hash=%s device_id=%d", devices.OpenTransactionID, devices.DeviceIDHash, devices.DeviceID) 1172 1173 // A device id might have already been deleted before transaction 1174 // closed. In that case this call will fail. Just leave a message 1175 // in case of failure. 1176 if err := devicemapper.DeleteDevice(devices.getPoolDevName(), devices.DeviceID); err != nil { 1177 logrus.Errorf("Unable to delete device: %s", err) 1178 } 1179 1180 dinfo := &devInfo{Hash: devices.DeviceIDHash} 1181 if err := devices.removeMetadata(dinfo); err != nil { 1182 logrus.Errorf("Unable to remove metadata: %s", err) 1183 } else { 1184 devices.markDeviceIDFree(devices.DeviceID) 1185 } 1186 1187 if err := devices.removeTransactionMetaData(); err != nil { 1188 logrus.Errorf("Unable to remove transaction meta file %s: %s", devices.transactionMetaFile(), err) 1189 } 1190 1191 return nil 1192 } 1193 1194 func (devices *DeviceSet) processPendingTransaction() error { 1195 if err := devices.loadTransactionMetaData(); err != nil { 1196 return err 1197 } 1198 1199 // If there was open transaction but pool transaction ID is same 1200 // as open transaction ID, nothing to roll back. 1201 if devices.TransactionID == devices.OpenTransactionID { 1202 return nil 1203 } 1204 1205 // If open transaction ID is less than pool transaction ID, something 1206 // is wrong. Bail out. 1207 if devices.OpenTransactionID < devices.TransactionID { 1208 logrus.Errorf("Open Transaction id %d is less than pool transaction id %d", devices.OpenTransactionID, devices.TransactionID) 1209 return nil 1210 } 1211 1212 // Pool transaction ID is not same as open transaction. There is 1213 // a transaction which was not completed. 1214 if err := devices.rollbackTransaction(); err != nil { 1215 return fmt.Errorf("Rolling back open transaction failed: %s", err) 1216 } 1217 1218 devices.OpenTransactionID = devices.TransactionID 1219 return nil 1220 } 1221 1222 func (devices *DeviceSet) loadDeviceSetMetaData() error { 1223 jsonData, err := ioutil.ReadFile(devices.deviceSetMetaFile()) 1224 if err != nil { 1225 // For backward compatibility return success if file does 1226 // not exist. 1227 if os.IsNotExist(err) { 1228 return nil 1229 } 1230 return err 1231 } 1232 1233 return json.Unmarshal(jsonData, devices) 1234 } 1235 1236 func (devices *DeviceSet) saveDeviceSetMetaData() error { 1237 jsonData, err := json.Marshal(devices) 1238 if err != nil { 1239 return fmt.Errorf("Error encoding metadata to json: %s", err) 1240 } 1241 1242 return devices.writeMetaFile(jsonData, devices.deviceSetMetaFile()) 1243 } 1244 1245 func (devices *DeviceSet) openTransaction(hash string, DeviceID int) error { 1246 devices.allocateTransactionID() 1247 devices.DeviceIDHash = hash 1248 devices.DeviceID = DeviceID 1249 if err := devices.saveTransactionMetaData(); err != nil { 1250 return fmt.Errorf("Error saving transaction metadata: %s", err) 1251 } 1252 return nil 1253 } 1254 1255 func (devices *DeviceSet) refreshTransaction(DeviceID int) error { 1256 devices.DeviceID = DeviceID 1257 if err := devices.saveTransactionMetaData(); err != nil { 1258 return fmt.Errorf("Error saving transaction metadata: %s", err) 1259 } 1260 return nil 1261 } 1262 1263 func (devices *DeviceSet) closeTransaction() error { 1264 if err := devices.updatePoolTransactionID(); err != nil { 1265 logrus.Debugf("Failed to close Transaction") 1266 return err 1267 } 1268 return nil 1269 } 1270 1271 func determineDriverCapabilities(version string) error { 1272 /* 1273 * Driver version 4.27.0 and greater support deferred activation 1274 * feature. 1275 */ 1276 1277 logrus.Debugf("devicemapper: driver version is %s", version) 1278 1279 versionSplit := strings.Split(version, ".") 1280 major, err := strconv.Atoi(versionSplit[0]) 1281 if err != nil { 1282 return graphdriver.ErrNotSupported 1283 } 1284 1285 if major > 4 { 1286 driverDeferredRemovalSupport = true 1287 return nil 1288 } 1289 1290 if major < 4 { 1291 return nil 1292 } 1293 1294 minor, err := strconv.Atoi(versionSplit[1]) 1295 if err != nil { 1296 return graphdriver.ErrNotSupported 1297 } 1298 1299 /* 1300 * If major is 4 and minor is 27, then there is no need to 1301 * check for patch level as it can not be less than 0. 1302 */ 1303 if minor >= 27 { 1304 driverDeferredRemovalSupport = true 1305 return nil 1306 } 1307 1308 return nil 1309 } 1310 1311 // Determine the major and minor number of loopback device 1312 func getDeviceMajorMinor(file *os.File) (uint64, uint64, error) { 1313 stat, err := file.Stat() 1314 if err != nil { 1315 return 0, 0, err 1316 } 1317 1318 dev := stat.Sys().(*syscall.Stat_t).Rdev 1319 majorNum := major(dev) 1320 minorNum := minor(dev) 1321 1322 logrus.Debugf("[devmapper]: Major:Minor for device: %s is:%v:%v", file.Name(), majorNum, minorNum) 1323 return majorNum, minorNum, nil 1324 } 1325 1326 // Given a file which is backing file of a loop back device, find the 1327 // loopback device name and its major/minor number. 1328 func getLoopFileDeviceMajMin(filename string) (string, uint64, uint64, error) { 1329 file, err := os.Open(filename) 1330 if err != nil { 1331 logrus.Debugf("[devmapper]: Failed to open file %s", filename) 1332 return "", 0, 0, err 1333 } 1334 1335 defer file.Close() 1336 loopbackDevice := devicemapper.FindLoopDeviceFor(file) 1337 if loopbackDevice == nil { 1338 return "", 0, 0, fmt.Errorf("[devmapper]: Unable to find loopback mount for: %s", filename) 1339 } 1340 defer loopbackDevice.Close() 1341 1342 Major, Minor, err := getDeviceMajorMinor(loopbackDevice) 1343 if err != nil { 1344 return "", 0, 0, err 1345 } 1346 return loopbackDevice.Name(), Major, Minor, nil 1347 } 1348 1349 // Get the major/minor numbers of thin pool data and metadata devices 1350 func (devices *DeviceSet) getThinPoolDataMetaMajMin() (uint64, uint64, uint64, uint64, error) { 1351 var params, poolDataMajMin, poolMetadataMajMin string 1352 1353 _, _, _, params, err := devicemapper.GetTable(devices.getPoolName()) 1354 if err != nil { 1355 return 0, 0, 0, 0, err 1356 } 1357 1358 if _, err = fmt.Sscanf(params, "%s %s", &poolMetadataMajMin, &poolDataMajMin); err != nil { 1359 return 0, 0, 0, 0, err 1360 } 1361 1362 logrus.Debugf("[devmapper]: poolDataMajMin=%s poolMetaMajMin=%s\n", poolDataMajMin, poolMetadataMajMin) 1363 1364 poolDataMajMinorSplit := strings.Split(poolDataMajMin, ":") 1365 poolDataMajor, err := strconv.ParseUint(poolDataMajMinorSplit[0], 10, 32) 1366 if err != nil { 1367 return 0, 0, 0, 0, err 1368 } 1369 1370 poolDataMinor, err := strconv.ParseUint(poolDataMajMinorSplit[1], 10, 32) 1371 if err != nil { 1372 return 0, 0, 0, 0, err 1373 } 1374 1375 poolMetadataMajMinorSplit := strings.Split(poolMetadataMajMin, ":") 1376 poolMetadataMajor, err := strconv.ParseUint(poolMetadataMajMinorSplit[0], 10, 32) 1377 if err != nil { 1378 return 0, 0, 0, 0, err 1379 } 1380 1381 poolMetadataMinor, err := strconv.ParseUint(poolMetadataMajMinorSplit[1], 10, 32) 1382 if err != nil { 1383 return 0, 0, 0, 0, err 1384 } 1385 1386 return poolDataMajor, poolDataMinor, poolMetadataMajor, poolMetadataMinor, nil 1387 } 1388 1389 func (devices *DeviceSet) loadThinPoolLoopBackInfo() error { 1390 poolDataMajor, poolDataMinor, poolMetadataMajor, poolMetadataMinor, err := devices.getThinPoolDataMetaMajMin() 1391 if err != nil { 1392 return err 1393 } 1394 1395 dirname := devices.loopbackDir() 1396 1397 // data device has not been passed in. So there should be a data file 1398 // which is being mounted as loop device. 1399 if devices.dataDevice == "" { 1400 datafilename := path.Join(dirname, "data") 1401 dataLoopDevice, dataMajor, dataMinor, err := getLoopFileDeviceMajMin(datafilename) 1402 if err != nil { 1403 return err 1404 } 1405 1406 // Compare the two 1407 if poolDataMajor == dataMajor && poolDataMinor == dataMinor { 1408 devices.dataDevice = dataLoopDevice 1409 devices.dataLoopFile = datafilename 1410 } 1411 1412 } 1413 1414 // metadata device has not been passed in. So there should be a 1415 // metadata file which is being mounted as loop device. 1416 if devices.metadataDevice == "" { 1417 metadatafilename := path.Join(dirname, "metadata") 1418 metadataLoopDevice, metadataMajor, metadataMinor, err := getLoopFileDeviceMajMin(metadatafilename) 1419 if err != nil { 1420 return err 1421 } 1422 if poolMetadataMajor == metadataMajor && poolMetadataMinor == metadataMinor { 1423 devices.metadataDevice = metadataLoopDevice 1424 devices.metadataLoopFile = metadatafilename 1425 } 1426 } 1427 1428 return nil 1429 } 1430 1431 func (devices *DeviceSet) initDevmapper(doInit bool) error { 1432 // give ourselves to libdm as a log handler 1433 devicemapper.LogInit(devices) 1434 1435 version, err := devicemapper.GetDriverVersion() 1436 if err != nil { 1437 // Can't even get driver version, assume not supported 1438 return graphdriver.ErrNotSupported 1439 } 1440 1441 if err := determineDriverCapabilities(version); err != nil { 1442 return graphdriver.ErrNotSupported 1443 } 1444 1445 // If user asked for deferred removal then check both libdm library 1446 // and kernel driver support deferred removal otherwise error out. 1447 if enableDeferredRemoval { 1448 if !driverDeferredRemovalSupport { 1449 return fmt.Errorf("devmapper: Deferred removal can not be enabled as kernel does not support it") 1450 } 1451 if !devicemapper.LibraryDeferredRemovalSupport { 1452 return fmt.Errorf("devmapper: Deferred removal can not be enabled as libdm does not support it") 1453 } 1454 logrus.Debugf("devmapper: Deferred removal support enabled.") 1455 devices.deferredRemove = true 1456 } 1457 1458 if enableDeferredDeletion { 1459 if !devices.deferredRemove { 1460 return fmt.Errorf("devmapper: Deferred deletion can not be enabled as deferred removal is not enabled. Enable deferred removal using --storage-opt dm.use_deferred_removal=true parameter") 1461 } 1462 logrus.Debugf("devmapper: Deferred deletion support enabled.") 1463 devices.deferredDelete = true 1464 } 1465 1466 // https://github.com/docker/docker/issues/4036 1467 if supported := devicemapper.UdevSetSyncSupport(true); !supported { 1468 logrus.Warn("Udev sync is not supported. This will lead to unexpected behavior, data loss and errors. For more information, see https://docs.docker.com/reference/commandline/daemon/#daemon-storage-driver-option") 1469 } 1470 1471 //create the root dir of the devmapper driver ownership to match this 1472 //daemon's remapped root uid/gid so containers can start properly 1473 uid, gid, err := idtools.GetRootUIDGID(devices.uidMaps, devices.gidMaps) 1474 if err != nil { 1475 return err 1476 } 1477 if err := idtools.MkdirAs(devices.root, 0700, uid, gid); err != nil && !os.IsExist(err) { 1478 return err 1479 } 1480 if err := os.MkdirAll(devices.metadataDir(), 0700); err != nil && !os.IsExist(err) { 1481 return err 1482 } 1483 1484 // Set the device prefix from the device id and inode of the docker root dir 1485 1486 st, err := os.Stat(devices.root) 1487 if err != nil { 1488 return fmt.Errorf("Error looking up dir %s: %s", devices.root, err) 1489 } 1490 sysSt := st.Sys().(*syscall.Stat_t) 1491 // "reg-" stands for "regular file". 1492 // In the future we might use "dev-" for "device file", etc. 1493 // docker-maj,min[-inode] stands for: 1494 // - Managed by docker 1495 // - The target of this device is at major <maj> and minor <min> 1496 // - If <inode> is defined, use that file inside the device as a loopback image. Otherwise use the device itself. 1497 devices.devicePrefix = fmt.Sprintf("docker-%d:%d-%d", major(sysSt.Dev), minor(sysSt.Dev), sysSt.Ino) 1498 logrus.Debugf("Generated prefix: %s", devices.devicePrefix) 1499 1500 // Check for the existence of the thin-pool device 1501 poolExists, err := devices.thinPoolExists(devices.getPoolName()) 1502 if err != nil { 1503 return err 1504 } 1505 1506 // It seems libdevmapper opens this without O_CLOEXEC, and go exec will not close files 1507 // that are not Close-on-exec, and lxc-start will die if it inherits any unexpected files, 1508 // so we add this badhack to make sure it closes itself 1509 setCloseOnExec("/dev/mapper/control") 1510 1511 // Make sure the sparse images exist in <root>/devicemapper/data and 1512 // <root>/devicemapper/metadata 1513 1514 createdLoopback := false 1515 1516 // If the pool doesn't exist, create it 1517 if !poolExists && devices.thinPoolDevice == "" { 1518 logrus.Debugf("Pool doesn't exist. Creating it.") 1519 1520 var ( 1521 dataFile *os.File 1522 metadataFile *os.File 1523 ) 1524 1525 if devices.dataDevice == "" { 1526 // Make sure the sparse images exist in <root>/devicemapper/data 1527 1528 hasData := devices.hasImage("data") 1529 1530 if !doInit && !hasData { 1531 return errors.New("Loopback data file not found") 1532 } 1533 1534 if !hasData { 1535 createdLoopback = true 1536 } 1537 1538 data, err := devices.ensureImage("data", devices.dataLoopbackSize) 1539 if err != nil { 1540 logrus.Debugf("Error device ensureImage (data): %s", err) 1541 return err 1542 } 1543 1544 dataFile, err = devicemapper.AttachLoopDevice(data) 1545 if err != nil { 1546 return err 1547 } 1548 devices.dataLoopFile = data 1549 devices.dataDevice = dataFile.Name() 1550 } else { 1551 dataFile, err = os.OpenFile(devices.dataDevice, os.O_RDWR, 0600) 1552 if err != nil { 1553 return err 1554 } 1555 } 1556 defer dataFile.Close() 1557 1558 if devices.metadataDevice == "" { 1559 // Make sure the sparse images exist in <root>/devicemapper/metadata 1560 1561 hasMetadata := devices.hasImage("metadata") 1562 1563 if !doInit && !hasMetadata { 1564 return errors.New("Loopback metadata file not found") 1565 } 1566 1567 if !hasMetadata { 1568 createdLoopback = true 1569 } 1570 1571 metadata, err := devices.ensureImage("metadata", devices.metaDataLoopbackSize) 1572 if err != nil { 1573 logrus.Debugf("Error device ensureImage (metadata): %s", err) 1574 return err 1575 } 1576 1577 metadataFile, err = devicemapper.AttachLoopDevice(metadata) 1578 if err != nil { 1579 return err 1580 } 1581 devices.metadataLoopFile = metadata 1582 devices.metadataDevice = metadataFile.Name() 1583 } else { 1584 metadataFile, err = os.OpenFile(devices.metadataDevice, os.O_RDWR, 0600) 1585 if err != nil { 1586 return err 1587 } 1588 } 1589 defer metadataFile.Close() 1590 1591 if err := devicemapper.CreatePool(devices.getPoolName(), dataFile, metadataFile, devices.thinpBlockSize); err != nil { 1592 return err 1593 } 1594 } 1595 1596 // Pool already exists and caller did not pass us a pool. That means 1597 // we probably created pool earlier and could not remove it as some 1598 // containers were still using it. Detect some of the properties of 1599 // pool, like is it using loop devices. 1600 if poolExists && devices.thinPoolDevice == "" { 1601 if err := devices.loadThinPoolLoopBackInfo(); err != nil { 1602 logrus.Debugf("Failed to load thin pool loopback device information:%v", err) 1603 return err 1604 } 1605 } 1606 1607 // If we didn't just create the data or metadata image, we need to 1608 // load the transaction id and migrate old metadata 1609 if !createdLoopback { 1610 if err := devices.initMetaData(); err != nil { 1611 return err 1612 } 1613 } 1614 1615 if devices.thinPoolDevice == "" { 1616 if devices.metadataLoopFile != "" || devices.dataLoopFile != "" { 1617 logrus.Warnf("Usage of loopback devices is strongly discouraged for production use. Please use `--storage-opt dm.thinpooldev` or use `man docker` to refer to dm.thinpooldev section.") 1618 } 1619 } 1620 1621 // Right now this loads only NextDeviceID. If there is more metadata 1622 // down the line, we might have to move it earlier. 1623 if err := devices.loadDeviceSetMetaData(); err != nil { 1624 return err 1625 } 1626 1627 // Setup the base image 1628 if doInit { 1629 if err := devices.setupBaseImage(); err != nil { 1630 logrus.Debugf("Error device setupBaseImage: %s", err) 1631 return err 1632 } 1633 } 1634 1635 return nil 1636 } 1637 1638 // AddDevice adds a device and registers in the hash. 1639 func (devices *DeviceSet) AddDevice(hash, baseHash string) error { 1640 logrus.Debugf("[deviceset] AddDevice(hash=%s basehash=%s)", hash, baseHash) 1641 defer logrus.Debugf("[deviceset] AddDevice(hash=%s basehash=%s) END", hash, baseHash) 1642 1643 // If a deleted device exists, return error. 1644 baseInfo, err := devices.lookupDeviceWithLock(baseHash) 1645 if err != nil { 1646 return err 1647 } 1648 1649 if baseInfo.Deleted { 1650 return fmt.Errorf("devmapper: Base device %v has been marked for deferred deletion", baseInfo.Hash) 1651 } 1652 1653 baseInfo.lock.Lock() 1654 defer baseInfo.lock.Unlock() 1655 1656 devices.Lock() 1657 defer devices.Unlock() 1658 1659 // Also include deleted devices in case hash of new device is 1660 // same as one of the deleted devices. 1661 if info, _ := devices.lookupDevice(hash); info != nil { 1662 return fmt.Errorf("device %s already exists. Deleted=%v", hash, info.Deleted) 1663 } 1664 1665 if err := devices.createRegisterSnapDevice(hash, baseInfo); err != nil { 1666 return err 1667 } 1668 1669 return nil 1670 } 1671 1672 func (devices *DeviceSet) markForDeferredDeletion(info *devInfo) error { 1673 // If device is already in deleted state, there is nothing to be done. 1674 if info.Deleted { 1675 return nil 1676 } 1677 1678 logrus.Debugf("devmapper: Marking device %s for deferred deletion.", info.Hash) 1679 1680 info.Deleted = true 1681 1682 // save device metadata to refelect deleted state. 1683 if err := devices.saveMetadata(info); err != nil { 1684 info.Deleted = false 1685 return err 1686 } 1687 1688 devices.nrDeletedDevices++ 1689 return nil 1690 } 1691 1692 // Should be caled with devices.Lock() held. 1693 func (devices *DeviceSet) deleteTransaction(info *devInfo, syncDelete bool) error { 1694 if err := devices.openTransaction(info.Hash, info.DeviceID); err != nil { 1695 logrus.Debugf("Error opening transaction hash = %s deviceId = %d", "", info.DeviceID) 1696 return err 1697 } 1698 1699 defer devices.closeTransaction() 1700 1701 err := devicemapper.DeleteDevice(devices.getPoolDevName(), info.DeviceID) 1702 if err != nil { 1703 // If syncDelete is true, we want to return error. If deferred 1704 // deletion is not enabled, we return an error. If error is 1705 // something other then EBUSY, return an error. 1706 if syncDelete || !devices.deferredDelete || err != devicemapper.ErrBusy { 1707 logrus.Debugf("Error deleting device: %s", err) 1708 return err 1709 } 1710 } 1711 1712 if err == nil { 1713 if err := devices.unregisterDevice(info.DeviceID, info.Hash); err != nil { 1714 return err 1715 } 1716 // If device was already in deferred delete state that means 1717 // deletion was being tried again later. Reduce the deleted 1718 // device count. 1719 if info.Deleted { 1720 devices.nrDeletedDevices-- 1721 } 1722 } else { 1723 if err := devices.markForDeferredDeletion(info); err != nil { 1724 return err 1725 } 1726 } 1727 1728 return nil 1729 } 1730 1731 // Issue discard only if device open count is zero. 1732 func (devices *DeviceSet) issueDiscard(info *devInfo) error { 1733 logrus.Debugf("devmapper: issueDiscard(device: %s). START", info.Hash) 1734 defer logrus.Debugf("devmapper: issueDiscard(device: %s). END", info.Hash) 1735 // This is a workaround for the kernel not discarding block so 1736 // on the thin pool when we remove a thinp device, so we do it 1737 // manually. 1738 // Even if device is deferred deleted, activate it and isue 1739 // discards. 1740 if err := devices.activateDeviceIfNeeded(info, true); err != nil { 1741 return err 1742 } 1743 1744 devinfo, err := devicemapper.GetInfo(info.Name()) 1745 if err != nil { 1746 return err 1747 } 1748 1749 if devinfo.OpenCount != 0 { 1750 logrus.Debugf("devmapper: Device: %s is in use. OpenCount=%d. Not issuing discards.", info.Hash, devinfo.OpenCount) 1751 return nil 1752 } 1753 1754 if err := devicemapper.BlockDeviceDiscard(info.DevName()); err != nil { 1755 logrus.Debugf("Error discarding block on device: %s (ignoring)", err) 1756 } 1757 return nil 1758 } 1759 1760 // Should be called with devices.Lock() held. 1761 func (devices *DeviceSet) deleteDevice(info *devInfo, syncDelete bool) error { 1762 if devices.doBlkDiscard { 1763 devices.issueDiscard(info) 1764 } 1765 1766 // Try to deactivate device in case it is active. 1767 if err := devices.deactivateDevice(info); err != nil { 1768 logrus.Debugf("Error deactivating device: %s", err) 1769 return err 1770 } 1771 1772 if err := devices.deleteTransaction(info, syncDelete); err != nil { 1773 return err 1774 } 1775 1776 devices.markDeviceIDFree(info.DeviceID) 1777 1778 return nil 1779 } 1780 1781 // DeleteDevice will return success if device has been marked for deferred 1782 // removal. If one wants to override that and want DeleteDevice() to fail if 1783 // device was busy and could not be deleted, set syncDelete=true. 1784 func (devices *DeviceSet) DeleteDevice(hash string, syncDelete bool) error { 1785 logrus.Debugf("devmapper: DeleteDevice(hash=%v syncDelete=%v) START", hash, syncDelete) 1786 defer logrus.Debugf("devmapper: DeleteDevice(hash=%v syncDelete=%v) END", hash, syncDelete) 1787 info, err := devices.lookupDeviceWithLock(hash) 1788 if err != nil { 1789 return err 1790 } 1791 1792 info.lock.Lock() 1793 defer info.lock.Unlock() 1794 1795 devices.Lock() 1796 defer devices.Unlock() 1797 1798 // If mountcount is not zero, that means devices is still in use 1799 // or has not been Put() properly. Fail device deletion. 1800 1801 if info.mountCount != 0 { 1802 return fmt.Errorf("devmapper: Can't delete device %v as it is still mounted. mntCount=%v", info.Hash, info.mountCount) 1803 } 1804 1805 return devices.deleteDevice(info, syncDelete) 1806 } 1807 1808 func (devices *DeviceSet) deactivatePool() error { 1809 logrus.Debugf("[devmapper] deactivatePool()") 1810 defer logrus.Debugf("[devmapper] deactivatePool END") 1811 devname := devices.getPoolDevName() 1812 1813 devinfo, err := devicemapper.GetInfo(devname) 1814 if err != nil { 1815 return err 1816 } 1817 1818 if devinfo.Exists == 0 { 1819 return nil 1820 } 1821 if err := devicemapper.RemoveDevice(devname); err != nil { 1822 return err 1823 } 1824 1825 if d, err := devicemapper.GetDeps(devname); err == nil { 1826 logrus.Warnf("[devmapper] device %s still has %d active dependents", devname, d.Count) 1827 } 1828 1829 return nil 1830 } 1831 1832 func (devices *DeviceSet) deactivateDevice(info *devInfo) error { 1833 logrus.Debugf("[devmapper] deactivateDevice(%s)", info.Hash) 1834 defer logrus.Debugf("[devmapper] deactivateDevice END(%s)", info.Hash) 1835 1836 devinfo, err := devicemapper.GetInfo(info.Name()) 1837 if err != nil { 1838 return err 1839 } 1840 1841 if devinfo.Exists == 0 { 1842 return nil 1843 } 1844 1845 if devices.deferredRemove { 1846 if err := devicemapper.RemoveDeviceDeferred(info.Name()); err != nil { 1847 return err 1848 } 1849 } else { 1850 if err := devices.removeDevice(info.Name()); err != nil { 1851 return err 1852 } 1853 } 1854 return nil 1855 } 1856 1857 // Issues the underlying dm remove operation. 1858 func (devices *DeviceSet) removeDevice(devname string) error { 1859 var err error 1860 1861 logrus.Debugf("[devmapper] removeDevice START(%s)", devname) 1862 defer logrus.Debugf("[devmapper] removeDevice END(%s)", devname) 1863 1864 for i := 0; i < 200; i++ { 1865 err = devicemapper.RemoveDevice(devname) 1866 if err == nil { 1867 break 1868 } 1869 if err != devicemapper.ErrBusy { 1870 return err 1871 } 1872 1873 // If we see EBUSY it may be a transient error, 1874 // sleep a bit a retry a few times. 1875 devices.Unlock() 1876 time.Sleep(100 * time.Millisecond) 1877 devices.Lock() 1878 } 1879 1880 return err 1881 } 1882 1883 func (devices *DeviceSet) cancelDeferredRemoval(info *devInfo) error { 1884 if !devices.deferredRemove { 1885 return nil 1886 } 1887 1888 logrus.Debugf("[devmapper] cancelDeferredRemoval START(%s)", info.Name()) 1889 defer logrus.Debugf("[devmapper] cancelDeferredRemoval END(%s)", info.Name()) 1890 1891 devinfo, err := devicemapper.GetInfoWithDeferred(info.Name()) 1892 1893 if devinfo != nil && devinfo.DeferredRemove == 0 { 1894 return nil 1895 } 1896 1897 // Cancel deferred remove 1898 for i := 0; i < 100; i++ { 1899 err = devicemapper.CancelDeferredRemove(info.Name()) 1900 if err == nil { 1901 break 1902 } 1903 1904 if err == devicemapper.ErrEnxio { 1905 // Device is probably already gone. Return success. 1906 return nil 1907 } 1908 1909 if err != devicemapper.ErrBusy { 1910 return err 1911 } 1912 1913 // If we see EBUSY it may be a transient error, 1914 // sleep a bit a retry a few times. 1915 devices.Unlock() 1916 time.Sleep(100 * time.Millisecond) 1917 devices.Lock() 1918 } 1919 return err 1920 } 1921 1922 // Shutdown shuts down the device by unmounting the root. 1923 func (devices *DeviceSet) Shutdown() error { 1924 logrus.Debugf("[deviceset %s] Shutdown()", devices.devicePrefix) 1925 logrus.Debugf("[devmapper] Shutting down DeviceSet: %s", devices.root) 1926 defer logrus.Debugf("[deviceset %s] Shutdown() END", devices.devicePrefix) 1927 1928 var devs []*devInfo 1929 1930 // Stop deletion worker. This should start delivering new events to 1931 // ticker channel. That means no new instance of cleanupDeletedDevice() 1932 // will run after this call. If one instance is already running at 1933 // the time of the call, it must be holding devices.Lock() and 1934 // we will block on this lock till cleanup function exits. 1935 devices.deletionWorkerTicker.Stop() 1936 1937 devices.Lock() 1938 // Save DeviceSet Metadata first. Docker kills all threads if they 1939 // don't finish in certain time. It is possible that Shutdown() 1940 // routine does not finish in time as we loop trying to deactivate 1941 // some devices while these are busy. In that case shutdown() routine 1942 // will be killed and we will not get a chance to save deviceset 1943 // metadata. Hence save this early before trying to deactivate devices. 1944 devices.saveDeviceSetMetaData() 1945 1946 for _, info := range devices.Devices { 1947 devs = append(devs, info) 1948 } 1949 devices.Unlock() 1950 1951 for _, info := range devs { 1952 info.lock.Lock() 1953 if info.mountCount > 0 { 1954 // We use MNT_DETACH here in case it is still busy in some running 1955 // container. This means it'll go away from the global scope directly, 1956 // and the device will be released when that container dies. 1957 if err := syscall.Unmount(info.mountPath, syscall.MNT_DETACH); err != nil { 1958 logrus.Debugf("Shutdown unmounting %s, error: %s", info.mountPath, err) 1959 } 1960 1961 devices.Lock() 1962 if err := devices.deactivateDevice(info); err != nil { 1963 logrus.Debugf("Shutdown deactivate %s , error: %s", info.Hash, err) 1964 } 1965 devices.Unlock() 1966 } 1967 info.lock.Unlock() 1968 } 1969 1970 info, _ := devices.lookupDeviceWithLock("") 1971 if info != nil { 1972 info.lock.Lock() 1973 devices.Lock() 1974 if err := devices.deactivateDevice(info); err != nil { 1975 logrus.Debugf("Shutdown deactivate base , error: %s", err) 1976 } 1977 devices.Unlock() 1978 info.lock.Unlock() 1979 } 1980 1981 devices.Lock() 1982 if devices.thinPoolDevice == "" { 1983 if err := devices.deactivatePool(); err != nil { 1984 logrus.Debugf("Shutdown deactivate pool , error: %s", err) 1985 } 1986 } 1987 devices.Unlock() 1988 1989 return nil 1990 } 1991 1992 // MountDevice mounts the device if not already mounted. 1993 func (devices *DeviceSet) MountDevice(hash, path, mountLabel string) error { 1994 info, err := devices.lookupDeviceWithLock(hash) 1995 if err != nil { 1996 return err 1997 } 1998 1999 if info.Deleted { 2000 return fmt.Errorf("devmapper: Can't mount device %v as it has been marked for deferred deletion", info.Hash) 2001 } 2002 2003 info.lock.Lock() 2004 defer info.lock.Unlock() 2005 2006 devices.Lock() 2007 defer devices.Unlock() 2008 2009 if info.mountCount > 0 { 2010 if path != info.mountPath { 2011 return fmt.Errorf("Trying to mount devmapper device in multiple places (%s, %s)", info.mountPath, path) 2012 } 2013 2014 info.mountCount++ 2015 return nil 2016 } 2017 2018 if err := devices.activateDeviceIfNeeded(info, false); err != nil { 2019 return fmt.Errorf("Error activating devmapper device for '%s': %s", hash, err) 2020 } 2021 2022 fstype, err := ProbeFsType(info.DevName()) 2023 if err != nil { 2024 return err 2025 } 2026 2027 options := "" 2028 2029 if fstype == "xfs" { 2030 // XFS needs nouuid or it can't mount filesystems with the same fs 2031 options = joinMountOptions(options, "nouuid") 2032 } 2033 2034 options = joinMountOptions(options, devices.mountOptions) 2035 options = joinMountOptions(options, label.FormatMountLabel("", mountLabel)) 2036 2037 if err := mount.Mount(info.DevName(), path, fstype, options); err != nil { 2038 return fmt.Errorf("Error mounting '%s' on '%s': %s", info.DevName(), path, err) 2039 } 2040 2041 info.mountCount = 1 2042 info.mountPath = path 2043 2044 return nil 2045 } 2046 2047 // UnmountDevice unmounts the device and removes it from hash. 2048 func (devices *DeviceSet) UnmountDevice(hash string) error { 2049 logrus.Debugf("[devmapper] UnmountDevice(hash=%s)", hash) 2050 defer logrus.Debugf("[devmapper] UnmountDevice(hash=%s) END", hash) 2051 2052 info, err := devices.lookupDeviceWithLock(hash) 2053 if err != nil { 2054 return err 2055 } 2056 2057 info.lock.Lock() 2058 defer info.lock.Unlock() 2059 2060 devices.Lock() 2061 defer devices.Unlock() 2062 2063 if info.mountCount == 0 { 2064 return fmt.Errorf("UnmountDevice: device not-mounted id %s", hash) 2065 } 2066 2067 info.mountCount-- 2068 if info.mountCount > 0 { 2069 return nil 2070 } 2071 2072 logrus.Debugf("[devmapper] Unmount(%s)", info.mountPath) 2073 if err := syscall.Unmount(info.mountPath, syscall.MNT_DETACH); err != nil { 2074 return err 2075 } 2076 logrus.Debugf("[devmapper] Unmount done") 2077 2078 if err := devices.deactivateDevice(info); err != nil { 2079 return err 2080 } 2081 2082 info.mountPath = "" 2083 2084 return nil 2085 } 2086 2087 // HasDevice returns true if the device metadata exists. 2088 func (devices *DeviceSet) HasDevice(hash string) bool { 2089 info, _ := devices.lookupDeviceWithLock(hash) 2090 return info != nil 2091 } 2092 2093 // List returns a list of device ids. 2094 func (devices *DeviceSet) List() []string { 2095 devices.Lock() 2096 defer devices.Unlock() 2097 2098 ids := make([]string, len(devices.Devices)) 2099 i := 0 2100 for k := range devices.Devices { 2101 ids[i] = k 2102 i++ 2103 } 2104 return ids 2105 } 2106 2107 func (devices *DeviceSet) deviceStatus(devName string) (sizeInSectors, mappedSectors, highestMappedSector uint64, err error) { 2108 var params string 2109 _, sizeInSectors, _, params, err = devicemapper.GetStatus(devName) 2110 if err != nil { 2111 return 2112 } 2113 if _, err = fmt.Sscanf(params, "%d %d", &mappedSectors, &highestMappedSector); err == nil { 2114 return 2115 } 2116 return 2117 } 2118 2119 // GetDeviceStatus provides size, mapped sectors 2120 func (devices *DeviceSet) GetDeviceStatus(hash string) (*DevStatus, error) { 2121 info, err := devices.lookupDeviceWithLock(hash) 2122 if err != nil { 2123 return nil, err 2124 } 2125 2126 info.lock.Lock() 2127 defer info.lock.Unlock() 2128 2129 devices.Lock() 2130 defer devices.Unlock() 2131 2132 status := &DevStatus{ 2133 DeviceID: info.DeviceID, 2134 Size: info.Size, 2135 TransactionID: info.TransactionID, 2136 } 2137 2138 if err := devices.activateDeviceIfNeeded(info, false); err != nil { 2139 return nil, fmt.Errorf("Error activating devmapper device for '%s': %s", hash, err) 2140 } 2141 2142 sizeInSectors, mappedSectors, highestMappedSector, err := devices.deviceStatus(info.DevName()) 2143 2144 if err != nil { 2145 return nil, err 2146 } 2147 2148 status.SizeInSectors = sizeInSectors 2149 status.MappedSectors = mappedSectors 2150 status.HighestMappedSector = highestMappedSector 2151 2152 return status, nil 2153 } 2154 2155 func (devices *DeviceSet) poolStatus() (totalSizeInSectors, transactionID, dataUsed, dataTotal, metadataUsed, metadataTotal uint64, err error) { 2156 var params string 2157 if _, totalSizeInSectors, _, params, err = devicemapper.GetStatus(devices.getPoolName()); err == nil { 2158 _, err = fmt.Sscanf(params, "%d %d/%d %d/%d", &transactionID, &metadataUsed, &metadataTotal, &dataUsed, &dataTotal) 2159 } 2160 return 2161 } 2162 2163 // DataDevicePath returns the path to the data storage for this deviceset, 2164 // regardless of loopback or block device 2165 func (devices *DeviceSet) DataDevicePath() string { 2166 return devices.dataDevice 2167 } 2168 2169 // MetadataDevicePath returns the path to the metadata storage for this deviceset, 2170 // regardless of loopback or block device 2171 func (devices *DeviceSet) MetadataDevicePath() string { 2172 return devices.metadataDevice 2173 } 2174 2175 func (devices *DeviceSet) getUnderlyingAvailableSpace(loopFile string) (uint64, error) { 2176 buf := new(syscall.Statfs_t) 2177 if err := syscall.Statfs(loopFile, buf); err != nil { 2178 logrus.Warnf("Couldn't stat loopfile filesystem %v: %v", loopFile, err) 2179 return 0, err 2180 } 2181 return buf.Bfree * uint64(buf.Bsize), nil 2182 } 2183 2184 func (devices *DeviceSet) isRealFile(loopFile string) (bool, error) { 2185 if loopFile != "" { 2186 fi, err := os.Stat(loopFile) 2187 if err != nil { 2188 logrus.Warnf("Couldn't stat loopfile %v: %v", loopFile, err) 2189 return false, err 2190 } 2191 return fi.Mode().IsRegular(), nil 2192 } 2193 return false, nil 2194 } 2195 2196 // Status returns the current status of this deviceset 2197 func (devices *DeviceSet) Status() *Status { 2198 devices.Lock() 2199 defer devices.Unlock() 2200 2201 status := &Status{} 2202 2203 status.PoolName = devices.getPoolName() 2204 status.DataFile = devices.DataDevicePath() 2205 status.DataLoopback = devices.dataLoopFile 2206 status.MetadataFile = devices.MetadataDevicePath() 2207 status.MetadataLoopback = devices.metadataLoopFile 2208 status.UdevSyncSupported = devicemapper.UdevSyncSupported() 2209 status.DeferredRemoveEnabled = devices.deferredRemove 2210 status.DeferredDeleteEnabled = devices.deferredDelete 2211 status.DeferredDeletedDeviceCount = devices.nrDeletedDevices 2212 status.BaseDeviceSize = devices.getBaseDeviceSize() 2213 2214 totalSizeInSectors, _, dataUsed, dataTotal, metadataUsed, metadataTotal, err := devices.poolStatus() 2215 if err == nil { 2216 // Convert from blocks to bytes 2217 blockSizeInSectors := totalSizeInSectors / dataTotal 2218 2219 status.Data.Used = dataUsed * blockSizeInSectors * 512 2220 status.Data.Total = dataTotal * blockSizeInSectors * 512 2221 status.Data.Available = status.Data.Total - status.Data.Used 2222 2223 // metadata blocks are always 4k 2224 status.Metadata.Used = metadataUsed * 4096 2225 status.Metadata.Total = metadataTotal * 4096 2226 status.Metadata.Available = status.Metadata.Total - status.Metadata.Used 2227 2228 status.SectorSize = blockSizeInSectors * 512 2229 2230 if check, _ := devices.isRealFile(devices.dataLoopFile); check { 2231 actualSpace, err := devices.getUnderlyingAvailableSpace(devices.dataLoopFile) 2232 if err == nil && actualSpace < status.Data.Available { 2233 status.Data.Available = actualSpace 2234 } 2235 } 2236 2237 if check, _ := devices.isRealFile(devices.metadataLoopFile); check { 2238 actualSpace, err := devices.getUnderlyingAvailableSpace(devices.metadataLoopFile) 2239 if err == nil && actualSpace < status.Metadata.Available { 2240 status.Metadata.Available = actualSpace 2241 } 2242 } 2243 } 2244 2245 return status 2246 } 2247 2248 // Status returns the current status of this deviceset 2249 func (devices *DeviceSet) exportDeviceMetadata(hash string) (*deviceMetadata, error) { 2250 info, err := devices.lookupDeviceWithLock(hash) 2251 if err != nil { 2252 return nil, err 2253 } 2254 2255 info.lock.Lock() 2256 defer info.lock.Unlock() 2257 2258 metadata := &deviceMetadata{info.DeviceID, info.Size, info.Name()} 2259 return metadata, nil 2260 } 2261 2262 // NewDeviceSet creates the device set based on the options provided. 2263 func NewDeviceSet(root string, doInit bool, options []string, uidMaps, gidMaps []idtools.IDMap) (*DeviceSet, error) { 2264 devicemapper.SetDevDir("/dev") 2265 2266 devices := &DeviceSet{ 2267 root: root, 2268 metaData: metaData{Devices: make(map[string]*devInfo)}, 2269 dataLoopbackSize: defaultDataLoopbackSize, 2270 metaDataLoopbackSize: defaultMetaDataLoopbackSize, 2271 baseFsSize: defaultBaseFsSize, 2272 overrideUdevSyncCheck: defaultUdevSyncOverride, 2273 filesystem: "ext4", 2274 doBlkDiscard: true, 2275 thinpBlockSize: defaultThinpBlockSize, 2276 deviceIDMap: make([]byte, deviceIDMapSz), 2277 deletionWorkerTicker: time.NewTicker(time.Second * 30), 2278 uidMaps: uidMaps, 2279 gidMaps: gidMaps, 2280 } 2281 2282 foundBlkDiscard := false 2283 for _, option := range options { 2284 key, val, err := parsers.ParseKeyValueOpt(option) 2285 if err != nil { 2286 return nil, err 2287 } 2288 key = strings.ToLower(key) 2289 switch key { 2290 case "dm.basesize": 2291 size, err := units.RAMInBytes(val) 2292 if err != nil { 2293 return nil, err 2294 } 2295 devices.baseFsSize = uint64(size) 2296 case "dm.loopdatasize": 2297 size, err := units.RAMInBytes(val) 2298 if err != nil { 2299 return nil, err 2300 } 2301 devices.dataLoopbackSize = size 2302 case "dm.loopmetadatasize": 2303 size, err := units.RAMInBytes(val) 2304 if err != nil { 2305 return nil, err 2306 } 2307 devices.metaDataLoopbackSize = size 2308 case "dm.fs": 2309 if val != "ext4" && val != "xfs" { 2310 return nil, fmt.Errorf("Unsupported filesystem %s\n", val) 2311 } 2312 devices.filesystem = val 2313 case "dm.mkfsarg": 2314 devices.mkfsArgs = append(devices.mkfsArgs, val) 2315 case "dm.mountopt": 2316 devices.mountOptions = joinMountOptions(devices.mountOptions, val) 2317 case "dm.metadatadev": 2318 devices.metadataDevice = val 2319 case "dm.datadev": 2320 devices.dataDevice = val 2321 case "dm.thinpooldev": 2322 devices.thinPoolDevice = strings.TrimPrefix(val, "/dev/mapper/") 2323 case "dm.blkdiscard": 2324 foundBlkDiscard = true 2325 devices.doBlkDiscard, err = strconv.ParseBool(val) 2326 if err != nil { 2327 return nil, err 2328 } 2329 case "dm.blocksize": 2330 size, err := units.RAMInBytes(val) 2331 if err != nil { 2332 return nil, err 2333 } 2334 // convert to 512b sectors 2335 devices.thinpBlockSize = uint32(size) >> 9 2336 case "dm.override_udev_sync_check": 2337 devices.overrideUdevSyncCheck, err = strconv.ParseBool(val) 2338 if err != nil { 2339 return nil, err 2340 } 2341 2342 case "dm.use_deferred_removal": 2343 enableDeferredRemoval, err = strconv.ParseBool(val) 2344 if err != nil { 2345 return nil, err 2346 } 2347 2348 case "dm.use_deferred_deletion": 2349 enableDeferredDeletion, err = strconv.ParseBool(val) 2350 if err != nil { 2351 return nil, err 2352 } 2353 2354 default: 2355 return nil, fmt.Errorf("Unknown option %s\n", key) 2356 } 2357 } 2358 2359 // By default, don't do blk discard hack on raw devices, its rarely useful and is expensive 2360 if !foundBlkDiscard && (devices.dataDevice != "" || devices.thinPoolDevice != "") { 2361 devices.doBlkDiscard = false 2362 } 2363 2364 if err := devices.initDevmapper(doInit); err != nil { 2365 return nil, err 2366 } 2367 2368 return devices, nil 2369 }