github.com/daaku/docker@v1.5.0/daemon/graphdriver/devmapper/deviceset.go (about) 1 // +build linux 2 3 package devmapper 4 5 import ( 6 "encoding/json" 7 "errors" 8 "fmt" 9 "io" 10 "io/ioutil" 11 "os" 12 "os/exec" 13 "path" 14 "path/filepath" 15 "strconv" 16 "strings" 17 "sync" 18 "syscall" 19 "time" 20 21 log "github.com/Sirupsen/logrus" 22 "github.com/docker/docker/daemon/graphdriver" 23 "github.com/docker/docker/pkg/devicemapper" 24 "github.com/docker/docker/pkg/parsers" 25 "github.com/docker/docker/pkg/units" 26 "github.com/docker/libcontainer/label" 27 ) 28 29 var ( 30 DefaultDataLoopbackSize int64 = 100 * 1024 * 1024 * 1024 31 DefaultMetaDataLoopbackSize int64 = 2 * 1024 * 1024 * 1024 32 DefaultBaseFsSize uint64 = 10 * 1024 * 1024 * 1024 33 DefaultThinpBlockSize uint32 = 128 // 64K = 128 512b sectors 34 MaxDeviceId int = 0xffffff // 24 bit, pool limit 35 DeviceIdMapSz int = (MaxDeviceId + 1) / 8 36 ) 37 38 const deviceSetMetaFile string = "deviceset-metadata" 39 const transactionMetaFile string = "transaction-metadata" 40 41 type Transaction struct { 42 OpenTransactionId uint64 `json:"open_transaction_id"` 43 DeviceIdHash string `json:"device_hash"` 44 DeviceId int `json:"device_id"` 45 } 46 47 type DevInfo struct { 48 Hash string `json:"-"` 49 DeviceId int `json:"device_id"` 50 Size uint64 `json:"size"` 51 TransactionId uint64 `json:"transaction_id"` 52 Initialized bool `json:"initialized"` 53 devices *DeviceSet 54 55 mountCount int 56 mountPath string 57 58 // The global DeviceSet lock guarantees that we serialize all 59 // the calls to libdevmapper (which is not threadsafe), but we 60 // sometimes release that lock while sleeping. In that case 61 // this per-device lock is still held, protecting against 62 // other accesses to the device that we're doing the wait on. 63 // 64 // WARNING: In order to avoid AB-BA deadlocks when releasing 65 // the global lock while holding the per-device locks all 66 // device locks must be aquired *before* the device lock, and 67 // multiple device locks should be aquired parent before child. 68 lock sync.Mutex 69 } 70 71 type MetaData struct { 72 Devices map[string]*DevInfo `json:"Devices"` 73 devicesLock sync.Mutex // Protects all read/writes to Devices map 74 } 75 76 type DeviceSet struct { 77 MetaData `json:"-"` 78 sync.Mutex `json:"-"` // Protects Devices map and serializes calls into libdevmapper 79 root string 80 devicePrefix string 81 TransactionId uint64 `json:"-"` 82 NextDeviceId int `json:"next_device_id"` 83 deviceIdMap []byte 84 85 // Options 86 dataLoopbackSize int64 87 metaDataLoopbackSize int64 88 baseFsSize uint64 89 filesystem string 90 mountOptions string 91 mkfsArgs []string 92 dataDevice string // block or loop dev 93 dataLoopFile string // loopback file, if used 94 metadataDevice string // block or loop dev 95 metadataLoopFile string // loopback file, if used 96 doBlkDiscard bool 97 thinpBlockSize uint32 98 thinPoolDevice string 99 Transaction `json:"-"` 100 } 101 102 type DiskUsage struct { 103 Used uint64 104 Total uint64 105 } 106 107 type Status struct { 108 PoolName string 109 DataFile string // actual block device for data 110 DataLoopback string // loopback file, if used 111 MetadataFile string // actual block device for metadata 112 MetadataLoopback string // loopback file, if used 113 Data DiskUsage 114 Metadata DiskUsage 115 SectorSize uint64 116 UdevSyncSupported bool 117 } 118 119 type DevStatus struct { 120 DeviceId int 121 Size uint64 122 TransactionId uint64 123 SizeInSectors uint64 124 MappedSectors uint64 125 HighestMappedSector uint64 126 } 127 128 func getDevName(name string) string { 129 return "/dev/mapper/" + name 130 } 131 132 func (info *DevInfo) Name() string { 133 hash := info.Hash 134 if hash == "" { 135 hash = "base" 136 } 137 return fmt.Sprintf("%s-%s", info.devices.devicePrefix, hash) 138 } 139 140 func (info *DevInfo) DevName() string { 141 return getDevName(info.Name()) 142 } 143 144 func (devices *DeviceSet) loopbackDir() string { 145 return path.Join(devices.root, "devicemapper") 146 } 147 148 func (devices *DeviceSet) metadataDir() string { 149 return path.Join(devices.root, "metadata") 150 } 151 152 func (devices *DeviceSet) metadataFile(info *DevInfo) string { 153 file := info.Hash 154 if file == "" { 155 file = "base" 156 } 157 return path.Join(devices.metadataDir(), file) 158 } 159 160 func (devices *DeviceSet) transactionMetaFile() string { 161 return path.Join(devices.metadataDir(), transactionMetaFile) 162 } 163 164 func (devices *DeviceSet) deviceSetMetaFile() string { 165 return path.Join(devices.metadataDir(), deviceSetMetaFile) 166 } 167 168 func (devices *DeviceSet) oldMetadataFile() string { 169 return path.Join(devices.loopbackDir(), "json") 170 } 171 172 func (devices *DeviceSet) getPoolName() string { 173 if devices.thinPoolDevice == "" { 174 return devices.devicePrefix + "-pool" 175 } 176 return devices.thinPoolDevice 177 } 178 179 func (devices *DeviceSet) getPoolDevName() string { 180 return getDevName(devices.getPoolName()) 181 } 182 183 func (devices *DeviceSet) hasImage(name string) bool { 184 dirname := devices.loopbackDir() 185 filename := path.Join(dirname, name) 186 187 _, err := os.Stat(filename) 188 return err == nil 189 } 190 191 // ensureImage creates a sparse file of <size> bytes at the path 192 // <root>/devicemapper/<name>. 193 // If the file already exists, it does nothing. 194 // Either way it returns the full path. 195 func (devices *DeviceSet) ensureImage(name string, size int64) (string, error) { 196 dirname := devices.loopbackDir() 197 filename := path.Join(dirname, name) 198 199 if err := os.MkdirAll(dirname, 0700); err != nil && !os.IsExist(err) { 200 return "", err 201 } 202 203 if _, err := os.Stat(filename); err != nil { 204 if !os.IsNotExist(err) { 205 return "", err 206 } 207 log.Debugf("Creating loopback file %s for device-manage use", filename) 208 file, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE, 0600) 209 if err != nil { 210 return "", err 211 } 212 defer file.Close() 213 214 if err = file.Truncate(size); err != nil { 215 return "", err 216 } 217 } 218 return filename, nil 219 } 220 221 func (devices *DeviceSet) allocateTransactionId() uint64 { 222 devices.OpenTransactionId = devices.TransactionId + 1 223 return devices.OpenTransactionId 224 } 225 226 func (devices *DeviceSet) updatePoolTransactionId() error { 227 if err := devicemapper.SetTransactionId(devices.getPoolDevName(), devices.TransactionId, devices.OpenTransactionId); err != nil { 228 return fmt.Errorf("Error setting devmapper transaction ID: %s", err) 229 } 230 devices.TransactionId = devices.OpenTransactionId 231 return nil 232 } 233 234 func (devices *DeviceSet) removeMetadata(info *DevInfo) error { 235 if err := os.RemoveAll(devices.metadataFile(info)); err != nil { 236 return fmt.Errorf("Error removing metadata file %s: %s", devices.metadataFile(info), err) 237 } 238 return nil 239 } 240 241 // Given json data and file path, write it to disk 242 func (devices *DeviceSet) writeMetaFile(jsonData []byte, filePath string) error { 243 tmpFile, err := ioutil.TempFile(devices.metadataDir(), ".tmp") 244 if err != nil { 245 return fmt.Errorf("Error creating metadata file: %s", err) 246 } 247 248 n, err := tmpFile.Write(jsonData) 249 if err != nil { 250 return fmt.Errorf("Error writing metadata to %s: %s", tmpFile.Name(), err) 251 } 252 if n < len(jsonData) { 253 return io.ErrShortWrite 254 } 255 if err := tmpFile.Sync(); err != nil { 256 return fmt.Errorf("Error syncing metadata file %s: %s", tmpFile.Name(), err) 257 } 258 if err := tmpFile.Close(); err != nil { 259 return fmt.Errorf("Error closing metadata file %s: %s", tmpFile.Name(), err) 260 } 261 if err := os.Rename(tmpFile.Name(), filePath); err != nil { 262 return fmt.Errorf("Error committing metadata file %s: %s", tmpFile.Name(), err) 263 } 264 265 return nil 266 } 267 268 func (devices *DeviceSet) saveMetadata(info *DevInfo) error { 269 jsonData, err := json.Marshal(info) 270 if err != nil { 271 return fmt.Errorf("Error encoding metadata to json: %s", err) 272 } 273 if err := devices.writeMetaFile(jsonData, devices.metadataFile(info)); err != nil { 274 return err 275 } 276 return nil 277 } 278 279 func (devices *DeviceSet) markDeviceIdUsed(deviceId int) { 280 var mask byte 281 i := deviceId % 8 282 mask = 1 << uint(i) 283 devices.deviceIdMap[deviceId/8] = devices.deviceIdMap[deviceId/8] | mask 284 } 285 286 func (devices *DeviceSet) markDeviceIdFree(deviceId int) { 287 var mask byte 288 i := deviceId % 8 289 mask = ^(1 << uint(i)) 290 devices.deviceIdMap[deviceId/8] = devices.deviceIdMap[deviceId/8] & mask 291 } 292 293 func (devices *DeviceSet) isDeviceIdFree(deviceId int) bool { 294 var mask byte 295 i := deviceId % 8 296 mask = (1 << uint(i)) 297 if (devices.deviceIdMap[deviceId/8] & mask) != 0 { 298 return false 299 } 300 return true 301 } 302 303 func (devices *DeviceSet) lookupDevice(hash string) (*DevInfo, error) { 304 devices.devicesLock.Lock() 305 defer devices.devicesLock.Unlock() 306 info := devices.Devices[hash] 307 if info == nil { 308 info = devices.loadMetadata(hash) 309 if info == nil { 310 return nil, fmt.Errorf("Unknown device %s", hash) 311 } 312 313 devices.Devices[hash] = info 314 } 315 return info, nil 316 } 317 318 func (devices *DeviceSet) deviceFileWalkFunction(path string, finfo os.FileInfo) error { 319 320 // Skip some of the meta files which are not device files. 321 if strings.HasSuffix(finfo.Name(), ".migrated") { 322 log.Debugf("Skipping file %s", path) 323 return nil 324 } 325 326 if finfo.Name() == deviceSetMetaFile { 327 log.Debugf("Skipping file %s", path) 328 return nil 329 } 330 331 log.Debugf("Loading data for file %s", path) 332 333 hash := finfo.Name() 334 if hash == "base" { 335 hash = "" 336 } 337 338 dinfo := devices.loadMetadata(hash) 339 if dinfo == nil { 340 return fmt.Errorf("Error loading device metadata file %s", hash) 341 } 342 343 if dinfo.DeviceId > MaxDeviceId { 344 log.Errorf("Warning: Ignoring Invalid DeviceId=%d", dinfo.DeviceId) 345 return nil 346 } 347 348 devices.Lock() 349 devices.markDeviceIdUsed(dinfo.DeviceId) 350 devices.Unlock() 351 352 log.Debugf("Added deviceId=%d to DeviceIdMap", dinfo.DeviceId) 353 return nil 354 } 355 356 func (devices *DeviceSet) constructDeviceIdMap() error { 357 log.Debugf("[deviceset] constructDeviceIdMap()") 358 defer log.Debugf("[deviceset] constructDeviceIdMap() END") 359 360 var scan = func(path string, info os.FileInfo, err error) error { 361 if err != nil { 362 log.Debugf("Can't walk the file %s", path) 363 return nil 364 } 365 366 // Skip any directories 367 if info.IsDir() { 368 return nil 369 } 370 371 return devices.deviceFileWalkFunction(path, info) 372 } 373 374 return filepath.Walk(devices.metadataDir(), scan) 375 } 376 377 func (devices *DeviceSet) unregisterDevice(id int, hash string) error { 378 log.Debugf("unregisterDevice(%v, %v)", id, hash) 379 info := &DevInfo{ 380 Hash: hash, 381 DeviceId: id, 382 } 383 384 devices.devicesLock.Lock() 385 delete(devices.Devices, hash) 386 devices.devicesLock.Unlock() 387 388 if err := devices.removeMetadata(info); err != nil { 389 log.Debugf("Error removing meta data: %s", err) 390 return err 391 } 392 393 return nil 394 } 395 396 func (devices *DeviceSet) registerDevice(id int, hash string, size uint64, transactionId uint64) (*DevInfo, error) { 397 log.Debugf("registerDevice(%v, %v)", id, hash) 398 info := &DevInfo{ 399 Hash: hash, 400 DeviceId: id, 401 Size: size, 402 TransactionId: transactionId, 403 Initialized: false, 404 devices: devices, 405 } 406 407 devices.devicesLock.Lock() 408 devices.Devices[hash] = info 409 devices.devicesLock.Unlock() 410 411 if err := devices.saveMetadata(info); err != nil { 412 // Try to remove unused device 413 devices.devicesLock.Lock() 414 delete(devices.Devices, hash) 415 devices.devicesLock.Unlock() 416 return nil, err 417 } 418 419 return info, nil 420 } 421 422 func (devices *DeviceSet) activateDeviceIfNeeded(info *DevInfo) error { 423 log.Debugf("activateDeviceIfNeeded(%v)", info.Hash) 424 425 if devinfo, _ := devicemapper.GetInfo(info.Name()); devinfo != nil && devinfo.Exists != 0 { 426 return nil 427 } 428 429 return devicemapper.ActivateDevice(devices.getPoolDevName(), info.Name(), info.DeviceId, info.Size) 430 } 431 432 func (devices *DeviceSet) createFilesystem(info *DevInfo) error { 433 devname := info.DevName() 434 435 args := []string{} 436 for _, arg := range devices.mkfsArgs { 437 args = append(args, arg) 438 } 439 440 args = append(args, devname) 441 442 var err error 443 switch devices.filesystem { 444 case "xfs": 445 err = exec.Command("mkfs.xfs", args...).Run() 446 case "ext4": 447 err = exec.Command("mkfs.ext4", append([]string{"-E", "nodiscard,lazy_itable_init=0,lazy_journal_init=0"}, args...)...).Run() 448 if err != nil { 449 err = exec.Command("mkfs.ext4", append([]string{"-E", "nodiscard,lazy_itable_init=0"}, args...)...).Run() 450 } 451 if err != nil { 452 return err 453 } 454 err = exec.Command("tune2fs", append([]string{"-c", "-1", "-i", "0"}, devname)...).Run() 455 default: 456 err = fmt.Errorf("Unsupported filesystem type %s", devices.filesystem) 457 } 458 if err != nil { 459 return err 460 } 461 462 return nil 463 } 464 465 func (devices *DeviceSet) migrateOldMetaData() error { 466 // Migrate old metadatafile 467 jsonData, err := ioutil.ReadFile(devices.oldMetadataFile()) 468 if err != nil && !os.IsNotExist(err) { 469 return err 470 } 471 472 if jsonData != nil { 473 m := MetaData{Devices: make(map[string]*DevInfo)} 474 475 if err := json.Unmarshal(jsonData, &m); err != nil { 476 return err 477 } 478 479 for hash, info := range m.Devices { 480 info.Hash = hash 481 devices.saveMetadata(info) 482 } 483 if err := os.Rename(devices.oldMetadataFile(), devices.oldMetadataFile()+".migrated"); err != nil { 484 return err 485 } 486 487 } 488 489 return nil 490 } 491 492 func (devices *DeviceSet) initMetaData() error { 493 if err := devices.migrateOldMetaData(); err != nil { 494 return err 495 } 496 497 _, transactionId, _, _, _, _, err := devices.poolStatus() 498 if err != nil { 499 return err 500 } 501 502 devices.TransactionId = transactionId 503 504 if err := devices.constructDeviceIdMap(); err != nil { 505 return err 506 } 507 508 if err := devices.processPendingTransaction(); err != nil { 509 return err 510 } 511 return nil 512 } 513 514 func (devices *DeviceSet) incNextDeviceId() { 515 // Ids are 24bit, so wrap around 516 devices.NextDeviceId = (devices.NextDeviceId + 1) & MaxDeviceId 517 } 518 519 func (devices *DeviceSet) getNextFreeDeviceId() (int, error) { 520 devices.incNextDeviceId() 521 for i := 0; i <= MaxDeviceId; i++ { 522 if devices.isDeviceIdFree(devices.NextDeviceId) { 523 devices.markDeviceIdUsed(devices.NextDeviceId) 524 return devices.NextDeviceId, nil 525 } 526 devices.incNextDeviceId() 527 } 528 529 return 0, fmt.Errorf("Unable to find a free device Id") 530 } 531 532 func (devices *DeviceSet) createRegisterDevice(hash string) (*DevInfo, error) { 533 deviceId, err := devices.getNextFreeDeviceId() 534 if err != nil { 535 return nil, err 536 } 537 538 if err := devices.openTransaction(hash, deviceId); err != nil { 539 log.Debugf("Error opening transaction hash = %s deviceId = %d", hash, deviceId) 540 devices.markDeviceIdFree(deviceId) 541 return nil, err 542 } 543 544 for { 545 if err := devicemapper.CreateDevice(devices.getPoolDevName(), deviceId); err != nil { 546 if devicemapper.DeviceIdExists(err) { 547 // Device Id already exists. This should not 548 // happen. Now we have a mechianism to find 549 // a free device Id. So something is not right. 550 // Give a warning and continue. 551 log.Errorf("Warning: Device Id %d exists in pool but it is supposed to be unused", deviceId) 552 deviceId, err = devices.getNextFreeDeviceId() 553 if err != nil { 554 return nil, err 555 } 556 // Save new device id into transaction 557 devices.refreshTransaction(deviceId) 558 continue 559 } 560 log.Debugf("Error creating device: %s", err) 561 devices.markDeviceIdFree(deviceId) 562 return nil, err 563 } 564 break 565 } 566 567 log.Debugf("Registering device (id %v) with FS size %v", deviceId, devices.baseFsSize) 568 info, err := devices.registerDevice(deviceId, hash, devices.baseFsSize, devices.OpenTransactionId) 569 if err != nil { 570 _ = devicemapper.DeleteDevice(devices.getPoolDevName(), deviceId) 571 devices.markDeviceIdFree(deviceId) 572 return nil, err 573 } 574 575 if err := devices.closeTransaction(); err != nil { 576 devices.unregisterDevice(deviceId, hash) 577 devicemapper.DeleteDevice(devices.getPoolDevName(), deviceId) 578 devices.markDeviceIdFree(deviceId) 579 return nil, err 580 } 581 return info, nil 582 } 583 584 func (devices *DeviceSet) createRegisterSnapDevice(hash string, baseInfo *DevInfo) error { 585 deviceId, err := devices.getNextFreeDeviceId() 586 if err != nil { 587 return err 588 } 589 590 if err := devices.openTransaction(hash, deviceId); err != nil { 591 log.Debugf("Error opening transaction hash = %s deviceId = %d", hash, deviceId) 592 devices.markDeviceIdFree(deviceId) 593 return err 594 } 595 596 for { 597 if err := devicemapper.CreateSnapDevice(devices.getPoolDevName(), deviceId, baseInfo.Name(), baseInfo.DeviceId); err != nil { 598 if devicemapper.DeviceIdExists(err) { 599 // Device Id already exists. This should not 600 // happen. Now we have a mechianism to find 601 // a free device Id. So something is not right. 602 // Give a warning and continue. 603 log.Errorf("Warning: Device Id %d exists in pool but it is supposed to be unused", deviceId) 604 deviceId, err = devices.getNextFreeDeviceId() 605 if err != nil { 606 return err 607 } 608 // Save new device id into transaction 609 devices.refreshTransaction(deviceId) 610 continue 611 } 612 log.Debugf("Error creating snap device: %s", err) 613 devices.markDeviceIdFree(deviceId) 614 return err 615 } 616 break 617 } 618 619 if _, err := devices.registerDevice(deviceId, hash, baseInfo.Size, devices.OpenTransactionId); err != nil { 620 devicemapper.DeleteDevice(devices.getPoolDevName(), deviceId) 621 devices.markDeviceIdFree(deviceId) 622 log.Debugf("Error registering device: %s", err) 623 return err 624 } 625 626 if err := devices.closeTransaction(); err != nil { 627 devices.unregisterDevice(deviceId, hash) 628 devicemapper.DeleteDevice(devices.getPoolDevName(), deviceId) 629 devices.markDeviceIdFree(deviceId) 630 return err 631 } 632 return nil 633 } 634 635 func (devices *DeviceSet) loadMetadata(hash string) *DevInfo { 636 info := &DevInfo{Hash: hash, devices: devices} 637 638 jsonData, err := ioutil.ReadFile(devices.metadataFile(info)) 639 if err != nil { 640 return nil 641 } 642 643 if err := json.Unmarshal(jsonData, &info); err != nil { 644 return nil 645 } 646 647 return info 648 } 649 650 func (devices *DeviceSet) setupBaseImage() error { 651 oldInfo, _ := devices.lookupDevice("") 652 if oldInfo != nil && oldInfo.Initialized { 653 return nil 654 } 655 656 if oldInfo != nil && !oldInfo.Initialized { 657 log.Debugf("Removing uninitialized base image") 658 if err := devices.DeleteDevice(""); err != nil { 659 return err 660 } 661 } 662 663 if devices.thinPoolDevice != "" && oldInfo == nil { 664 _, transactionId, dataUsed, _, _, _, err := devices.poolStatus() 665 if err != nil { 666 return err 667 } 668 if dataUsed != 0 { 669 return fmt.Errorf("Unable to take ownership of thin-pool (%s) that already has used data blocks", 670 devices.thinPoolDevice) 671 } 672 if transactionId != 0 { 673 return fmt.Errorf("Unable to take ownership of thin-pool (%s) with non-zero transaction Id", 674 devices.thinPoolDevice) 675 } 676 } 677 678 log.Debugf("Initializing base device-mapper thin volume") 679 680 // Create initial device 681 info, err := devices.createRegisterDevice("") 682 if err != nil { 683 return err 684 } 685 686 log.Debugf("Creating filesystem on base device-mapper thin volume") 687 688 if err = devices.activateDeviceIfNeeded(info); err != nil { 689 return err 690 } 691 692 if err := devices.createFilesystem(info); err != nil { 693 return err 694 } 695 696 info.Initialized = true 697 if err = devices.saveMetadata(info); err != nil { 698 info.Initialized = false 699 return err 700 } 701 702 return nil 703 } 704 705 func setCloseOnExec(name string) { 706 if fileInfos, _ := ioutil.ReadDir("/proc/self/fd"); fileInfos != nil { 707 for _, i := range fileInfos { 708 link, _ := os.Readlink(filepath.Join("/proc/self/fd", i.Name())) 709 if link == name { 710 fd, err := strconv.Atoi(i.Name()) 711 if err == nil { 712 syscall.CloseOnExec(fd) 713 } 714 } 715 } 716 } 717 } 718 719 func (devices *DeviceSet) DMLog(level int, file string, line int, dmError int, message string) { 720 if level >= devicemapper.LogLevelDebug { 721 // (vbatts) libdm debug is very verbose. If you're debugging libdm, you can 722 // comment out this check yourself 723 level = devicemapper.LogLevelInfo 724 } 725 726 // FIXME(vbatts) push this back into ./pkg/devicemapper/ 727 log.Debugf("libdevmapper(%d): %s:%d (%d) %s", level, file, line, dmError, message) 728 } 729 730 func major(device uint64) uint64 { 731 return (device >> 8) & 0xfff 732 } 733 734 func minor(device uint64) uint64 { 735 return (device & 0xff) | ((device >> 12) & 0xfff00) 736 } 737 738 func (devices *DeviceSet) ResizePool(size int64) error { 739 dirname := devices.loopbackDir() 740 datafilename := path.Join(dirname, "data") 741 if len(devices.dataDevice) > 0 { 742 datafilename = devices.dataDevice 743 } 744 metadatafilename := path.Join(dirname, "metadata") 745 if len(devices.metadataDevice) > 0 { 746 metadatafilename = devices.metadataDevice 747 } 748 749 datafile, err := os.OpenFile(datafilename, os.O_RDWR, 0) 750 if datafile == nil { 751 return err 752 } 753 defer datafile.Close() 754 755 fi, err := datafile.Stat() 756 if fi == nil { 757 return err 758 } 759 760 if fi.Size() > size { 761 return fmt.Errorf("Can't shrink file") 762 } 763 764 dataloopback := devicemapper.FindLoopDeviceFor(datafile) 765 if dataloopback == nil { 766 return fmt.Errorf("Unable to find loopback mount for: %s", datafilename) 767 } 768 defer dataloopback.Close() 769 770 metadatafile, err := os.OpenFile(metadatafilename, os.O_RDWR, 0) 771 if metadatafile == nil { 772 return err 773 } 774 defer metadatafile.Close() 775 776 metadataloopback := devicemapper.FindLoopDeviceFor(metadatafile) 777 if metadataloopback == nil { 778 return fmt.Errorf("Unable to find loopback mount for: %s", metadatafilename) 779 } 780 defer metadataloopback.Close() 781 782 // Grow loopback file 783 if err := datafile.Truncate(size); err != nil { 784 return fmt.Errorf("Unable to grow loopback file: %s", err) 785 } 786 787 // Reload size for loopback device 788 if err := devicemapper.LoopbackSetCapacity(dataloopback); err != nil { 789 return fmt.Errorf("Unable to update loopback capacity: %s", err) 790 } 791 792 // Suspend the pool 793 if err := devicemapper.SuspendDevice(devices.getPoolName()); err != nil { 794 return fmt.Errorf("Unable to suspend pool: %s", err) 795 } 796 797 // Reload with the new block sizes 798 if err := devicemapper.ReloadPool(devices.getPoolName(), dataloopback, metadataloopback, devices.thinpBlockSize); err != nil { 799 return fmt.Errorf("Unable to reload pool: %s", err) 800 } 801 802 // Resume the pool 803 if err := devicemapper.ResumeDevice(devices.getPoolName()); err != nil { 804 return fmt.Errorf("Unable to resume pool: %s", err) 805 } 806 807 return nil 808 } 809 810 func (devices *DeviceSet) loadTransactionMetaData() error { 811 jsonData, err := ioutil.ReadFile(devices.transactionMetaFile()) 812 if err != nil { 813 // There is no active transaction. This will be the case 814 // during upgrade. 815 if os.IsNotExist(err) { 816 devices.OpenTransactionId = devices.TransactionId 817 return nil 818 } 819 return err 820 } 821 822 json.Unmarshal(jsonData, &devices.Transaction) 823 return nil 824 } 825 826 func (devices *DeviceSet) saveTransactionMetaData() error { 827 jsonData, err := json.Marshal(&devices.Transaction) 828 if err != nil { 829 return fmt.Errorf("Error encoding metadata to json: %s", err) 830 } 831 832 return devices.writeMetaFile(jsonData, devices.transactionMetaFile()) 833 } 834 835 func (devices *DeviceSet) removeTransactionMetaData() error { 836 if err := os.RemoveAll(devices.transactionMetaFile()); err != nil { 837 return err 838 } 839 return nil 840 } 841 842 func (devices *DeviceSet) rollbackTransaction() error { 843 log.Debugf("Rolling back open transaction: TransactionId=%d hash=%s device_id=%d", devices.OpenTransactionId, devices.DeviceIdHash, devices.DeviceId) 844 845 // A device id might have already been deleted before transaction 846 // closed. In that case this call will fail. Just leave a message 847 // in case of failure. 848 if err := devicemapper.DeleteDevice(devices.getPoolDevName(), devices.DeviceId); err != nil { 849 log.Errorf("Warning: Unable to delete device: %s", err) 850 } 851 852 dinfo := &DevInfo{Hash: devices.DeviceIdHash} 853 if err := devices.removeMetadata(dinfo); err != nil { 854 log.Errorf("Warning: Unable to remove meta data: %s", err) 855 } else { 856 devices.markDeviceIdFree(devices.DeviceId) 857 } 858 859 if err := devices.removeTransactionMetaData(); err != nil { 860 log.Errorf("Warning: Unable to remove transaction meta file %s: %s", devices.transactionMetaFile(), err) 861 } 862 863 return nil 864 } 865 866 func (devices *DeviceSet) processPendingTransaction() error { 867 if err := devices.loadTransactionMetaData(); err != nil { 868 return err 869 } 870 871 // If there was open transaction but pool transaction Id is same 872 // as open transaction Id, nothing to roll back. 873 if devices.TransactionId == devices.OpenTransactionId { 874 return nil 875 } 876 877 // If open transaction Id is less than pool transaction Id, something 878 // is wrong. Bail out. 879 if devices.OpenTransactionId < devices.TransactionId { 880 log.Errorf("Warning: Open Transaction id %d is less than pool transaction id %d", devices.OpenTransactionId, devices.TransactionId) 881 return nil 882 } 883 884 // Pool transaction Id is not same as open transaction. There is 885 // a transaction which was not completed. 886 if err := devices.rollbackTransaction(); err != nil { 887 return fmt.Errorf("Rolling back open transaction failed: %s", err) 888 } 889 890 devices.OpenTransactionId = devices.TransactionId 891 return nil 892 } 893 894 func (devices *DeviceSet) loadDeviceSetMetaData() error { 895 jsonData, err := ioutil.ReadFile(devices.deviceSetMetaFile()) 896 if err != nil { 897 // For backward compatibility return success if file does 898 // not exist. 899 if os.IsNotExist(err) { 900 return nil 901 } 902 return err 903 } 904 905 return json.Unmarshal(jsonData, devices) 906 } 907 908 func (devices *DeviceSet) saveDeviceSetMetaData() error { 909 jsonData, err := json.Marshal(devices) 910 if err != nil { 911 return fmt.Errorf("Error encoding metadata to json: %s", err) 912 } 913 914 return devices.writeMetaFile(jsonData, devices.deviceSetMetaFile()) 915 } 916 917 func (devices *DeviceSet) openTransaction(hash string, DeviceId int) error { 918 devices.allocateTransactionId() 919 devices.DeviceIdHash = hash 920 devices.DeviceId = DeviceId 921 if err := devices.saveTransactionMetaData(); err != nil { 922 return fmt.Errorf("Error saving transaction meta data: %s", err) 923 } 924 return nil 925 } 926 927 func (devices *DeviceSet) refreshTransaction(DeviceId int) error { 928 devices.DeviceId = DeviceId 929 if err := devices.saveTransactionMetaData(); err != nil { 930 return fmt.Errorf("Error saving transaction meta data: %s", err) 931 } 932 return nil 933 } 934 935 func (devices *DeviceSet) closeTransaction() error { 936 if err := devices.updatePoolTransactionId(); err != nil { 937 log.Debugf("Failed to close Transaction") 938 return err 939 } 940 return nil 941 } 942 943 func (devices *DeviceSet) initDevmapper(doInit bool) error { 944 if os.Getenv("DEBUG") != "" { 945 devicemapper.LogInitVerbose(devicemapper.LogLevelDebug) 946 } else { 947 devicemapper.LogInitVerbose(devicemapper.LogLevelWarn) 948 } 949 // give ourselves to libdm as a log handler 950 devicemapper.LogInit(devices) 951 952 _, err := devicemapper.GetDriverVersion() 953 if err != nil { 954 // Can't even get driver version, assume not supported 955 return graphdriver.ErrNotSupported 956 } 957 958 // https://github.com/docker/docker/issues/4036 959 if supported := devicemapper.UdevSetSyncSupport(true); !supported { 960 log.Warnf("WARNING: Udev sync is not supported. This will lead to unexpected behavior, data loss and errors") 961 } 962 log.Debugf("devicemapper: udev sync support: %v", devicemapper.UdevSyncSupported()) 963 964 if err := os.MkdirAll(devices.metadataDir(), 0700); err != nil && !os.IsExist(err) { 965 return err 966 } 967 968 // Set the device prefix from the device id and inode of the docker root dir 969 970 st, err := os.Stat(devices.root) 971 if err != nil { 972 return fmt.Errorf("Error looking up dir %s: %s", devices.root, err) 973 } 974 sysSt := st.Sys().(*syscall.Stat_t) 975 // "reg-" stands for "regular file". 976 // In the future we might use "dev-" for "device file", etc. 977 // docker-maj,min[-inode] stands for: 978 // - Managed by docker 979 // - The target of this device is at major <maj> and minor <min> 980 // - If <inode> is defined, use that file inside the device as a loopback image. Otherwise use the device itself. 981 devices.devicePrefix = fmt.Sprintf("docker-%d:%d-%d", major(sysSt.Dev), minor(sysSt.Dev), sysSt.Ino) 982 log.Debugf("Generated prefix: %s", devices.devicePrefix) 983 984 // Check for the existence of the thin-pool device 985 log.Debugf("Checking for existence of the pool '%s'", devices.getPoolName()) 986 info, err := devicemapper.GetInfo(devices.getPoolName()) 987 if info == nil { 988 log.Debugf("Error device devicemapper.GetInfo: %s", err) 989 return err 990 } 991 992 // It seems libdevmapper opens this without O_CLOEXEC, and go exec will not close files 993 // that are not Close-on-exec, and lxc-start will die if it inherits any unexpected files, 994 // so we add this badhack to make sure it closes itself 995 setCloseOnExec("/dev/mapper/control") 996 997 // Make sure the sparse images exist in <root>/devicemapper/data and 998 // <root>/devicemapper/metadata 999 1000 createdLoopback := false 1001 1002 // If the pool doesn't exist, create it 1003 if info.Exists == 0 && devices.thinPoolDevice == "" { 1004 log.Debugf("Pool doesn't exist. Creating it.") 1005 1006 var ( 1007 dataFile *os.File 1008 metadataFile *os.File 1009 ) 1010 1011 if devices.dataDevice == "" { 1012 // Make sure the sparse images exist in <root>/devicemapper/data 1013 1014 hasData := devices.hasImage("data") 1015 1016 if !doInit && !hasData { 1017 return errors.New("Loopback data file not found") 1018 } 1019 1020 if !hasData { 1021 createdLoopback = true 1022 } 1023 1024 data, err := devices.ensureImage("data", devices.dataLoopbackSize) 1025 if err != nil { 1026 log.Debugf("Error device ensureImage (data): %s", err) 1027 return err 1028 } 1029 1030 dataFile, err = devicemapper.AttachLoopDevice(data) 1031 if err != nil { 1032 return err 1033 } 1034 devices.dataLoopFile = data 1035 devices.dataDevice = dataFile.Name() 1036 } else { 1037 dataFile, err = os.OpenFile(devices.dataDevice, os.O_RDWR, 0600) 1038 if err != nil { 1039 return err 1040 } 1041 } 1042 defer dataFile.Close() 1043 1044 if devices.metadataDevice == "" { 1045 // Make sure the sparse images exist in <root>/devicemapper/metadata 1046 1047 hasMetadata := devices.hasImage("metadata") 1048 1049 if !doInit && !hasMetadata { 1050 return errors.New("Loopback metadata file not found") 1051 } 1052 1053 if !hasMetadata { 1054 createdLoopback = true 1055 } 1056 1057 metadata, err := devices.ensureImage("metadata", devices.metaDataLoopbackSize) 1058 if err != nil { 1059 log.Debugf("Error device ensureImage (metadata): %s", err) 1060 return err 1061 } 1062 1063 metadataFile, err = devicemapper.AttachLoopDevice(metadata) 1064 if err != nil { 1065 return err 1066 } 1067 devices.metadataLoopFile = metadata 1068 devices.metadataDevice = metadataFile.Name() 1069 } else { 1070 metadataFile, err = os.OpenFile(devices.metadataDevice, os.O_RDWR, 0600) 1071 if err != nil { 1072 return err 1073 } 1074 } 1075 defer metadataFile.Close() 1076 1077 if err := devicemapper.CreatePool(devices.getPoolName(), dataFile, metadataFile, devices.thinpBlockSize); err != nil { 1078 return err 1079 } 1080 } 1081 1082 // If we didn't just create the data or metadata image, we need to 1083 // load the transaction id and migrate old metadata 1084 if !createdLoopback { 1085 if err = devices.initMetaData(); err != nil { 1086 return err 1087 } 1088 } 1089 1090 // Right now this loads only NextDeviceId. If there is more metatadata 1091 // down the line, we might have to move it earlier. 1092 if err = devices.loadDeviceSetMetaData(); err != nil { 1093 return err 1094 } 1095 1096 // Setup the base image 1097 if doInit { 1098 if err := devices.setupBaseImage(); err != nil { 1099 log.Debugf("Error device setupBaseImage: %s", err) 1100 return err 1101 } 1102 } 1103 1104 return nil 1105 } 1106 1107 func (devices *DeviceSet) AddDevice(hash, baseHash string) error { 1108 log.Debugf("[deviceset] AddDevice() hash=%s basehash=%s", hash, baseHash) 1109 defer log.Debugf("[deviceset] AddDevice(hash=%s basehash=%s) END", hash, baseHash) 1110 1111 baseInfo, err := devices.lookupDevice(baseHash) 1112 if err != nil { 1113 return err 1114 } 1115 1116 baseInfo.lock.Lock() 1117 defer baseInfo.lock.Unlock() 1118 1119 devices.Lock() 1120 defer devices.Unlock() 1121 1122 if info, _ := devices.lookupDevice(hash); info != nil { 1123 return fmt.Errorf("device %s already exists", hash) 1124 } 1125 1126 if err := devices.createRegisterSnapDevice(hash, baseInfo); err != nil { 1127 return err 1128 } 1129 1130 return nil 1131 } 1132 1133 func (devices *DeviceSet) deleteDevice(info *DevInfo) error { 1134 if devices.doBlkDiscard { 1135 // This is a workaround for the kernel not discarding block so 1136 // on the thin pool when we remove a thinp device, so we do it 1137 // manually 1138 if err := devices.activateDeviceIfNeeded(info); err == nil { 1139 if err := devicemapper.BlockDeviceDiscard(info.DevName()); err != nil { 1140 log.Debugf("Error discarding block on device: %s (ignoring)", err) 1141 } 1142 } 1143 } 1144 1145 devinfo, _ := devicemapper.GetInfo(info.Name()) 1146 if devinfo != nil && devinfo.Exists != 0 { 1147 if err := devices.removeDeviceAndWait(info.Name()); err != nil { 1148 log.Debugf("Error removing device: %s", err) 1149 return err 1150 } 1151 } 1152 1153 if err := devices.openTransaction(info.Hash, info.DeviceId); err != nil { 1154 log.Debugf("Error opening transaction hash = %s deviceId = %d", "", info.DeviceId) 1155 return err 1156 } 1157 1158 if err := devicemapper.DeleteDevice(devices.getPoolDevName(), info.DeviceId); err != nil { 1159 log.Debugf("Error deleting device: %s", err) 1160 return err 1161 } 1162 1163 if err := devices.unregisterDevice(info.DeviceId, info.Hash); err != nil { 1164 return err 1165 } 1166 1167 if err := devices.closeTransaction(); err != nil { 1168 return err 1169 } 1170 1171 devices.markDeviceIdFree(info.DeviceId) 1172 1173 return nil 1174 } 1175 1176 func (devices *DeviceSet) DeleteDevice(hash string) error { 1177 info, err := devices.lookupDevice(hash) 1178 if err != nil { 1179 return err 1180 } 1181 1182 info.lock.Lock() 1183 defer info.lock.Unlock() 1184 1185 devices.Lock() 1186 defer devices.Unlock() 1187 1188 return devices.deleteDevice(info) 1189 } 1190 1191 func (devices *DeviceSet) deactivatePool() error { 1192 log.Debugf("[devmapper] deactivatePool()") 1193 defer log.Debugf("[devmapper] deactivatePool END") 1194 devname := devices.getPoolDevName() 1195 1196 devinfo, err := devicemapper.GetInfo(devname) 1197 if err != nil { 1198 return err 1199 } 1200 if d, err := devicemapper.GetDeps(devname); err == nil { 1201 // Access to more Debug output 1202 log.Debugf("[devmapper] devicemapper.GetDeps() %s: %#v", devname, d) 1203 } 1204 if devinfo.Exists != 0 { 1205 return devicemapper.RemoveDevice(devname) 1206 } 1207 1208 return nil 1209 } 1210 1211 func (devices *DeviceSet) deactivateDevice(info *DevInfo) error { 1212 log.Debugf("[devmapper] deactivateDevice(%s)", info.Hash) 1213 defer log.Debugf("[devmapper] deactivateDevice END(%s)", info.Hash) 1214 1215 // Wait for the unmount to be effective, 1216 // by watching the value of Info.OpenCount for the device 1217 if err := devices.waitClose(info); err != nil { 1218 log.Errorf("Warning: error waiting for device %s to close: %s", info.Hash, err) 1219 } 1220 1221 devinfo, err := devicemapper.GetInfo(info.Name()) 1222 if err != nil { 1223 return err 1224 } 1225 if devinfo.Exists != 0 { 1226 if err := devices.removeDeviceAndWait(info.Name()); err != nil { 1227 return err 1228 } 1229 } 1230 1231 return nil 1232 } 1233 1234 // Issues the underlying dm remove operation and then waits 1235 // for it to finish. 1236 func (devices *DeviceSet) removeDeviceAndWait(devname string) error { 1237 var err error 1238 1239 for i := 0; i < 1000; i++ { 1240 err = devicemapper.RemoveDevice(devname) 1241 if err == nil { 1242 break 1243 } 1244 if err != devicemapper.ErrBusy { 1245 return err 1246 } 1247 1248 // If we see EBUSY it may be a transient error, 1249 // sleep a bit a retry a few times. 1250 devices.Unlock() 1251 time.Sleep(10 * time.Millisecond) 1252 devices.Lock() 1253 } 1254 if err != nil { 1255 return err 1256 } 1257 1258 if err := devices.waitRemove(devname); err != nil { 1259 return err 1260 } 1261 return nil 1262 } 1263 1264 // waitRemove blocks until either: 1265 // a) the device registered at <device_set_prefix>-<hash> is removed, 1266 // or b) the 10 second timeout expires. 1267 func (devices *DeviceSet) waitRemove(devname string) error { 1268 log.Debugf("[deviceset %s] waitRemove(%s)", devices.devicePrefix, devname) 1269 defer log.Debugf("[deviceset %s] waitRemove(%s) END", devices.devicePrefix, devname) 1270 i := 0 1271 for ; i < 1000; i++ { 1272 devinfo, err := devicemapper.GetInfo(devname) 1273 if err != nil { 1274 // If there is an error we assume the device doesn't exist. 1275 // The error might actually be something else, but we can't differentiate. 1276 return nil 1277 } 1278 if i%100 == 0 { 1279 log.Debugf("Waiting for removal of %s: exists=%d", devname, devinfo.Exists) 1280 } 1281 if devinfo.Exists == 0 { 1282 break 1283 } 1284 1285 devices.Unlock() 1286 time.Sleep(10 * time.Millisecond) 1287 devices.Lock() 1288 } 1289 if i == 1000 { 1290 return fmt.Errorf("Timeout while waiting for device %s to be removed", devname) 1291 } 1292 return nil 1293 } 1294 1295 // waitClose blocks until either: 1296 // a) the device registered at <device_set_prefix>-<hash> is closed, 1297 // or b) the 10 second timeout expires. 1298 func (devices *DeviceSet) waitClose(info *DevInfo) error { 1299 i := 0 1300 for ; i < 1000; i++ { 1301 devinfo, err := devicemapper.GetInfo(info.Name()) 1302 if err != nil { 1303 return err 1304 } 1305 if i%100 == 0 { 1306 log.Debugf("Waiting for unmount of %s: opencount=%d", info.Hash, devinfo.OpenCount) 1307 } 1308 if devinfo.OpenCount == 0 { 1309 break 1310 } 1311 devices.Unlock() 1312 time.Sleep(10 * time.Millisecond) 1313 devices.Lock() 1314 } 1315 if i == 1000 { 1316 return fmt.Errorf("Timeout while waiting for device %s to close", info.Hash) 1317 } 1318 return nil 1319 } 1320 1321 func (devices *DeviceSet) Shutdown() error { 1322 log.Debugf("[deviceset %s] shutdown()", devices.devicePrefix) 1323 log.Debugf("[devmapper] Shutting down DeviceSet: %s", devices.root) 1324 defer log.Debugf("[deviceset %s] shutdown END", devices.devicePrefix) 1325 1326 var devs []*DevInfo 1327 1328 devices.devicesLock.Lock() 1329 for _, info := range devices.Devices { 1330 devs = append(devs, info) 1331 } 1332 devices.devicesLock.Unlock() 1333 1334 for _, info := range devs { 1335 info.lock.Lock() 1336 if info.mountCount > 0 { 1337 // We use MNT_DETACH here in case it is still busy in some running 1338 // container. This means it'll go away from the global scope directly, 1339 // and the device will be released when that container dies. 1340 if err := syscall.Unmount(info.mountPath, syscall.MNT_DETACH); err != nil { 1341 log.Debugf("Shutdown unmounting %s, error: %s", info.mountPath, err) 1342 } 1343 1344 devices.Lock() 1345 if err := devices.deactivateDevice(info); err != nil { 1346 log.Debugf("Shutdown deactivate %s , error: %s", info.Hash, err) 1347 } 1348 devices.Unlock() 1349 } 1350 info.lock.Unlock() 1351 } 1352 1353 info, _ := devices.lookupDevice("") 1354 if info != nil { 1355 info.lock.Lock() 1356 devices.Lock() 1357 if err := devices.deactivateDevice(info); err != nil { 1358 log.Debugf("Shutdown deactivate base , error: %s", err) 1359 } 1360 devices.Unlock() 1361 info.lock.Unlock() 1362 } 1363 1364 devices.Lock() 1365 if devices.thinPoolDevice == "" { 1366 if err := devices.deactivatePool(); err != nil { 1367 log.Debugf("Shutdown deactivate pool , error: %s", err) 1368 } 1369 } 1370 1371 devices.saveDeviceSetMetaData() 1372 devices.Unlock() 1373 1374 return nil 1375 } 1376 1377 func (devices *DeviceSet) MountDevice(hash, path, mountLabel string) error { 1378 info, err := devices.lookupDevice(hash) 1379 if err != nil { 1380 return err 1381 } 1382 1383 info.lock.Lock() 1384 defer info.lock.Unlock() 1385 1386 devices.Lock() 1387 defer devices.Unlock() 1388 1389 if info.mountCount > 0 { 1390 if path != info.mountPath { 1391 return fmt.Errorf("Trying to mount devmapper device in multple places (%s, %s)", info.mountPath, path) 1392 } 1393 1394 info.mountCount++ 1395 return nil 1396 } 1397 1398 if err := devices.activateDeviceIfNeeded(info); err != nil { 1399 return fmt.Errorf("Error activating devmapper device for '%s': %s", hash, err) 1400 } 1401 1402 var flags uintptr = syscall.MS_MGC_VAL 1403 1404 fstype, err := ProbeFsType(info.DevName()) 1405 if err != nil { 1406 return err 1407 } 1408 1409 options := "" 1410 1411 if fstype == "xfs" { 1412 // XFS needs nouuid or it can't mount filesystems with the same fs 1413 options = joinMountOptions(options, "nouuid") 1414 } 1415 1416 options = joinMountOptions(options, devices.mountOptions) 1417 options = joinMountOptions(options, label.FormatMountLabel("", mountLabel)) 1418 1419 err = syscall.Mount(info.DevName(), path, fstype, flags, joinMountOptions("discard", options)) 1420 if err != nil && err == syscall.EINVAL { 1421 err = syscall.Mount(info.DevName(), path, fstype, flags, options) 1422 } 1423 if err != nil { 1424 return fmt.Errorf("Error mounting '%s' on '%s': %s", info.DevName(), path, err) 1425 } 1426 1427 info.mountCount = 1 1428 info.mountPath = path 1429 1430 return nil 1431 } 1432 1433 func (devices *DeviceSet) UnmountDevice(hash string) error { 1434 log.Debugf("[devmapper] UnmountDevice(hash=%s)", hash) 1435 defer log.Debugf("[devmapper] UnmountDevice(hash=%s) END", hash) 1436 1437 info, err := devices.lookupDevice(hash) 1438 if err != nil { 1439 return err 1440 } 1441 1442 info.lock.Lock() 1443 defer info.lock.Unlock() 1444 1445 devices.Lock() 1446 defer devices.Unlock() 1447 1448 if info.mountCount == 0 { 1449 return fmt.Errorf("UnmountDevice: device not-mounted id %s", hash) 1450 } 1451 1452 info.mountCount-- 1453 if info.mountCount > 0 { 1454 return nil 1455 } 1456 1457 log.Debugf("[devmapper] Unmount(%s)", info.mountPath) 1458 if err := syscall.Unmount(info.mountPath, syscall.MNT_DETACH); err != nil { 1459 return err 1460 } 1461 log.Debugf("[devmapper] Unmount done") 1462 1463 if err := devices.deactivateDevice(info); err != nil { 1464 return err 1465 } 1466 1467 info.mountPath = "" 1468 1469 return nil 1470 } 1471 1472 func (devices *DeviceSet) HasDevice(hash string) bool { 1473 devices.Lock() 1474 defer devices.Unlock() 1475 1476 info, _ := devices.lookupDevice(hash) 1477 return info != nil 1478 } 1479 1480 func (devices *DeviceSet) HasActivatedDevice(hash string) bool { 1481 info, _ := devices.lookupDevice(hash) 1482 if info == nil { 1483 return false 1484 } 1485 1486 info.lock.Lock() 1487 defer info.lock.Unlock() 1488 1489 devices.Lock() 1490 defer devices.Unlock() 1491 1492 devinfo, _ := devicemapper.GetInfo(info.Name()) 1493 return devinfo != nil && devinfo.Exists != 0 1494 } 1495 1496 func (devices *DeviceSet) List() []string { 1497 devices.Lock() 1498 defer devices.Unlock() 1499 1500 devices.devicesLock.Lock() 1501 ids := make([]string, len(devices.Devices)) 1502 i := 0 1503 for k := range devices.Devices { 1504 ids[i] = k 1505 i++ 1506 } 1507 devices.devicesLock.Unlock() 1508 1509 return ids 1510 } 1511 1512 func (devices *DeviceSet) deviceStatus(devName string) (sizeInSectors, mappedSectors, highestMappedSector uint64, err error) { 1513 var params string 1514 _, sizeInSectors, _, params, err = devicemapper.GetStatus(devName) 1515 if err != nil { 1516 return 1517 } 1518 if _, err = fmt.Sscanf(params, "%d %d", &mappedSectors, &highestMappedSector); err == nil { 1519 return 1520 } 1521 return 1522 } 1523 1524 func (devices *DeviceSet) GetDeviceStatus(hash string) (*DevStatus, error) { 1525 info, err := devices.lookupDevice(hash) 1526 if err != nil { 1527 return nil, err 1528 } 1529 1530 info.lock.Lock() 1531 defer info.lock.Unlock() 1532 1533 devices.Lock() 1534 defer devices.Unlock() 1535 1536 status := &DevStatus{ 1537 DeviceId: info.DeviceId, 1538 Size: info.Size, 1539 TransactionId: info.TransactionId, 1540 } 1541 1542 if err := devices.activateDeviceIfNeeded(info); err != nil { 1543 return nil, fmt.Errorf("Error activating devmapper device for '%s': %s", hash, err) 1544 } 1545 1546 if sizeInSectors, mappedSectors, highestMappedSector, err := devices.deviceStatus(info.DevName()); err != nil { 1547 return nil, err 1548 } else { 1549 status.SizeInSectors = sizeInSectors 1550 status.MappedSectors = mappedSectors 1551 status.HighestMappedSector = highestMappedSector 1552 } 1553 1554 return status, nil 1555 } 1556 1557 func (devices *DeviceSet) poolStatus() (totalSizeInSectors, transactionId, dataUsed, dataTotal, metadataUsed, metadataTotal uint64, err error) { 1558 var params string 1559 if _, totalSizeInSectors, _, params, err = devicemapper.GetStatus(devices.getPoolName()); err == nil { 1560 _, err = fmt.Sscanf(params, "%d %d/%d %d/%d", &transactionId, &metadataUsed, &metadataTotal, &dataUsed, &dataTotal) 1561 } 1562 return 1563 } 1564 1565 // MetadataDevicePath returns the path to the metadata storage for this deviceset, 1566 // regardless of loopback or block device 1567 func (devices *DeviceSet) DataDevicePath() string { 1568 return devices.dataDevice 1569 } 1570 1571 // MetadataDevicePath returns the path to the metadata storage for this deviceset, 1572 // regardless of loopback or block device 1573 func (devices *DeviceSet) MetadataDevicePath() string { 1574 return devices.metadataDevice 1575 } 1576 1577 // Status returns the current status of this deviceset 1578 func (devices *DeviceSet) Status() *Status { 1579 devices.Lock() 1580 defer devices.Unlock() 1581 1582 status := &Status{} 1583 1584 status.PoolName = devices.getPoolName() 1585 status.DataFile = devices.DataDevicePath() 1586 status.DataLoopback = devices.dataLoopFile 1587 status.MetadataFile = devices.MetadataDevicePath() 1588 status.MetadataLoopback = devices.metadataLoopFile 1589 status.UdevSyncSupported = devicemapper.UdevSyncSupported() 1590 1591 totalSizeInSectors, _, dataUsed, dataTotal, metadataUsed, metadataTotal, err := devices.poolStatus() 1592 if err == nil { 1593 // Convert from blocks to bytes 1594 blockSizeInSectors := totalSizeInSectors / dataTotal 1595 1596 status.Data.Used = dataUsed * blockSizeInSectors * 512 1597 status.Data.Total = dataTotal * blockSizeInSectors * 512 1598 1599 // metadata blocks are always 4k 1600 status.Metadata.Used = metadataUsed * 4096 1601 status.Metadata.Total = metadataTotal * 4096 1602 1603 status.SectorSize = blockSizeInSectors * 512 1604 } 1605 1606 return status 1607 } 1608 1609 func NewDeviceSet(root string, doInit bool, options []string) (*DeviceSet, error) { 1610 devicemapper.SetDevDir("/dev") 1611 1612 devices := &DeviceSet{ 1613 root: root, 1614 MetaData: MetaData{Devices: make(map[string]*DevInfo)}, 1615 dataLoopbackSize: DefaultDataLoopbackSize, 1616 metaDataLoopbackSize: DefaultMetaDataLoopbackSize, 1617 baseFsSize: DefaultBaseFsSize, 1618 filesystem: "ext4", 1619 doBlkDiscard: true, 1620 thinpBlockSize: DefaultThinpBlockSize, 1621 deviceIdMap: make([]byte, DeviceIdMapSz), 1622 } 1623 1624 foundBlkDiscard := false 1625 for _, option := range options { 1626 key, val, err := parsers.ParseKeyValueOpt(option) 1627 if err != nil { 1628 return nil, err 1629 } 1630 key = strings.ToLower(key) 1631 switch key { 1632 case "dm.basesize": 1633 size, err := units.RAMInBytes(val) 1634 if err != nil { 1635 return nil, err 1636 } 1637 devices.baseFsSize = uint64(size) 1638 case "dm.loopdatasize": 1639 size, err := units.RAMInBytes(val) 1640 if err != nil { 1641 return nil, err 1642 } 1643 devices.dataLoopbackSize = size 1644 case "dm.loopmetadatasize": 1645 size, err := units.RAMInBytes(val) 1646 if err != nil { 1647 return nil, err 1648 } 1649 devices.metaDataLoopbackSize = size 1650 case "dm.fs": 1651 if val != "ext4" && val != "xfs" { 1652 return nil, fmt.Errorf("Unsupported filesystem %s\n", val) 1653 } 1654 devices.filesystem = val 1655 case "dm.mkfsarg": 1656 devices.mkfsArgs = append(devices.mkfsArgs, val) 1657 case "dm.mountopt": 1658 devices.mountOptions = joinMountOptions(devices.mountOptions, val) 1659 case "dm.metadatadev": 1660 devices.metadataDevice = val 1661 case "dm.datadev": 1662 devices.dataDevice = val 1663 case "dm.thinpooldev": 1664 devices.thinPoolDevice = strings.TrimPrefix(val, "/dev/mapper/") 1665 case "dm.blkdiscard": 1666 foundBlkDiscard = true 1667 devices.doBlkDiscard, err = strconv.ParseBool(val) 1668 if err != nil { 1669 return nil, err 1670 } 1671 case "dm.blocksize": 1672 size, err := units.RAMInBytes(val) 1673 if err != nil { 1674 return nil, err 1675 } 1676 // convert to 512b sectors 1677 devices.thinpBlockSize = uint32(size) >> 9 1678 default: 1679 return nil, fmt.Errorf("Unknown option %s\n", key) 1680 } 1681 } 1682 1683 // By default, don't do blk discard hack on raw devices, its rarely useful and is expensive 1684 if !foundBlkDiscard && (devices.dataDevice != "" || devices.thinPoolDevice != "") { 1685 devices.doBlkDiscard = false 1686 } 1687 1688 if err := devices.initDevmapper(doInit); err != nil { 1689 return nil, err 1690 } 1691 1692 return devices, nil 1693 }