github.com/scorpionis/docker@v1.6.0-rc7/daemon/graphdriver/devmapper/deviceset.go (about) 1 // +build linux 2 3 package devmapper 4 5 import ( 6 "encoding/json" 7 "errors" 8 "fmt" 9 "io" 10 "io/ioutil" 11 "os" 12 "os/exec" 13 "path" 14 "path/filepath" 15 "strconv" 16 "strings" 17 "sync" 18 "syscall" 19 "time" 20 21 log "github.com/Sirupsen/logrus" 22 "github.com/docker/docker/daemon/graphdriver" 23 "github.com/docker/docker/pkg/devicemapper" 24 "github.com/docker/docker/pkg/parsers" 25 "github.com/docker/docker/pkg/units" 26 "github.com/docker/libcontainer/label" 27 ) 28 29 var ( 30 DefaultDataLoopbackSize int64 = 100 * 1024 * 1024 * 1024 31 DefaultMetaDataLoopbackSize int64 = 2 * 1024 * 1024 * 1024 32 DefaultBaseFsSize uint64 = 10 * 1024 * 1024 * 1024 33 DefaultThinpBlockSize uint32 = 128 // 64K = 128 512b sectors 34 MaxDeviceId int = 0xffffff // 24 bit, pool limit 35 DeviceIdMapSz int = (MaxDeviceId + 1) / 8 36 ) 37 38 const deviceSetMetaFile string = "deviceset-metadata" 39 const transactionMetaFile string = "transaction-metadata" 40 41 type Transaction struct { 42 OpenTransactionId uint64 `json:"open_transaction_id"` 43 DeviceIdHash string `json:"device_hash"` 44 DeviceId int `json:"device_id"` 45 } 46 47 type DevInfo struct { 48 Hash string `json:"-"` 49 DeviceId int `json:"device_id"` 50 Size uint64 `json:"size"` 51 TransactionId uint64 `json:"transaction_id"` 52 Initialized bool `json:"initialized"` 53 devices *DeviceSet 54 55 mountCount int 56 mountPath string 57 58 // The global DeviceSet lock guarantees that we serialize all 59 // the calls to libdevmapper (which is not threadsafe), but we 60 // sometimes release that lock while sleeping. In that case 61 // this per-device lock is still held, protecting against 62 // other accesses to the device that we're doing the wait on. 63 // 64 // WARNING: In order to avoid AB-BA deadlocks when releasing 65 // the global lock while holding the per-device locks all 66 // device locks must be aquired *before* the device lock, and 67 // multiple device locks should be aquired parent before child. 68 lock sync.Mutex 69 } 70 71 type MetaData struct { 72 Devices map[string]*DevInfo `json:"Devices"` 73 devicesLock sync.Mutex // Protects all read/writes to Devices map 74 } 75 76 type DeviceSet struct { 77 MetaData `json:"-"` 78 sync.Mutex `json:"-"` // Protects Devices map and serializes calls into libdevmapper 79 root string 80 devicePrefix string 81 TransactionId uint64 `json:"-"` 82 NextDeviceId int `json:"next_device_id"` 83 deviceIdMap []byte 84 85 // Options 86 dataLoopbackSize int64 87 metaDataLoopbackSize int64 88 baseFsSize uint64 89 filesystem string 90 mountOptions string 91 mkfsArgs []string 92 dataDevice string // block or loop dev 93 dataLoopFile string // loopback file, if used 94 metadataDevice string // block or loop dev 95 metadataLoopFile string // loopback file, if used 96 doBlkDiscard bool 97 thinpBlockSize uint32 98 thinPoolDevice string 99 Transaction `json:"-"` 100 } 101 102 type DiskUsage struct { 103 Used uint64 104 Total uint64 105 Available uint64 106 } 107 108 type Status struct { 109 PoolName string 110 DataFile string // actual block device for data 111 DataLoopback string // loopback file, if used 112 MetadataFile string // actual block device for metadata 113 MetadataLoopback string // loopback file, if used 114 Data DiskUsage 115 Metadata DiskUsage 116 SectorSize uint64 117 UdevSyncSupported bool 118 } 119 120 type DevStatus struct { 121 DeviceId int 122 Size uint64 123 TransactionId uint64 124 SizeInSectors uint64 125 MappedSectors uint64 126 HighestMappedSector uint64 127 } 128 129 func getDevName(name string) string { 130 return "/dev/mapper/" + name 131 } 132 133 func (info *DevInfo) Name() string { 134 hash := info.Hash 135 if hash == "" { 136 hash = "base" 137 } 138 return fmt.Sprintf("%s-%s", info.devices.devicePrefix, hash) 139 } 140 141 func (info *DevInfo) DevName() string { 142 return getDevName(info.Name()) 143 } 144 145 func (devices *DeviceSet) loopbackDir() string { 146 return path.Join(devices.root, "devicemapper") 147 } 148 149 func (devices *DeviceSet) metadataDir() string { 150 return path.Join(devices.root, "metadata") 151 } 152 153 func (devices *DeviceSet) metadataFile(info *DevInfo) string { 154 file := info.Hash 155 if file == "" { 156 file = "base" 157 } 158 return path.Join(devices.metadataDir(), file) 159 } 160 161 func (devices *DeviceSet) transactionMetaFile() string { 162 return path.Join(devices.metadataDir(), transactionMetaFile) 163 } 164 165 func (devices *DeviceSet) deviceSetMetaFile() string { 166 return path.Join(devices.metadataDir(), deviceSetMetaFile) 167 } 168 169 func (devices *DeviceSet) oldMetadataFile() string { 170 return path.Join(devices.loopbackDir(), "json") 171 } 172 173 func (devices *DeviceSet) getPoolName() string { 174 if devices.thinPoolDevice == "" { 175 return devices.devicePrefix + "-pool" 176 } 177 return devices.thinPoolDevice 178 } 179 180 func (devices *DeviceSet) getPoolDevName() string { 181 return getDevName(devices.getPoolName()) 182 } 183 184 func (devices *DeviceSet) hasImage(name string) bool { 185 dirname := devices.loopbackDir() 186 filename := path.Join(dirname, name) 187 188 _, err := os.Stat(filename) 189 return err == nil 190 } 191 192 // ensureImage creates a sparse file of <size> bytes at the path 193 // <root>/devicemapper/<name>. 194 // If the file already exists, it does nothing. 195 // Either way it returns the full path. 196 func (devices *DeviceSet) ensureImage(name string, size int64) (string, error) { 197 dirname := devices.loopbackDir() 198 filename := path.Join(dirname, name) 199 200 if err := os.MkdirAll(dirname, 0700); err != nil && !os.IsExist(err) { 201 return "", err 202 } 203 204 if _, err := os.Stat(filename); err != nil { 205 if !os.IsNotExist(err) { 206 return "", err 207 } 208 log.Debugf("Creating loopback file %s for device-manage use", filename) 209 file, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE, 0600) 210 if err != nil { 211 return "", err 212 } 213 defer file.Close() 214 215 if err = file.Truncate(size); err != nil { 216 return "", err 217 } 218 } 219 return filename, nil 220 } 221 222 func (devices *DeviceSet) allocateTransactionId() uint64 { 223 devices.OpenTransactionId = devices.TransactionId + 1 224 return devices.OpenTransactionId 225 } 226 227 func (devices *DeviceSet) updatePoolTransactionId() error { 228 if err := devicemapper.SetTransactionId(devices.getPoolDevName(), devices.TransactionId, devices.OpenTransactionId); err != nil { 229 return fmt.Errorf("Error setting devmapper transaction ID: %s", err) 230 } 231 devices.TransactionId = devices.OpenTransactionId 232 return nil 233 } 234 235 func (devices *DeviceSet) removeMetadata(info *DevInfo) error { 236 if err := os.RemoveAll(devices.metadataFile(info)); err != nil { 237 return fmt.Errorf("Error removing metadata file %s: %s", devices.metadataFile(info), err) 238 } 239 return nil 240 } 241 242 // Given json data and file path, write it to disk 243 func (devices *DeviceSet) writeMetaFile(jsonData []byte, filePath string) error { 244 tmpFile, err := ioutil.TempFile(devices.metadataDir(), ".tmp") 245 if err != nil { 246 return fmt.Errorf("Error creating metadata file: %s", err) 247 } 248 249 n, err := tmpFile.Write(jsonData) 250 if err != nil { 251 return fmt.Errorf("Error writing metadata to %s: %s", tmpFile.Name(), err) 252 } 253 if n < len(jsonData) { 254 return io.ErrShortWrite 255 } 256 if err := tmpFile.Sync(); err != nil { 257 return fmt.Errorf("Error syncing metadata file %s: %s", tmpFile.Name(), err) 258 } 259 if err := tmpFile.Close(); err != nil { 260 return fmt.Errorf("Error closing metadata file %s: %s", tmpFile.Name(), err) 261 } 262 if err := os.Rename(tmpFile.Name(), filePath); err != nil { 263 return fmt.Errorf("Error committing metadata file %s: %s", tmpFile.Name(), err) 264 } 265 266 return nil 267 } 268 269 func (devices *DeviceSet) saveMetadata(info *DevInfo) error { 270 jsonData, err := json.Marshal(info) 271 if err != nil { 272 return fmt.Errorf("Error encoding metadata to json: %s", err) 273 } 274 if err := devices.writeMetaFile(jsonData, devices.metadataFile(info)); err != nil { 275 return err 276 } 277 return nil 278 } 279 280 func (devices *DeviceSet) markDeviceIdUsed(deviceId int) { 281 var mask byte 282 i := deviceId % 8 283 mask = 1 << uint(i) 284 devices.deviceIdMap[deviceId/8] = devices.deviceIdMap[deviceId/8] | mask 285 } 286 287 func (devices *DeviceSet) markDeviceIdFree(deviceId int) { 288 var mask byte 289 i := deviceId % 8 290 mask = ^(1 << uint(i)) 291 devices.deviceIdMap[deviceId/8] = devices.deviceIdMap[deviceId/8] & mask 292 } 293 294 func (devices *DeviceSet) isDeviceIdFree(deviceId int) bool { 295 var mask byte 296 i := deviceId % 8 297 mask = (1 << uint(i)) 298 if (devices.deviceIdMap[deviceId/8] & mask) != 0 { 299 return false 300 } 301 return true 302 } 303 304 func (devices *DeviceSet) lookupDevice(hash string) (*DevInfo, error) { 305 devices.devicesLock.Lock() 306 defer devices.devicesLock.Unlock() 307 info := devices.Devices[hash] 308 if info == nil { 309 info = devices.loadMetadata(hash) 310 if info == nil { 311 return nil, fmt.Errorf("Unknown device %s", hash) 312 } 313 314 devices.Devices[hash] = info 315 } 316 return info, nil 317 } 318 319 func (devices *DeviceSet) deviceFileWalkFunction(path string, finfo os.FileInfo) error { 320 321 // Skip some of the meta files which are not device files. 322 if strings.HasSuffix(finfo.Name(), ".migrated") { 323 log.Debugf("Skipping file %s", path) 324 return nil 325 } 326 327 if strings.HasPrefix(finfo.Name(), ".") { 328 log.Debugf("Skipping file %s", path) 329 return nil 330 } 331 332 if finfo.Name() == deviceSetMetaFile { 333 log.Debugf("Skipping file %s", path) 334 return nil 335 } 336 337 log.Debugf("Loading data for file %s", path) 338 339 hash := finfo.Name() 340 if hash == "base" { 341 hash = "" 342 } 343 344 dinfo := devices.loadMetadata(hash) 345 if dinfo == nil { 346 return fmt.Errorf("Error loading device metadata file %s", hash) 347 } 348 349 if dinfo.DeviceId > MaxDeviceId { 350 log.Errorf("Ignoring Invalid DeviceId=%d", dinfo.DeviceId) 351 return nil 352 } 353 354 devices.Lock() 355 devices.markDeviceIdUsed(dinfo.DeviceId) 356 devices.Unlock() 357 358 log.Debugf("Added deviceId=%d to DeviceIdMap", dinfo.DeviceId) 359 return nil 360 } 361 362 func (devices *DeviceSet) constructDeviceIdMap() error { 363 log.Debugf("[deviceset] constructDeviceIdMap()") 364 defer log.Debugf("[deviceset] constructDeviceIdMap() END") 365 366 var scan = func(path string, info os.FileInfo, err error) error { 367 if err != nil { 368 log.Debugf("Can't walk the file %s", path) 369 return nil 370 } 371 372 // Skip any directories 373 if info.IsDir() { 374 return nil 375 } 376 377 return devices.deviceFileWalkFunction(path, info) 378 } 379 380 return filepath.Walk(devices.metadataDir(), scan) 381 } 382 383 func (devices *DeviceSet) unregisterDevice(id int, hash string) error { 384 log.Debugf("unregisterDevice(%v, %v)", id, hash) 385 info := &DevInfo{ 386 Hash: hash, 387 DeviceId: id, 388 } 389 390 devices.devicesLock.Lock() 391 delete(devices.Devices, hash) 392 devices.devicesLock.Unlock() 393 394 if err := devices.removeMetadata(info); err != nil { 395 log.Debugf("Error removing metadata: %s", err) 396 return err 397 } 398 399 return nil 400 } 401 402 func (devices *DeviceSet) registerDevice(id int, hash string, size uint64, transactionId uint64) (*DevInfo, error) { 403 log.Debugf("registerDevice(%v, %v)", id, hash) 404 info := &DevInfo{ 405 Hash: hash, 406 DeviceId: id, 407 Size: size, 408 TransactionId: transactionId, 409 Initialized: false, 410 devices: devices, 411 } 412 413 devices.devicesLock.Lock() 414 devices.Devices[hash] = info 415 devices.devicesLock.Unlock() 416 417 if err := devices.saveMetadata(info); err != nil { 418 // Try to remove unused device 419 devices.devicesLock.Lock() 420 delete(devices.Devices, hash) 421 devices.devicesLock.Unlock() 422 return nil, err 423 } 424 425 return info, nil 426 } 427 428 func (devices *DeviceSet) activateDeviceIfNeeded(info *DevInfo) error { 429 log.Debugf("activateDeviceIfNeeded(%v)", info.Hash) 430 431 if devinfo, _ := devicemapper.GetInfo(info.Name()); devinfo != nil && devinfo.Exists != 0 { 432 return nil 433 } 434 435 return devicemapper.ActivateDevice(devices.getPoolDevName(), info.Name(), info.DeviceId, info.Size) 436 } 437 438 func (devices *DeviceSet) createFilesystem(info *DevInfo) error { 439 devname := info.DevName() 440 441 args := []string{} 442 for _, arg := range devices.mkfsArgs { 443 args = append(args, arg) 444 } 445 446 args = append(args, devname) 447 448 var err error 449 switch devices.filesystem { 450 case "xfs": 451 err = exec.Command("mkfs.xfs", args...).Run() 452 case "ext4": 453 err = exec.Command("mkfs.ext4", append([]string{"-E", "nodiscard,lazy_itable_init=0,lazy_journal_init=0"}, args...)...).Run() 454 if err != nil { 455 err = exec.Command("mkfs.ext4", append([]string{"-E", "nodiscard,lazy_itable_init=0"}, args...)...).Run() 456 } 457 if err != nil { 458 return err 459 } 460 err = exec.Command("tune2fs", append([]string{"-c", "-1", "-i", "0"}, devname)...).Run() 461 default: 462 err = fmt.Errorf("Unsupported filesystem type %s", devices.filesystem) 463 } 464 if err != nil { 465 return err 466 } 467 468 return nil 469 } 470 471 func (devices *DeviceSet) migrateOldMetaData() error { 472 // Migrate old metadata file 473 jsonData, err := ioutil.ReadFile(devices.oldMetadataFile()) 474 if err != nil && !os.IsNotExist(err) { 475 return err 476 } 477 478 if jsonData != nil { 479 m := MetaData{Devices: make(map[string]*DevInfo)} 480 481 if err := json.Unmarshal(jsonData, &m); err != nil { 482 return err 483 } 484 485 for hash, info := range m.Devices { 486 info.Hash = hash 487 devices.saveMetadata(info) 488 } 489 if err := os.Rename(devices.oldMetadataFile(), devices.oldMetadataFile()+".migrated"); err != nil { 490 return err 491 } 492 493 } 494 495 return nil 496 } 497 498 func (devices *DeviceSet) initMetaData() error { 499 if err := devices.migrateOldMetaData(); err != nil { 500 return err 501 } 502 503 _, transactionId, _, _, _, _, err := devices.poolStatus() 504 if err != nil { 505 return err 506 } 507 508 devices.TransactionId = transactionId 509 510 if err := devices.constructDeviceIdMap(); err != nil { 511 return err 512 } 513 514 if err := devices.processPendingTransaction(); err != nil { 515 return err 516 } 517 return nil 518 } 519 520 func (devices *DeviceSet) incNextDeviceId() { 521 // Ids are 24bit, so wrap around 522 devices.NextDeviceId = (devices.NextDeviceId + 1) & MaxDeviceId 523 } 524 525 func (devices *DeviceSet) getNextFreeDeviceId() (int, error) { 526 devices.incNextDeviceId() 527 for i := 0; i <= MaxDeviceId; i++ { 528 if devices.isDeviceIdFree(devices.NextDeviceId) { 529 devices.markDeviceIdUsed(devices.NextDeviceId) 530 return devices.NextDeviceId, nil 531 } 532 devices.incNextDeviceId() 533 } 534 535 return 0, fmt.Errorf("Unable to find a free device Id") 536 } 537 538 func (devices *DeviceSet) createRegisterDevice(hash string) (*DevInfo, error) { 539 deviceId, err := devices.getNextFreeDeviceId() 540 if err != nil { 541 return nil, err 542 } 543 544 if err := devices.openTransaction(hash, deviceId); err != nil { 545 log.Debugf("Error opening transaction hash = %s deviceId = %d", hash, deviceId) 546 devices.markDeviceIdFree(deviceId) 547 return nil, err 548 } 549 550 for { 551 if err := devicemapper.CreateDevice(devices.getPoolDevName(), deviceId); err != nil { 552 if devicemapper.DeviceIdExists(err) { 553 // Device Id already exists. This should not 554 // happen. Now we have a mechianism to find 555 // a free device Id. So something is not right. 556 // Give a warning and continue. 557 log.Errorf("Device Id %d exists in pool but it is supposed to be unused", deviceId) 558 deviceId, err = devices.getNextFreeDeviceId() 559 if err != nil { 560 return nil, err 561 } 562 // Save new device id into transaction 563 devices.refreshTransaction(deviceId) 564 continue 565 } 566 log.Debugf("Error creating device: %s", err) 567 devices.markDeviceIdFree(deviceId) 568 return nil, err 569 } 570 break 571 } 572 573 log.Debugf("Registering device (id %v) with FS size %v", deviceId, devices.baseFsSize) 574 info, err := devices.registerDevice(deviceId, hash, devices.baseFsSize, devices.OpenTransactionId) 575 if err != nil { 576 _ = devicemapper.DeleteDevice(devices.getPoolDevName(), deviceId) 577 devices.markDeviceIdFree(deviceId) 578 return nil, err 579 } 580 581 if err := devices.closeTransaction(); err != nil { 582 devices.unregisterDevice(deviceId, hash) 583 devicemapper.DeleteDevice(devices.getPoolDevName(), deviceId) 584 devices.markDeviceIdFree(deviceId) 585 return nil, err 586 } 587 return info, nil 588 } 589 590 func (devices *DeviceSet) createRegisterSnapDevice(hash string, baseInfo *DevInfo) error { 591 deviceId, err := devices.getNextFreeDeviceId() 592 if err != nil { 593 return err 594 } 595 596 if err := devices.openTransaction(hash, deviceId); err != nil { 597 log.Debugf("Error opening transaction hash = %s deviceId = %d", hash, deviceId) 598 devices.markDeviceIdFree(deviceId) 599 return err 600 } 601 602 for { 603 if err := devicemapper.CreateSnapDevice(devices.getPoolDevName(), deviceId, baseInfo.Name(), baseInfo.DeviceId); err != nil { 604 if devicemapper.DeviceIdExists(err) { 605 // Device Id already exists. This should not 606 // happen. Now we have a mechianism to find 607 // a free device Id. So something is not right. 608 // Give a warning and continue. 609 log.Errorf("Device Id %d exists in pool but it is supposed to be unused", deviceId) 610 deviceId, err = devices.getNextFreeDeviceId() 611 if err != nil { 612 return err 613 } 614 // Save new device id into transaction 615 devices.refreshTransaction(deviceId) 616 continue 617 } 618 log.Debugf("Error creating snap device: %s", err) 619 devices.markDeviceIdFree(deviceId) 620 return err 621 } 622 break 623 } 624 625 if _, err := devices.registerDevice(deviceId, hash, baseInfo.Size, devices.OpenTransactionId); err != nil { 626 devicemapper.DeleteDevice(devices.getPoolDevName(), deviceId) 627 devices.markDeviceIdFree(deviceId) 628 log.Debugf("Error registering device: %s", err) 629 return err 630 } 631 632 if err := devices.closeTransaction(); err != nil { 633 devices.unregisterDevice(deviceId, hash) 634 devicemapper.DeleteDevice(devices.getPoolDevName(), deviceId) 635 devices.markDeviceIdFree(deviceId) 636 return err 637 } 638 return nil 639 } 640 641 func (devices *DeviceSet) loadMetadata(hash string) *DevInfo { 642 info := &DevInfo{Hash: hash, devices: devices} 643 644 jsonData, err := ioutil.ReadFile(devices.metadataFile(info)) 645 if err != nil { 646 return nil 647 } 648 649 if err := json.Unmarshal(jsonData, &info); err != nil { 650 return nil 651 } 652 653 return info 654 } 655 656 func (devices *DeviceSet) setupBaseImage() error { 657 oldInfo, _ := devices.lookupDevice("") 658 if oldInfo != nil && oldInfo.Initialized { 659 return nil 660 } 661 662 if oldInfo != nil && !oldInfo.Initialized { 663 log.Debugf("Removing uninitialized base image") 664 if err := devices.DeleteDevice(""); err != nil { 665 return err 666 } 667 } 668 669 if devices.thinPoolDevice != "" && oldInfo == nil { 670 _, transactionId, dataUsed, _, _, _, err := devices.poolStatus() 671 if err != nil { 672 return err 673 } 674 if dataUsed != 0 { 675 return fmt.Errorf("Unable to take ownership of thin-pool (%s) that already has used data blocks", 676 devices.thinPoolDevice) 677 } 678 if transactionId != 0 { 679 return fmt.Errorf("Unable to take ownership of thin-pool (%s) with non-zero transaction Id", 680 devices.thinPoolDevice) 681 } 682 } 683 684 log.Debugf("Initializing base device-mapper thin volume") 685 686 // Create initial device 687 info, err := devices.createRegisterDevice("") 688 if err != nil { 689 return err 690 } 691 692 log.Debugf("Creating filesystem on base device-mapper thin volume") 693 694 if err = devices.activateDeviceIfNeeded(info); err != nil { 695 return err 696 } 697 698 if err := devices.createFilesystem(info); err != nil { 699 return err 700 } 701 702 info.Initialized = true 703 if err = devices.saveMetadata(info); err != nil { 704 info.Initialized = false 705 return err 706 } 707 708 return nil 709 } 710 711 func setCloseOnExec(name string) { 712 if fileInfos, _ := ioutil.ReadDir("/proc/self/fd"); fileInfos != nil { 713 for _, i := range fileInfos { 714 link, _ := os.Readlink(filepath.Join("/proc/self/fd", i.Name())) 715 if link == name { 716 fd, err := strconv.Atoi(i.Name()) 717 if err == nil { 718 syscall.CloseOnExec(fd) 719 } 720 } 721 } 722 } 723 } 724 725 func (devices *DeviceSet) DMLog(level int, file string, line int, dmError int, message string) { 726 if level >= devicemapper.LogLevelDebug { 727 // (vbatts) libdm debug is very verbose. If you're debugging libdm, you can 728 // comment out this check yourself 729 level = devicemapper.LogLevelInfo 730 } 731 732 // FIXME(vbatts) push this back into ./pkg/devicemapper/ 733 log.Debugf("libdevmapper(%d): %s:%d (%d) %s", level, file, line, dmError, message) 734 } 735 736 func major(device uint64) uint64 { 737 return (device >> 8) & 0xfff 738 } 739 740 func minor(device uint64) uint64 { 741 return (device & 0xff) | ((device >> 12) & 0xfff00) 742 } 743 744 func (devices *DeviceSet) ResizePool(size int64) error { 745 dirname := devices.loopbackDir() 746 datafilename := path.Join(dirname, "data") 747 if len(devices.dataDevice) > 0 { 748 datafilename = devices.dataDevice 749 } 750 metadatafilename := path.Join(dirname, "metadata") 751 if len(devices.metadataDevice) > 0 { 752 metadatafilename = devices.metadataDevice 753 } 754 755 datafile, err := os.OpenFile(datafilename, os.O_RDWR, 0) 756 if datafile == nil { 757 return err 758 } 759 defer datafile.Close() 760 761 fi, err := datafile.Stat() 762 if fi == nil { 763 return err 764 } 765 766 if fi.Size() > size { 767 return fmt.Errorf("Can't shrink file") 768 } 769 770 dataloopback := devicemapper.FindLoopDeviceFor(datafile) 771 if dataloopback == nil { 772 return fmt.Errorf("Unable to find loopback mount for: %s", datafilename) 773 } 774 defer dataloopback.Close() 775 776 metadatafile, err := os.OpenFile(metadatafilename, os.O_RDWR, 0) 777 if metadatafile == nil { 778 return err 779 } 780 defer metadatafile.Close() 781 782 metadataloopback := devicemapper.FindLoopDeviceFor(metadatafile) 783 if metadataloopback == nil { 784 return fmt.Errorf("Unable to find loopback mount for: %s", metadatafilename) 785 } 786 defer metadataloopback.Close() 787 788 // Grow loopback file 789 if err := datafile.Truncate(size); err != nil { 790 return fmt.Errorf("Unable to grow loopback file: %s", err) 791 } 792 793 // Reload size for loopback device 794 if err := devicemapper.LoopbackSetCapacity(dataloopback); err != nil { 795 return fmt.Errorf("Unable to update loopback capacity: %s", err) 796 } 797 798 // Suspend the pool 799 if err := devicemapper.SuspendDevice(devices.getPoolName()); err != nil { 800 return fmt.Errorf("Unable to suspend pool: %s", err) 801 } 802 803 // Reload with the new block sizes 804 if err := devicemapper.ReloadPool(devices.getPoolName(), dataloopback, metadataloopback, devices.thinpBlockSize); err != nil { 805 return fmt.Errorf("Unable to reload pool: %s", err) 806 } 807 808 // Resume the pool 809 if err := devicemapper.ResumeDevice(devices.getPoolName()); err != nil { 810 return fmt.Errorf("Unable to resume pool: %s", err) 811 } 812 813 return nil 814 } 815 816 func (devices *DeviceSet) loadTransactionMetaData() error { 817 jsonData, err := ioutil.ReadFile(devices.transactionMetaFile()) 818 if err != nil { 819 // There is no active transaction. This will be the case 820 // during upgrade. 821 if os.IsNotExist(err) { 822 devices.OpenTransactionId = devices.TransactionId 823 return nil 824 } 825 return err 826 } 827 828 json.Unmarshal(jsonData, &devices.Transaction) 829 return nil 830 } 831 832 func (devices *DeviceSet) saveTransactionMetaData() error { 833 jsonData, err := json.Marshal(&devices.Transaction) 834 if err != nil { 835 return fmt.Errorf("Error encoding metadata to json: %s", err) 836 } 837 838 return devices.writeMetaFile(jsonData, devices.transactionMetaFile()) 839 } 840 841 func (devices *DeviceSet) removeTransactionMetaData() error { 842 if err := os.RemoveAll(devices.transactionMetaFile()); err != nil { 843 return err 844 } 845 return nil 846 } 847 848 func (devices *DeviceSet) rollbackTransaction() error { 849 log.Debugf("Rolling back open transaction: TransactionId=%d hash=%s device_id=%d", devices.OpenTransactionId, devices.DeviceIdHash, devices.DeviceId) 850 851 // A device id might have already been deleted before transaction 852 // closed. In that case this call will fail. Just leave a message 853 // in case of failure. 854 if err := devicemapper.DeleteDevice(devices.getPoolDevName(), devices.DeviceId); err != nil { 855 log.Errorf("Unable to delete device: %s", err) 856 } 857 858 dinfo := &DevInfo{Hash: devices.DeviceIdHash} 859 if err := devices.removeMetadata(dinfo); err != nil { 860 log.Errorf("Unable to remove metadata: %s", err) 861 } else { 862 devices.markDeviceIdFree(devices.DeviceId) 863 } 864 865 if err := devices.removeTransactionMetaData(); err != nil { 866 log.Errorf("Unable to remove transaction meta file %s: %s", devices.transactionMetaFile(), err) 867 } 868 869 return nil 870 } 871 872 func (devices *DeviceSet) processPendingTransaction() error { 873 if err := devices.loadTransactionMetaData(); err != nil { 874 return err 875 } 876 877 // If there was open transaction but pool transaction Id is same 878 // as open transaction Id, nothing to roll back. 879 if devices.TransactionId == devices.OpenTransactionId { 880 return nil 881 } 882 883 // If open transaction Id is less than pool transaction Id, something 884 // is wrong. Bail out. 885 if devices.OpenTransactionId < devices.TransactionId { 886 log.Errorf("Open Transaction id %d is less than pool transaction id %d", devices.OpenTransactionId, devices.TransactionId) 887 return nil 888 } 889 890 // Pool transaction Id is not same as open transaction. There is 891 // a transaction which was not completed. 892 if err := devices.rollbackTransaction(); err != nil { 893 return fmt.Errorf("Rolling back open transaction failed: %s", err) 894 } 895 896 devices.OpenTransactionId = devices.TransactionId 897 return nil 898 } 899 900 func (devices *DeviceSet) loadDeviceSetMetaData() error { 901 jsonData, err := ioutil.ReadFile(devices.deviceSetMetaFile()) 902 if err != nil { 903 // For backward compatibility return success if file does 904 // not exist. 905 if os.IsNotExist(err) { 906 return nil 907 } 908 return err 909 } 910 911 return json.Unmarshal(jsonData, devices) 912 } 913 914 func (devices *DeviceSet) saveDeviceSetMetaData() error { 915 jsonData, err := json.Marshal(devices) 916 if err != nil { 917 return fmt.Errorf("Error encoding metadata to json: %s", err) 918 } 919 920 return devices.writeMetaFile(jsonData, devices.deviceSetMetaFile()) 921 } 922 923 func (devices *DeviceSet) openTransaction(hash string, DeviceId int) error { 924 devices.allocateTransactionId() 925 devices.DeviceIdHash = hash 926 devices.DeviceId = DeviceId 927 if err := devices.saveTransactionMetaData(); err != nil { 928 return fmt.Errorf("Error saving transaction metadata: %s", err) 929 } 930 return nil 931 } 932 933 func (devices *DeviceSet) refreshTransaction(DeviceId int) error { 934 devices.DeviceId = DeviceId 935 if err := devices.saveTransactionMetaData(); err != nil { 936 return fmt.Errorf("Error saving transaction metadata: %s", err) 937 } 938 return nil 939 } 940 941 func (devices *DeviceSet) closeTransaction() error { 942 if err := devices.updatePoolTransactionId(); err != nil { 943 log.Debugf("Failed to close Transaction") 944 return err 945 } 946 return nil 947 } 948 949 func (devices *DeviceSet) initDevmapper(doInit bool) error { 950 if os.Getenv("DEBUG") != "" { 951 devicemapper.LogInitVerbose(devicemapper.LogLevelDebug) 952 } else { 953 devicemapper.LogInitVerbose(devicemapper.LogLevelWarn) 954 } 955 // give ourselves to libdm as a log handler 956 devicemapper.LogInit(devices) 957 958 _, err := devicemapper.GetDriverVersion() 959 if err != nil { 960 // Can't even get driver version, assume not supported 961 return graphdriver.ErrNotSupported 962 } 963 964 // https://github.com/docker/docker/issues/4036 965 if supported := devicemapper.UdevSetSyncSupport(true); !supported { 966 log.Warnf("Udev sync is not supported. This will lead to unexpected behavior, data loss and errors") 967 } 968 log.Debugf("devicemapper: udev sync support: %v", devicemapper.UdevSyncSupported()) 969 970 if err := os.MkdirAll(devices.metadataDir(), 0700); err != nil && !os.IsExist(err) { 971 return err 972 } 973 974 // Set the device prefix from the device id and inode of the docker root dir 975 976 st, err := os.Stat(devices.root) 977 if err != nil { 978 return fmt.Errorf("Error looking up dir %s: %s", devices.root, err) 979 } 980 sysSt := st.Sys().(*syscall.Stat_t) 981 // "reg-" stands for "regular file". 982 // In the future we might use "dev-" for "device file", etc. 983 // docker-maj,min[-inode] stands for: 984 // - Managed by docker 985 // - The target of this device is at major <maj> and minor <min> 986 // - If <inode> is defined, use that file inside the device as a loopback image. Otherwise use the device itself. 987 devices.devicePrefix = fmt.Sprintf("docker-%d:%d-%d", major(sysSt.Dev), minor(sysSt.Dev), sysSt.Ino) 988 log.Debugf("Generated prefix: %s", devices.devicePrefix) 989 990 // Check for the existence of the thin-pool device 991 log.Debugf("Checking for existence of the pool '%s'", devices.getPoolName()) 992 info, err := devicemapper.GetInfo(devices.getPoolName()) 993 if info == nil { 994 log.Debugf("Error device devicemapper.GetInfo: %s", err) 995 return err 996 } 997 998 // It seems libdevmapper opens this without O_CLOEXEC, and go exec will not close files 999 // that are not Close-on-exec, and lxc-start will die if it inherits any unexpected files, 1000 // so we add this badhack to make sure it closes itself 1001 setCloseOnExec("/dev/mapper/control") 1002 1003 // Make sure the sparse images exist in <root>/devicemapper/data and 1004 // <root>/devicemapper/metadata 1005 1006 createdLoopback := false 1007 1008 // If the pool doesn't exist, create it 1009 if info.Exists == 0 && devices.thinPoolDevice == "" { 1010 log.Debugf("Pool doesn't exist. Creating it.") 1011 1012 var ( 1013 dataFile *os.File 1014 metadataFile *os.File 1015 ) 1016 1017 if devices.dataDevice == "" { 1018 // Make sure the sparse images exist in <root>/devicemapper/data 1019 1020 hasData := devices.hasImage("data") 1021 1022 if !doInit && !hasData { 1023 return errors.New("Loopback data file not found") 1024 } 1025 1026 if !hasData { 1027 createdLoopback = true 1028 } 1029 1030 data, err := devices.ensureImage("data", devices.dataLoopbackSize) 1031 if err != nil { 1032 log.Debugf("Error device ensureImage (data): %s", err) 1033 return err 1034 } 1035 1036 dataFile, err = devicemapper.AttachLoopDevice(data) 1037 if err != nil { 1038 return err 1039 } 1040 devices.dataLoopFile = data 1041 devices.dataDevice = dataFile.Name() 1042 } else { 1043 dataFile, err = os.OpenFile(devices.dataDevice, os.O_RDWR, 0600) 1044 if err != nil { 1045 return err 1046 } 1047 } 1048 defer dataFile.Close() 1049 1050 if devices.metadataDevice == "" { 1051 // Make sure the sparse images exist in <root>/devicemapper/metadata 1052 1053 hasMetadata := devices.hasImage("metadata") 1054 1055 if !doInit && !hasMetadata { 1056 return errors.New("Loopback metadata file not found") 1057 } 1058 1059 if !hasMetadata { 1060 createdLoopback = true 1061 } 1062 1063 metadata, err := devices.ensureImage("metadata", devices.metaDataLoopbackSize) 1064 if err != nil { 1065 log.Debugf("Error device ensureImage (metadata): %s", err) 1066 return err 1067 } 1068 1069 metadataFile, err = devicemapper.AttachLoopDevice(metadata) 1070 if err != nil { 1071 return err 1072 } 1073 devices.metadataLoopFile = metadata 1074 devices.metadataDevice = metadataFile.Name() 1075 } else { 1076 metadataFile, err = os.OpenFile(devices.metadataDevice, os.O_RDWR, 0600) 1077 if err != nil { 1078 return err 1079 } 1080 } 1081 defer metadataFile.Close() 1082 1083 if err := devicemapper.CreatePool(devices.getPoolName(), dataFile, metadataFile, devices.thinpBlockSize); err != nil { 1084 return err 1085 } 1086 } 1087 1088 // If we didn't just create the data or metadata image, we need to 1089 // load the transaction id and migrate old metadata 1090 if !createdLoopback { 1091 if err = devices.initMetaData(); err != nil { 1092 return err 1093 } 1094 } 1095 1096 // Right now this loads only NextDeviceId. If there is more metadata 1097 // down the line, we might have to move it earlier. 1098 if err = devices.loadDeviceSetMetaData(); err != nil { 1099 return err 1100 } 1101 1102 // Setup the base image 1103 if doInit { 1104 if err := devices.setupBaseImage(); err != nil { 1105 log.Debugf("Error device setupBaseImage: %s", err) 1106 return err 1107 } 1108 } 1109 1110 return nil 1111 } 1112 1113 func (devices *DeviceSet) AddDevice(hash, baseHash string) error { 1114 log.Debugf("[deviceset] AddDevice(hash=%s basehash=%s)", hash, baseHash) 1115 defer log.Debugf("[deviceset] AddDevice(hash=%s basehash=%s) END", hash, baseHash) 1116 1117 baseInfo, err := devices.lookupDevice(baseHash) 1118 if err != nil { 1119 return err 1120 } 1121 1122 baseInfo.lock.Lock() 1123 defer baseInfo.lock.Unlock() 1124 1125 devices.Lock() 1126 defer devices.Unlock() 1127 1128 if info, _ := devices.lookupDevice(hash); info != nil { 1129 return fmt.Errorf("device %s already exists", hash) 1130 } 1131 1132 if err := devices.createRegisterSnapDevice(hash, baseInfo); err != nil { 1133 return err 1134 } 1135 1136 return nil 1137 } 1138 1139 func (devices *DeviceSet) deleteDevice(info *DevInfo) error { 1140 if devices.doBlkDiscard { 1141 // This is a workaround for the kernel not discarding block so 1142 // on the thin pool when we remove a thinp device, so we do it 1143 // manually 1144 if err := devices.activateDeviceIfNeeded(info); err == nil { 1145 if err := devicemapper.BlockDeviceDiscard(info.DevName()); err != nil { 1146 log.Debugf("Error discarding block on device: %s (ignoring)", err) 1147 } 1148 } 1149 } 1150 1151 devinfo, _ := devicemapper.GetInfo(info.Name()) 1152 if devinfo != nil && devinfo.Exists != 0 { 1153 if err := devices.removeDeviceAndWait(info.Name()); err != nil { 1154 log.Debugf("Error removing device: %s", err) 1155 return err 1156 } 1157 } 1158 1159 if err := devices.openTransaction(info.Hash, info.DeviceId); err != nil { 1160 log.Debugf("Error opening transaction hash = %s deviceId = %d", "", info.DeviceId) 1161 return err 1162 } 1163 1164 if err := devicemapper.DeleteDevice(devices.getPoolDevName(), info.DeviceId); err != nil { 1165 log.Debugf("Error deleting device: %s", err) 1166 return err 1167 } 1168 1169 if err := devices.unregisterDevice(info.DeviceId, info.Hash); err != nil { 1170 return err 1171 } 1172 1173 if err := devices.closeTransaction(); err != nil { 1174 return err 1175 } 1176 1177 devices.markDeviceIdFree(info.DeviceId) 1178 1179 return nil 1180 } 1181 1182 func (devices *DeviceSet) DeleteDevice(hash string) error { 1183 info, err := devices.lookupDevice(hash) 1184 if err != nil { 1185 return err 1186 } 1187 1188 info.lock.Lock() 1189 defer info.lock.Unlock() 1190 1191 devices.Lock() 1192 defer devices.Unlock() 1193 1194 return devices.deleteDevice(info) 1195 } 1196 1197 func (devices *DeviceSet) deactivatePool() error { 1198 log.Debugf("[devmapper] deactivatePool()") 1199 defer log.Debugf("[devmapper] deactivatePool END") 1200 devname := devices.getPoolDevName() 1201 1202 devinfo, err := devicemapper.GetInfo(devname) 1203 if err != nil { 1204 return err 1205 } 1206 if d, err := devicemapper.GetDeps(devname); err == nil { 1207 // Access to more Debug output 1208 log.Debugf("[devmapper] devicemapper.GetDeps() %s: %#v", devname, d) 1209 } 1210 if devinfo.Exists != 0 { 1211 return devicemapper.RemoveDevice(devname) 1212 } 1213 1214 return nil 1215 } 1216 1217 func (devices *DeviceSet) deactivateDevice(info *DevInfo) error { 1218 log.Debugf("[devmapper] deactivateDevice(%s)", info.Hash) 1219 defer log.Debugf("[devmapper] deactivateDevice END(%s)", info.Hash) 1220 1221 // Wait for the unmount to be effective, 1222 // by watching the value of Info.OpenCount for the device 1223 if err := devices.waitClose(info); err != nil { 1224 log.Errorf("Error waiting for device %s to close: %s", info.Hash, err) 1225 } 1226 1227 devinfo, err := devicemapper.GetInfo(info.Name()) 1228 if err != nil { 1229 return err 1230 } 1231 if devinfo.Exists != 0 { 1232 if err := devices.removeDeviceAndWait(info.Name()); err != nil { 1233 return err 1234 } 1235 } 1236 1237 return nil 1238 } 1239 1240 // Issues the underlying dm remove operation and then waits 1241 // for it to finish. 1242 func (devices *DeviceSet) removeDeviceAndWait(devname string) error { 1243 var err error 1244 1245 for i := 0; i < 1000; i++ { 1246 err = devicemapper.RemoveDevice(devname) 1247 if err == nil { 1248 break 1249 } 1250 if err != devicemapper.ErrBusy { 1251 return err 1252 } 1253 1254 // If we see EBUSY it may be a transient error, 1255 // sleep a bit a retry a few times. 1256 devices.Unlock() 1257 time.Sleep(10 * time.Millisecond) 1258 devices.Lock() 1259 } 1260 if err != nil { 1261 return err 1262 } 1263 1264 if err := devices.waitRemove(devname); err != nil { 1265 return err 1266 } 1267 return nil 1268 } 1269 1270 // waitRemove blocks until either: 1271 // a) the device registered at <device_set_prefix>-<hash> is removed, 1272 // or b) the 10 second timeout expires. 1273 func (devices *DeviceSet) waitRemove(devname string) error { 1274 log.Debugf("[deviceset %s] waitRemove(%s)", devices.devicePrefix, devname) 1275 defer log.Debugf("[deviceset %s] waitRemove(%s) END", devices.devicePrefix, devname) 1276 i := 0 1277 for ; i < 1000; i++ { 1278 devinfo, err := devicemapper.GetInfo(devname) 1279 if err != nil { 1280 // If there is an error we assume the device doesn't exist. 1281 // The error might actually be something else, but we can't differentiate. 1282 return nil 1283 } 1284 if i%100 == 0 { 1285 log.Debugf("Waiting for removal of %s: exists=%d", devname, devinfo.Exists) 1286 } 1287 if devinfo.Exists == 0 { 1288 break 1289 } 1290 1291 devices.Unlock() 1292 time.Sleep(10 * time.Millisecond) 1293 devices.Lock() 1294 } 1295 if i == 1000 { 1296 return fmt.Errorf("Timeout while waiting for device %s to be removed", devname) 1297 } 1298 return nil 1299 } 1300 1301 // waitClose blocks until either: 1302 // a) the device registered at <device_set_prefix>-<hash> is closed, 1303 // or b) the 10 second timeout expires. 1304 func (devices *DeviceSet) waitClose(info *DevInfo) error { 1305 i := 0 1306 for ; i < 1000; i++ { 1307 devinfo, err := devicemapper.GetInfo(info.Name()) 1308 if err != nil { 1309 return err 1310 } 1311 if i%100 == 0 { 1312 log.Debugf("Waiting for unmount of %s: opencount=%d", info.Hash, devinfo.OpenCount) 1313 } 1314 if devinfo.OpenCount == 0 { 1315 break 1316 } 1317 devices.Unlock() 1318 time.Sleep(10 * time.Millisecond) 1319 devices.Lock() 1320 } 1321 if i == 1000 { 1322 return fmt.Errorf("Timeout while waiting for device %s to close", info.Hash) 1323 } 1324 return nil 1325 } 1326 1327 func (devices *DeviceSet) Shutdown() error { 1328 log.Debugf("[deviceset %s] Shutdown()", devices.devicePrefix) 1329 log.Debugf("[devmapper] Shutting down DeviceSet: %s", devices.root) 1330 defer log.Debugf("[deviceset %s] Shutdown() END", devices.devicePrefix) 1331 1332 var devs []*DevInfo 1333 1334 devices.devicesLock.Lock() 1335 for _, info := range devices.Devices { 1336 devs = append(devs, info) 1337 } 1338 devices.devicesLock.Unlock() 1339 1340 for _, info := range devs { 1341 info.lock.Lock() 1342 if info.mountCount > 0 { 1343 // We use MNT_DETACH here in case it is still busy in some running 1344 // container. This means it'll go away from the global scope directly, 1345 // and the device will be released when that container dies. 1346 if err := syscall.Unmount(info.mountPath, syscall.MNT_DETACH); err != nil { 1347 log.Debugf("Shutdown unmounting %s, error: %s", info.mountPath, err) 1348 } 1349 1350 devices.Lock() 1351 if err := devices.deactivateDevice(info); err != nil { 1352 log.Debugf("Shutdown deactivate %s , error: %s", info.Hash, err) 1353 } 1354 devices.Unlock() 1355 } 1356 info.lock.Unlock() 1357 } 1358 1359 info, _ := devices.lookupDevice("") 1360 if info != nil { 1361 info.lock.Lock() 1362 devices.Lock() 1363 if err := devices.deactivateDevice(info); err != nil { 1364 log.Debugf("Shutdown deactivate base , error: %s", err) 1365 } 1366 devices.Unlock() 1367 info.lock.Unlock() 1368 } 1369 1370 devices.Lock() 1371 if devices.thinPoolDevice == "" { 1372 if err := devices.deactivatePool(); err != nil { 1373 log.Debugf("Shutdown deactivate pool , error: %s", err) 1374 } 1375 } 1376 1377 devices.saveDeviceSetMetaData() 1378 devices.Unlock() 1379 1380 return nil 1381 } 1382 1383 func (devices *DeviceSet) MountDevice(hash, path, mountLabel string) error { 1384 info, err := devices.lookupDevice(hash) 1385 if err != nil { 1386 return err 1387 } 1388 1389 info.lock.Lock() 1390 defer info.lock.Unlock() 1391 1392 devices.Lock() 1393 defer devices.Unlock() 1394 1395 if info.mountCount > 0 { 1396 if path != info.mountPath { 1397 return fmt.Errorf("Trying to mount devmapper device in multiple places (%s, %s)", info.mountPath, path) 1398 } 1399 1400 info.mountCount++ 1401 return nil 1402 } 1403 1404 if err := devices.activateDeviceIfNeeded(info); err != nil { 1405 return fmt.Errorf("Error activating devmapper device for '%s': %s", hash, err) 1406 } 1407 1408 var flags uintptr = syscall.MS_MGC_VAL 1409 1410 fstype, err := ProbeFsType(info.DevName()) 1411 if err != nil { 1412 return err 1413 } 1414 1415 options := "" 1416 1417 if fstype == "xfs" { 1418 // XFS needs nouuid or it can't mount filesystems with the same fs 1419 options = joinMountOptions(options, "nouuid") 1420 } 1421 1422 options = joinMountOptions(options, devices.mountOptions) 1423 options = joinMountOptions(options, label.FormatMountLabel("", mountLabel)) 1424 1425 err = syscall.Mount(info.DevName(), path, fstype, flags, joinMountOptions("discard", options)) 1426 if err != nil && err == syscall.EINVAL { 1427 err = syscall.Mount(info.DevName(), path, fstype, flags, options) 1428 } 1429 if err != nil { 1430 return fmt.Errorf("Error mounting '%s' on '%s': %s", info.DevName(), path, err) 1431 } 1432 1433 info.mountCount = 1 1434 info.mountPath = path 1435 1436 return nil 1437 } 1438 1439 func (devices *DeviceSet) UnmountDevice(hash string) error { 1440 log.Debugf("[devmapper] UnmountDevice(hash=%s)", hash) 1441 defer log.Debugf("[devmapper] UnmountDevice(hash=%s) END", hash) 1442 1443 info, err := devices.lookupDevice(hash) 1444 if err != nil { 1445 return err 1446 } 1447 1448 info.lock.Lock() 1449 defer info.lock.Unlock() 1450 1451 devices.Lock() 1452 defer devices.Unlock() 1453 1454 if info.mountCount == 0 { 1455 return fmt.Errorf("UnmountDevice: device not-mounted id %s", hash) 1456 } 1457 1458 info.mountCount-- 1459 if info.mountCount > 0 { 1460 return nil 1461 } 1462 1463 log.Debugf("[devmapper] Unmount(%s)", info.mountPath) 1464 if err := syscall.Unmount(info.mountPath, syscall.MNT_DETACH); err != nil { 1465 return err 1466 } 1467 log.Debugf("[devmapper] Unmount done") 1468 1469 if err := devices.deactivateDevice(info); err != nil { 1470 return err 1471 } 1472 1473 info.mountPath = "" 1474 1475 return nil 1476 } 1477 1478 func (devices *DeviceSet) HasDevice(hash string) bool { 1479 devices.Lock() 1480 defer devices.Unlock() 1481 1482 info, _ := devices.lookupDevice(hash) 1483 return info != nil 1484 } 1485 1486 func (devices *DeviceSet) HasActivatedDevice(hash string) bool { 1487 info, _ := devices.lookupDevice(hash) 1488 if info == nil { 1489 return false 1490 } 1491 1492 info.lock.Lock() 1493 defer info.lock.Unlock() 1494 1495 devices.Lock() 1496 defer devices.Unlock() 1497 1498 devinfo, _ := devicemapper.GetInfo(info.Name()) 1499 return devinfo != nil && devinfo.Exists != 0 1500 } 1501 1502 func (devices *DeviceSet) List() []string { 1503 devices.Lock() 1504 defer devices.Unlock() 1505 1506 devices.devicesLock.Lock() 1507 ids := make([]string, len(devices.Devices)) 1508 i := 0 1509 for k := range devices.Devices { 1510 ids[i] = k 1511 i++ 1512 } 1513 devices.devicesLock.Unlock() 1514 1515 return ids 1516 } 1517 1518 func (devices *DeviceSet) deviceStatus(devName string) (sizeInSectors, mappedSectors, highestMappedSector uint64, err error) { 1519 var params string 1520 _, sizeInSectors, _, params, err = devicemapper.GetStatus(devName) 1521 if err != nil { 1522 return 1523 } 1524 if _, err = fmt.Sscanf(params, "%d %d", &mappedSectors, &highestMappedSector); err == nil { 1525 return 1526 } 1527 return 1528 } 1529 1530 func (devices *DeviceSet) GetDeviceStatus(hash string) (*DevStatus, error) { 1531 info, err := devices.lookupDevice(hash) 1532 if err != nil { 1533 return nil, err 1534 } 1535 1536 info.lock.Lock() 1537 defer info.lock.Unlock() 1538 1539 devices.Lock() 1540 defer devices.Unlock() 1541 1542 status := &DevStatus{ 1543 DeviceId: info.DeviceId, 1544 Size: info.Size, 1545 TransactionId: info.TransactionId, 1546 } 1547 1548 if err := devices.activateDeviceIfNeeded(info); err != nil { 1549 return nil, fmt.Errorf("Error activating devmapper device for '%s': %s", hash, err) 1550 } 1551 1552 if sizeInSectors, mappedSectors, highestMappedSector, err := devices.deviceStatus(info.DevName()); err != nil { 1553 return nil, err 1554 } else { 1555 status.SizeInSectors = sizeInSectors 1556 status.MappedSectors = mappedSectors 1557 status.HighestMappedSector = highestMappedSector 1558 } 1559 1560 return status, nil 1561 } 1562 1563 func (devices *DeviceSet) poolStatus() (totalSizeInSectors, transactionId, dataUsed, dataTotal, metadataUsed, metadataTotal uint64, err error) { 1564 var params string 1565 if _, totalSizeInSectors, _, params, err = devicemapper.GetStatus(devices.getPoolName()); err == nil { 1566 _, err = fmt.Sscanf(params, "%d %d/%d %d/%d", &transactionId, &metadataUsed, &metadataTotal, &dataUsed, &dataTotal) 1567 } 1568 return 1569 } 1570 1571 // DataDevicePath returns the path to the data storage for this deviceset, 1572 // regardless of loopback or block device 1573 func (devices *DeviceSet) DataDevicePath() string { 1574 return devices.dataDevice 1575 } 1576 1577 // MetadataDevicePath returns the path to the metadata storage for this deviceset, 1578 // regardless of loopback or block device 1579 func (devices *DeviceSet) MetadataDevicePath() string { 1580 return devices.metadataDevice 1581 } 1582 1583 func (devices *DeviceSet) getUnderlyingAvailableSpace(loopFile string) (uint64, error) { 1584 buf := new(syscall.Statfs_t) 1585 err := syscall.Statfs(loopFile, buf) 1586 if err != nil { 1587 log.Warnf("Couldn't stat loopfile filesystem %v: %v", loopFile, err) 1588 return 0, err 1589 } 1590 return buf.Bfree * uint64(buf.Bsize), nil 1591 } 1592 1593 func (devices *DeviceSet) isRealFile(loopFile string) (bool, error) { 1594 if loopFile != "" { 1595 fi, err := os.Stat(loopFile) 1596 if err != nil { 1597 log.Warnf("Couldn't stat loopfile %v: %v", loopFile, err) 1598 return false, err 1599 } 1600 return fi.Mode().IsRegular(), nil 1601 } 1602 return false, nil 1603 } 1604 1605 // Status returns the current status of this deviceset 1606 func (devices *DeviceSet) Status() *Status { 1607 devices.Lock() 1608 defer devices.Unlock() 1609 1610 status := &Status{} 1611 1612 status.PoolName = devices.getPoolName() 1613 status.DataFile = devices.DataDevicePath() 1614 status.DataLoopback = devices.dataLoopFile 1615 status.MetadataFile = devices.MetadataDevicePath() 1616 status.MetadataLoopback = devices.metadataLoopFile 1617 status.UdevSyncSupported = devicemapper.UdevSyncSupported() 1618 1619 totalSizeInSectors, _, dataUsed, dataTotal, metadataUsed, metadataTotal, err := devices.poolStatus() 1620 if err == nil { 1621 // Convert from blocks to bytes 1622 blockSizeInSectors := totalSizeInSectors / dataTotal 1623 1624 status.Data.Used = dataUsed * blockSizeInSectors * 512 1625 status.Data.Total = dataTotal * blockSizeInSectors * 512 1626 status.Data.Available = status.Data.Total - status.Data.Used 1627 1628 // metadata blocks are always 4k 1629 status.Metadata.Used = metadataUsed * 4096 1630 status.Metadata.Total = metadataTotal * 4096 1631 status.Metadata.Available = status.Metadata.Total - status.Metadata.Used 1632 1633 status.SectorSize = blockSizeInSectors * 512 1634 1635 if check, _ := devices.isRealFile(devices.dataLoopFile); check { 1636 actualSpace, err := devices.getUnderlyingAvailableSpace(devices.dataLoopFile) 1637 if err == nil && actualSpace < status.Data.Available { 1638 status.Data.Available = actualSpace 1639 } 1640 } 1641 1642 if check, _ := devices.isRealFile(devices.metadataLoopFile); check { 1643 actualSpace, err := devices.getUnderlyingAvailableSpace(devices.metadataLoopFile) 1644 if err == nil && actualSpace < status.Metadata.Available { 1645 status.Metadata.Available = actualSpace 1646 } 1647 } 1648 } 1649 1650 return status 1651 } 1652 1653 func NewDeviceSet(root string, doInit bool, options []string) (*DeviceSet, error) { 1654 devicemapper.SetDevDir("/dev") 1655 1656 devices := &DeviceSet{ 1657 root: root, 1658 MetaData: MetaData{Devices: make(map[string]*DevInfo)}, 1659 dataLoopbackSize: DefaultDataLoopbackSize, 1660 metaDataLoopbackSize: DefaultMetaDataLoopbackSize, 1661 baseFsSize: DefaultBaseFsSize, 1662 filesystem: "ext4", 1663 doBlkDiscard: true, 1664 thinpBlockSize: DefaultThinpBlockSize, 1665 deviceIdMap: make([]byte, DeviceIdMapSz), 1666 } 1667 1668 foundBlkDiscard := false 1669 for _, option := range options { 1670 key, val, err := parsers.ParseKeyValueOpt(option) 1671 if err != nil { 1672 return nil, err 1673 } 1674 key = strings.ToLower(key) 1675 switch key { 1676 case "dm.basesize": 1677 size, err := units.RAMInBytes(val) 1678 if err != nil { 1679 return nil, err 1680 } 1681 devices.baseFsSize = uint64(size) 1682 case "dm.loopdatasize": 1683 size, err := units.RAMInBytes(val) 1684 if err != nil { 1685 return nil, err 1686 } 1687 devices.dataLoopbackSize = size 1688 case "dm.loopmetadatasize": 1689 size, err := units.RAMInBytes(val) 1690 if err != nil { 1691 return nil, err 1692 } 1693 devices.metaDataLoopbackSize = size 1694 case "dm.fs": 1695 if val != "ext4" && val != "xfs" { 1696 return nil, fmt.Errorf("Unsupported filesystem %s\n", val) 1697 } 1698 devices.filesystem = val 1699 case "dm.mkfsarg": 1700 devices.mkfsArgs = append(devices.mkfsArgs, val) 1701 case "dm.mountopt": 1702 devices.mountOptions = joinMountOptions(devices.mountOptions, val) 1703 case "dm.metadatadev": 1704 devices.metadataDevice = val 1705 case "dm.datadev": 1706 devices.dataDevice = val 1707 case "dm.thinpooldev": 1708 devices.thinPoolDevice = strings.TrimPrefix(val, "/dev/mapper/") 1709 case "dm.blkdiscard": 1710 foundBlkDiscard = true 1711 devices.doBlkDiscard, err = strconv.ParseBool(val) 1712 if err != nil { 1713 return nil, err 1714 } 1715 case "dm.blocksize": 1716 size, err := units.RAMInBytes(val) 1717 if err != nil { 1718 return nil, err 1719 } 1720 // convert to 512b sectors 1721 devices.thinpBlockSize = uint32(size) >> 9 1722 default: 1723 return nil, fmt.Errorf("Unknown option %s\n", key) 1724 } 1725 } 1726 1727 // By default, don't do blk discard hack on raw devices, its rarely useful and is expensive 1728 if !foundBlkDiscard && (devices.dataDevice != "" || devices.thinPoolDevice != "") { 1729 devices.doBlkDiscard = false 1730 } 1731 1732 if err := devices.initDevmapper(doInit); err != nil { 1733 return nil, err 1734 } 1735 1736 return devices, nil 1737 }