github.com/adxhyt/docker@v1.4.2-0.20150117221845-467b7c821390/daemon/graphdriver/devmapper/deviceset.go (about) 1 // +build linux 2 3 package devmapper 4 5 import ( 6 "encoding/json" 7 "errors" 8 "fmt" 9 "io" 10 "io/ioutil" 11 "os" 12 "os/exec" 13 "path" 14 "path/filepath" 15 "strconv" 16 "strings" 17 "sync" 18 "syscall" 19 "time" 20 21 log "github.com/Sirupsen/logrus" 22 "github.com/docker/docker/daemon/graphdriver" 23 "github.com/docker/docker/pkg/devicemapper" 24 "github.com/docker/docker/pkg/parsers" 25 "github.com/docker/docker/pkg/units" 26 "github.com/docker/libcontainer/label" 27 ) 28 29 var ( 30 DefaultDataLoopbackSize int64 = 100 * 1024 * 1024 * 1024 31 DefaultMetaDataLoopbackSize int64 = 2 * 1024 * 1024 * 1024 32 DefaultBaseFsSize uint64 = 10 * 1024 * 1024 * 1024 33 DefaultThinpBlockSize uint32 = 128 // 64K = 128 512b sectors 34 MaxDeviceId int = 0xffffff // 24 bit, pool limit 35 DeviceIdMapSz int = (MaxDeviceId + 1) / 8 36 ) 37 38 const deviceSetMetaFile string = "deviceset-metadata" 39 const transactionMetaFile string = "transaction-metadata" 40 41 type Transaction struct { 42 OpenTransactionId uint64 `json:"open_transaction_id"` 43 DeviceIdHash string `json:"device_hash"` 44 DeviceId int `json:"device_id"` 45 } 46 47 type DevInfo struct { 48 Hash string `json:"-"` 49 DeviceId int `json:"device_id"` 50 Size uint64 `json:"size"` 51 TransactionId uint64 `json:"transaction_id"` 52 Initialized bool `json:"initialized"` 53 devices *DeviceSet 54 55 mountCount int 56 mountPath string 57 58 // The global DeviceSet lock guarantees that we serialize all 59 // the calls to libdevmapper (which is not threadsafe), but we 60 // sometimes release that lock while sleeping. In that case 61 // this per-device lock is still held, protecting against 62 // other accesses to the device that we're doing the wait on. 63 // 64 // WARNING: In order to avoid AB-BA deadlocks when releasing 65 // the global lock while holding the per-device locks all 66 // device locks must be aquired *before* the device lock, and 67 // multiple device locks should be aquired parent before child. 68 lock sync.Mutex 69 } 70 71 type MetaData struct { 72 Devices map[string]*DevInfo `json:"Devices"` 73 devicesLock sync.Mutex // Protects all read/writes to Devices map 74 } 75 76 type DeviceSet struct { 77 MetaData `json:"-"` 78 sync.Mutex `json:"-"` // Protects Devices map and serializes calls into libdevmapper 79 root string 80 devicePrefix string 81 TransactionId uint64 `json:"-"` 82 NextDeviceId int `json:"next_device_id"` 83 deviceIdMap []byte 84 85 // Options 86 dataLoopbackSize int64 87 metaDataLoopbackSize int64 88 baseFsSize uint64 89 filesystem string 90 mountOptions string 91 mkfsArgs []string 92 dataDevice string // block or loop dev 93 dataLoopFile string // loopback file, if used 94 metadataDevice string // block or loop dev 95 metadataLoopFile string // loopback file, if used 96 doBlkDiscard bool 97 thinpBlockSize uint32 98 thinPoolDevice string 99 Transaction `json:"-"` 100 } 101 102 type DiskUsage struct { 103 Used uint64 104 Total uint64 105 } 106 107 type Status struct { 108 PoolName string 109 DataFile string // actual block device for data 110 DataLoopback string // loopback file, if used 111 MetadataFile string // actual block device for metadata 112 MetadataLoopback string // loopback file, if used 113 Data DiskUsage 114 Metadata DiskUsage 115 SectorSize uint64 116 } 117 118 type DevStatus struct { 119 DeviceId int 120 Size uint64 121 TransactionId uint64 122 SizeInSectors uint64 123 MappedSectors uint64 124 HighestMappedSector uint64 125 } 126 127 func getDevName(name string) string { 128 return "/dev/mapper/" + name 129 } 130 131 func (info *DevInfo) Name() string { 132 hash := info.Hash 133 if hash == "" { 134 hash = "base" 135 } 136 return fmt.Sprintf("%s-%s", info.devices.devicePrefix, hash) 137 } 138 139 func (info *DevInfo) DevName() string { 140 return getDevName(info.Name()) 141 } 142 143 func (devices *DeviceSet) loopbackDir() string { 144 return path.Join(devices.root, "devicemapper") 145 } 146 147 func (devices *DeviceSet) metadataDir() string { 148 return path.Join(devices.root, "metadata") 149 } 150 151 func (devices *DeviceSet) metadataFile(info *DevInfo) string { 152 file := info.Hash 153 if file == "" { 154 file = "base" 155 } 156 return path.Join(devices.metadataDir(), file) 157 } 158 159 func (devices *DeviceSet) transactionMetaFile() string { 160 return path.Join(devices.metadataDir(), transactionMetaFile) 161 } 162 163 func (devices *DeviceSet) deviceSetMetaFile() string { 164 return path.Join(devices.metadataDir(), deviceSetMetaFile) 165 } 166 167 func (devices *DeviceSet) oldMetadataFile() string { 168 return path.Join(devices.loopbackDir(), "json") 169 } 170 171 func (devices *DeviceSet) getPoolName() string { 172 if devices.thinPoolDevice == "" { 173 return devices.devicePrefix + "-pool" 174 } 175 return devices.thinPoolDevice 176 } 177 178 func (devices *DeviceSet) getPoolDevName() string { 179 return getDevName(devices.getPoolName()) 180 } 181 182 func (devices *DeviceSet) hasImage(name string) bool { 183 dirname := devices.loopbackDir() 184 filename := path.Join(dirname, name) 185 186 _, err := os.Stat(filename) 187 return err == nil 188 } 189 190 // ensureImage creates a sparse file of <size> bytes at the path 191 // <root>/devicemapper/<name>. 192 // If the file already exists, it does nothing. 193 // Either way it returns the full path. 194 func (devices *DeviceSet) ensureImage(name string, size int64) (string, error) { 195 dirname := devices.loopbackDir() 196 filename := path.Join(dirname, name) 197 198 if err := os.MkdirAll(dirname, 0700); err != nil && !os.IsExist(err) { 199 return "", err 200 } 201 202 if _, err := os.Stat(filename); err != nil { 203 if !os.IsNotExist(err) { 204 return "", err 205 } 206 log.Debugf("Creating loopback file %s for device-manage use", filename) 207 file, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE, 0600) 208 if err != nil { 209 return "", err 210 } 211 defer file.Close() 212 213 if err = file.Truncate(size); err != nil { 214 return "", err 215 } 216 } 217 return filename, nil 218 } 219 220 func (devices *DeviceSet) allocateTransactionId() uint64 { 221 devices.OpenTransactionId = devices.TransactionId + 1 222 return devices.OpenTransactionId 223 } 224 225 func (devices *DeviceSet) updatePoolTransactionId() error { 226 if err := devicemapper.SetTransactionId(devices.getPoolDevName(), devices.TransactionId, devices.OpenTransactionId); err != nil { 227 return fmt.Errorf("Error setting devmapper transaction ID: %s", err) 228 } 229 devices.TransactionId = devices.OpenTransactionId 230 return nil 231 } 232 233 func (devices *DeviceSet) removeMetadata(info *DevInfo) error { 234 if err := os.RemoveAll(devices.metadataFile(info)); err != nil { 235 return fmt.Errorf("Error removing metadata file %s: %s", devices.metadataFile(info), err) 236 } 237 return nil 238 } 239 240 // Given json data and file path, write it to disk 241 func (devices *DeviceSet) writeMetaFile(jsonData []byte, filePath string) error { 242 tmpFile, err := ioutil.TempFile(devices.metadataDir(), ".tmp") 243 if err != nil { 244 return fmt.Errorf("Error creating metadata file: %s", err) 245 } 246 247 n, err := tmpFile.Write(jsonData) 248 if err != nil { 249 return fmt.Errorf("Error writing metadata to %s: %s", tmpFile.Name(), err) 250 } 251 if n < len(jsonData) { 252 return io.ErrShortWrite 253 } 254 if err := tmpFile.Sync(); err != nil { 255 return fmt.Errorf("Error syncing metadata file %s: %s", tmpFile.Name(), err) 256 } 257 if err := tmpFile.Close(); err != nil { 258 return fmt.Errorf("Error closing metadata file %s: %s", tmpFile.Name(), err) 259 } 260 if err := os.Rename(tmpFile.Name(), filePath); err != nil { 261 return fmt.Errorf("Error committing metadata file %s: %s", tmpFile.Name(), err) 262 } 263 264 return nil 265 } 266 267 func (devices *DeviceSet) saveMetadata(info *DevInfo) error { 268 jsonData, err := json.Marshal(info) 269 if err != nil { 270 return fmt.Errorf("Error encoding metadata to json: %s", err) 271 } 272 if err := devices.writeMetaFile(jsonData, devices.metadataFile(info)); err != nil { 273 return err 274 } 275 return nil 276 } 277 278 func (devices *DeviceSet) markDeviceIdUsed(deviceId int) { 279 var mask byte 280 i := deviceId % 8 281 mask = 1 << uint(i) 282 devices.deviceIdMap[deviceId/8] = devices.deviceIdMap[deviceId/8] | mask 283 } 284 285 func (devices *DeviceSet) markDeviceIdFree(deviceId int) { 286 var mask byte 287 i := deviceId % 8 288 mask = ^(1 << uint(i)) 289 devices.deviceIdMap[deviceId/8] = devices.deviceIdMap[deviceId/8] & mask 290 } 291 292 func (devices *DeviceSet) isDeviceIdFree(deviceId int) bool { 293 var mask byte 294 i := deviceId % 8 295 mask = (1 << uint(i)) 296 if (devices.deviceIdMap[deviceId/8] & mask) != 0 { 297 return false 298 } 299 return true 300 } 301 302 func (devices *DeviceSet) lookupDevice(hash string) (*DevInfo, error) { 303 devices.devicesLock.Lock() 304 defer devices.devicesLock.Unlock() 305 info := devices.Devices[hash] 306 if info == nil { 307 info = devices.loadMetadata(hash) 308 if info == nil { 309 return nil, fmt.Errorf("Unknown device %s", hash) 310 } 311 312 devices.Devices[hash] = info 313 } 314 return info, nil 315 } 316 317 func (devices *DeviceSet) deviceFileWalkFunction(path string, finfo os.FileInfo) error { 318 319 // Skip some of the meta files which are not device files. 320 if strings.HasSuffix(finfo.Name(), ".migrated") { 321 log.Debugf("Skipping file %s", path) 322 return nil 323 } 324 325 if finfo.Name() == deviceSetMetaFile { 326 log.Debugf("Skipping file %s", path) 327 return nil 328 } 329 330 log.Debugf("Loading data for file %s", path) 331 332 hash := finfo.Name() 333 if hash == "base" { 334 hash = "" 335 } 336 337 dinfo := devices.loadMetadata(hash) 338 if dinfo == nil { 339 return fmt.Errorf("Error loading device metadata file %s", hash) 340 } 341 342 if dinfo.DeviceId > MaxDeviceId { 343 log.Errorf("Warning: Ignoring Invalid DeviceId=%d", dinfo.DeviceId) 344 return nil 345 } 346 347 devices.Lock() 348 devices.markDeviceIdUsed(dinfo.DeviceId) 349 devices.Unlock() 350 351 log.Debugf("Added deviceId=%d to DeviceIdMap", dinfo.DeviceId) 352 return nil 353 } 354 355 func (devices *DeviceSet) constructDeviceIdMap() error { 356 log.Debugf("[deviceset] constructDeviceIdMap()") 357 defer log.Debugf("[deviceset] constructDeviceIdMap() END") 358 359 var scan = func(path string, info os.FileInfo, err error) error { 360 if err != nil { 361 log.Debugf("Can't walk the file %s", path) 362 return nil 363 } 364 365 // Skip any directories 366 if info.IsDir() { 367 return nil 368 } 369 370 return devices.deviceFileWalkFunction(path, info) 371 } 372 373 return filepath.Walk(devices.metadataDir(), scan) 374 } 375 376 func (devices *DeviceSet) unregisterDevice(id int, hash string) error { 377 log.Debugf("unregisterDevice(%v, %v)", id, hash) 378 info := &DevInfo{ 379 Hash: hash, 380 DeviceId: id, 381 } 382 383 devices.devicesLock.Lock() 384 delete(devices.Devices, hash) 385 devices.devicesLock.Unlock() 386 387 if err := devices.removeMetadata(info); err != nil { 388 log.Debugf("Error removing meta data: %s", err) 389 return err 390 } 391 392 return nil 393 } 394 395 func (devices *DeviceSet) registerDevice(id int, hash string, size uint64, transactionId uint64) (*DevInfo, error) { 396 log.Debugf("registerDevice(%v, %v)", id, hash) 397 info := &DevInfo{ 398 Hash: hash, 399 DeviceId: id, 400 Size: size, 401 TransactionId: transactionId, 402 Initialized: false, 403 devices: devices, 404 } 405 406 devices.devicesLock.Lock() 407 devices.Devices[hash] = info 408 devices.devicesLock.Unlock() 409 410 if err := devices.saveMetadata(info); err != nil { 411 // Try to remove unused device 412 devices.devicesLock.Lock() 413 delete(devices.Devices, hash) 414 devices.devicesLock.Unlock() 415 return nil, err 416 } 417 418 return info, nil 419 } 420 421 func (devices *DeviceSet) activateDeviceIfNeeded(info *DevInfo) error { 422 log.Debugf("activateDeviceIfNeeded(%v)", info.Hash) 423 424 if devinfo, _ := devicemapper.GetInfo(info.Name()); devinfo != nil && devinfo.Exists != 0 { 425 return nil 426 } 427 428 return devicemapper.ActivateDevice(devices.getPoolDevName(), info.Name(), info.DeviceId, info.Size) 429 } 430 431 func (devices *DeviceSet) createFilesystem(info *DevInfo) error { 432 devname := info.DevName() 433 434 args := []string{} 435 for _, arg := range devices.mkfsArgs { 436 args = append(args, arg) 437 } 438 439 args = append(args, devname) 440 441 var err error 442 switch devices.filesystem { 443 case "xfs": 444 err = exec.Command("mkfs.xfs", args...).Run() 445 case "ext4": 446 err = exec.Command("mkfs.ext4", append([]string{"-E", "nodiscard,lazy_itable_init=0,lazy_journal_init=0"}, args...)...).Run() 447 if err != nil { 448 err = exec.Command("mkfs.ext4", append([]string{"-E", "nodiscard,lazy_itable_init=0"}, args...)...).Run() 449 } 450 if err != nil { 451 return err 452 } 453 err = exec.Command("tune2fs", append([]string{"-c", "-1", "-i", "0"}, devname)...).Run() 454 default: 455 err = fmt.Errorf("Unsupported filesystem type %s", devices.filesystem) 456 } 457 if err != nil { 458 return err 459 } 460 461 return nil 462 } 463 464 func (devices *DeviceSet) migrateOldMetaData() error { 465 // Migrate old metadatafile 466 jsonData, err := ioutil.ReadFile(devices.oldMetadataFile()) 467 if err != nil && !os.IsNotExist(err) { 468 return err 469 } 470 471 if jsonData != nil { 472 m := MetaData{Devices: make(map[string]*DevInfo)} 473 474 if err := json.Unmarshal(jsonData, &m); err != nil { 475 return err 476 } 477 478 for hash, info := range m.Devices { 479 info.Hash = hash 480 devices.saveMetadata(info) 481 } 482 if err := os.Rename(devices.oldMetadataFile(), devices.oldMetadataFile()+".migrated"); err != nil { 483 return err 484 } 485 486 } 487 488 return nil 489 } 490 491 func (devices *DeviceSet) initMetaData() error { 492 if err := devices.migrateOldMetaData(); err != nil { 493 return err 494 } 495 496 _, transactionId, _, _, _, _, err := devices.poolStatus() 497 if err != nil { 498 return err 499 } 500 501 devices.TransactionId = transactionId 502 503 if err := devices.constructDeviceIdMap(); err != nil { 504 return err 505 } 506 507 if err := devices.processPendingTransaction(); err != nil { 508 return err 509 } 510 return nil 511 } 512 513 func (devices *DeviceSet) incNextDeviceId() { 514 // Ids are 24bit, so wrap around 515 devices.NextDeviceId = (devices.NextDeviceId + 1) & MaxDeviceId 516 } 517 518 func (devices *DeviceSet) getNextFreeDeviceId() (int, error) { 519 devices.incNextDeviceId() 520 for i := 0; i <= MaxDeviceId; i++ { 521 if devices.isDeviceIdFree(devices.NextDeviceId) { 522 devices.markDeviceIdUsed(devices.NextDeviceId) 523 return devices.NextDeviceId, nil 524 } 525 devices.incNextDeviceId() 526 } 527 528 return 0, fmt.Errorf("Unable to find a free device Id") 529 } 530 531 func (devices *DeviceSet) createRegisterDevice(hash string) (*DevInfo, error) { 532 deviceId, err := devices.getNextFreeDeviceId() 533 if err != nil { 534 return nil, err 535 } 536 537 if err := devices.openTransaction(hash, deviceId); err != nil { 538 log.Debugf("Error opening transaction hash = %s deviceId = %d", hash, deviceId) 539 devices.markDeviceIdFree(deviceId) 540 return nil, err 541 } 542 543 for { 544 if err := devicemapper.CreateDevice(devices.getPoolDevName(), deviceId); err != nil { 545 if devicemapper.DeviceIdExists(err) { 546 // Device Id already exists. This should not 547 // happen. Now we have a mechianism to find 548 // a free device Id. So something is not right. 549 // Give a warning and continue. 550 log.Errorf("Warning: Device Id %d exists in pool but it is supposed to be unused", deviceId) 551 deviceId, err = devices.getNextFreeDeviceId() 552 if err != nil { 553 return nil, err 554 } 555 // Save new device id into transaction 556 devices.refreshTransaction(deviceId) 557 continue 558 } 559 log.Debugf("Error creating device: %s", err) 560 devices.markDeviceIdFree(deviceId) 561 return nil, err 562 } 563 break 564 } 565 566 log.Debugf("Registering device (id %v) with FS size %v", deviceId, devices.baseFsSize) 567 info, err := devices.registerDevice(deviceId, hash, devices.baseFsSize, devices.OpenTransactionId) 568 if err != nil { 569 _ = devicemapper.DeleteDevice(devices.getPoolDevName(), deviceId) 570 devices.markDeviceIdFree(deviceId) 571 return nil, err 572 } 573 574 if err := devices.closeTransaction(); err != nil { 575 devices.unregisterDevice(deviceId, hash) 576 devicemapper.DeleteDevice(devices.getPoolDevName(), deviceId) 577 devices.markDeviceIdFree(deviceId) 578 return nil, err 579 } 580 return info, nil 581 } 582 583 func (devices *DeviceSet) createRegisterSnapDevice(hash string, baseInfo *DevInfo) error { 584 deviceId, err := devices.getNextFreeDeviceId() 585 if err != nil { 586 return err 587 } 588 589 if err := devices.openTransaction(hash, deviceId); err != nil { 590 log.Debugf("Error opening transaction hash = %s deviceId = %d", hash, deviceId) 591 devices.markDeviceIdFree(deviceId) 592 return err 593 } 594 595 for { 596 if err := devicemapper.CreateSnapDevice(devices.getPoolDevName(), deviceId, baseInfo.Name(), baseInfo.DeviceId); err != nil { 597 if devicemapper.DeviceIdExists(err) { 598 // Device Id already exists. This should not 599 // happen. Now we have a mechianism to find 600 // a free device Id. So something is not right. 601 // Give a warning and continue. 602 log.Errorf("Warning: Device Id %d exists in pool but it is supposed to be unused", deviceId) 603 deviceId, err = devices.getNextFreeDeviceId() 604 if err != nil { 605 return err 606 } 607 // Save new device id into transaction 608 devices.refreshTransaction(deviceId) 609 continue 610 } 611 log.Debugf("Error creating snap device: %s", err) 612 devices.markDeviceIdFree(deviceId) 613 return err 614 } 615 break 616 } 617 618 if _, err := devices.registerDevice(deviceId, hash, baseInfo.Size, devices.OpenTransactionId); err != nil { 619 devicemapper.DeleteDevice(devices.getPoolDevName(), deviceId) 620 devices.markDeviceIdFree(deviceId) 621 log.Debugf("Error registering device: %s", err) 622 return err 623 } 624 625 if err := devices.closeTransaction(); err != nil { 626 devices.unregisterDevice(deviceId, hash) 627 devicemapper.DeleteDevice(devices.getPoolDevName(), deviceId) 628 devices.markDeviceIdFree(deviceId) 629 return err 630 } 631 return nil 632 } 633 634 func (devices *DeviceSet) loadMetadata(hash string) *DevInfo { 635 info := &DevInfo{Hash: hash, devices: devices} 636 637 jsonData, err := ioutil.ReadFile(devices.metadataFile(info)) 638 if err != nil { 639 return nil 640 } 641 642 if err := json.Unmarshal(jsonData, &info); err != nil { 643 return nil 644 } 645 646 return info 647 } 648 649 func (devices *DeviceSet) setupBaseImage() error { 650 oldInfo, _ := devices.lookupDevice("") 651 if oldInfo != nil && oldInfo.Initialized { 652 return nil 653 } 654 655 if oldInfo != nil && !oldInfo.Initialized { 656 log.Debugf("Removing uninitialized base image") 657 if err := devices.DeleteDevice(""); err != nil { 658 return err 659 } 660 } 661 662 if devices.thinPoolDevice != "" && oldInfo == nil { 663 _, transactionId, dataUsed, _, _, _, err := devices.poolStatus() 664 if err != nil { 665 return err 666 } 667 if dataUsed != 0 { 668 return fmt.Errorf("Unable to take ownership of thin-pool (%s) that already has used data blocks", 669 devices.thinPoolDevice) 670 } 671 if transactionId != 0 { 672 return fmt.Errorf("Unable to take ownership of thin-pool (%s) with non-zero transaction Id", 673 devices.thinPoolDevice) 674 } 675 } 676 677 log.Debugf("Initializing base device-mapper thin volume") 678 679 // Create initial device 680 info, err := devices.createRegisterDevice("") 681 if err != nil { 682 return err 683 } 684 685 log.Debugf("Creating filesystem on base device-mapper thin volume") 686 687 if err = devices.activateDeviceIfNeeded(info); err != nil { 688 return err 689 } 690 691 if err := devices.createFilesystem(info); err != nil { 692 return err 693 } 694 695 info.Initialized = true 696 if err = devices.saveMetadata(info); err != nil { 697 info.Initialized = false 698 return err 699 } 700 701 return nil 702 } 703 704 func setCloseOnExec(name string) { 705 if fileInfos, _ := ioutil.ReadDir("/proc/self/fd"); fileInfos != nil { 706 for _, i := range fileInfos { 707 link, _ := os.Readlink(filepath.Join("/proc/self/fd", i.Name())) 708 if link == name { 709 fd, err := strconv.Atoi(i.Name()) 710 if err == nil { 711 syscall.CloseOnExec(fd) 712 } 713 } 714 } 715 } 716 } 717 718 func (devices *DeviceSet) DMLog(level int, file string, line int, dmError int, message string) { 719 if level >= 7 { 720 return // Ignore _LOG_DEBUG 721 } 722 723 // FIXME(vbatts) push this back into ./pkg/devicemapper/ 724 log.Debugf("libdevmapper(%d): %s:%d (%d) %s", level, file, line, dmError, message) 725 } 726 727 func major(device uint64) uint64 { 728 return (device >> 8) & 0xfff 729 } 730 731 func minor(device uint64) uint64 { 732 return (device & 0xff) | ((device >> 12) & 0xfff00) 733 } 734 735 func (devices *DeviceSet) ResizePool(size int64) error { 736 dirname := devices.loopbackDir() 737 datafilename := path.Join(dirname, "data") 738 if len(devices.dataDevice) > 0 { 739 datafilename = devices.dataDevice 740 } 741 metadatafilename := path.Join(dirname, "metadata") 742 if len(devices.metadataDevice) > 0 { 743 metadatafilename = devices.metadataDevice 744 } 745 746 datafile, err := os.OpenFile(datafilename, os.O_RDWR, 0) 747 if datafile == nil { 748 return err 749 } 750 defer datafile.Close() 751 752 fi, err := datafile.Stat() 753 if fi == nil { 754 return err 755 } 756 757 if fi.Size() > size { 758 return fmt.Errorf("Can't shrink file") 759 } 760 761 dataloopback := devicemapper.FindLoopDeviceFor(datafile) 762 if dataloopback == nil { 763 return fmt.Errorf("Unable to find loopback mount for: %s", datafilename) 764 } 765 defer dataloopback.Close() 766 767 metadatafile, err := os.OpenFile(metadatafilename, os.O_RDWR, 0) 768 if metadatafile == nil { 769 return err 770 } 771 defer metadatafile.Close() 772 773 metadataloopback := devicemapper.FindLoopDeviceFor(metadatafile) 774 if metadataloopback == nil { 775 return fmt.Errorf("Unable to find loopback mount for: %s", metadatafilename) 776 } 777 defer metadataloopback.Close() 778 779 // Grow loopback file 780 if err := datafile.Truncate(size); err != nil { 781 return fmt.Errorf("Unable to grow loopback file: %s", err) 782 } 783 784 // Reload size for loopback device 785 if err := devicemapper.LoopbackSetCapacity(dataloopback); err != nil { 786 return fmt.Errorf("Unable to update loopback capacity: %s", err) 787 } 788 789 // Suspend the pool 790 if err := devicemapper.SuspendDevice(devices.getPoolName()); err != nil { 791 return fmt.Errorf("Unable to suspend pool: %s", err) 792 } 793 794 // Reload with the new block sizes 795 if err := devicemapper.ReloadPool(devices.getPoolName(), dataloopback, metadataloopback, devices.thinpBlockSize); err != nil { 796 return fmt.Errorf("Unable to reload pool: %s", err) 797 } 798 799 // Resume the pool 800 if err := devicemapper.ResumeDevice(devices.getPoolName()); err != nil { 801 return fmt.Errorf("Unable to resume pool: %s", err) 802 } 803 804 return nil 805 } 806 807 func (devices *DeviceSet) loadTransactionMetaData() error { 808 jsonData, err := ioutil.ReadFile(devices.transactionMetaFile()) 809 if err != nil { 810 // There is no active transaction. This will be the case 811 // during upgrade. 812 if os.IsNotExist(err) { 813 devices.OpenTransactionId = devices.TransactionId 814 return nil 815 } 816 return err 817 } 818 819 json.Unmarshal(jsonData, &devices.Transaction) 820 return nil 821 } 822 823 func (devices *DeviceSet) saveTransactionMetaData() error { 824 jsonData, err := json.Marshal(&devices.Transaction) 825 if err != nil { 826 return fmt.Errorf("Error encoding metadata to json: %s", err) 827 } 828 829 return devices.writeMetaFile(jsonData, devices.transactionMetaFile()) 830 } 831 832 func (devices *DeviceSet) removeTransactionMetaData() error { 833 if err := os.RemoveAll(devices.transactionMetaFile()); err != nil { 834 return err 835 } 836 return nil 837 } 838 839 func (devices *DeviceSet) rollbackTransaction() error { 840 log.Debugf("Rolling back open transaction: TransactionId=%d hash=%s device_id=%d", devices.OpenTransactionId, devices.DeviceIdHash, devices.DeviceId) 841 842 // A device id might have already been deleted before transaction 843 // closed. In that case this call will fail. Just leave a message 844 // in case of failure. 845 if err := devicemapper.DeleteDevice(devices.getPoolDevName(), devices.DeviceId); err != nil { 846 log.Errorf("Warning: Unable to delete device: %s", err) 847 } 848 849 dinfo := &DevInfo{Hash: devices.DeviceIdHash} 850 if err := devices.removeMetadata(dinfo); err != nil { 851 log.Errorf("Warning: Unable to remove meta data: %s", err) 852 } else { 853 devices.markDeviceIdFree(devices.DeviceId) 854 } 855 856 if err := devices.removeTransactionMetaData(); err != nil { 857 log.Errorf("Warning: Unable to remove transaction meta file %s: %s", devices.transactionMetaFile(), err) 858 } 859 860 return nil 861 } 862 863 func (devices *DeviceSet) processPendingTransaction() error { 864 if err := devices.loadTransactionMetaData(); err != nil { 865 return err 866 } 867 868 // If there was open transaction but pool transaction Id is same 869 // as open transaction Id, nothing to roll back. 870 if devices.TransactionId == devices.OpenTransactionId { 871 return nil 872 } 873 874 // If open transaction Id is less than pool transaction Id, something 875 // is wrong. Bail out. 876 if devices.OpenTransactionId < devices.TransactionId { 877 log.Errorf("Warning: Open Transaction id %d is less than pool transaction id %d", devices.OpenTransactionId, devices.TransactionId) 878 return nil 879 } 880 881 // Pool transaction Id is not same as open transaction. There is 882 // a transaction which was not completed. 883 if err := devices.rollbackTransaction(); err != nil { 884 return fmt.Errorf("Rolling back open transaction failed: %s", err) 885 } 886 887 devices.OpenTransactionId = devices.TransactionId 888 return nil 889 } 890 891 func (devices *DeviceSet) loadDeviceSetMetaData() error { 892 jsonData, err := ioutil.ReadFile(devices.deviceSetMetaFile()) 893 if err != nil { 894 // For backward compatibility return success if file does 895 // not exist. 896 if os.IsNotExist(err) { 897 return nil 898 } 899 return err 900 } 901 902 return json.Unmarshal(jsonData, devices) 903 } 904 905 func (devices *DeviceSet) saveDeviceSetMetaData() error { 906 jsonData, err := json.Marshal(devices) 907 if err != nil { 908 return fmt.Errorf("Error encoding metadata to json: %s", err) 909 } 910 911 return devices.writeMetaFile(jsonData, devices.deviceSetMetaFile()) 912 } 913 914 func (devices *DeviceSet) openTransaction(hash string, DeviceId int) error { 915 devices.allocateTransactionId() 916 devices.DeviceIdHash = hash 917 devices.DeviceId = DeviceId 918 if err := devices.saveTransactionMetaData(); err != nil { 919 return fmt.Errorf("Error saving transaction meta data: %s", err) 920 } 921 return nil 922 } 923 924 func (devices *DeviceSet) refreshTransaction(DeviceId int) error { 925 devices.DeviceId = DeviceId 926 if err := devices.saveTransactionMetaData(); err != nil { 927 return fmt.Errorf("Error saving transaction meta data: %s", err) 928 } 929 return nil 930 } 931 932 func (devices *DeviceSet) closeTransaction() error { 933 if err := devices.updatePoolTransactionId(); err != nil { 934 log.Debugf("Failed to close Transaction") 935 return err 936 } 937 return nil 938 } 939 940 func (devices *DeviceSet) initDevmapper(doInit bool) error { 941 // give ourselves to libdm as a log handler 942 devicemapper.LogInit(devices) 943 944 _, err := devicemapper.GetDriverVersion() 945 if err != nil { 946 // Can't even get driver version, assume not supported 947 return graphdriver.ErrNotSupported 948 } 949 950 if err := os.MkdirAll(devices.metadataDir(), 0700); err != nil && !os.IsExist(err) { 951 return err 952 } 953 954 // Set the device prefix from the device id and inode of the docker root dir 955 956 st, err := os.Stat(devices.root) 957 if err != nil { 958 return fmt.Errorf("Error looking up dir %s: %s", devices.root, err) 959 } 960 sysSt := st.Sys().(*syscall.Stat_t) 961 // "reg-" stands for "regular file". 962 // In the future we might use "dev-" for "device file", etc. 963 // docker-maj,min[-inode] stands for: 964 // - Managed by docker 965 // - The target of this device is at major <maj> and minor <min> 966 // - If <inode> is defined, use that file inside the device as a loopback image. Otherwise use the device itself. 967 devices.devicePrefix = fmt.Sprintf("docker-%d:%d-%d", major(sysSt.Dev), minor(sysSt.Dev), sysSt.Ino) 968 log.Debugf("Generated prefix: %s", devices.devicePrefix) 969 970 // Check for the existence of the thin-pool device 971 log.Debugf("Checking for existence of the pool '%s'", devices.getPoolName()) 972 info, err := devicemapper.GetInfo(devices.getPoolName()) 973 if info == nil { 974 log.Debugf("Error device devicemapper.GetInfo: %s", err) 975 return err 976 } 977 978 // It seems libdevmapper opens this without O_CLOEXEC, and go exec will not close files 979 // that are not Close-on-exec, and lxc-start will die if it inherits any unexpected files, 980 // so we add this badhack to make sure it closes itself 981 setCloseOnExec("/dev/mapper/control") 982 983 // Make sure the sparse images exist in <root>/devicemapper/data and 984 // <root>/devicemapper/metadata 985 986 createdLoopback := false 987 988 // If the pool doesn't exist, create it 989 if info.Exists == 0 && devices.thinPoolDevice == "" { 990 log.Debugf("Pool doesn't exist. Creating it.") 991 992 var ( 993 dataFile *os.File 994 metadataFile *os.File 995 ) 996 997 if devices.dataDevice == "" { 998 // Make sure the sparse images exist in <root>/devicemapper/data 999 1000 hasData := devices.hasImage("data") 1001 1002 if !doInit && !hasData { 1003 return errors.New("Loopback data file not found") 1004 } 1005 1006 if !hasData { 1007 createdLoopback = true 1008 } 1009 1010 data, err := devices.ensureImage("data", devices.dataLoopbackSize) 1011 if err != nil { 1012 log.Debugf("Error device ensureImage (data): %s", err) 1013 return err 1014 } 1015 1016 dataFile, err = devicemapper.AttachLoopDevice(data) 1017 if err != nil { 1018 return err 1019 } 1020 devices.dataLoopFile = data 1021 devices.dataDevice = dataFile.Name() 1022 } else { 1023 dataFile, err = os.OpenFile(devices.dataDevice, os.O_RDWR, 0600) 1024 if err != nil { 1025 return err 1026 } 1027 } 1028 defer dataFile.Close() 1029 1030 if devices.metadataDevice == "" { 1031 // Make sure the sparse images exist in <root>/devicemapper/metadata 1032 1033 hasMetadata := devices.hasImage("metadata") 1034 1035 if !doInit && !hasMetadata { 1036 return errors.New("Loopback metadata file not found") 1037 } 1038 1039 if !hasMetadata { 1040 createdLoopback = true 1041 } 1042 1043 metadata, err := devices.ensureImage("metadata", devices.metaDataLoopbackSize) 1044 if err != nil { 1045 log.Debugf("Error device ensureImage (metadata): %s", err) 1046 return err 1047 } 1048 1049 metadataFile, err = devicemapper.AttachLoopDevice(metadata) 1050 if err != nil { 1051 return err 1052 } 1053 devices.metadataLoopFile = metadata 1054 devices.metadataDevice = metadataFile.Name() 1055 } else { 1056 metadataFile, err = os.OpenFile(devices.metadataDevice, os.O_RDWR, 0600) 1057 if err != nil { 1058 return err 1059 } 1060 } 1061 defer metadataFile.Close() 1062 1063 if err := devicemapper.CreatePool(devices.getPoolName(), dataFile, metadataFile, devices.thinpBlockSize); err != nil { 1064 return err 1065 } 1066 } 1067 1068 // If we didn't just create the data or metadata image, we need to 1069 // load the transaction id and migrate old metadata 1070 if !createdLoopback { 1071 if err = devices.initMetaData(); err != nil { 1072 return err 1073 } 1074 } 1075 1076 // Right now this loads only NextDeviceId. If there is more metatadata 1077 // down the line, we might have to move it earlier. 1078 if err = devices.loadDeviceSetMetaData(); err != nil { 1079 return err 1080 } 1081 1082 // Setup the base image 1083 if doInit { 1084 if err := devices.setupBaseImage(); err != nil { 1085 log.Debugf("Error device setupBaseImage: %s", err) 1086 return err 1087 } 1088 } 1089 1090 return nil 1091 } 1092 1093 func (devices *DeviceSet) AddDevice(hash, baseHash string) error { 1094 log.Debugf("[deviceset] AddDevice() hash=%s basehash=%s", hash, baseHash) 1095 defer log.Debugf("[deviceset] AddDevice END") 1096 1097 baseInfo, err := devices.lookupDevice(baseHash) 1098 if err != nil { 1099 return err 1100 } 1101 1102 baseInfo.lock.Lock() 1103 defer baseInfo.lock.Unlock() 1104 1105 devices.Lock() 1106 defer devices.Unlock() 1107 1108 if info, _ := devices.lookupDevice(hash); info != nil { 1109 return fmt.Errorf("device %s already exists", hash) 1110 } 1111 1112 if err := devices.createRegisterSnapDevice(hash, baseInfo); err != nil { 1113 return err 1114 } 1115 1116 return nil 1117 } 1118 1119 func (devices *DeviceSet) deleteDevice(info *DevInfo) error { 1120 if devices.doBlkDiscard { 1121 // This is a workaround for the kernel not discarding block so 1122 // on the thin pool when we remove a thinp device, so we do it 1123 // manually 1124 if err := devices.activateDeviceIfNeeded(info); err == nil { 1125 if err := devicemapper.BlockDeviceDiscard(info.DevName()); err != nil { 1126 log.Debugf("Error discarding block on device: %s (ignoring)", err) 1127 } 1128 } 1129 } 1130 1131 devinfo, _ := devicemapper.GetInfo(info.Name()) 1132 if devinfo != nil && devinfo.Exists != 0 { 1133 if err := devices.removeDeviceAndWait(info.Name()); err != nil { 1134 log.Debugf("Error removing device: %s", err) 1135 return err 1136 } 1137 } 1138 1139 if err := devices.openTransaction(info.Hash, info.DeviceId); err != nil { 1140 log.Debugf("Error opening transaction hash = %s deviceId = %d", "", info.DeviceId) 1141 return err 1142 } 1143 1144 if err := devicemapper.DeleteDevice(devices.getPoolDevName(), info.DeviceId); err != nil { 1145 log.Debugf("Error deleting device: %s", err) 1146 return err 1147 } 1148 1149 if err := devices.unregisterDevice(info.DeviceId, info.Hash); err != nil { 1150 return err 1151 } 1152 1153 if err := devices.closeTransaction(); err != nil { 1154 return err 1155 } 1156 1157 devices.markDeviceIdFree(info.DeviceId) 1158 1159 return nil 1160 } 1161 1162 func (devices *DeviceSet) DeleteDevice(hash string) error { 1163 info, err := devices.lookupDevice(hash) 1164 if err != nil { 1165 return err 1166 } 1167 1168 info.lock.Lock() 1169 defer info.lock.Unlock() 1170 1171 devices.Lock() 1172 defer devices.Unlock() 1173 1174 return devices.deleteDevice(info) 1175 } 1176 1177 func (devices *DeviceSet) deactivatePool() error { 1178 log.Debugf("[devmapper] deactivatePool()") 1179 defer log.Debugf("[devmapper] deactivatePool END") 1180 devname := devices.getPoolDevName() 1181 1182 devinfo, err := devicemapper.GetInfo(devname) 1183 if err != nil { 1184 return err 1185 } 1186 if d, err := devicemapper.GetDeps(devname); err == nil { 1187 // Access to more Debug output 1188 log.Debugf("[devmapper] devicemapper.GetDeps() %s: %#v", devname, d) 1189 } 1190 if devinfo.Exists != 0 { 1191 return devicemapper.RemoveDevice(devname) 1192 } 1193 1194 return nil 1195 } 1196 1197 func (devices *DeviceSet) deactivateDevice(info *DevInfo) error { 1198 log.Debugf("[devmapper] deactivateDevice(%s)", info.Hash) 1199 defer log.Debugf("[devmapper] deactivateDevice END") 1200 1201 // Wait for the unmount to be effective, 1202 // by watching the value of Info.OpenCount for the device 1203 if err := devices.waitClose(info); err != nil { 1204 log.Errorf("Warning: error waiting for device %s to close: %s", info.Hash, err) 1205 } 1206 1207 devinfo, err := devicemapper.GetInfo(info.Name()) 1208 if err != nil { 1209 return err 1210 } 1211 if devinfo.Exists != 0 { 1212 if err := devices.removeDeviceAndWait(info.Name()); err != nil { 1213 return err 1214 } 1215 } 1216 1217 return nil 1218 } 1219 1220 // Issues the underlying dm remove operation and then waits 1221 // for it to finish. 1222 func (devices *DeviceSet) removeDeviceAndWait(devname string) error { 1223 var err error 1224 1225 for i := 0; i < 1000; i++ { 1226 err = devicemapper.RemoveDevice(devname) 1227 if err == nil { 1228 break 1229 } 1230 if err != devicemapper.ErrBusy { 1231 return err 1232 } 1233 1234 // If we see EBUSY it may be a transient error, 1235 // sleep a bit a retry a few times. 1236 devices.Unlock() 1237 time.Sleep(10 * time.Millisecond) 1238 devices.Lock() 1239 } 1240 if err != nil { 1241 return err 1242 } 1243 1244 if err := devices.waitRemove(devname); err != nil { 1245 return err 1246 } 1247 return nil 1248 } 1249 1250 // waitRemove blocks until either: 1251 // a) the device registered at <device_set_prefix>-<hash> is removed, 1252 // or b) the 10 second timeout expires. 1253 func (devices *DeviceSet) waitRemove(devname string) error { 1254 log.Debugf("[deviceset %s] waitRemove(%s)", devices.devicePrefix, devname) 1255 defer log.Debugf("[deviceset %s] waitRemove(%s) END", devices.devicePrefix, devname) 1256 i := 0 1257 for ; i < 1000; i++ { 1258 devinfo, err := devicemapper.GetInfo(devname) 1259 if err != nil { 1260 // If there is an error we assume the device doesn't exist. 1261 // The error might actually be something else, but we can't differentiate. 1262 return nil 1263 } 1264 if i%100 == 0 { 1265 log.Debugf("Waiting for removal of %s: exists=%d", devname, devinfo.Exists) 1266 } 1267 if devinfo.Exists == 0 { 1268 break 1269 } 1270 1271 devices.Unlock() 1272 time.Sleep(10 * time.Millisecond) 1273 devices.Lock() 1274 } 1275 if i == 1000 { 1276 return fmt.Errorf("Timeout while waiting for device %s to be removed", devname) 1277 } 1278 return nil 1279 } 1280 1281 // waitClose blocks until either: 1282 // a) the device registered at <device_set_prefix>-<hash> is closed, 1283 // or b) the 10 second timeout expires. 1284 func (devices *DeviceSet) waitClose(info *DevInfo) error { 1285 i := 0 1286 for ; i < 1000; i++ { 1287 devinfo, err := devicemapper.GetInfo(info.Name()) 1288 if err != nil { 1289 return err 1290 } 1291 if i%100 == 0 { 1292 log.Debugf("Waiting for unmount of %s: opencount=%d", info.Hash, devinfo.OpenCount) 1293 } 1294 if devinfo.OpenCount == 0 { 1295 break 1296 } 1297 devices.Unlock() 1298 time.Sleep(10 * time.Millisecond) 1299 devices.Lock() 1300 } 1301 if i == 1000 { 1302 return fmt.Errorf("Timeout while waiting for device %s to close", info.Hash) 1303 } 1304 return nil 1305 } 1306 1307 func (devices *DeviceSet) Shutdown() error { 1308 log.Debugf("[deviceset %s] shutdown()", devices.devicePrefix) 1309 log.Debugf("[devmapper] Shutting down DeviceSet: %s", devices.root) 1310 defer log.Debugf("[deviceset %s] shutdown END", devices.devicePrefix) 1311 1312 var devs []*DevInfo 1313 1314 devices.devicesLock.Lock() 1315 for _, info := range devices.Devices { 1316 devs = append(devs, info) 1317 } 1318 devices.devicesLock.Unlock() 1319 1320 for _, info := range devs { 1321 info.lock.Lock() 1322 if info.mountCount > 0 { 1323 // We use MNT_DETACH here in case it is still busy in some running 1324 // container. This means it'll go away from the global scope directly, 1325 // and the device will be released when that container dies. 1326 if err := syscall.Unmount(info.mountPath, syscall.MNT_DETACH); err != nil { 1327 log.Debugf("Shutdown unmounting %s, error: %s", info.mountPath, err) 1328 } 1329 1330 devices.Lock() 1331 if err := devices.deactivateDevice(info); err != nil { 1332 log.Debugf("Shutdown deactivate %s , error: %s", info.Hash, err) 1333 } 1334 devices.Unlock() 1335 } 1336 info.lock.Unlock() 1337 } 1338 1339 info, _ := devices.lookupDevice("") 1340 if info != nil { 1341 info.lock.Lock() 1342 devices.Lock() 1343 if err := devices.deactivateDevice(info); err != nil { 1344 log.Debugf("Shutdown deactivate base , error: %s", err) 1345 } 1346 devices.Unlock() 1347 info.lock.Unlock() 1348 } 1349 1350 devices.Lock() 1351 if devices.thinPoolDevice == "" { 1352 if err := devices.deactivatePool(); err != nil { 1353 log.Debugf("Shutdown deactivate pool , error: %s", err) 1354 } 1355 } 1356 1357 devices.saveDeviceSetMetaData() 1358 devices.Unlock() 1359 1360 return nil 1361 } 1362 1363 func (devices *DeviceSet) MountDevice(hash, path, mountLabel string) error { 1364 info, err := devices.lookupDevice(hash) 1365 if err != nil { 1366 return err 1367 } 1368 1369 info.lock.Lock() 1370 defer info.lock.Unlock() 1371 1372 devices.Lock() 1373 defer devices.Unlock() 1374 1375 if info.mountCount > 0 { 1376 if path != info.mountPath { 1377 return fmt.Errorf("Trying to mount devmapper device in multple places (%s, %s)", info.mountPath, path) 1378 } 1379 1380 info.mountCount++ 1381 return nil 1382 } 1383 1384 if err := devices.activateDeviceIfNeeded(info); err != nil { 1385 return fmt.Errorf("Error activating devmapper device for '%s': %s", hash, err) 1386 } 1387 1388 var flags uintptr = syscall.MS_MGC_VAL 1389 1390 fstype, err := ProbeFsType(info.DevName()) 1391 if err != nil { 1392 return err 1393 } 1394 1395 options := "" 1396 1397 if fstype == "xfs" { 1398 // XFS needs nouuid or it can't mount filesystems with the same fs 1399 options = joinMountOptions(options, "nouuid") 1400 } 1401 1402 options = joinMountOptions(options, devices.mountOptions) 1403 options = joinMountOptions(options, label.FormatMountLabel("", mountLabel)) 1404 1405 err = syscall.Mount(info.DevName(), path, fstype, flags, joinMountOptions("discard", options)) 1406 if err != nil && err == syscall.EINVAL { 1407 err = syscall.Mount(info.DevName(), path, fstype, flags, options) 1408 } 1409 if err != nil { 1410 return fmt.Errorf("Error mounting '%s' on '%s': %s", info.DevName(), path, err) 1411 } 1412 1413 info.mountCount = 1 1414 info.mountPath = path 1415 1416 return nil 1417 } 1418 1419 func (devices *DeviceSet) UnmountDevice(hash string) error { 1420 log.Debugf("[devmapper] UnmountDevice(hash=%s)", hash) 1421 defer log.Debugf("[devmapper] UnmountDevice END") 1422 1423 info, err := devices.lookupDevice(hash) 1424 if err != nil { 1425 return err 1426 } 1427 1428 info.lock.Lock() 1429 defer info.lock.Unlock() 1430 1431 devices.Lock() 1432 defer devices.Unlock() 1433 1434 if info.mountCount == 0 { 1435 return fmt.Errorf("UnmountDevice: device not-mounted id %s", hash) 1436 } 1437 1438 info.mountCount-- 1439 if info.mountCount > 0 { 1440 return nil 1441 } 1442 1443 log.Debugf("[devmapper] Unmount(%s)", info.mountPath) 1444 if err := syscall.Unmount(info.mountPath, syscall.MNT_DETACH); err != nil { 1445 return err 1446 } 1447 log.Debugf("[devmapper] Unmount done") 1448 1449 if err := devices.deactivateDevice(info); err != nil { 1450 return err 1451 } 1452 1453 info.mountPath = "" 1454 1455 return nil 1456 } 1457 1458 func (devices *DeviceSet) HasDevice(hash string) bool { 1459 devices.Lock() 1460 defer devices.Unlock() 1461 1462 info, _ := devices.lookupDevice(hash) 1463 return info != nil 1464 } 1465 1466 func (devices *DeviceSet) HasActivatedDevice(hash string) bool { 1467 info, _ := devices.lookupDevice(hash) 1468 if info == nil { 1469 return false 1470 } 1471 1472 info.lock.Lock() 1473 defer info.lock.Unlock() 1474 1475 devices.Lock() 1476 defer devices.Unlock() 1477 1478 devinfo, _ := devicemapper.GetInfo(info.Name()) 1479 return devinfo != nil && devinfo.Exists != 0 1480 } 1481 1482 func (devices *DeviceSet) List() []string { 1483 devices.Lock() 1484 defer devices.Unlock() 1485 1486 devices.devicesLock.Lock() 1487 ids := make([]string, len(devices.Devices)) 1488 i := 0 1489 for k := range devices.Devices { 1490 ids[i] = k 1491 i++ 1492 } 1493 devices.devicesLock.Unlock() 1494 1495 return ids 1496 } 1497 1498 func (devices *DeviceSet) deviceStatus(devName string) (sizeInSectors, mappedSectors, highestMappedSector uint64, err error) { 1499 var params string 1500 _, sizeInSectors, _, params, err = devicemapper.GetStatus(devName) 1501 if err != nil { 1502 return 1503 } 1504 if _, err = fmt.Sscanf(params, "%d %d", &mappedSectors, &highestMappedSector); err == nil { 1505 return 1506 } 1507 return 1508 } 1509 1510 func (devices *DeviceSet) GetDeviceStatus(hash string) (*DevStatus, error) { 1511 info, err := devices.lookupDevice(hash) 1512 if err != nil { 1513 return nil, err 1514 } 1515 1516 info.lock.Lock() 1517 defer info.lock.Unlock() 1518 1519 devices.Lock() 1520 defer devices.Unlock() 1521 1522 status := &DevStatus{ 1523 DeviceId: info.DeviceId, 1524 Size: info.Size, 1525 TransactionId: info.TransactionId, 1526 } 1527 1528 if err := devices.activateDeviceIfNeeded(info); err != nil { 1529 return nil, fmt.Errorf("Error activating devmapper device for '%s': %s", hash, err) 1530 } 1531 1532 if sizeInSectors, mappedSectors, highestMappedSector, err := devices.deviceStatus(info.DevName()); err != nil { 1533 return nil, err 1534 } else { 1535 status.SizeInSectors = sizeInSectors 1536 status.MappedSectors = mappedSectors 1537 status.HighestMappedSector = highestMappedSector 1538 } 1539 1540 return status, nil 1541 } 1542 1543 func (devices *DeviceSet) poolStatus() (totalSizeInSectors, transactionId, dataUsed, dataTotal, metadataUsed, metadataTotal uint64, err error) { 1544 var params string 1545 if _, totalSizeInSectors, _, params, err = devicemapper.GetStatus(devices.getPoolName()); err == nil { 1546 _, err = fmt.Sscanf(params, "%d %d/%d %d/%d", &transactionId, &metadataUsed, &metadataTotal, &dataUsed, &dataTotal) 1547 } 1548 return 1549 } 1550 1551 // MetadataDevicePath returns the path to the metadata storage for this deviceset, 1552 // regardless of loopback or block device 1553 func (devices *DeviceSet) DataDevicePath() string { 1554 return devices.dataDevice 1555 } 1556 1557 // MetadataDevicePath returns the path to the metadata storage for this deviceset, 1558 // regardless of loopback or block device 1559 func (devices *DeviceSet) MetadataDevicePath() string { 1560 return devices.metadataDevice 1561 } 1562 1563 // Status returns the current status of this deviceset 1564 func (devices *DeviceSet) Status() *Status { 1565 devices.Lock() 1566 defer devices.Unlock() 1567 1568 status := &Status{} 1569 1570 status.PoolName = devices.getPoolName() 1571 status.DataFile = devices.DataDevicePath() 1572 status.DataLoopback = devices.dataLoopFile 1573 status.MetadataFile = devices.MetadataDevicePath() 1574 status.MetadataLoopback = devices.metadataLoopFile 1575 1576 totalSizeInSectors, _, dataUsed, dataTotal, metadataUsed, metadataTotal, err := devices.poolStatus() 1577 if err == nil { 1578 // Convert from blocks to bytes 1579 blockSizeInSectors := totalSizeInSectors / dataTotal 1580 1581 status.Data.Used = dataUsed * blockSizeInSectors * 512 1582 status.Data.Total = dataTotal * blockSizeInSectors * 512 1583 1584 // metadata blocks are always 4k 1585 status.Metadata.Used = metadataUsed * 4096 1586 status.Metadata.Total = metadataTotal * 4096 1587 1588 status.SectorSize = blockSizeInSectors * 512 1589 } 1590 1591 return status 1592 } 1593 1594 func NewDeviceSet(root string, doInit bool, options []string) (*DeviceSet, error) { 1595 devicemapper.SetDevDir("/dev") 1596 1597 devices := &DeviceSet{ 1598 root: root, 1599 MetaData: MetaData{Devices: make(map[string]*DevInfo)}, 1600 dataLoopbackSize: DefaultDataLoopbackSize, 1601 metaDataLoopbackSize: DefaultMetaDataLoopbackSize, 1602 baseFsSize: DefaultBaseFsSize, 1603 filesystem: "ext4", 1604 doBlkDiscard: true, 1605 thinpBlockSize: DefaultThinpBlockSize, 1606 deviceIdMap: make([]byte, DeviceIdMapSz), 1607 } 1608 1609 foundBlkDiscard := false 1610 for _, option := range options { 1611 key, val, err := parsers.ParseKeyValueOpt(option) 1612 if err != nil { 1613 return nil, err 1614 } 1615 key = strings.ToLower(key) 1616 switch key { 1617 case "dm.basesize": 1618 size, err := units.RAMInBytes(val) 1619 if err != nil { 1620 return nil, err 1621 } 1622 devices.baseFsSize = uint64(size) 1623 case "dm.loopdatasize": 1624 size, err := units.RAMInBytes(val) 1625 if err != nil { 1626 return nil, err 1627 } 1628 devices.dataLoopbackSize = size 1629 case "dm.loopmetadatasize": 1630 size, err := units.RAMInBytes(val) 1631 if err != nil { 1632 return nil, err 1633 } 1634 devices.metaDataLoopbackSize = size 1635 case "dm.fs": 1636 if val != "ext4" && val != "xfs" { 1637 return nil, fmt.Errorf("Unsupported filesystem %s\n", val) 1638 } 1639 devices.filesystem = val 1640 case "dm.mkfsarg": 1641 devices.mkfsArgs = append(devices.mkfsArgs, val) 1642 case "dm.mountopt": 1643 devices.mountOptions = joinMountOptions(devices.mountOptions, val) 1644 case "dm.metadatadev": 1645 devices.metadataDevice = val 1646 case "dm.datadev": 1647 devices.dataDevice = val 1648 case "dm.thinpooldev": 1649 devices.thinPoolDevice = strings.TrimPrefix(val, "/dev/mapper/") 1650 case "dm.blkdiscard": 1651 foundBlkDiscard = true 1652 devices.doBlkDiscard, err = strconv.ParseBool(val) 1653 if err != nil { 1654 return nil, err 1655 } 1656 case "dm.blocksize": 1657 size, err := units.RAMInBytes(val) 1658 if err != nil { 1659 return nil, err 1660 } 1661 // convert to 512b sectors 1662 devices.thinpBlockSize = uint32(size) >> 9 1663 default: 1664 return nil, fmt.Errorf("Unknown option %s\n", key) 1665 } 1666 } 1667 1668 // By default, don't do blk discard hack on raw devices, its rarely useful and is expensive 1669 if !foundBlkDiscard && (devices.dataDevice != "" || devices.thinPoolDevice != "") { 1670 devices.doBlkDiscard = false 1671 } 1672 1673 if err := devices.initDevmapper(doInit); err != nil { 1674 return nil, err 1675 } 1676 1677 return devices, nil 1678 }