github.com/pritambaral/docker@v1.4.2-0.20150120174542-b2fe1b3dd952/daemon/graphdriver/devmapper/deviceset.go (about) 1 // +build linux 2 3 package devmapper 4 5 import ( 6 "encoding/json" 7 "errors" 8 "fmt" 9 "io" 10 "io/ioutil" 11 "os" 12 "os/exec" 13 "path" 14 "path/filepath" 15 "strconv" 16 "strings" 17 "sync" 18 "syscall" 19 "time" 20 21 log "github.com/Sirupsen/logrus" 22 "github.com/docker/docker/daemon/graphdriver" 23 "github.com/docker/docker/pkg/devicemapper" 24 "github.com/docker/docker/pkg/parsers" 25 "github.com/docker/docker/pkg/units" 26 "github.com/docker/libcontainer/label" 27 ) 28 29 var ( 30 DefaultDataLoopbackSize int64 = 100 * 1024 * 1024 * 1024 31 DefaultMetaDataLoopbackSize int64 = 2 * 1024 * 1024 * 1024 32 DefaultBaseFsSize uint64 = 10 * 1024 * 1024 * 1024 33 DefaultThinpBlockSize uint32 = 128 // 64K = 128 512b sectors 34 MaxDeviceId int = 0xffffff // 24 bit, pool limit 35 DeviceIdMapSz int = (MaxDeviceId + 1) / 8 36 ) 37 38 const deviceSetMetaFile string = "deviceset-metadata" 39 const transactionMetaFile string = "transaction-metadata" 40 41 type Transaction struct { 42 OpenTransactionId uint64 `json:"open_transaction_id"` 43 DeviceIdHash string `json:"device_hash"` 44 DeviceId int `json:"device_id"` 45 } 46 47 type DevInfo struct { 48 Hash string `json:"-"` 49 DeviceId int `json:"device_id"` 50 Size uint64 `json:"size"` 51 TransactionId uint64 `json:"transaction_id"` 52 Initialized bool `json:"initialized"` 53 devices *DeviceSet 54 55 mountCount int 56 mountPath string 57 58 // The global DeviceSet lock guarantees that we serialize all 59 // the calls to libdevmapper (which is not threadsafe), but we 60 // sometimes release that lock while sleeping. In that case 61 // this per-device lock is still held, protecting against 62 // other accesses to the device that we're doing the wait on. 63 // 64 // WARNING: In order to avoid AB-BA deadlocks when releasing 65 // the global lock while holding the per-device locks all 66 // device locks must be aquired *before* the device lock, and 67 // multiple device locks should be aquired parent before child. 68 lock sync.Mutex 69 } 70 71 type MetaData struct { 72 Devices map[string]*DevInfo `json:"Devices"` 73 devicesLock sync.Mutex // Protects all read/writes to Devices map 74 } 75 76 type DeviceSet struct { 77 MetaData `json:"-"` 78 sync.Mutex `json:"-"` // Protects Devices map and serializes calls into libdevmapper 79 root string 80 devicePrefix string 81 TransactionId uint64 `json:"-"` 82 NextDeviceId int `json:"next_device_id"` 83 deviceIdMap []byte 84 85 // Options 86 dataLoopbackSize int64 87 metaDataLoopbackSize int64 88 baseFsSize uint64 89 filesystem string 90 mountOptions string 91 mkfsArgs []string 92 dataDevice string // block or loop dev 93 dataLoopFile string // loopback file, if used 94 metadataDevice string // block or loop dev 95 metadataLoopFile string // loopback file, if used 96 doBlkDiscard bool 97 thinpBlockSize uint32 98 thinPoolDevice string 99 Transaction `json:"-"` 100 } 101 102 type DiskUsage struct { 103 Used uint64 104 Total uint64 105 } 106 107 type Status struct { 108 PoolName string 109 DataFile string // actual block device for data 110 DataLoopback string // loopback file, if used 111 MetadataFile string // actual block device for metadata 112 MetadataLoopback string // loopback file, if used 113 Data DiskUsage 114 Metadata DiskUsage 115 SectorSize uint64 116 UdevSyncSupported bool 117 } 118 119 type DevStatus struct { 120 DeviceId int 121 Size uint64 122 TransactionId uint64 123 SizeInSectors uint64 124 MappedSectors uint64 125 HighestMappedSector uint64 126 } 127 128 func getDevName(name string) string { 129 return "/dev/mapper/" + name 130 } 131 132 func (info *DevInfo) Name() string { 133 hash := info.Hash 134 if hash == "" { 135 hash = "base" 136 } 137 return fmt.Sprintf("%s-%s", info.devices.devicePrefix, hash) 138 } 139 140 func (info *DevInfo) DevName() string { 141 return getDevName(info.Name()) 142 } 143 144 func (devices *DeviceSet) loopbackDir() string { 145 return path.Join(devices.root, "devicemapper") 146 } 147 148 func (devices *DeviceSet) metadataDir() string { 149 return path.Join(devices.root, "metadata") 150 } 151 152 func (devices *DeviceSet) metadataFile(info *DevInfo) string { 153 file := info.Hash 154 if file == "" { 155 file = "base" 156 } 157 return path.Join(devices.metadataDir(), file) 158 } 159 160 func (devices *DeviceSet) transactionMetaFile() string { 161 return path.Join(devices.metadataDir(), transactionMetaFile) 162 } 163 164 func (devices *DeviceSet) deviceSetMetaFile() string { 165 return path.Join(devices.metadataDir(), deviceSetMetaFile) 166 } 167 168 func (devices *DeviceSet) oldMetadataFile() string { 169 return path.Join(devices.loopbackDir(), "json") 170 } 171 172 func (devices *DeviceSet) getPoolName() string { 173 if devices.thinPoolDevice == "" { 174 return devices.devicePrefix + "-pool" 175 } 176 return devices.thinPoolDevice 177 } 178 179 func (devices *DeviceSet) getPoolDevName() string { 180 return getDevName(devices.getPoolName()) 181 } 182 183 func (devices *DeviceSet) hasImage(name string) bool { 184 dirname := devices.loopbackDir() 185 filename := path.Join(dirname, name) 186 187 _, err := os.Stat(filename) 188 return err == nil 189 } 190 191 // ensureImage creates a sparse file of <size> bytes at the path 192 // <root>/devicemapper/<name>. 193 // If the file already exists, it does nothing. 194 // Either way it returns the full path. 195 func (devices *DeviceSet) ensureImage(name string, size int64) (string, error) { 196 dirname := devices.loopbackDir() 197 filename := path.Join(dirname, name) 198 199 if err := os.MkdirAll(dirname, 0700); err != nil && !os.IsExist(err) { 200 return "", err 201 } 202 203 if _, err := os.Stat(filename); err != nil { 204 if !os.IsNotExist(err) { 205 return "", err 206 } 207 log.Debugf("Creating loopback file %s for device-manage use", filename) 208 file, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE, 0600) 209 if err != nil { 210 return "", err 211 } 212 defer file.Close() 213 214 if err = file.Truncate(size); err != nil { 215 return "", err 216 } 217 } 218 return filename, nil 219 } 220 221 func (devices *DeviceSet) allocateTransactionId() uint64 { 222 devices.OpenTransactionId = devices.TransactionId + 1 223 return devices.OpenTransactionId 224 } 225 226 func (devices *DeviceSet) updatePoolTransactionId() error { 227 if err := devicemapper.SetTransactionId(devices.getPoolDevName(), devices.TransactionId, devices.OpenTransactionId); err != nil { 228 return fmt.Errorf("Error setting devmapper transaction ID: %s", err) 229 } 230 devices.TransactionId = devices.OpenTransactionId 231 return nil 232 } 233 234 func (devices *DeviceSet) removeMetadata(info *DevInfo) error { 235 if err := os.RemoveAll(devices.metadataFile(info)); err != nil { 236 return fmt.Errorf("Error removing metadata file %s: %s", devices.metadataFile(info), err) 237 } 238 return nil 239 } 240 241 // Given json data and file path, write it to disk 242 func (devices *DeviceSet) writeMetaFile(jsonData []byte, filePath string) error { 243 tmpFile, err := ioutil.TempFile(devices.metadataDir(), ".tmp") 244 if err != nil { 245 return fmt.Errorf("Error creating metadata file: %s", err) 246 } 247 248 n, err := tmpFile.Write(jsonData) 249 if err != nil { 250 return fmt.Errorf("Error writing metadata to %s: %s", tmpFile.Name(), err) 251 } 252 if n < len(jsonData) { 253 return io.ErrShortWrite 254 } 255 if err := tmpFile.Sync(); err != nil { 256 return fmt.Errorf("Error syncing metadata file %s: %s", tmpFile.Name(), err) 257 } 258 if err := tmpFile.Close(); err != nil { 259 return fmt.Errorf("Error closing metadata file %s: %s", tmpFile.Name(), err) 260 } 261 if err := os.Rename(tmpFile.Name(), filePath); err != nil { 262 return fmt.Errorf("Error committing metadata file %s: %s", tmpFile.Name(), err) 263 } 264 265 return nil 266 } 267 268 func (devices *DeviceSet) saveMetadata(info *DevInfo) error { 269 jsonData, err := json.Marshal(info) 270 if err != nil { 271 return fmt.Errorf("Error encoding metadata to json: %s", err) 272 } 273 if err := devices.writeMetaFile(jsonData, devices.metadataFile(info)); err != nil { 274 return err 275 } 276 return nil 277 } 278 279 func (devices *DeviceSet) markDeviceIdUsed(deviceId int) { 280 var mask byte 281 i := deviceId % 8 282 mask = 1 << uint(i) 283 devices.deviceIdMap[deviceId/8] = devices.deviceIdMap[deviceId/8] | mask 284 } 285 286 func (devices *DeviceSet) markDeviceIdFree(deviceId int) { 287 var mask byte 288 i := deviceId % 8 289 mask = ^(1 << uint(i)) 290 devices.deviceIdMap[deviceId/8] = devices.deviceIdMap[deviceId/8] & mask 291 } 292 293 func (devices *DeviceSet) isDeviceIdFree(deviceId int) bool { 294 var mask byte 295 i := deviceId % 8 296 mask = (1 << uint(i)) 297 if (devices.deviceIdMap[deviceId/8] & mask) != 0 { 298 return false 299 } 300 return true 301 } 302 303 func (devices *DeviceSet) lookupDevice(hash string) (*DevInfo, error) { 304 devices.devicesLock.Lock() 305 defer devices.devicesLock.Unlock() 306 info := devices.Devices[hash] 307 if info == nil { 308 info = devices.loadMetadata(hash) 309 if info == nil { 310 return nil, fmt.Errorf("Unknown device %s", hash) 311 } 312 313 devices.Devices[hash] = info 314 } 315 return info, nil 316 } 317 318 func (devices *DeviceSet) deviceFileWalkFunction(path string, finfo os.FileInfo) error { 319 320 // Skip some of the meta files which are not device files. 321 if strings.HasSuffix(finfo.Name(), ".migrated") { 322 log.Debugf("Skipping file %s", path) 323 return nil 324 } 325 326 if finfo.Name() == deviceSetMetaFile { 327 log.Debugf("Skipping file %s", path) 328 return nil 329 } 330 331 log.Debugf("Loading data for file %s", path) 332 333 hash := finfo.Name() 334 if hash == "base" { 335 hash = "" 336 } 337 338 dinfo := devices.loadMetadata(hash) 339 if dinfo == nil { 340 return fmt.Errorf("Error loading device metadata file %s", hash) 341 } 342 343 if dinfo.DeviceId > MaxDeviceId { 344 log.Errorf("Warning: Ignoring Invalid DeviceId=%d", dinfo.DeviceId) 345 return nil 346 } 347 348 devices.Lock() 349 devices.markDeviceIdUsed(dinfo.DeviceId) 350 devices.Unlock() 351 352 log.Debugf("Added deviceId=%d to DeviceIdMap", dinfo.DeviceId) 353 return nil 354 } 355 356 func (devices *DeviceSet) constructDeviceIdMap() error { 357 log.Debugf("[deviceset] constructDeviceIdMap()") 358 defer log.Debugf("[deviceset] constructDeviceIdMap() END") 359 360 var scan = func(path string, info os.FileInfo, err error) error { 361 if err != nil { 362 log.Debugf("Can't walk the file %s", path) 363 return nil 364 } 365 366 // Skip any directories 367 if info.IsDir() { 368 return nil 369 } 370 371 return devices.deviceFileWalkFunction(path, info) 372 } 373 374 return filepath.Walk(devices.metadataDir(), scan) 375 } 376 377 func (devices *DeviceSet) unregisterDevice(id int, hash string) error { 378 log.Debugf("unregisterDevice(%v, %v)", id, hash) 379 info := &DevInfo{ 380 Hash: hash, 381 DeviceId: id, 382 } 383 384 devices.devicesLock.Lock() 385 delete(devices.Devices, hash) 386 devices.devicesLock.Unlock() 387 388 if err := devices.removeMetadata(info); err != nil { 389 log.Debugf("Error removing meta data: %s", err) 390 return err 391 } 392 393 return nil 394 } 395 396 func (devices *DeviceSet) registerDevice(id int, hash string, size uint64, transactionId uint64) (*DevInfo, error) { 397 log.Debugf("registerDevice(%v, %v)", id, hash) 398 info := &DevInfo{ 399 Hash: hash, 400 DeviceId: id, 401 Size: size, 402 TransactionId: transactionId, 403 Initialized: false, 404 devices: devices, 405 } 406 407 devices.devicesLock.Lock() 408 devices.Devices[hash] = info 409 devices.devicesLock.Unlock() 410 411 if err := devices.saveMetadata(info); err != nil { 412 // Try to remove unused device 413 devices.devicesLock.Lock() 414 delete(devices.Devices, hash) 415 devices.devicesLock.Unlock() 416 return nil, err 417 } 418 419 return info, nil 420 } 421 422 func (devices *DeviceSet) activateDeviceIfNeeded(info *DevInfo) error { 423 log.Debugf("activateDeviceIfNeeded(%v)", info.Hash) 424 425 if devinfo, _ := devicemapper.GetInfo(info.Name()); devinfo != nil && devinfo.Exists != 0 { 426 return nil 427 } 428 429 return devicemapper.ActivateDevice(devices.getPoolDevName(), info.Name(), info.DeviceId, info.Size) 430 } 431 432 func (devices *DeviceSet) createFilesystem(info *DevInfo) error { 433 devname := info.DevName() 434 435 args := []string{} 436 for _, arg := range devices.mkfsArgs { 437 args = append(args, arg) 438 } 439 440 args = append(args, devname) 441 442 var err error 443 switch devices.filesystem { 444 case "xfs": 445 err = exec.Command("mkfs.xfs", args...).Run() 446 case "ext4": 447 err = exec.Command("mkfs.ext4", append([]string{"-E", "nodiscard,lazy_itable_init=0,lazy_journal_init=0"}, args...)...).Run() 448 if err != nil { 449 err = exec.Command("mkfs.ext4", append([]string{"-E", "nodiscard,lazy_itable_init=0"}, args...)...).Run() 450 } 451 if err != nil { 452 return err 453 } 454 err = exec.Command("tune2fs", append([]string{"-c", "-1", "-i", "0"}, devname)...).Run() 455 default: 456 err = fmt.Errorf("Unsupported filesystem type %s", devices.filesystem) 457 } 458 if err != nil { 459 return err 460 } 461 462 return nil 463 } 464 465 func (devices *DeviceSet) migrateOldMetaData() error { 466 // Migrate old metadatafile 467 jsonData, err := ioutil.ReadFile(devices.oldMetadataFile()) 468 if err != nil && !os.IsNotExist(err) { 469 return err 470 } 471 472 if jsonData != nil { 473 m := MetaData{Devices: make(map[string]*DevInfo)} 474 475 if err := json.Unmarshal(jsonData, &m); err != nil { 476 return err 477 } 478 479 for hash, info := range m.Devices { 480 info.Hash = hash 481 devices.saveMetadata(info) 482 } 483 if err := os.Rename(devices.oldMetadataFile(), devices.oldMetadataFile()+".migrated"); err != nil { 484 return err 485 } 486 487 } 488 489 return nil 490 } 491 492 func (devices *DeviceSet) initMetaData() error { 493 if err := devices.migrateOldMetaData(); err != nil { 494 return err 495 } 496 497 _, transactionId, _, _, _, _, err := devices.poolStatus() 498 if err != nil { 499 return err 500 } 501 502 devices.TransactionId = transactionId 503 504 if err := devices.constructDeviceIdMap(); err != nil { 505 return err 506 } 507 508 if err := devices.processPendingTransaction(); err != nil { 509 return err 510 } 511 return nil 512 } 513 514 func (devices *DeviceSet) incNextDeviceId() { 515 // Ids are 24bit, so wrap around 516 devices.NextDeviceId = (devices.NextDeviceId + 1) & MaxDeviceId 517 } 518 519 func (devices *DeviceSet) getNextFreeDeviceId() (int, error) { 520 devices.incNextDeviceId() 521 for i := 0; i <= MaxDeviceId; i++ { 522 if devices.isDeviceIdFree(devices.NextDeviceId) { 523 devices.markDeviceIdUsed(devices.NextDeviceId) 524 return devices.NextDeviceId, nil 525 } 526 devices.incNextDeviceId() 527 } 528 529 return 0, fmt.Errorf("Unable to find a free device Id") 530 } 531 532 func (devices *DeviceSet) createRegisterDevice(hash string) (*DevInfo, error) { 533 deviceId, err := devices.getNextFreeDeviceId() 534 if err != nil { 535 return nil, err 536 } 537 538 if err := devices.openTransaction(hash, deviceId); err != nil { 539 log.Debugf("Error opening transaction hash = %s deviceId = %d", hash, deviceId) 540 devices.markDeviceIdFree(deviceId) 541 return nil, err 542 } 543 544 for { 545 if err := devicemapper.CreateDevice(devices.getPoolDevName(), deviceId); err != nil { 546 if devicemapper.DeviceIdExists(err) { 547 // Device Id already exists. This should not 548 // happen. Now we have a mechianism to find 549 // a free device Id. So something is not right. 550 // Give a warning and continue. 551 log.Errorf("Warning: Device Id %d exists in pool but it is supposed to be unused", deviceId) 552 deviceId, err = devices.getNextFreeDeviceId() 553 if err != nil { 554 return nil, err 555 } 556 // Save new device id into transaction 557 devices.refreshTransaction(deviceId) 558 continue 559 } 560 log.Debugf("Error creating device: %s", err) 561 devices.markDeviceIdFree(deviceId) 562 return nil, err 563 } 564 break 565 } 566 567 log.Debugf("Registering device (id %v) with FS size %v", deviceId, devices.baseFsSize) 568 info, err := devices.registerDevice(deviceId, hash, devices.baseFsSize, devices.OpenTransactionId) 569 if err != nil { 570 _ = devicemapper.DeleteDevice(devices.getPoolDevName(), deviceId) 571 devices.markDeviceIdFree(deviceId) 572 return nil, err 573 } 574 575 if err := devices.closeTransaction(); err != nil { 576 devices.unregisterDevice(deviceId, hash) 577 devicemapper.DeleteDevice(devices.getPoolDevName(), deviceId) 578 devices.markDeviceIdFree(deviceId) 579 return nil, err 580 } 581 return info, nil 582 } 583 584 func (devices *DeviceSet) createRegisterSnapDevice(hash string, baseInfo *DevInfo) error { 585 deviceId, err := devices.getNextFreeDeviceId() 586 if err != nil { 587 return err 588 } 589 590 if err := devices.openTransaction(hash, deviceId); err != nil { 591 log.Debugf("Error opening transaction hash = %s deviceId = %d", hash, deviceId) 592 devices.markDeviceIdFree(deviceId) 593 return err 594 } 595 596 for { 597 if err := devicemapper.CreateSnapDevice(devices.getPoolDevName(), deviceId, baseInfo.Name(), baseInfo.DeviceId); err != nil { 598 if devicemapper.DeviceIdExists(err) { 599 // Device Id already exists. This should not 600 // happen. Now we have a mechianism to find 601 // a free device Id. So something is not right. 602 // Give a warning and continue. 603 log.Errorf("Warning: Device Id %d exists in pool but it is supposed to be unused", deviceId) 604 deviceId, err = devices.getNextFreeDeviceId() 605 if err != nil { 606 return err 607 } 608 // Save new device id into transaction 609 devices.refreshTransaction(deviceId) 610 continue 611 } 612 log.Debugf("Error creating snap device: %s", err) 613 devices.markDeviceIdFree(deviceId) 614 return err 615 } 616 break 617 } 618 619 if _, err := devices.registerDevice(deviceId, hash, baseInfo.Size, devices.OpenTransactionId); err != nil { 620 devicemapper.DeleteDevice(devices.getPoolDevName(), deviceId) 621 devices.markDeviceIdFree(deviceId) 622 log.Debugf("Error registering device: %s", err) 623 return err 624 } 625 626 if err := devices.closeTransaction(); err != nil { 627 devices.unregisterDevice(deviceId, hash) 628 devicemapper.DeleteDevice(devices.getPoolDevName(), deviceId) 629 devices.markDeviceIdFree(deviceId) 630 return err 631 } 632 return nil 633 } 634 635 func (devices *DeviceSet) loadMetadata(hash string) *DevInfo { 636 info := &DevInfo{Hash: hash, devices: devices} 637 638 jsonData, err := ioutil.ReadFile(devices.metadataFile(info)) 639 if err != nil { 640 return nil 641 } 642 643 if err := json.Unmarshal(jsonData, &info); err != nil { 644 return nil 645 } 646 647 return info 648 } 649 650 func (devices *DeviceSet) setupBaseImage() error { 651 oldInfo, _ := devices.lookupDevice("") 652 if oldInfo != nil && oldInfo.Initialized { 653 return nil 654 } 655 656 if oldInfo != nil && !oldInfo.Initialized { 657 log.Debugf("Removing uninitialized base image") 658 if err := devices.DeleteDevice(""); err != nil { 659 return err 660 } 661 } 662 663 if devices.thinPoolDevice != "" && oldInfo == nil { 664 _, transactionId, dataUsed, _, _, _, err := devices.poolStatus() 665 if err != nil { 666 return err 667 } 668 if dataUsed != 0 { 669 return fmt.Errorf("Unable to take ownership of thin-pool (%s) that already has used data blocks", 670 devices.thinPoolDevice) 671 } 672 if transactionId != 0 { 673 return fmt.Errorf("Unable to take ownership of thin-pool (%s) with non-zero transaction Id", 674 devices.thinPoolDevice) 675 } 676 } 677 678 log.Debugf("Initializing base device-mapper thin volume") 679 680 // Create initial device 681 info, err := devices.createRegisterDevice("") 682 if err != nil { 683 return err 684 } 685 686 log.Debugf("Creating filesystem on base device-mapper thin volume") 687 688 if err = devices.activateDeviceIfNeeded(info); err != nil { 689 return err 690 } 691 692 if err := devices.createFilesystem(info); err != nil { 693 return err 694 } 695 696 info.Initialized = true 697 if err = devices.saveMetadata(info); err != nil { 698 info.Initialized = false 699 return err 700 } 701 702 return nil 703 } 704 705 func setCloseOnExec(name string) { 706 if fileInfos, _ := ioutil.ReadDir("/proc/self/fd"); fileInfos != nil { 707 for _, i := range fileInfos { 708 link, _ := os.Readlink(filepath.Join("/proc/self/fd", i.Name())) 709 if link == name { 710 fd, err := strconv.Atoi(i.Name()) 711 if err == nil { 712 syscall.CloseOnExec(fd) 713 } 714 } 715 } 716 } 717 } 718 719 func (devices *DeviceSet) DMLog(level int, file string, line int, dmError int, message string) { 720 if level >= 7 { 721 return // Ignore _LOG_DEBUG 722 } 723 724 // FIXME(vbatts) push this back into ./pkg/devicemapper/ 725 log.Debugf("libdevmapper(%d): %s:%d (%d) %s", level, file, line, dmError, message) 726 } 727 728 func major(device uint64) uint64 { 729 return (device >> 8) & 0xfff 730 } 731 732 func minor(device uint64) uint64 { 733 return (device & 0xff) | ((device >> 12) & 0xfff00) 734 } 735 736 func (devices *DeviceSet) ResizePool(size int64) error { 737 dirname := devices.loopbackDir() 738 datafilename := path.Join(dirname, "data") 739 if len(devices.dataDevice) > 0 { 740 datafilename = devices.dataDevice 741 } 742 metadatafilename := path.Join(dirname, "metadata") 743 if len(devices.metadataDevice) > 0 { 744 metadatafilename = devices.metadataDevice 745 } 746 747 datafile, err := os.OpenFile(datafilename, os.O_RDWR, 0) 748 if datafile == nil { 749 return err 750 } 751 defer datafile.Close() 752 753 fi, err := datafile.Stat() 754 if fi == nil { 755 return err 756 } 757 758 if fi.Size() > size { 759 return fmt.Errorf("Can't shrink file") 760 } 761 762 dataloopback := devicemapper.FindLoopDeviceFor(datafile) 763 if dataloopback == nil { 764 return fmt.Errorf("Unable to find loopback mount for: %s", datafilename) 765 } 766 defer dataloopback.Close() 767 768 metadatafile, err := os.OpenFile(metadatafilename, os.O_RDWR, 0) 769 if metadatafile == nil { 770 return err 771 } 772 defer metadatafile.Close() 773 774 metadataloopback := devicemapper.FindLoopDeviceFor(metadatafile) 775 if metadataloopback == nil { 776 return fmt.Errorf("Unable to find loopback mount for: %s", metadatafilename) 777 } 778 defer metadataloopback.Close() 779 780 // Grow loopback file 781 if err := datafile.Truncate(size); err != nil { 782 return fmt.Errorf("Unable to grow loopback file: %s", err) 783 } 784 785 // Reload size for loopback device 786 if err := devicemapper.LoopbackSetCapacity(dataloopback); err != nil { 787 return fmt.Errorf("Unable to update loopback capacity: %s", err) 788 } 789 790 // Suspend the pool 791 if err := devicemapper.SuspendDevice(devices.getPoolName()); err != nil { 792 return fmt.Errorf("Unable to suspend pool: %s", err) 793 } 794 795 // Reload with the new block sizes 796 if err := devicemapper.ReloadPool(devices.getPoolName(), dataloopback, metadataloopback, devices.thinpBlockSize); err != nil { 797 return fmt.Errorf("Unable to reload pool: %s", err) 798 } 799 800 // Resume the pool 801 if err := devicemapper.ResumeDevice(devices.getPoolName()); err != nil { 802 return fmt.Errorf("Unable to resume pool: %s", err) 803 } 804 805 return nil 806 } 807 808 func (devices *DeviceSet) loadTransactionMetaData() error { 809 jsonData, err := ioutil.ReadFile(devices.transactionMetaFile()) 810 if err != nil { 811 // There is no active transaction. This will be the case 812 // during upgrade. 813 if os.IsNotExist(err) { 814 devices.OpenTransactionId = devices.TransactionId 815 return nil 816 } 817 return err 818 } 819 820 json.Unmarshal(jsonData, &devices.Transaction) 821 return nil 822 } 823 824 func (devices *DeviceSet) saveTransactionMetaData() error { 825 jsonData, err := json.Marshal(&devices.Transaction) 826 if err != nil { 827 return fmt.Errorf("Error encoding metadata to json: %s", err) 828 } 829 830 return devices.writeMetaFile(jsonData, devices.transactionMetaFile()) 831 } 832 833 func (devices *DeviceSet) removeTransactionMetaData() error { 834 if err := os.RemoveAll(devices.transactionMetaFile()); err != nil { 835 return err 836 } 837 return nil 838 } 839 840 func (devices *DeviceSet) rollbackTransaction() error { 841 log.Debugf("Rolling back open transaction: TransactionId=%d hash=%s device_id=%d", devices.OpenTransactionId, devices.DeviceIdHash, devices.DeviceId) 842 843 // A device id might have already been deleted before transaction 844 // closed. In that case this call will fail. Just leave a message 845 // in case of failure. 846 if err := devicemapper.DeleteDevice(devices.getPoolDevName(), devices.DeviceId); err != nil { 847 log.Errorf("Warning: Unable to delete device: %s", err) 848 } 849 850 dinfo := &DevInfo{Hash: devices.DeviceIdHash} 851 if err := devices.removeMetadata(dinfo); err != nil { 852 log.Errorf("Warning: Unable to remove meta data: %s", err) 853 } else { 854 devices.markDeviceIdFree(devices.DeviceId) 855 } 856 857 if err := devices.removeTransactionMetaData(); err != nil { 858 log.Errorf("Warning: Unable to remove transaction meta file %s: %s", devices.transactionMetaFile(), err) 859 } 860 861 return nil 862 } 863 864 func (devices *DeviceSet) processPendingTransaction() error { 865 if err := devices.loadTransactionMetaData(); err != nil { 866 return err 867 } 868 869 // If there was open transaction but pool transaction Id is same 870 // as open transaction Id, nothing to roll back. 871 if devices.TransactionId == devices.OpenTransactionId { 872 return nil 873 } 874 875 // If open transaction Id is less than pool transaction Id, something 876 // is wrong. Bail out. 877 if devices.OpenTransactionId < devices.TransactionId { 878 log.Errorf("Warning: Open Transaction id %d is less than pool transaction id %d", devices.OpenTransactionId, devices.TransactionId) 879 return nil 880 } 881 882 // Pool transaction Id is not same as open transaction. There is 883 // a transaction which was not completed. 884 if err := devices.rollbackTransaction(); err != nil { 885 return fmt.Errorf("Rolling back open transaction failed: %s", err) 886 } 887 888 devices.OpenTransactionId = devices.TransactionId 889 return nil 890 } 891 892 func (devices *DeviceSet) loadDeviceSetMetaData() error { 893 jsonData, err := ioutil.ReadFile(devices.deviceSetMetaFile()) 894 if err != nil { 895 // For backward compatibility return success if file does 896 // not exist. 897 if os.IsNotExist(err) { 898 return nil 899 } 900 return err 901 } 902 903 return json.Unmarshal(jsonData, devices) 904 } 905 906 func (devices *DeviceSet) saveDeviceSetMetaData() error { 907 jsonData, err := json.Marshal(devices) 908 if err != nil { 909 return fmt.Errorf("Error encoding metadata to json: %s", err) 910 } 911 912 return devices.writeMetaFile(jsonData, devices.deviceSetMetaFile()) 913 } 914 915 func (devices *DeviceSet) openTransaction(hash string, DeviceId int) error { 916 devices.allocateTransactionId() 917 devices.DeviceIdHash = hash 918 devices.DeviceId = DeviceId 919 if err := devices.saveTransactionMetaData(); err != nil { 920 return fmt.Errorf("Error saving transaction meta data: %s", err) 921 } 922 return nil 923 } 924 925 func (devices *DeviceSet) refreshTransaction(DeviceId int) error { 926 devices.DeviceId = DeviceId 927 if err := devices.saveTransactionMetaData(); err != nil { 928 return fmt.Errorf("Error saving transaction meta data: %s", err) 929 } 930 return nil 931 } 932 933 func (devices *DeviceSet) closeTransaction() error { 934 if err := devices.updatePoolTransactionId(); err != nil { 935 log.Debugf("Failed to close Transaction") 936 return err 937 } 938 return nil 939 } 940 941 func (devices *DeviceSet) initDevmapper(doInit bool) error { 942 // give ourselves to libdm as a log handler 943 devicemapper.LogInit(devices) 944 945 _, err := devicemapper.GetDriverVersion() 946 if err != nil { 947 // Can't even get driver version, assume not supported 948 return graphdriver.ErrNotSupported 949 } 950 951 // https://github.com/docker/docker/issues/4036 952 if supported := devicemapper.UdevSetSyncSupport(true); !supported { 953 log.Warnf("WARNING: Udev sync is not supported. This will lead to unexpected behavior, data loss and errors") 954 } 955 log.Debugf("devicemapper: udev sync support: %v", devicemapper.UdevSyncSupported()) 956 957 if err := os.MkdirAll(devices.metadataDir(), 0700); err != nil && !os.IsExist(err) { 958 return err 959 } 960 961 // Set the device prefix from the device id and inode of the docker root dir 962 963 st, err := os.Stat(devices.root) 964 if err != nil { 965 return fmt.Errorf("Error looking up dir %s: %s", devices.root, err) 966 } 967 sysSt := st.Sys().(*syscall.Stat_t) 968 // "reg-" stands for "regular file". 969 // In the future we might use "dev-" for "device file", etc. 970 // docker-maj,min[-inode] stands for: 971 // - Managed by docker 972 // - The target of this device is at major <maj> and minor <min> 973 // - If <inode> is defined, use that file inside the device as a loopback image. Otherwise use the device itself. 974 devices.devicePrefix = fmt.Sprintf("docker-%d:%d-%d", major(sysSt.Dev), minor(sysSt.Dev), sysSt.Ino) 975 log.Debugf("Generated prefix: %s", devices.devicePrefix) 976 977 // Check for the existence of the thin-pool device 978 log.Debugf("Checking for existence of the pool '%s'", devices.getPoolName()) 979 info, err := devicemapper.GetInfo(devices.getPoolName()) 980 if info == nil { 981 log.Debugf("Error device devicemapper.GetInfo: %s", err) 982 return err 983 } 984 985 // It seems libdevmapper opens this without O_CLOEXEC, and go exec will not close files 986 // that are not Close-on-exec, and lxc-start will die if it inherits any unexpected files, 987 // so we add this badhack to make sure it closes itself 988 setCloseOnExec("/dev/mapper/control") 989 990 // Make sure the sparse images exist in <root>/devicemapper/data and 991 // <root>/devicemapper/metadata 992 993 createdLoopback := false 994 995 // If the pool doesn't exist, create it 996 if info.Exists == 0 && devices.thinPoolDevice == "" { 997 log.Debugf("Pool doesn't exist. Creating it.") 998 999 var ( 1000 dataFile *os.File 1001 metadataFile *os.File 1002 ) 1003 1004 if devices.dataDevice == "" { 1005 // Make sure the sparse images exist in <root>/devicemapper/data 1006 1007 hasData := devices.hasImage("data") 1008 1009 if !doInit && !hasData { 1010 return errors.New("Loopback data file not found") 1011 } 1012 1013 if !hasData { 1014 createdLoopback = true 1015 } 1016 1017 data, err := devices.ensureImage("data", devices.dataLoopbackSize) 1018 if err != nil { 1019 log.Debugf("Error device ensureImage (data): %s", err) 1020 return err 1021 } 1022 1023 dataFile, err = devicemapper.AttachLoopDevice(data) 1024 if err != nil { 1025 return err 1026 } 1027 devices.dataLoopFile = data 1028 devices.dataDevice = dataFile.Name() 1029 } else { 1030 dataFile, err = os.OpenFile(devices.dataDevice, os.O_RDWR, 0600) 1031 if err != nil { 1032 return err 1033 } 1034 } 1035 defer dataFile.Close() 1036 1037 if devices.metadataDevice == "" { 1038 // Make sure the sparse images exist in <root>/devicemapper/metadata 1039 1040 hasMetadata := devices.hasImage("metadata") 1041 1042 if !doInit && !hasMetadata { 1043 return errors.New("Loopback metadata file not found") 1044 } 1045 1046 if !hasMetadata { 1047 createdLoopback = true 1048 } 1049 1050 metadata, err := devices.ensureImage("metadata", devices.metaDataLoopbackSize) 1051 if err != nil { 1052 log.Debugf("Error device ensureImage (metadata): %s", err) 1053 return err 1054 } 1055 1056 metadataFile, err = devicemapper.AttachLoopDevice(metadata) 1057 if err != nil { 1058 return err 1059 } 1060 devices.metadataLoopFile = metadata 1061 devices.metadataDevice = metadataFile.Name() 1062 } else { 1063 metadataFile, err = os.OpenFile(devices.metadataDevice, os.O_RDWR, 0600) 1064 if err != nil { 1065 return err 1066 } 1067 } 1068 defer metadataFile.Close() 1069 1070 if err := devicemapper.CreatePool(devices.getPoolName(), dataFile, metadataFile, devices.thinpBlockSize); err != nil { 1071 return err 1072 } 1073 } 1074 1075 // If we didn't just create the data or metadata image, we need to 1076 // load the transaction id and migrate old metadata 1077 if !createdLoopback { 1078 if err = devices.initMetaData(); err != nil { 1079 return err 1080 } 1081 } 1082 1083 // Right now this loads only NextDeviceId. If there is more metatadata 1084 // down the line, we might have to move it earlier. 1085 if err = devices.loadDeviceSetMetaData(); err != nil { 1086 return err 1087 } 1088 1089 // Setup the base image 1090 if doInit { 1091 if err := devices.setupBaseImage(); err != nil { 1092 log.Debugf("Error device setupBaseImage: %s", err) 1093 return err 1094 } 1095 } 1096 1097 return nil 1098 } 1099 1100 func (devices *DeviceSet) AddDevice(hash, baseHash string) error { 1101 log.Debugf("[deviceset] AddDevice() hash=%s basehash=%s", hash, baseHash) 1102 defer log.Debugf("[deviceset] AddDevice END") 1103 1104 baseInfo, err := devices.lookupDevice(baseHash) 1105 if err != nil { 1106 return err 1107 } 1108 1109 baseInfo.lock.Lock() 1110 defer baseInfo.lock.Unlock() 1111 1112 devices.Lock() 1113 defer devices.Unlock() 1114 1115 if info, _ := devices.lookupDevice(hash); info != nil { 1116 return fmt.Errorf("device %s already exists", hash) 1117 } 1118 1119 if err := devices.createRegisterSnapDevice(hash, baseInfo); err != nil { 1120 return err 1121 } 1122 1123 return nil 1124 } 1125 1126 func (devices *DeviceSet) deleteDevice(info *DevInfo) error { 1127 if devices.doBlkDiscard { 1128 // This is a workaround for the kernel not discarding block so 1129 // on the thin pool when we remove a thinp device, so we do it 1130 // manually 1131 if err := devices.activateDeviceIfNeeded(info); err == nil { 1132 if err := devicemapper.BlockDeviceDiscard(info.DevName()); err != nil { 1133 log.Debugf("Error discarding block on device: %s (ignoring)", err) 1134 } 1135 } 1136 } 1137 1138 devinfo, _ := devicemapper.GetInfo(info.Name()) 1139 if devinfo != nil && devinfo.Exists != 0 { 1140 if err := devices.removeDeviceAndWait(info.Name()); err != nil { 1141 log.Debugf("Error removing device: %s", err) 1142 return err 1143 } 1144 } 1145 1146 if err := devices.openTransaction(info.Hash, info.DeviceId); err != nil { 1147 log.Debugf("Error opening transaction hash = %s deviceId = %d", "", info.DeviceId) 1148 return err 1149 } 1150 1151 if err := devicemapper.DeleteDevice(devices.getPoolDevName(), info.DeviceId); err != nil { 1152 log.Debugf("Error deleting device: %s", err) 1153 return err 1154 } 1155 1156 if err := devices.unregisterDevice(info.DeviceId, info.Hash); err != nil { 1157 return err 1158 } 1159 1160 if err := devices.closeTransaction(); err != nil { 1161 return err 1162 } 1163 1164 devices.markDeviceIdFree(info.DeviceId) 1165 1166 return nil 1167 } 1168 1169 func (devices *DeviceSet) DeleteDevice(hash string) error { 1170 info, err := devices.lookupDevice(hash) 1171 if err != nil { 1172 return err 1173 } 1174 1175 info.lock.Lock() 1176 defer info.lock.Unlock() 1177 1178 devices.Lock() 1179 defer devices.Unlock() 1180 1181 return devices.deleteDevice(info) 1182 } 1183 1184 func (devices *DeviceSet) deactivatePool() error { 1185 log.Debugf("[devmapper] deactivatePool()") 1186 defer log.Debugf("[devmapper] deactivatePool END") 1187 devname := devices.getPoolDevName() 1188 1189 devinfo, err := devicemapper.GetInfo(devname) 1190 if err != nil { 1191 return err 1192 } 1193 if d, err := devicemapper.GetDeps(devname); err == nil { 1194 // Access to more Debug output 1195 log.Debugf("[devmapper] devicemapper.GetDeps() %s: %#v", devname, d) 1196 } 1197 if devinfo.Exists != 0 { 1198 return devicemapper.RemoveDevice(devname) 1199 } 1200 1201 return nil 1202 } 1203 1204 func (devices *DeviceSet) deactivateDevice(info *DevInfo) error { 1205 log.Debugf("[devmapper] deactivateDevice(%s)", info.Hash) 1206 defer log.Debugf("[devmapper] deactivateDevice END") 1207 1208 // Wait for the unmount to be effective, 1209 // by watching the value of Info.OpenCount for the device 1210 if err := devices.waitClose(info); err != nil { 1211 log.Errorf("Warning: error waiting for device %s to close: %s", info.Hash, err) 1212 } 1213 1214 devinfo, err := devicemapper.GetInfo(info.Name()) 1215 if err != nil { 1216 return err 1217 } 1218 if devinfo.Exists != 0 { 1219 if err := devices.removeDeviceAndWait(info.Name()); err != nil { 1220 return err 1221 } 1222 } 1223 1224 return nil 1225 } 1226 1227 // Issues the underlying dm remove operation and then waits 1228 // for it to finish. 1229 func (devices *DeviceSet) removeDeviceAndWait(devname string) error { 1230 var err error 1231 1232 for i := 0; i < 1000; i++ { 1233 err = devicemapper.RemoveDevice(devname) 1234 if err == nil { 1235 break 1236 } 1237 if err != devicemapper.ErrBusy { 1238 return err 1239 } 1240 1241 // If we see EBUSY it may be a transient error, 1242 // sleep a bit a retry a few times. 1243 devices.Unlock() 1244 time.Sleep(10 * time.Millisecond) 1245 devices.Lock() 1246 } 1247 if err != nil { 1248 return err 1249 } 1250 1251 if err := devices.waitRemove(devname); err != nil { 1252 return err 1253 } 1254 return nil 1255 } 1256 1257 // waitRemove blocks until either: 1258 // a) the device registered at <device_set_prefix>-<hash> is removed, 1259 // or b) the 10 second timeout expires. 1260 func (devices *DeviceSet) waitRemove(devname string) error { 1261 log.Debugf("[deviceset %s] waitRemove(%s)", devices.devicePrefix, devname) 1262 defer log.Debugf("[deviceset %s] waitRemove(%s) END", devices.devicePrefix, devname) 1263 i := 0 1264 for ; i < 1000; i++ { 1265 devinfo, err := devicemapper.GetInfo(devname) 1266 if err != nil { 1267 // If there is an error we assume the device doesn't exist. 1268 // The error might actually be something else, but we can't differentiate. 1269 return nil 1270 } 1271 if i%100 == 0 { 1272 log.Debugf("Waiting for removal of %s: exists=%d", devname, devinfo.Exists) 1273 } 1274 if devinfo.Exists == 0 { 1275 break 1276 } 1277 1278 devices.Unlock() 1279 time.Sleep(10 * time.Millisecond) 1280 devices.Lock() 1281 } 1282 if i == 1000 { 1283 return fmt.Errorf("Timeout while waiting for device %s to be removed", devname) 1284 } 1285 return nil 1286 } 1287 1288 // waitClose blocks until either: 1289 // a) the device registered at <device_set_prefix>-<hash> is closed, 1290 // or b) the 10 second timeout expires. 1291 func (devices *DeviceSet) waitClose(info *DevInfo) error { 1292 i := 0 1293 for ; i < 1000; i++ { 1294 devinfo, err := devicemapper.GetInfo(info.Name()) 1295 if err != nil { 1296 return err 1297 } 1298 if i%100 == 0 { 1299 log.Debugf("Waiting for unmount of %s: opencount=%d", info.Hash, devinfo.OpenCount) 1300 } 1301 if devinfo.OpenCount == 0 { 1302 break 1303 } 1304 devices.Unlock() 1305 time.Sleep(10 * time.Millisecond) 1306 devices.Lock() 1307 } 1308 if i == 1000 { 1309 return fmt.Errorf("Timeout while waiting for device %s to close", info.Hash) 1310 } 1311 return nil 1312 } 1313 1314 func (devices *DeviceSet) Shutdown() error { 1315 log.Debugf("[deviceset %s] shutdown()", devices.devicePrefix) 1316 log.Debugf("[devmapper] Shutting down DeviceSet: %s", devices.root) 1317 defer log.Debugf("[deviceset %s] shutdown END", devices.devicePrefix) 1318 1319 var devs []*DevInfo 1320 1321 devices.devicesLock.Lock() 1322 for _, info := range devices.Devices { 1323 devs = append(devs, info) 1324 } 1325 devices.devicesLock.Unlock() 1326 1327 for _, info := range devs { 1328 info.lock.Lock() 1329 if info.mountCount > 0 { 1330 // We use MNT_DETACH here in case it is still busy in some running 1331 // container. This means it'll go away from the global scope directly, 1332 // and the device will be released when that container dies. 1333 if err := syscall.Unmount(info.mountPath, syscall.MNT_DETACH); err != nil { 1334 log.Debugf("Shutdown unmounting %s, error: %s", info.mountPath, err) 1335 } 1336 1337 devices.Lock() 1338 if err := devices.deactivateDevice(info); err != nil { 1339 log.Debugf("Shutdown deactivate %s , error: %s", info.Hash, err) 1340 } 1341 devices.Unlock() 1342 } 1343 info.lock.Unlock() 1344 } 1345 1346 info, _ := devices.lookupDevice("") 1347 if info != nil { 1348 info.lock.Lock() 1349 devices.Lock() 1350 if err := devices.deactivateDevice(info); err != nil { 1351 log.Debugf("Shutdown deactivate base , error: %s", err) 1352 } 1353 devices.Unlock() 1354 info.lock.Unlock() 1355 } 1356 1357 devices.Lock() 1358 if devices.thinPoolDevice == "" { 1359 if err := devices.deactivatePool(); err != nil { 1360 log.Debugf("Shutdown deactivate pool , error: %s", err) 1361 } 1362 } 1363 1364 devices.saveDeviceSetMetaData() 1365 devices.Unlock() 1366 1367 return nil 1368 } 1369 1370 func (devices *DeviceSet) MountDevice(hash, path, mountLabel string) error { 1371 info, err := devices.lookupDevice(hash) 1372 if err != nil { 1373 return err 1374 } 1375 1376 info.lock.Lock() 1377 defer info.lock.Unlock() 1378 1379 devices.Lock() 1380 defer devices.Unlock() 1381 1382 if info.mountCount > 0 { 1383 if path != info.mountPath { 1384 return fmt.Errorf("Trying to mount devmapper device in multple places (%s, %s)", info.mountPath, path) 1385 } 1386 1387 info.mountCount++ 1388 return nil 1389 } 1390 1391 if err := devices.activateDeviceIfNeeded(info); err != nil { 1392 return fmt.Errorf("Error activating devmapper device for '%s': %s", hash, err) 1393 } 1394 1395 var flags uintptr = syscall.MS_MGC_VAL 1396 1397 fstype, err := ProbeFsType(info.DevName()) 1398 if err != nil { 1399 return err 1400 } 1401 1402 options := "" 1403 1404 if fstype == "xfs" { 1405 // XFS needs nouuid or it can't mount filesystems with the same fs 1406 options = joinMountOptions(options, "nouuid") 1407 } 1408 1409 options = joinMountOptions(options, devices.mountOptions) 1410 options = joinMountOptions(options, label.FormatMountLabel("", mountLabel)) 1411 1412 err = syscall.Mount(info.DevName(), path, fstype, flags, joinMountOptions("discard", options)) 1413 if err != nil && err == syscall.EINVAL { 1414 err = syscall.Mount(info.DevName(), path, fstype, flags, options) 1415 } 1416 if err != nil { 1417 return fmt.Errorf("Error mounting '%s' on '%s': %s", info.DevName(), path, err) 1418 } 1419 1420 info.mountCount = 1 1421 info.mountPath = path 1422 1423 return nil 1424 } 1425 1426 func (devices *DeviceSet) UnmountDevice(hash string) error { 1427 log.Debugf("[devmapper] UnmountDevice(hash=%s)", hash) 1428 defer log.Debugf("[devmapper] UnmountDevice END") 1429 1430 info, err := devices.lookupDevice(hash) 1431 if err != nil { 1432 return err 1433 } 1434 1435 info.lock.Lock() 1436 defer info.lock.Unlock() 1437 1438 devices.Lock() 1439 defer devices.Unlock() 1440 1441 if info.mountCount == 0 { 1442 return fmt.Errorf("UnmountDevice: device not-mounted id %s", hash) 1443 } 1444 1445 info.mountCount-- 1446 if info.mountCount > 0 { 1447 return nil 1448 } 1449 1450 log.Debugf("[devmapper] Unmount(%s)", info.mountPath) 1451 if err := syscall.Unmount(info.mountPath, syscall.MNT_DETACH); err != nil { 1452 return err 1453 } 1454 log.Debugf("[devmapper] Unmount done") 1455 1456 if err := devices.deactivateDevice(info); err != nil { 1457 return err 1458 } 1459 1460 info.mountPath = "" 1461 1462 return nil 1463 } 1464 1465 func (devices *DeviceSet) HasDevice(hash string) bool { 1466 devices.Lock() 1467 defer devices.Unlock() 1468 1469 info, _ := devices.lookupDevice(hash) 1470 return info != nil 1471 } 1472 1473 func (devices *DeviceSet) HasActivatedDevice(hash string) bool { 1474 info, _ := devices.lookupDevice(hash) 1475 if info == nil { 1476 return false 1477 } 1478 1479 info.lock.Lock() 1480 defer info.lock.Unlock() 1481 1482 devices.Lock() 1483 defer devices.Unlock() 1484 1485 devinfo, _ := devicemapper.GetInfo(info.Name()) 1486 return devinfo != nil && devinfo.Exists != 0 1487 } 1488 1489 func (devices *DeviceSet) List() []string { 1490 devices.Lock() 1491 defer devices.Unlock() 1492 1493 devices.devicesLock.Lock() 1494 ids := make([]string, len(devices.Devices)) 1495 i := 0 1496 for k := range devices.Devices { 1497 ids[i] = k 1498 i++ 1499 } 1500 devices.devicesLock.Unlock() 1501 1502 return ids 1503 } 1504 1505 func (devices *DeviceSet) deviceStatus(devName string) (sizeInSectors, mappedSectors, highestMappedSector uint64, err error) { 1506 var params string 1507 _, sizeInSectors, _, params, err = devicemapper.GetStatus(devName) 1508 if err != nil { 1509 return 1510 } 1511 if _, err = fmt.Sscanf(params, "%d %d", &mappedSectors, &highestMappedSector); err == nil { 1512 return 1513 } 1514 return 1515 } 1516 1517 func (devices *DeviceSet) GetDeviceStatus(hash string) (*DevStatus, error) { 1518 info, err := devices.lookupDevice(hash) 1519 if err != nil { 1520 return nil, err 1521 } 1522 1523 info.lock.Lock() 1524 defer info.lock.Unlock() 1525 1526 devices.Lock() 1527 defer devices.Unlock() 1528 1529 status := &DevStatus{ 1530 DeviceId: info.DeviceId, 1531 Size: info.Size, 1532 TransactionId: info.TransactionId, 1533 } 1534 1535 if err := devices.activateDeviceIfNeeded(info); err != nil { 1536 return nil, fmt.Errorf("Error activating devmapper device for '%s': %s", hash, err) 1537 } 1538 1539 if sizeInSectors, mappedSectors, highestMappedSector, err := devices.deviceStatus(info.DevName()); err != nil { 1540 return nil, err 1541 } else { 1542 status.SizeInSectors = sizeInSectors 1543 status.MappedSectors = mappedSectors 1544 status.HighestMappedSector = highestMappedSector 1545 } 1546 1547 return status, nil 1548 } 1549 1550 func (devices *DeviceSet) poolStatus() (totalSizeInSectors, transactionId, dataUsed, dataTotal, metadataUsed, metadataTotal uint64, err error) { 1551 var params string 1552 if _, totalSizeInSectors, _, params, err = devicemapper.GetStatus(devices.getPoolName()); err == nil { 1553 _, err = fmt.Sscanf(params, "%d %d/%d %d/%d", &transactionId, &metadataUsed, &metadataTotal, &dataUsed, &dataTotal) 1554 } 1555 return 1556 } 1557 1558 // MetadataDevicePath returns the path to the metadata storage for this deviceset, 1559 // regardless of loopback or block device 1560 func (devices *DeviceSet) DataDevicePath() string { 1561 return devices.dataDevice 1562 } 1563 1564 // MetadataDevicePath returns the path to the metadata storage for this deviceset, 1565 // regardless of loopback or block device 1566 func (devices *DeviceSet) MetadataDevicePath() string { 1567 return devices.metadataDevice 1568 } 1569 1570 // Status returns the current status of this deviceset 1571 func (devices *DeviceSet) Status() *Status { 1572 devices.Lock() 1573 defer devices.Unlock() 1574 1575 status := &Status{} 1576 1577 status.PoolName = devices.getPoolName() 1578 status.DataFile = devices.DataDevicePath() 1579 status.DataLoopback = devices.dataLoopFile 1580 status.MetadataFile = devices.MetadataDevicePath() 1581 status.MetadataLoopback = devices.metadataLoopFile 1582 status.UdevSyncSupported = devicemapper.UdevSyncSupported() 1583 1584 totalSizeInSectors, _, dataUsed, dataTotal, metadataUsed, metadataTotal, err := devices.poolStatus() 1585 if err == nil { 1586 // Convert from blocks to bytes 1587 blockSizeInSectors := totalSizeInSectors / dataTotal 1588 1589 status.Data.Used = dataUsed * blockSizeInSectors * 512 1590 status.Data.Total = dataTotal * blockSizeInSectors * 512 1591 1592 // metadata blocks are always 4k 1593 status.Metadata.Used = metadataUsed * 4096 1594 status.Metadata.Total = metadataTotal * 4096 1595 1596 status.SectorSize = blockSizeInSectors * 512 1597 } 1598 1599 return status 1600 } 1601 1602 func NewDeviceSet(root string, doInit bool, options []string) (*DeviceSet, error) { 1603 devicemapper.SetDevDir("/dev") 1604 1605 devices := &DeviceSet{ 1606 root: root, 1607 MetaData: MetaData{Devices: make(map[string]*DevInfo)}, 1608 dataLoopbackSize: DefaultDataLoopbackSize, 1609 metaDataLoopbackSize: DefaultMetaDataLoopbackSize, 1610 baseFsSize: DefaultBaseFsSize, 1611 filesystem: "ext4", 1612 doBlkDiscard: true, 1613 thinpBlockSize: DefaultThinpBlockSize, 1614 deviceIdMap: make([]byte, DeviceIdMapSz), 1615 } 1616 1617 foundBlkDiscard := false 1618 for _, option := range options { 1619 key, val, err := parsers.ParseKeyValueOpt(option) 1620 if err != nil { 1621 return nil, err 1622 } 1623 key = strings.ToLower(key) 1624 switch key { 1625 case "dm.basesize": 1626 size, err := units.RAMInBytes(val) 1627 if err != nil { 1628 return nil, err 1629 } 1630 devices.baseFsSize = uint64(size) 1631 case "dm.loopdatasize": 1632 size, err := units.RAMInBytes(val) 1633 if err != nil { 1634 return nil, err 1635 } 1636 devices.dataLoopbackSize = size 1637 case "dm.loopmetadatasize": 1638 size, err := units.RAMInBytes(val) 1639 if err != nil { 1640 return nil, err 1641 } 1642 devices.metaDataLoopbackSize = size 1643 case "dm.fs": 1644 if val != "ext4" && val != "xfs" { 1645 return nil, fmt.Errorf("Unsupported filesystem %s\n", val) 1646 } 1647 devices.filesystem = val 1648 case "dm.mkfsarg": 1649 devices.mkfsArgs = append(devices.mkfsArgs, val) 1650 case "dm.mountopt": 1651 devices.mountOptions = joinMountOptions(devices.mountOptions, val) 1652 case "dm.metadatadev": 1653 devices.metadataDevice = val 1654 case "dm.datadev": 1655 devices.dataDevice = val 1656 case "dm.thinpooldev": 1657 devices.thinPoolDevice = strings.TrimPrefix(val, "/dev/mapper/") 1658 case "dm.blkdiscard": 1659 foundBlkDiscard = true 1660 devices.doBlkDiscard, err = strconv.ParseBool(val) 1661 if err != nil { 1662 return nil, err 1663 } 1664 case "dm.blocksize": 1665 size, err := units.RAMInBytes(val) 1666 if err != nil { 1667 return nil, err 1668 } 1669 // convert to 512b sectors 1670 devices.thinpBlockSize = uint32(size) >> 9 1671 default: 1672 return nil, fmt.Errorf("Unknown option %s\n", key) 1673 } 1674 } 1675 1676 // By default, don't do blk discard hack on raw devices, its rarely useful and is expensive 1677 if !foundBlkDiscard && (devices.dataDevice != "" || devices.thinPoolDevice != "") { 1678 devices.doBlkDiscard = false 1679 } 1680 1681 if err := devices.initDevmapper(doInit); err != nil { 1682 return nil, err 1683 } 1684 1685 return devices, nil 1686 }