github.com/demonoid81/moby@v0.0.0-20200517203328-62dd8e17c460/daemon/graphdriver/devmapper/deviceset.go (about)

     1  // +build linux
     2  
     3  package devmapper // import "github.com/demonoid81/moby/daemon/graphdriver/devmapper"
     4  
     5  import (
     6  	"bufio"
     7  	"encoding/json"
     8  	"fmt"
     9  	"io"
    10  	"io/ioutil"
    11  	"os"
    12  	"os/exec"
    13  	"path"
    14  	"path/filepath"
    15  	"reflect"
    16  	"strconv"
    17  	"strings"
    18  	"sync"
    19  	"time"
    20  
    21  	"github.com/demonoid81/moby/daemon/graphdriver"
    22  	"github.com/demonoid81/moby/dockerversion"
    23  	"github.com/demonoid81/moby/pkg/devicemapper"
    24  	"github.com/demonoid81/moby/pkg/dmesg"
    25  	"github.com/demonoid81/moby/pkg/idtools"
    26  	"github.com/demonoid81/moby/pkg/loopback"
    27  	"github.com/demonoid81/moby/pkg/parsers"
    28  	"github.com/demonoid81/moby/pkg/parsers/kernel"
    29  	units "github.com/docker/go-units"
    30  	"github.com/moby/sys/mount"
    31  	"github.com/opencontainers/selinux/go-selinux/label"
    32  	"github.com/pkg/errors"
    33  	"github.com/sirupsen/logrus"
    34  	"golang.org/x/sys/unix"
    35  )
    36  
    37  var (
    38  	defaultDataLoopbackSize      int64  = 100 * 1024 * 1024 * 1024
    39  	defaultMetaDataLoopbackSize  int64  = 2 * 1024 * 1024 * 1024
    40  	defaultBaseFsSize            uint64 = 10 * 1024 * 1024 * 1024
    41  	defaultThinpBlockSize        uint32 = 128 // 64K = 128 512b sectors
    42  	defaultUdevSyncOverride             = false
    43  	maxDeviceID                         = 0xffffff // 24 bit, pool limit
    44  	deviceIDMapSz                       = (maxDeviceID + 1) / 8
    45  	driverDeferredRemovalSupport        = false
    46  	enableDeferredRemoval               = false
    47  	enableDeferredDeletion              = false
    48  	userBaseSize                        = false
    49  	defaultMinFreeSpacePercent   uint32 = 10
    50  	lvmSetupConfigForce          bool
    51  )
    52  
    53  const deviceSetMetaFile = "deviceset-metadata"
    54  const transactionMetaFile = "transaction-metadata"
    55  
    56  type transaction struct {
    57  	OpenTransactionID uint64 `json:"open_transaction_id"`
    58  	DeviceIDHash      string `json:"device_hash"`
    59  	DeviceID          int    `json:"device_id"`
    60  }
    61  
    62  type devInfo struct {
    63  	Hash          string `json:"-"`
    64  	DeviceID      int    `json:"device_id"`
    65  	Size          uint64 `json:"size"`
    66  	TransactionID uint64 `json:"transaction_id"`
    67  	Initialized   bool   `json:"initialized"`
    68  	Deleted       bool   `json:"deleted"`
    69  	devices       *DeviceSet
    70  
    71  	// The global DeviceSet lock guarantees that we serialize all
    72  	// the calls to libdevmapper (which is not threadsafe), but we
    73  	// sometimes release that lock while sleeping. In that case
    74  	// this per-device lock is still held, protecting against
    75  	// other accesses to the device that we're doing the wait on.
    76  	//
    77  	// WARNING: In order to avoid AB-BA deadlocks when releasing
    78  	// the global lock while holding the per-device locks all
    79  	// device locks must be acquired *before* the device lock, and
    80  	// multiple device locks should be acquired parent before child.
    81  	lock sync.Mutex
    82  }
    83  
    84  type metaData struct {
    85  	Devices map[string]*devInfo `json:"Devices"`
    86  }
    87  
    88  // DeviceSet holds information about list of devices
    89  type DeviceSet struct {
    90  	metaData      `json:"-"`
    91  	sync.Mutex    `json:"-"` // Protects all fields of DeviceSet and serializes calls into libdevmapper
    92  	root          string
    93  	devicePrefix  string
    94  	TransactionID uint64 `json:"-"`
    95  	NextDeviceID  int    `json:"next_device_id"`
    96  	deviceIDMap   []byte
    97  
    98  	// Options
    99  	dataLoopbackSize      int64
   100  	metaDataLoopbackSize  int64
   101  	baseFsSize            uint64
   102  	filesystem            string
   103  	mountOptions          string
   104  	mkfsArgs              []string
   105  	dataDevice            string // block or loop dev
   106  	dataLoopFile          string // loopback file, if used
   107  	metadataDevice        string // block or loop dev
   108  	metadataLoopFile      string // loopback file, if used
   109  	doBlkDiscard          bool
   110  	thinpBlockSize        uint32
   111  	thinPoolDevice        string
   112  	transaction           `json:"-"`
   113  	overrideUdevSyncCheck bool
   114  	deferredRemove        bool   // use deferred removal
   115  	deferredDelete        bool   // use deferred deletion
   116  	BaseDeviceUUID        string // save UUID of base device
   117  	BaseDeviceFilesystem  string // save filesystem of base device
   118  	nrDeletedDevices      uint   // number of deleted devices
   119  	deletionWorkerTicker  *time.Ticker
   120  	uidMaps               []idtools.IDMap
   121  	gidMaps               []idtools.IDMap
   122  	minFreeSpacePercent   uint32 // min free space percentage in thinpool
   123  	xfsNospaceRetries     string // max retries when xfs receives ENOSPC
   124  	lvmSetupConfig        directLVMConfig
   125  }
   126  
   127  // DiskUsage contains information about disk usage and is used when reporting Status of a device.
   128  type DiskUsage struct {
   129  	// Used bytes on the disk.
   130  	Used uint64
   131  	// Total bytes on the disk.
   132  	Total uint64
   133  	// Available bytes on the disk.
   134  	Available uint64
   135  }
   136  
   137  // Status returns the information about the device.
   138  type Status struct {
   139  	// PoolName is the name of the data pool.
   140  	PoolName string
   141  	// DataFile is the actual block device for data.
   142  	DataFile string
   143  	// DataLoopback loopback file, if used.
   144  	DataLoopback string
   145  	// MetadataFile is the actual block device for metadata.
   146  	MetadataFile string
   147  	// MetadataLoopback is the loopback file, if used.
   148  	MetadataLoopback string
   149  	// Data is the disk used for data.
   150  	Data DiskUsage
   151  	// Metadata is the disk used for meta data.
   152  	Metadata DiskUsage
   153  	// BaseDeviceSize is base size of container and image
   154  	BaseDeviceSize uint64
   155  	// BaseDeviceFS is backing filesystem.
   156  	BaseDeviceFS string
   157  	// SectorSize size of the vector.
   158  	SectorSize uint64
   159  	// UdevSyncSupported is true if sync is supported.
   160  	UdevSyncSupported bool
   161  	// DeferredRemoveEnabled is true then the device is not unmounted.
   162  	DeferredRemoveEnabled bool
   163  	// True if deferred deletion is enabled. This is different from
   164  	// deferred removal. "removal" means that device mapper device is
   165  	// deactivated. Thin device is still in thin pool and can be activated
   166  	// again. But "deletion" means that thin device will be deleted from
   167  	// thin pool and it can't be activated again.
   168  	DeferredDeleteEnabled      bool
   169  	DeferredDeletedDeviceCount uint
   170  	MinFreeSpace               uint64
   171  }
   172  
   173  // Structure used to export image/container metadata in docker inspect.
   174  type deviceMetadata struct {
   175  	deviceID   int
   176  	deviceSize uint64 // size in bytes
   177  	deviceName string // Device name as used during activation
   178  }
   179  
   180  // DevStatus returns information about device mounted containing its id, size and sector information.
   181  type DevStatus struct {
   182  	// DeviceID is the id of the device.
   183  	DeviceID int
   184  	// Size is the size of the filesystem.
   185  	Size uint64
   186  	// TransactionID is a unique integer per device set used to identify an operation on the file system, this number is incremental.
   187  	TransactionID uint64
   188  	// SizeInSectors indicates the size of the sectors allocated.
   189  	SizeInSectors uint64
   190  	// MappedSectors indicates number of mapped sectors.
   191  	MappedSectors uint64
   192  	// HighestMappedSector is the pointer to the highest mapped sector.
   193  	HighestMappedSector uint64
   194  }
   195  
   196  func getDevName(name string) string {
   197  	return "/dev/mapper/" + name
   198  }
   199  
   200  func (info *devInfo) Name() string {
   201  	hash := info.Hash
   202  	if hash == "" {
   203  		hash = "base"
   204  	}
   205  	return fmt.Sprintf("%s-%s", info.devices.devicePrefix, hash)
   206  }
   207  
   208  func (info *devInfo) DevName() string {
   209  	return getDevName(info.Name())
   210  }
   211  
   212  func (devices *DeviceSet) loopbackDir() string {
   213  	return path.Join(devices.root, "devicemapper")
   214  }
   215  
   216  func (devices *DeviceSet) metadataDir() string {
   217  	return path.Join(devices.root, "metadata")
   218  }
   219  
   220  func (devices *DeviceSet) metadataFile(info *devInfo) string {
   221  	file := info.Hash
   222  	if file == "" {
   223  		file = "base"
   224  	}
   225  	return path.Join(devices.metadataDir(), file)
   226  }
   227  
   228  func (devices *DeviceSet) transactionMetaFile() string {
   229  	return path.Join(devices.metadataDir(), transactionMetaFile)
   230  }
   231  
   232  func (devices *DeviceSet) deviceSetMetaFile() string {
   233  	return path.Join(devices.metadataDir(), deviceSetMetaFile)
   234  }
   235  
   236  func (devices *DeviceSet) oldMetadataFile() string {
   237  	return path.Join(devices.loopbackDir(), "json")
   238  }
   239  
   240  func (devices *DeviceSet) getPoolName() string {
   241  	if devices.thinPoolDevice == "" {
   242  		return devices.devicePrefix + "-pool"
   243  	}
   244  	return devices.thinPoolDevice
   245  }
   246  
   247  func (devices *DeviceSet) getPoolDevName() string {
   248  	return getDevName(devices.getPoolName())
   249  }
   250  
   251  func (devices *DeviceSet) hasImage(name string) bool {
   252  	dirname := devices.loopbackDir()
   253  	filename := path.Join(dirname, name)
   254  
   255  	_, err := os.Stat(filename)
   256  	return err == nil
   257  }
   258  
   259  // ensureImage creates a sparse file of <size> bytes at the path
   260  // <root>/devicemapper/<name>.
   261  // If the file already exists and new size is larger than its current size, it grows to the new size.
   262  // Either way it returns the full path.
   263  func (devices *DeviceSet) ensureImage(name string, size int64) (string, error) {
   264  	dirname := devices.loopbackDir()
   265  	filename := path.Join(dirname, name)
   266  
   267  	uid, gid, err := idtools.GetRootUIDGID(devices.uidMaps, devices.gidMaps)
   268  	if err != nil {
   269  		return "", err
   270  	}
   271  	if err := idtools.MkdirAllAndChown(dirname, 0700, idtools.Identity{UID: uid, GID: gid}); err != nil {
   272  		return "", err
   273  	}
   274  
   275  	if fi, err := os.Stat(filename); err != nil {
   276  		if !os.IsNotExist(err) {
   277  			return "", err
   278  		}
   279  		logrus.WithField("storage-driver", "devicemapper").Debugf("Creating loopback file %s for device-manage use", filename)
   280  		file, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE, 0600)
   281  		if err != nil {
   282  			return "", err
   283  		}
   284  		defer file.Close()
   285  
   286  		if err := file.Truncate(size); err != nil {
   287  			return "", err
   288  		}
   289  	} else {
   290  		if fi.Size() < size {
   291  			file, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE, 0600)
   292  			if err != nil {
   293  				return "", err
   294  			}
   295  			defer file.Close()
   296  			if err := file.Truncate(size); err != nil {
   297  				return "", fmt.Errorf("devmapper: Unable to grow loopback file %s: %v", filename, err)
   298  			}
   299  		} else if fi.Size() > size {
   300  			logrus.WithField("storage-driver", "devicemapper").Warnf("Can't shrink loopback file %s", filename)
   301  		}
   302  	}
   303  	return filename, nil
   304  }
   305  
   306  func (devices *DeviceSet) allocateTransactionID() uint64 {
   307  	devices.OpenTransactionID = devices.TransactionID + 1
   308  	return devices.OpenTransactionID
   309  }
   310  
   311  func (devices *DeviceSet) updatePoolTransactionID() error {
   312  	if err := devicemapper.SetTransactionID(devices.getPoolDevName(), devices.TransactionID, devices.OpenTransactionID); err != nil {
   313  		return fmt.Errorf("devmapper: Error setting devmapper transaction ID: %s", err)
   314  	}
   315  	devices.TransactionID = devices.OpenTransactionID
   316  	return nil
   317  }
   318  
   319  func (devices *DeviceSet) removeMetadata(info *devInfo) error {
   320  	if err := os.RemoveAll(devices.metadataFile(info)); err != nil {
   321  		return fmt.Errorf("devmapper: Error removing metadata file %s: %s", devices.metadataFile(info), err)
   322  	}
   323  	return nil
   324  }
   325  
   326  // Given json data and file path, write it to disk
   327  func (devices *DeviceSet) writeMetaFile(jsonData []byte, filePath string) error {
   328  	tmpFile, err := ioutil.TempFile(devices.metadataDir(), ".tmp")
   329  	if err != nil {
   330  		return fmt.Errorf("devmapper: Error creating metadata file: %s", err)
   331  	}
   332  
   333  	n, err := tmpFile.Write(jsonData)
   334  	if err != nil {
   335  		return fmt.Errorf("devmapper: Error writing metadata to %s: %s", tmpFile.Name(), err)
   336  	}
   337  	if n < len(jsonData) {
   338  		return io.ErrShortWrite
   339  	}
   340  	if err := tmpFile.Sync(); err != nil {
   341  		return fmt.Errorf("devmapper: Error syncing metadata file %s: %s", tmpFile.Name(), err)
   342  	}
   343  	if err := tmpFile.Close(); err != nil {
   344  		return fmt.Errorf("devmapper: Error closing metadata file %s: %s", tmpFile.Name(), err)
   345  	}
   346  	if err := os.Rename(tmpFile.Name(), filePath); err != nil {
   347  		return fmt.Errorf("devmapper: Error committing metadata file %s: %s", tmpFile.Name(), err)
   348  	}
   349  
   350  	return nil
   351  }
   352  
   353  func (devices *DeviceSet) saveMetadata(info *devInfo) error {
   354  	jsonData, err := json.Marshal(info)
   355  	if err != nil {
   356  		return fmt.Errorf("devmapper: Error encoding metadata to json: %s", err)
   357  	}
   358  	return devices.writeMetaFile(jsonData, devices.metadataFile(info))
   359  }
   360  
   361  func (devices *DeviceSet) markDeviceIDUsed(deviceID int) {
   362  	var mask byte
   363  	i := deviceID % 8
   364  	mask = 1 << uint(i)
   365  	devices.deviceIDMap[deviceID/8] = devices.deviceIDMap[deviceID/8] | mask
   366  }
   367  
   368  func (devices *DeviceSet) markDeviceIDFree(deviceID int) {
   369  	var mask byte
   370  	i := deviceID % 8
   371  	mask = ^(1 << uint(i))
   372  	devices.deviceIDMap[deviceID/8] = devices.deviceIDMap[deviceID/8] & mask
   373  }
   374  
   375  func (devices *DeviceSet) isDeviceIDFree(deviceID int) bool {
   376  	var mask byte
   377  	i := deviceID % 8
   378  	mask = (1 << uint(i))
   379  	return (devices.deviceIDMap[deviceID/8] & mask) == 0
   380  }
   381  
   382  // Should be called with devices.Lock() held.
   383  func (devices *DeviceSet) lookupDevice(hash string) (*devInfo, error) {
   384  	info := devices.Devices[hash]
   385  	if info == nil {
   386  		info = devices.loadMetadata(hash)
   387  		if info == nil {
   388  			return nil, fmt.Errorf("devmapper: Unknown device %s", hash)
   389  		}
   390  
   391  		devices.Devices[hash] = info
   392  	}
   393  	return info, nil
   394  }
   395  
   396  func (devices *DeviceSet) lookupDeviceWithLock(hash string) (*devInfo, error) {
   397  	devices.Lock()
   398  	defer devices.Unlock()
   399  	info, err := devices.lookupDevice(hash)
   400  	return info, err
   401  }
   402  
   403  // This function relies on that device hash map has been loaded in advance.
   404  // Should be called with devices.Lock() held.
   405  func (devices *DeviceSet) constructDeviceIDMap() {
   406  	logrus.WithField("storage-driver", "devicemapper").Debug("constructDeviceIDMap()")
   407  	defer logrus.WithField("storage-driver", "devicemapper").Debug("constructDeviceIDMap() END")
   408  
   409  	for _, info := range devices.Devices {
   410  		devices.markDeviceIDUsed(info.DeviceID)
   411  		logrus.WithField("storage-driver", "devicemapper").Debugf("Added deviceId=%d to DeviceIdMap", info.DeviceID)
   412  	}
   413  }
   414  
   415  func (devices *DeviceSet) deviceFileWalkFunction(path string, finfo os.FileInfo) error {
   416  	logger := logrus.WithField("storage-driver", "devicemapper")
   417  
   418  	// Skip some of the meta files which are not device files.
   419  	if strings.HasSuffix(finfo.Name(), ".migrated") {
   420  		logger.Debugf("Skipping file %s", path)
   421  		return nil
   422  	}
   423  
   424  	if strings.HasPrefix(finfo.Name(), ".") {
   425  		logger.Debugf("Skipping file %s", path)
   426  		return nil
   427  	}
   428  
   429  	if finfo.Name() == deviceSetMetaFile {
   430  		logger.Debugf("Skipping file %s", path)
   431  		return nil
   432  	}
   433  
   434  	if finfo.Name() == transactionMetaFile {
   435  		logger.Debugf("Skipping file %s", path)
   436  		return nil
   437  	}
   438  
   439  	logger.Debugf("Loading data for file %s", path)
   440  
   441  	hash := finfo.Name()
   442  	if hash == "base" {
   443  		hash = ""
   444  	}
   445  
   446  	// Include deleted devices also as cleanup delete device logic
   447  	// will go through it and see if there are any deleted devices.
   448  	if _, err := devices.lookupDevice(hash); err != nil {
   449  		return fmt.Errorf("devmapper: Error looking up device %s:%v", hash, err)
   450  	}
   451  
   452  	return nil
   453  }
   454  
   455  func (devices *DeviceSet) loadDeviceFilesOnStart() error {
   456  	logrus.WithField("storage-driver", "devicemapper").Debug("loadDeviceFilesOnStart()")
   457  	defer logrus.WithField("storage-driver", "devicemapper").Debug("loadDeviceFilesOnStart() END")
   458  
   459  	var scan = func(path string, info os.FileInfo, err error) error {
   460  		if err != nil {
   461  			logrus.WithField("storage-driver", "devicemapper").Debugf("Can't walk the file %s", path)
   462  			return nil
   463  		}
   464  
   465  		// Skip any directories
   466  		if info.IsDir() {
   467  			return nil
   468  		}
   469  
   470  		return devices.deviceFileWalkFunction(path, info)
   471  	}
   472  
   473  	return filepath.Walk(devices.metadataDir(), scan)
   474  }
   475  
   476  // Should be called with devices.Lock() held.
   477  func (devices *DeviceSet) unregisterDevice(hash string) error {
   478  	logrus.WithField("storage-driver", "devicemapper").Debugf("unregisterDevice(%v)", hash)
   479  	info := &devInfo{
   480  		Hash: hash,
   481  	}
   482  
   483  	delete(devices.Devices, hash)
   484  
   485  	if err := devices.removeMetadata(info); err != nil {
   486  		logrus.WithField("storage-driver", "devicemapper").Debugf("Error removing metadata: %s", err)
   487  		return err
   488  	}
   489  
   490  	return nil
   491  }
   492  
   493  // Should be called with devices.Lock() held.
   494  func (devices *DeviceSet) registerDevice(id int, hash string, size uint64, transactionID uint64) (*devInfo, error) {
   495  	logrus.WithField("storage-driver", "devicemapper").Debugf("registerDevice(%v, %v)", id, hash)
   496  	info := &devInfo{
   497  		Hash:          hash,
   498  		DeviceID:      id,
   499  		Size:          size,
   500  		TransactionID: transactionID,
   501  		Initialized:   false,
   502  		devices:       devices,
   503  	}
   504  
   505  	devices.Devices[hash] = info
   506  
   507  	if err := devices.saveMetadata(info); err != nil {
   508  		// Try to remove unused device
   509  		delete(devices.Devices, hash)
   510  		return nil, err
   511  	}
   512  
   513  	return info, nil
   514  }
   515  
   516  func (devices *DeviceSet) activateDeviceIfNeeded(info *devInfo, ignoreDeleted bool) error {
   517  	logrus.WithField("storage-driver", "devicemapper").Debugf("activateDeviceIfNeeded(%v)", info.Hash)
   518  
   519  	if info.Deleted && !ignoreDeleted {
   520  		return fmt.Errorf("devmapper: Can't activate device %v as it is marked for deletion", info.Hash)
   521  	}
   522  
   523  	// Make sure deferred removal on device is canceled, if one was
   524  	// scheduled.
   525  	if err := devices.cancelDeferredRemovalIfNeeded(info); err != nil {
   526  		return fmt.Errorf("devmapper: Device Deferred Removal Cancellation Failed: %s", err)
   527  	}
   528  
   529  	if devinfo, _ := devicemapper.GetInfo(info.Name()); devinfo != nil && devinfo.Exists != 0 {
   530  		return nil
   531  	}
   532  
   533  	return devicemapper.ActivateDevice(devices.getPoolDevName(), info.Name(), info.DeviceID, info.Size)
   534  }
   535  
   536  // xfsSupported checks if xfs is supported, returns nil if it is, otherwise an error
   537  func xfsSupported() error {
   538  	// Make sure mkfs.xfs is available
   539  	if _, err := exec.LookPath("mkfs.xfs"); err != nil {
   540  		return err // error text is descriptive enough
   541  	}
   542  
   543  	mountTarget, err := ioutil.TempDir("", "supportsXFS")
   544  	if err != nil {
   545  		return errors.Wrapf(err, "error checking for xfs support")
   546  	}
   547  
   548  	/* The mounting will fail--after the module has been loaded.*/
   549  	defer os.RemoveAll(mountTarget)
   550  	unix.Mount("none", mountTarget, "xfs", 0, "")
   551  
   552  	f, err := os.Open("/proc/filesystems")
   553  	if err != nil {
   554  		return errors.Wrapf(err, "error checking for xfs support")
   555  	}
   556  	defer f.Close()
   557  
   558  	s := bufio.NewScanner(f)
   559  	for s.Scan() {
   560  		if strings.HasSuffix(s.Text(), "\txfs") {
   561  			return nil
   562  		}
   563  	}
   564  
   565  	if err := s.Err(); err != nil {
   566  		return errors.Wrapf(err, "error checking for xfs support")
   567  	}
   568  
   569  	return errors.New(`kernel does not support xfs, or "modprobe xfs" failed`)
   570  }
   571  
   572  func determineDefaultFS() string {
   573  	err := xfsSupported()
   574  	if err == nil {
   575  		return "xfs"
   576  	}
   577  
   578  	logrus.WithField("storage-driver", "devicemapper").Warnf("XFS is not supported in your system (%v). Defaulting to ext4 filesystem", err)
   579  	return "ext4"
   580  }
   581  
   582  // mkfsOptions tries to figure out whether some additional mkfs options are required
   583  func mkfsOptions(fs string) []string {
   584  	if fs == "xfs" && !kernel.CheckKernelVersion(3, 16, 0) {
   585  		// For kernels earlier than 3.16 (and newer xfsutils),
   586  		// some xfs features need to be explicitly disabled.
   587  		return []string{"-m", "crc=0,finobt=0"}
   588  	}
   589  
   590  	return []string{}
   591  }
   592  
   593  func (devices *DeviceSet) createFilesystem(info *devInfo) (err error) {
   594  	devname := info.DevName()
   595  
   596  	if devices.filesystem == "" {
   597  		devices.filesystem = determineDefaultFS()
   598  	}
   599  	if err := devices.saveBaseDeviceFilesystem(devices.filesystem); err != nil {
   600  		return err
   601  	}
   602  
   603  	args := mkfsOptions(devices.filesystem)
   604  	args = append(args, devices.mkfsArgs...)
   605  	args = append(args, devname)
   606  
   607  	logrus.WithField("storage-driver", "devicemapper").Infof("Creating filesystem %s on device %s, mkfs args: %v", devices.filesystem, info.Name(), args)
   608  	defer func() {
   609  		if err != nil {
   610  			logrus.WithField("storage-driver", "devicemapper").Infof("Error while creating filesystem %s on device %s: %v", devices.filesystem, info.Name(), err)
   611  		} else {
   612  			logrus.WithField("storage-driver", "devicemapper").Infof("Successfully created filesystem %s on device %s", devices.filesystem, info.Name())
   613  		}
   614  	}()
   615  
   616  	switch devices.filesystem {
   617  	case "xfs":
   618  		err = exec.Command("mkfs.xfs", args...).Run()
   619  	case "ext4":
   620  		err = exec.Command("mkfs.ext4", append([]string{"-E", "nodiscard,lazy_itable_init=0,lazy_journal_init=0"}, args...)...).Run()
   621  		if err != nil {
   622  			err = exec.Command("mkfs.ext4", append([]string{"-E", "nodiscard,lazy_itable_init=0"}, args...)...).Run()
   623  		}
   624  		if err != nil {
   625  			return err
   626  		}
   627  		err = exec.Command("tune2fs", append([]string{"-c", "-1", "-i", "0"}, devname)...).Run()
   628  	default:
   629  		err = fmt.Errorf("devmapper: Unsupported filesystem type %s", devices.filesystem)
   630  	}
   631  	return
   632  }
   633  
   634  func (devices *DeviceSet) migrateOldMetaData() error {
   635  	// Migrate old metadata file
   636  	jsonData, err := ioutil.ReadFile(devices.oldMetadataFile())
   637  	if err != nil && !os.IsNotExist(err) {
   638  		return err
   639  	}
   640  
   641  	if jsonData != nil {
   642  		m := metaData{Devices: make(map[string]*devInfo)}
   643  
   644  		if err := json.Unmarshal(jsonData, &m); err != nil {
   645  			return err
   646  		}
   647  
   648  		for hash, info := range m.Devices {
   649  			info.Hash = hash
   650  			devices.saveMetadata(info)
   651  		}
   652  		if err := os.Rename(devices.oldMetadataFile(), devices.oldMetadataFile()+".migrated"); err != nil {
   653  			return err
   654  		}
   655  
   656  	}
   657  
   658  	return nil
   659  }
   660  
   661  // Cleanup deleted devices. It assumes that all the devices have been
   662  // loaded in the hash table.
   663  func (devices *DeviceSet) cleanupDeletedDevices() error {
   664  	devices.Lock()
   665  
   666  	// If there are no deleted devices, there is nothing to do.
   667  	if devices.nrDeletedDevices == 0 {
   668  		devices.Unlock()
   669  		return nil
   670  	}
   671  
   672  	var deletedDevices []*devInfo
   673  
   674  	for _, info := range devices.Devices {
   675  		if !info.Deleted {
   676  			continue
   677  		}
   678  		logrus.WithField("storage-driver", "devicemapper").Debugf("Found deleted device %s.", info.Hash)
   679  		deletedDevices = append(deletedDevices, info)
   680  	}
   681  
   682  	// Delete the deleted devices. DeleteDevice() first takes the info lock
   683  	// and then devices.Lock(). So drop it to avoid deadlock.
   684  	devices.Unlock()
   685  
   686  	for _, info := range deletedDevices {
   687  		// This will again try deferred deletion.
   688  		if err := devices.DeleteDevice(info.Hash, false); err != nil {
   689  			logrus.WithField("storage-driver", "devicemapper").Warnf("Deletion of device %s, device_id=%v failed:%v", info.Hash, info.DeviceID, err)
   690  		}
   691  	}
   692  
   693  	return nil
   694  }
   695  
   696  func (devices *DeviceSet) countDeletedDevices() {
   697  	for _, info := range devices.Devices {
   698  		if !info.Deleted {
   699  			continue
   700  		}
   701  		devices.nrDeletedDevices++
   702  	}
   703  }
   704  
   705  func (devices *DeviceSet) startDeviceDeletionWorker() {
   706  	// Deferred deletion is not enabled. Don't do anything.
   707  	if !devices.deferredDelete {
   708  		return
   709  	}
   710  
   711  	logrus.WithField("storage-driver", "devicemapper").Debug("Worker to cleanup deleted devices started")
   712  	for range devices.deletionWorkerTicker.C {
   713  		devices.cleanupDeletedDevices()
   714  	}
   715  }
   716  
   717  func (devices *DeviceSet) initMetaData() error {
   718  	devices.Lock()
   719  	defer devices.Unlock()
   720  
   721  	if err := devices.migrateOldMetaData(); err != nil {
   722  		return err
   723  	}
   724  
   725  	_, transactionID, _, _, _, _, err := devices.poolStatus()
   726  	if err != nil {
   727  		return err
   728  	}
   729  
   730  	devices.TransactionID = transactionID
   731  
   732  	if err := devices.loadDeviceFilesOnStart(); err != nil {
   733  		return fmt.Errorf("devmapper: Failed to load device files:%v", err)
   734  	}
   735  
   736  	devices.constructDeviceIDMap()
   737  	devices.countDeletedDevices()
   738  
   739  	if err := devices.processPendingTransaction(); err != nil {
   740  		return err
   741  	}
   742  
   743  	// Start a goroutine to cleanup Deleted Devices
   744  	go devices.startDeviceDeletionWorker()
   745  	return nil
   746  }
   747  
   748  func (devices *DeviceSet) incNextDeviceID() {
   749  	// IDs are 24bit, so wrap around
   750  	devices.NextDeviceID = (devices.NextDeviceID + 1) & maxDeviceID
   751  }
   752  
   753  func (devices *DeviceSet) getNextFreeDeviceID() (int, error) {
   754  	devices.incNextDeviceID()
   755  	for i := 0; i <= maxDeviceID; i++ {
   756  		if devices.isDeviceIDFree(devices.NextDeviceID) {
   757  			devices.markDeviceIDUsed(devices.NextDeviceID)
   758  			return devices.NextDeviceID, nil
   759  		}
   760  		devices.incNextDeviceID()
   761  	}
   762  
   763  	return 0, fmt.Errorf("devmapper: Unable to find a free device ID")
   764  }
   765  
   766  func (devices *DeviceSet) poolHasFreeSpace() error {
   767  	if devices.minFreeSpacePercent == 0 {
   768  		return nil
   769  	}
   770  
   771  	_, _, dataUsed, dataTotal, metadataUsed, metadataTotal, err := devices.poolStatus()
   772  	if err != nil {
   773  		return err
   774  	}
   775  
   776  	minFreeData := (dataTotal * uint64(devices.minFreeSpacePercent)) / 100
   777  	if minFreeData < 1 {
   778  		minFreeData = 1
   779  	}
   780  	dataFree := dataTotal - dataUsed
   781  	if dataFree < minFreeData {
   782  		return fmt.Errorf("devmapper: Thin Pool has %v free data blocks which is less than minimum required %v free data blocks. Create more free space in thin pool or use dm.min_free_space option to change behavior", (dataTotal - dataUsed), minFreeData)
   783  	}
   784  
   785  	minFreeMetadata := (metadataTotal * uint64(devices.minFreeSpacePercent)) / 100
   786  	if minFreeMetadata < 1 {
   787  		minFreeMetadata = 1
   788  	}
   789  
   790  	metadataFree := metadataTotal - metadataUsed
   791  	if metadataFree < minFreeMetadata {
   792  		return fmt.Errorf("devmapper: Thin Pool has %v free metadata blocks which is less than minimum required %v free metadata blocks. Create more free metadata space in thin pool or use dm.min_free_space option to change behavior", (metadataTotal - metadataUsed), minFreeMetadata)
   793  	}
   794  
   795  	return nil
   796  }
   797  
   798  func (devices *DeviceSet) createRegisterDevice(hash string) (*devInfo, error) {
   799  	devices.Lock()
   800  	defer devices.Unlock()
   801  
   802  	deviceID, err := devices.getNextFreeDeviceID()
   803  	if err != nil {
   804  		return nil, err
   805  	}
   806  
   807  	logger := logrus.WithField("storage-driver", "devicemapper")
   808  
   809  	if err := devices.openTransaction(hash, deviceID); err != nil {
   810  		logger.Debugf("Error opening transaction hash = %s deviceID = %d", hash, deviceID)
   811  		devices.markDeviceIDFree(deviceID)
   812  		return nil, err
   813  	}
   814  
   815  	for {
   816  		if err := devicemapper.CreateDevice(devices.getPoolDevName(), deviceID); err != nil {
   817  			if devicemapper.DeviceIDExists(err) {
   818  				// Device ID already exists. This should not
   819  				// happen. Now we have a mechanism to find
   820  				// a free device ID. So something is not right.
   821  				// Give a warning and continue.
   822  				logger.Errorf("Device ID %d exists in pool but it is supposed to be unused", deviceID)
   823  				deviceID, err = devices.getNextFreeDeviceID()
   824  				if err != nil {
   825  					return nil, err
   826  				}
   827  				// Save new device id into transaction
   828  				devices.refreshTransaction(deviceID)
   829  				continue
   830  			}
   831  			logger.Debugf("Error creating device: %s", err)
   832  			devices.markDeviceIDFree(deviceID)
   833  			return nil, err
   834  		}
   835  		break
   836  	}
   837  
   838  	logger.Debugf("Registering device (id %v) with FS size %v", deviceID, devices.baseFsSize)
   839  	info, err := devices.registerDevice(deviceID, hash, devices.baseFsSize, devices.OpenTransactionID)
   840  	if err != nil {
   841  		_ = devicemapper.DeleteDevice(devices.getPoolDevName(), deviceID)
   842  		devices.markDeviceIDFree(deviceID)
   843  		return nil, err
   844  	}
   845  
   846  	if err := devices.closeTransaction(); err != nil {
   847  		devices.unregisterDevice(hash)
   848  		devicemapper.DeleteDevice(devices.getPoolDevName(), deviceID)
   849  		devices.markDeviceIDFree(deviceID)
   850  		return nil, err
   851  	}
   852  	return info, nil
   853  }
   854  
   855  func (devices *DeviceSet) takeSnapshot(hash string, baseInfo *devInfo, size uint64) error {
   856  	var (
   857  		devinfo *devicemapper.Info
   858  		err     error
   859  	)
   860  
   861  	if err = devices.poolHasFreeSpace(); err != nil {
   862  		return err
   863  	}
   864  
   865  	if devices.deferredRemove {
   866  		devinfo, err = devicemapper.GetInfoWithDeferred(baseInfo.Name())
   867  		if err != nil {
   868  			return err
   869  		}
   870  		if devinfo != nil && devinfo.DeferredRemove != 0 {
   871  			err = devices.cancelDeferredRemoval(baseInfo)
   872  			if err != nil {
   873  				// If Error is ErrEnxio. Device is probably already gone. Continue.
   874  				if err != devicemapper.ErrEnxio {
   875  					return err
   876  				}
   877  				devinfo = nil
   878  			} else {
   879  				defer devices.deactivateDevice(baseInfo)
   880  			}
   881  		}
   882  	} else {
   883  		devinfo, err = devicemapper.GetInfo(baseInfo.Name())
   884  		if err != nil {
   885  			return err
   886  		}
   887  	}
   888  
   889  	doSuspend := devinfo != nil && devinfo.Exists != 0
   890  
   891  	if doSuspend {
   892  		if err = devicemapper.SuspendDevice(baseInfo.Name()); err != nil {
   893  			return err
   894  		}
   895  		defer devicemapper.ResumeDevice(baseInfo.Name())
   896  	}
   897  
   898  	return devices.createRegisterSnapDevice(hash, baseInfo, size)
   899  }
   900  
   901  func (devices *DeviceSet) createRegisterSnapDevice(hash string, baseInfo *devInfo, size uint64) error {
   902  	deviceID, err := devices.getNextFreeDeviceID()
   903  	if err != nil {
   904  		return err
   905  	}
   906  
   907  	logger := logrus.WithField("storage-driver", "devicemapper")
   908  
   909  	if err := devices.openTransaction(hash, deviceID); err != nil {
   910  		logger.Debugf("Error opening transaction hash = %s deviceID = %d", hash, deviceID)
   911  		devices.markDeviceIDFree(deviceID)
   912  		return err
   913  	}
   914  
   915  	for {
   916  		if err := devicemapper.CreateSnapDeviceRaw(devices.getPoolDevName(), deviceID, baseInfo.DeviceID); err != nil {
   917  			if devicemapper.DeviceIDExists(err) {
   918  				// Device ID already exists. This should not
   919  				// happen. Now we have a mechanism to find
   920  				// a free device ID. So something is not right.
   921  				// Give a warning and continue.
   922  				logger.Errorf("Device ID %d exists in pool but it is supposed to be unused", deviceID)
   923  				deviceID, err = devices.getNextFreeDeviceID()
   924  				if err != nil {
   925  					return err
   926  				}
   927  				// Save new device id into transaction
   928  				devices.refreshTransaction(deviceID)
   929  				continue
   930  			}
   931  			logger.Debugf("Error creating snap device: %s", err)
   932  			devices.markDeviceIDFree(deviceID)
   933  			return err
   934  		}
   935  		break
   936  	}
   937  
   938  	if _, err := devices.registerDevice(deviceID, hash, size, devices.OpenTransactionID); err != nil {
   939  		devicemapper.DeleteDevice(devices.getPoolDevName(), deviceID)
   940  		devices.markDeviceIDFree(deviceID)
   941  		logger.Debugf("Error registering device: %s", err)
   942  		return err
   943  	}
   944  
   945  	if err := devices.closeTransaction(); err != nil {
   946  		devices.unregisterDevice(hash)
   947  		devicemapper.DeleteDevice(devices.getPoolDevName(), deviceID)
   948  		devices.markDeviceIDFree(deviceID)
   949  		return err
   950  	}
   951  	return nil
   952  }
   953  
   954  func (devices *DeviceSet) loadMetadata(hash string) *devInfo {
   955  	info := &devInfo{Hash: hash, devices: devices}
   956  	logger := logrus.WithField("storage-driver", "devicemapper")
   957  
   958  	jsonData, err := ioutil.ReadFile(devices.metadataFile(info))
   959  	if err != nil {
   960  		logger.Debugf("Failed to read %s with err: %v", devices.metadataFile(info), err)
   961  		return nil
   962  	}
   963  
   964  	if err := json.Unmarshal(jsonData, &info); err != nil {
   965  		logger.Debugf("Failed to unmarshal devInfo from %s with err: %v", devices.metadataFile(info), err)
   966  		return nil
   967  	}
   968  
   969  	if info.DeviceID > maxDeviceID {
   970  		logger.Errorf("Ignoring Invalid DeviceId=%d", info.DeviceID)
   971  		return nil
   972  	}
   973  
   974  	return info
   975  }
   976  
   977  func getDeviceUUID(device string) (string, error) {
   978  	out, err := exec.Command("blkid", "-s", "UUID", "-o", "value", device).Output()
   979  	if err != nil {
   980  		return "", fmt.Errorf("devmapper: Failed to find uuid for device %s:%v", device, err)
   981  	}
   982  
   983  	uuid := strings.TrimSuffix(string(out), "\n")
   984  	uuid = strings.TrimSpace(uuid)
   985  	logrus.WithField("storage-driver", "devicemapper").Debugf("UUID for device: %s is:%s", device, uuid)
   986  	return uuid, nil
   987  }
   988  
   989  func (devices *DeviceSet) getBaseDeviceSize() uint64 {
   990  	info, _ := devices.lookupDevice("")
   991  	if info == nil {
   992  		return 0
   993  	}
   994  	return info.Size
   995  }
   996  
   997  func (devices *DeviceSet) getBaseDeviceFS() string {
   998  	return devices.BaseDeviceFilesystem
   999  }
  1000  
  1001  func (devices *DeviceSet) verifyBaseDeviceUUIDFS(baseInfo *devInfo) error {
  1002  	devices.Lock()
  1003  	defer devices.Unlock()
  1004  
  1005  	if err := devices.activateDeviceIfNeeded(baseInfo, false); err != nil {
  1006  		return err
  1007  	}
  1008  	defer devices.deactivateDevice(baseInfo)
  1009  
  1010  	uuid, err := getDeviceUUID(baseInfo.DevName())
  1011  	if err != nil {
  1012  		return err
  1013  	}
  1014  
  1015  	if devices.BaseDeviceUUID != uuid {
  1016  		return fmt.Errorf("devmapper: Current Base Device UUID:%s does not match with stored UUID:%s. Possibly using a different thin pool than last invocation", uuid, devices.BaseDeviceUUID)
  1017  	}
  1018  
  1019  	if devices.BaseDeviceFilesystem == "" {
  1020  		fsType, err := ProbeFsType(baseInfo.DevName())
  1021  		if err != nil {
  1022  			return err
  1023  		}
  1024  		if err := devices.saveBaseDeviceFilesystem(fsType); err != nil {
  1025  			return err
  1026  		}
  1027  	}
  1028  
  1029  	// If user specified a filesystem using dm.fs option and current
  1030  	// file system of base image is not same, warn user that dm.fs
  1031  	// will be ignored.
  1032  	if devices.BaseDeviceFilesystem != devices.filesystem {
  1033  		logrus.WithField("storage-driver", "devicemapper").Warnf("Base device already exists and has filesystem %s on it. User specified filesystem %s will be ignored.", devices.BaseDeviceFilesystem, devices.filesystem)
  1034  		devices.filesystem = devices.BaseDeviceFilesystem
  1035  	}
  1036  	return nil
  1037  }
  1038  
  1039  func (devices *DeviceSet) saveBaseDeviceFilesystem(fs string) error {
  1040  	devices.BaseDeviceFilesystem = fs
  1041  	return devices.saveDeviceSetMetaData()
  1042  }
  1043  
  1044  func (devices *DeviceSet) saveBaseDeviceUUID(baseInfo *devInfo) error {
  1045  	devices.Lock()
  1046  	defer devices.Unlock()
  1047  
  1048  	if err := devices.activateDeviceIfNeeded(baseInfo, false); err != nil {
  1049  		return err
  1050  	}
  1051  	defer devices.deactivateDevice(baseInfo)
  1052  
  1053  	uuid, err := getDeviceUUID(baseInfo.DevName())
  1054  	if err != nil {
  1055  		return err
  1056  	}
  1057  
  1058  	devices.BaseDeviceUUID = uuid
  1059  	return devices.saveDeviceSetMetaData()
  1060  }
  1061  
  1062  func (devices *DeviceSet) createBaseImage() error {
  1063  	logrus.WithField("storage-driver", "devicemapper").Debug("Initializing base device-mapper thin volume")
  1064  
  1065  	// Create initial device
  1066  	info, err := devices.createRegisterDevice("")
  1067  	if err != nil {
  1068  		return err
  1069  	}
  1070  
  1071  	logrus.WithField("storage-driver", "devicemapper").Debug("Creating filesystem on base device-mapper thin volume")
  1072  
  1073  	if err := devices.activateDeviceIfNeeded(info, false); err != nil {
  1074  		return err
  1075  	}
  1076  
  1077  	if err := devices.createFilesystem(info); err != nil {
  1078  		return err
  1079  	}
  1080  
  1081  	info.Initialized = true
  1082  	if err := devices.saveMetadata(info); err != nil {
  1083  		info.Initialized = false
  1084  		return err
  1085  	}
  1086  
  1087  	if err := devices.saveBaseDeviceUUID(info); err != nil {
  1088  		return fmt.Errorf("devmapper: Could not query and save base device UUID:%v", err)
  1089  	}
  1090  
  1091  	return nil
  1092  }
  1093  
  1094  // Returns if thin pool device exists or not. If device exists, also makes
  1095  // sure it is a thin pool device and not some other type of device.
  1096  func (devices *DeviceSet) thinPoolExists(thinPoolDevice string) (bool, error) {
  1097  	logrus.WithField("storage-driver", "devicemapper").Debugf("Checking for existence of the pool %s", thinPoolDevice)
  1098  
  1099  	info, err := devicemapper.GetInfo(thinPoolDevice)
  1100  	if err != nil {
  1101  		return false, fmt.Errorf("devmapper: GetInfo() on device %s failed: %v", thinPoolDevice, err)
  1102  	}
  1103  
  1104  	// Device does not exist.
  1105  	if info.Exists == 0 {
  1106  		return false, nil
  1107  	}
  1108  
  1109  	_, _, deviceType, _, err := devicemapper.GetStatus(thinPoolDevice)
  1110  	if err != nil {
  1111  		return false, fmt.Errorf("devmapper: GetStatus() on device %s failed: %v", thinPoolDevice, err)
  1112  	}
  1113  
  1114  	if deviceType != "thin-pool" {
  1115  		return false, fmt.Errorf("devmapper: Device %s is not a thin pool", thinPoolDevice)
  1116  	}
  1117  
  1118  	return true, nil
  1119  }
  1120  
  1121  func (devices *DeviceSet) checkThinPool() error {
  1122  	_, transactionID, dataUsed, _, _, _, err := devices.poolStatus()
  1123  	if err != nil {
  1124  		return err
  1125  	}
  1126  	if dataUsed != 0 {
  1127  		return fmt.Errorf("devmapper: Unable to take ownership of thin-pool (%s) that already has used data blocks",
  1128  			devices.thinPoolDevice)
  1129  	}
  1130  	if transactionID != 0 {
  1131  		return fmt.Errorf("devmapper: Unable to take ownership of thin-pool (%s) with non-zero transaction ID",
  1132  			devices.thinPoolDevice)
  1133  	}
  1134  	return nil
  1135  }
  1136  
  1137  // Base image is initialized properly. Either save UUID for first time (for
  1138  // upgrade case or verify UUID.
  1139  func (devices *DeviceSet) setupVerifyBaseImageUUIDFS(baseInfo *devInfo) error {
  1140  	// If BaseDeviceUUID is nil (upgrade case), save it and return success.
  1141  	if devices.BaseDeviceUUID == "" {
  1142  		if err := devices.saveBaseDeviceUUID(baseInfo); err != nil {
  1143  			return fmt.Errorf("devmapper: Could not query and save base device UUID:%v", err)
  1144  		}
  1145  		return nil
  1146  	}
  1147  
  1148  	if err := devices.verifyBaseDeviceUUIDFS(baseInfo); err != nil {
  1149  		return fmt.Errorf("devmapper: Base Device UUID and Filesystem verification failed: %v", err)
  1150  	}
  1151  
  1152  	return nil
  1153  }
  1154  
  1155  func (devices *DeviceSet) checkGrowBaseDeviceFS(info *devInfo) error {
  1156  
  1157  	if !userBaseSize {
  1158  		return nil
  1159  	}
  1160  
  1161  	if devices.baseFsSize < devices.getBaseDeviceSize() {
  1162  		return fmt.Errorf("devmapper: Base device size cannot be smaller than %s", units.HumanSize(float64(devices.getBaseDeviceSize())))
  1163  	}
  1164  
  1165  	if devices.baseFsSize == devices.getBaseDeviceSize() {
  1166  		return nil
  1167  	}
  1168  
  1169  	info.lock.Lock()
  1170  	defer info.lock.Unlock()
  1171  
  1172  	devices.Lock()
  1173  	defer devices.Unlock()
  1174  
  1175  	info.Size = devices.baseFsSize
  1176  
  1177  	if err := devices.saveMetadata(info); err != nil {
  1178  		// Try to remove unused device
  1179  		delete(devices.Devices, info.Hash)
  1180  		return err
  1181  	}
  1182  
  1183  	return devices.growFS(info)
  1184  }
  1185  
  1186  func (devices *DeviceSet) growFS(info *devInfo) error {
  1187  	if err := devices.activateDeviceIfNeeded(info, false); err != nil {
  1188  		return fmt.Errorf("Error activating devmapper device: %s", err)
  1189  	}
  1190  
  1191  	defer devices.deactivateDevice(info)
  1192  
  1193  	fsMountPoint := "/run/docker/mnt"
  1194  	if _, err := os.Stat(fsMountPoint); os.IsNotExist(err) {
  1195  		if err := os.MkdirAll(fsMountPoint, 0700); err != nil {
  1196  			return err
  1197  		}
  1198  		defer os.RemoveAll(fsMountPoint)
  1199  	}
  1200  
  1201  	options := ""
  1202  	if devices.BaseDeviceFilesystem == "xfs" {
  1203  		// XFS needs nouuid or it can't mount filesystems with the same fs
  1204  		options = joinMountOptions(options, "nouuid")
  1205  	}
  1206  	options = joinMountOptions(options, devices.mountOptions)
  1207  
  1208  	if err := mount.Mount(info.DevName(), fsMountPoint, devices.BaseDeviceFilesystem, options); err != nil {
  1209  		return errors.Wrapf(err, "Failed to mount; dmesg: %s", string(dmesg.Dmesg(256)))
  1210  	}
  1211  
  1212  	defer unix.Unmount(fsMountPoint, unix.MNT_DETACH)
  1213  
  1214  	switch devices.BaseDeviceFilesystem {
  1215  	case "ext4":
  1216  		if out, err := exec.Command("resize2fs", info.DevName()).CombinedOutput(); err != nil {
  1217  			return fmt.Errorf("Failed to grow rootfs:%v:%s", err, string(out))
  1218  		}
  1219  	case "xfs":
  1220  		if out, err := exec.Command("xfs_growfs", info.DevName()).CombinedOutput(); err != nil {
  1221  			return fmt.Errorf("Failed to grow rootfs:%v:%s", err, string(out))
  1222  		}
  1223  	default:
  1224  		return fmt.Errorf("Unsupported filesystem type %s", devices.BaseDeviceFilesystem)
  1225  	}
  1226  	return nil
  1227  }
  1228  
  1229  func (devices *DeviceSet) setupBaseImage() error {
  1230  	oldInfo, _ := devices.lookupDeviceWithLock("")
  1231  
  1232  	// base image already exists. If it is initialized properly, do UUID
  1233  	// verification and return. Otherwise remove image and set it up
  1234  	// fresh.
  1235  
  1236  	if oldInfo != nil {
  1237  		if oldInfo.Initialized && !oldInfo.Deleted {
  1238  			if err := devices.setupVerifyBaseImageUUIDFS(oldInfo); err != nil {
  1239  				return err
  1240  			}
  1241  			return devices.checkGrowBaseDeviceFS(oldInfo)
  1242  		}
  1243  
  1244  		logrus.WithField("storage-driver", "devicemapper").Debug("Removing uninitialized base image")
  1245  		// If previous base device is in deferred delete state,
  1246  		// that needs to be cleaned up first. So don't try
  1247  		// deferred deletion.
  1248  		if err := devices.DeleteDevice("", true); err != nil {
  1249  			return err
  1250  		}
  1251  	}
  1252  
  1253  	// If we are setting up base image for the first time, make sure
  1254  	// thin pool is empty.
  1255  	if devices.thinPoolDevice != "" && oldInfo == nil {
  1256  		if err := devices.checkThinPool(); err != nil {
  1257  			return err
  1258  		}
  1259  	}
  1260  
  1261  	// Create new base image device
  1262  	return devices.createBaseImage()
  1263  }
  1264  
  1265  func setCloseOnExec(name string) {
  1266  	fileInfos, _ := ioutil.ReadDir("/proc/self/fd")
  1267  	for _, i := range fileInfos {
  1268  		link, _ := os.Readlink(filepath.Join("/proc/self/fd", i.Name()))
  1269  		if link == name {
  1270  			fd, err := strconv.Atoi(i.Name())
  1271  			if err == nil {
  1272  				unix.CloseOnExec(fd)
  1273  			}
  1274  		}
  1275  	}
  1276  }
  1277  
  1278  func major(device uint64) uint64 {
  1279  	return (device >> 8) & 0xfff
  1280  }
  1281  
  1282  func minor(device uint64) uint64 {
  1283  	return (device & 0xff) | ((device >> 12) & 0xfff00)
  1284  }
  1285  
  1286  // ResizePool increases the size of the pool.
  1287  func (devices *DeviceSet) ResizePool(size int64) error {
  1288  	dirname := devices.loopbackDir()
  1289  	datafilename := path.Join(dirname, "data")
  1290  	if len(devices.dataDevice) > 0 {
  1291  		datafilename = devices.dataDevice
  1292  	}
  1293  	metadatafilename := path.Join(dirname, "metadata")
  1294  	if len(devices.metadataDevice) > 0 {
  1295  		metadatafilename = devices.metadataDevice
  1296  	}
  1297  
  1298  	datafile, err := os.OpenFile(datafilename, os.O_RDWR, 0)
  1299  	if datafile == nil {
  1300  		return err
  1301  	}
  1302  	defer datafile.Close()
  1303  
  1304  	fi, err := datafile.Stat()
  1305  	if fi == nil {
  1306  		return err
  1307  	}
  1308  
  1309  	if fi.Size() > size {
  1310  		return fmt.Errorf("devmapper: Can't shrink file")
  1311  	}
  1312  
  1313  	dataloopback := loopback.FindLoopDeviceFor(datafile)
  1314  	if dataloopback == nil {
  1315  		return fmt.Errorf("devmapper: Unable to find loopback mount for: %s", datafilename)
  1316  	}
  1317  	defer dataloopback.Close()
  1318  
  1319  	metadatafile, err := os.OpenFile(metadatafilename, os.O_RDWR, 0)
  1320  	if metadatafile == nil {
  1321  		return err
  1322  	}
  1323  	defer metadatafile.Close()
  1324  
  1325  	metadataloopback := loopback.FindLoopDeviceFor(metadatafile)
  1326  	if metadataloopback == nil {
  1327  		return fmt.Errorf("devmapper: Unable to find loopback mount for: %s", metadatafilename)
  1328  	}
  1329  	defer metadataloopback.Close()
  1330  
  1331  	// Grow loopback file
  1332  	if err := datafile.Truncate(size); err != nil {
  1333  		return fmt.Errorf("devmapper: Unable to grow loopback file: %s", err)
  1334  	}
  1335  
  1336  	// Reload size for loopback device
  1337  	if err := loopback.SetCapacity(dataloopback); err != nil {
  1338  		return fmt.Errorf("Unable to update loopback capacity: %s", err)
  1339  	}
  1340  
  1341  	// Suspend the pool
  1342  	if err := devicemapper.SuspendDevice(devices.getPoolName()); err != nil {
  1343  		return fmt.Errorf("devmapper: Unable to suspend pool: %s", err)
  1344  	}
  1345  
  1346  	// Reload with the new block sizes
  1347  	if err := devicemapper.ReloadPool(devices.getPoolName(), dataloopback, metadataloopback, devices.thinpBlockSize); err != nil {
  1348  		return fmt.Errorf("devmapper: Unable to reload pool: %s", err)
  1349  	}
  1350  
  1351  	// Resume the pool
  1352  	if err := devicemapper.ResumeDevice(devices.getPoolName()); err != nil {
  1353  		return fmt.Errorf("devmapper: Unable to resume pool: %s", err)
  1354  	}
  1355  
  1356  	return nil
  1357  }
  1358  
  1359  func (devices *DeviceSet) loadTransactionMetaData() error {
  1360  	jsonData, err := ioutil.ReadFile(devices.transactionMetaFile())
  1361  	if err != nil {
  1362  		// There is no active transaction. This will be the case
  1363  		// during upgrade.
  1364  		if os.IsNotExist(err) {
  1365  			devices.OpenTransactionID = devices.TransactionID
  1366  			return nil
  1367  		}
  1368  		return err
  1369  	}
  1370  
  1371  	json.Unmarshal(jsonData, &devices.transaction)
  1372  	return nil
  1373  }
  1374  
  1375  func (devices *DeviceSet) saveTransactionMetaData() error {
  1376  	jsonData, err := json.Marshal(&devices.transaction)
  1377  	if err != nil {
  1378  		return fmt.Errorf("devmapper: Error encoding metadata to json: %s", err)
  1379  	}
  1380  
  1381  	return devices.writeMetaFile(jsonData, devices.transactionMetaFile())
  1382  }
  1383  
  1384  func (devices *DeviceSet) removeTransactionMetaData() error {
  1385  	return os.RemoveAll(devices.transactionMetaFile())
  1386  }
  1387  
  1388  func (devices *DeviceSet) rollbackTransaction() error {
  1389  	logger := logrus.WithField("storage-driver", "devicemapper")
  1390  
  1391  	logger.Debugf("Rolling back open transaction: TransactionID=%d hash=%s device_id=%d", devices.OpenTransactionID, devices.DeviceIDHash, devices.DeviceID)
  1392  
  1393  	// A device id might have already been deleted before transaction
  1394  	// closed. In that case this call will fail. Just leave a message
  1395  	// in case of failure.
  1396  	if err := devicemapper.DeleteDevice(devices.getPoolDevName(), devices.DeviceID); err != nil {
  1397  		logger.Errorf("Unable to delete device: %s", err)
  1398  	}
  1399  
  1400  	dinfo := &devInfo{Hash: devices.DeviceIDHash}
  1401  	if err := devices.removeMetadata(dinfo); err != nil {
  1402  		logger.Errorf("Unable to remove metadata: %s", err)
  1403  	} else {
  1404  		devices.markDeviceIDFree(devices.DeviceID)
  1405  	}
  1406  
  1407  	if err := devices.removeTransactionMetaData(); err != nil {
  1408  		logger.Errorf("Unable to remove transaction meta file %s: %s", devices.transactionMetaFile(), err)
  1409  	}
  1410  
  1411  	return nil
  1412  }
  1413  
  1414  func (devices *DeviceSet) processPendingTransaction() error {
  1415  	if err := devices.loadTransactionMetaData(); err != nil {
  1416  		return err
  1417  	}
  1418  
  1419  	// If there was open transaction but pool transaction ID is same
  1420  	// as open transaction ID, nothing to roll back.
  1421  	if devices.TransactionID == devices.OpenTransactionID {
  1422  		return nil
  1423  	}
  1424  
  1425  	// If open transaction ID is less than pool transaction ID, something
  1426  	// is wrong. Bail out.
  1427  	if devices.OpenTransactionID < devices.TransactionID {
  1428  		logrus.WithField("storage-driver", "devicemapper").Errorf("Open Transaction id %d is less than pool transaction id %d", devices.OpenTransactionID, devices.TransactionID)
  1429  		return nil
  1430  	}
  1431  
  1432  	// Pool transaction ID is not same as open transaction. There is
  1433  	// a transaction which was not completed.
  1434  	if err := devices.rollbackTransaction(); err != nil {
  1435  		return fmt.Errorf("devmapper: Rolling back open transaction failed: %s", err)
  1436  	}
  1437  
  1438  	devices.OpenTransactionID = devices.TransactionID
  1439  	return nil
  1440  }
  1441  
  1442  func (devices *DeviceSet) loadDeviceSetMetaData() error {
  1443  	jsonData, err := ioutil.ReadFile(devices.deviceSetMetaFile())
  1444  	if err != nil {
  1445  		// For backward compatibility return success if file does
  1446  		// not exist.
  1447  		if os.IsNotExist(err) {
  1448  			return nil
  1449  		}
  1450  		return err
  1451  	}
  1452  
  1453  	return json.Unmarshal(jsonData, devices)
  1454  }
  1455  
  1456  func (devices *DeviceSet) saveDeviceSetMetaData() error {
  1457  	jsonData, err := json.Marshal(devices)
  1458  	if err != nil {
  1459  		return fmt.Errorf("devmapper: Error encoding metadata to json: %s", err)
  1460  	}
  1461  
  1462  	return devices.writeMetaFile(jsonData, devices.deviceSetMetaFile())
  1463  }
  1464  
  1465  func (devices *DeviceSet) openTransaction(hash string, DeviceID int) error {
  1466  	devices.allocateTransactionID()
  1467  	devices.DeviceIDHash = hash
  1468  	devices.DeviceID = DeviceID
  1469  	if err := devices.saveTransactionMetaData(); err != nil {
  1470  		return fmt.Errorf("devmapper: Error saving transaction metadata: %s", err)
  1471  	}
  1472  	return nil
  1473  }
  1474  
  1475  func (devices *DeviceSet) refreshTransaction(DeviceID int) error {
  1476  	devices.DeviceID = DeviceID
  1477  	if err := devices.saveTransactionMetaData(); err != nil {
  1478  		return fmt.Errorf("devmapper: Error saving transaction metadata: %s", err)
  1479  	}
  1480  	return nil
  1481  }
  1482  
  1483  func (devices *DeviceSet) closeTransaction() error {
  1484  	if err := devices.updatePoolTransactionID(); err != nil {
  1485  		logrus.WithField("storage-driver", "devicemapper").Debug("Failed to close Transaction")
  1486  		return err
  1487  	}
  1488  	return nil
  1489  }
  1490  
  1491  func determineDriverCapabilities(version string) error {
  1492  	// Kernel driver version >= 4.27.0 support deferred removal
  1493  
  1494  	logrus.WithField("storage-driver", "devicemapper").Debugf("kernel dm driver version is %s", version)
  1495  
  1496  	versionSplit := strings.Split(version, ".")
  1497  	major, err := strconv.Atoi(versionSplit[0])
  1498  	if err != nil {
  1499  		return graphdriver.ErrNotSupported
  1500  	}
  1501  
  1502  	if major > 4 {
  1503  		driverDeferredRemovalSupport = true
  1504  		return nil
  1505  	}
  1506  
  1507  	if major < 4 {
  1508  		return nil
  1509  	}
  1510  
  1511  	minor, err := strconv.Atoi(versionSplit[1])
  1512  	if err != nil {
  1513  		return graphdriver.ErrNotSupported
  1514  	}
  1515  
  1516  	/*
  1517  	 * If major is 4 and minor is 27, then there is no need to
  1518  	 * check for patch level as it can not be less than 0.
  1519  	 */
  1520  	if minor >= 27 {
  1521  		driverDeferredRemovalSupport = true
  1522  		return nil
  1523  	}
  1524  
  1525  	return nil
  1526  }
  1527  
  1528  // Determine the major and minor number of loopback device
  1529  func getDeviceMajorMinor(file *os.File) (uint64, uint64, error) {
  1530  	var stat unix.Stat_t
  1531  	err := unix.Stat(file.Name(), &stat)
  1532  	if err != nil {
  1533  		return 0, 0, err
  1534  	}
  1535  
  1536  	// the type is 32bit on mips
  1537  	dev := uint64(stat.Rdev) // nolint: unconvert
  1538  	majorNum := major(dev)
  1539  	minorNum := minor(dev)
  1540  
  1541  	logrus.WithField("storage-driver", "devicemapper").Debugf("Major:Minor for device: %s is:%v:%v", file.Name(), majorNum, minorNum)
  1542  	return majorNum, minorNum, nil
  1543  }
  1544  
  1545  // Given a file which is backing file of a loop back device, find the
  1546  // loopback device name and its major/minor number.
  1547  func getLoopFileDeviceMajMin(filename string) (string, uint64, uint64, error) {
  1548  	file, err := os.Open(filename)
  1549  	if err != nil {
  1550  		logrus.WithField("storage-driver", "devicemapper").Debugf("Failed to open file %s", filename)
  1551  		return "", 0, 0, err
  1552  	}
  1553  
  1554  	defer file.Close()
  1555  	loopbackDevice := loopback.FindLoopDeviceFor(file)
  1556  	if loopbackDevice == nil {
  1557  		return "", 0, 0, fmt.Errorf("devmapper: Unable to find loopback mount for: %s", filename)
  1558  	}
  1559  	defer loopbackDevice.Close()
  1560  
  1561  	Major, Minor, err := getDeviceMajorMinor(loopbackDevice)
  1562  	if err != nil {
  1563  		return "", 0, 0, err
  1564  	}
  1565  	return loopbackDevice.Name(), Major, Minor, nil
  1566  }
  1567  
  1568  // Get the major/minor numbers of thin pool data and metadata devices
  1569  func (devices *DeviceSet) getThinPoolDataMetaMajMin() (uint64, uint64, uint64, uint64, error) {
  1570  	var params, poolDataMajMin, poolMetadataMajMin string
  1571  
  1572  	_, _, _, params, err := devicemapper.GetTable(devices.getPoolName())
  1573  	if err != nil {
  1574  		return 0, 0, 0, 0, err
  1575  	}
  1576  
  1577  	if _, err = fmt.Sscanf(params, "%s %s", &poolMetadataMajMin, &poolDataMajMin); err != nil {
  1578  		return 0, 0, 0, 0, err
  1579  	}
  1580  
  1581  	logrus.WithField("storage-driver", "devicemapper").Debugf("poolDataMajMin=%s poolMetaMajMin=%s\n", poolDataMajMin, poolMetadataMajMin)
  1582  
  1583  	poolDataMajMinorSplit := strings.Split(poolDataMajMin, ":")
  1584  	poolDataMajor, err := strconv.ParseUint(poolDataMajMinorSplit[0], 10, 32)
  1585  	if err != nil {
  1586  		return 0, 0, 0, 0, err
  1587  	}
  1588  
  1589  	poolDataMinor, err := strconv.ParseUint(poolDataMajMinorSplit[1], 10, 32)
  1590  	if err != nil {
  1591  		return 0, 0, 0, 0, err
  1592  	}
  1593  
  1594  	poolMetadataMajMinorSplit := strings.Split(poolMetadataMajMin, ":")
  1595  	poolMetadataMajor, err := strconv.ParseUint(poolMetadataMajMinorSplit[0], 10, 32)
  1596  	if err != nil {
  1597  		return 0, 0, 0, 0, err
  1598  	}
  1599  
  1600  	poolMetadataMinor, err := strconv.ParseUint(poolMetadataMajMinorSplit[1], 10, 32)
  1601  	if err != nil {
  1602  		return 0, 0, 0, 0, err
  1603  	}
  1604  
  1605  	return poolDataMajor, poolDataMinor, poolMetadataMajor, poolMetadataMinor, nil
  1606  }
  1607  
  1608  func (devices *DeviceSet) loadThinPoolLoopBackInfo() error {
  1609  	poolDataMajor, poolDataMinor, poolMetadataMajor, poolMetadataMinor, err := devices.getThinPoolDataMetaMajMin()
  1610  	if err != nil {
  1611  		return err
  1612  	}
  1613  
  1614  	dirname := devices.loopbackDir()
  1615  
  1616  	// data device has not been passed in. So there should be a data file
  1617  	// which is being mounted as loop device.
  1618  	if devices.dataDevice == "" {
  1619  		datafilename := path.Join(dirname, "data")
  1620  		dataLoopDevice, dataMajor, dataMinor, err := getLoopFileDeviceMajMin(datafilename)
  1621  		if err != nil {
  1622  			return err
  1623  		}
  1624  
  1625  		// Compare the two
  1626  		if poolDataMajor == dataMajor && poolDataMinor == dataMinor {
  1627  			devices.dataDevice = dataLoopDevice
  1628  			devices.dataLoopFile = datafilename
  1629  		}
  1630  
  1631  	}
  1632  
  1633  	// metadata device has not been passed in. So there should be a
  1634  	// metadata file which is being mounted as loop device.
  1635  	if devices.metadataDevice == "" {
  1636  		metadatafilename := path.Join(dirname, "metadata")
  1637  		metadataLoopDevice, metadataMajor, metadataMinor, err := getLoopFileDeviceMajMin(metadatafilename)
  1638  		if err != nil {
  1639  			return err
  1640  		}
  1641  		if poolMetadataMajor == metadataMajor && poolMetadataMinor == metadataMinor {
  1642  			devices.metadataDevice = metadataLoopDevice
  1643  			devices.metadataLoopFile = metadatafilename
  1644  		}
  1645  	}
  1646  
  1647  	return nil
  1648  }
  1649  
  1650  func (devices *DeviceSet) enableDeferredRemovalDeletion() error {
  1651  
  1652  	// If user asked for deferred removal then check both libdm library
  1653  	// and kernel driver support deferred removal otherwise error out.
  1654  	if enableDeferredRemoval {
  1655  		if !driverDeferredRemovalSupport {
  1656  			return fmt.Errorf("devmapper: Deferred removal can not be enabled as kernel does not support it")
  1657  		}
  1658  		if !devicemapper.LibraryDeferredRemovalSupport {
  1659  			return fmt.Errorf("devmapper: Deferred removal can not be enabled as libdm does not support it")
  1660  		}
  1661  		logrus.WithField("storage-driver", "devicemapper").Debug("Deferred removal support enabled.")
  1662  		devices.deferredRemove = true
  1663  	}
  1664  
  1665  	if enableDeferredDeletion {
  1666  		if !devices.deferredRemove {
  1667  			return fmt.Errorf("devmapper: Deferred deletion can not be enabled as deferred removal is not enabled. Enable deferred removal using --storage-opt dm.use_deferred_removal=true parameter")
  1668  		}
  1669  		logrus.WithField("storage-driver", "devicemapper").Debug("Deferred deletion support enabled.")
  1670  		devices.deferredDelete = true
  1671  	}
  1672  	return nil
  1673  }
  1674  
  1675  func (devices *DeviceSet) initDevmapper(doInit bool) (retErr error) {
  1676  	if err := devices.enableDeferredRemovalDeletion(); err != nil {
  1677  		return err
  1678  	}
  1679  
  1680  	logger := logrus.WithField("storage-driver", "devicemapper")
  1681  
  1682  	// https://github.com/demonoid81/moby/issues/4036
  1683  	if supported := devicemapper.UdevSetSyncSupport(true); !supported {
  1684  		if dockerversion.IAmStatic == "true" {
  1685  			logger.Error("Udev sync is not supported. This will lead to data loss and unexpected behavior. Install a dynamic binary to use devicemapper or select a different storage driver. For more information, see https://docs.docker.com/engine/reference/commandline/dockerd/#storage-driver-options")
  1686  		} else {
  1687  			logger.Error("Udev sync is not supported. This will lead to data loss and unexpected behavior. Install a more recent version of libdevmapper or select a different storage driver. For more information, see https://docs.docker.com/engine/reference/commandline/dockerd/#storage-driver-options")
  1688  		}
  1689  
  1690  		if !devices.overrideUdevSyncCheck {
  1691  			return graphdriver.ErrNotSupported
  1692  		}
  1693  	}
  1694  
  1695  	// create the root dir of the devmapper driver ownership to match this
  1696  	// daemon's remapped root uid/gid so containers can start properly
  1697  	uid, gid, err := idtools.GetRootUIDGID(devices.uidMaps, devices.gidMaps)
  1698  	if err != nil {
  1699  		return err
  1700  	}
  1701  	if err := idtools.MkdirAndChown(devices.root, 0700, idtools.Identity{UID: uid, GID: gid}); err != nil {
  1702  		return err
  1703  	}
  1704  	if err := os.MkdirAll(devices.metadataDir(), 0700); err != nil {
  1705  		return err
  1706  	}
  1707  
  1708  	prevSetupConfig, err := readLVMConfig(devices.root)
  1709  	if err != nil {
  1710  		return err
  1711  	}
  1712  
  1713  	if !reflect.DeepEqual(devices.lvmSetupConfig, directLVMConfig{}) {
  1714  		if devices.thinPoolDevice != "" {
  1715  			return errors.New("cannot setup direct-lvm when `dm.thinpooldev` is also specified")
  1716  		}
  1717  
  1718  		if !reflect.DeepEqual(prevSetupConfig, devices.lvmSetupConfig) {
  1719  			if !reflect.DeepEqual(prevSetupConfig, directLVMConfig{}) {
  1720  				return errors.New("changing direct-lvm config is not supported")
  1721  			}
  1722  			logger.WithField("direct-lvm-config", devices.lvmSetupConfig).Debugf("Setting up direct lvm mode")
  1723  			if err := verifyBlockDevice(devices.lvmSetupConfig.Device, lvmSetupConfigForce); err != nil {
  1724  				return err
  1725  			}
  1726  			if err := setupDirectLVM(devices.lvmSetupConfig); err != nil {
  1727  				return err
  1728  			}
  1729  			if err := writeLVMConfig(devices.root, devices.lvmSetupConfig); err != nil {
  1730  				return err
  1731  			}
  1732  		}
  1733  		devices.thinPoolDevice = "docker-thinpool"
  1734  		logger.Debugf("Setting dm.thinpooldev to %q", devices.thinPoolDevice)
  1735  	}
  1736  
  1737  	// Set the device prefix from the device id and inode of the docker root dir
  1738  	var st unix.Stat_t
  1739  	if err := unix.Stat(devices.root, &st); err != nil {
  1740  		return fmt.Errorf("devmapper: Error looking up dir %s: %s", devices.root, err)
  1741  	}
  1742  	// "reg-" stands for "regular file".
  1743  	// In the future we might use "dev-" for "device file", etc.
  1744  	// docker-maj,min[-inode] stands for:
  1745  	//	- Managed by docker
  1746  	//	- The target of this device is at major <maj> and minor <min>
  1747  	//	- If <inode> is defined, use that file inside the device as a loopback image. Otherwise use the device itself.
  1748  	// The type Dev in Stat_t is 32bit on mips.
  1749  	devices.devicePrefix = fmt.Sprintf("docker-%d:%d-%d", major(uint64(st.Dev)), minor(uint64(st.Dev)), st.Ino) // nolint: unconvert
  1750  	logger.Debugf("Generated prefix: %s", devices.devicePrefix)
  1751  
  1752  	// Check for the existence of the thin-pool device
  1753  	poolExists, err := devices.thinPoolExists(devices.getPoolName())
  1754  	if err != nil {
  1755  		return err
  1756  	}
  1757  
  1758  	// It seems libdevmapper opens this without O_CLOEXEC, and go exec will not close files
  1759  	// that are not Close-on-exec,
  1760  	// so we add this badhack to make sure it closes itself
  1761  	setCloseOnExec("/dev/mapper/control")
  1762  
  1763  	// Make sure the sparse images exist in <root>/devicemapper/data and
  1764  	// <root>/devicemapper/metadata
  1765  
  1766  	createdLoopback := false
  1767  
  1768  	// If the pool doesn't exist, create it
  1769  	if !poolExists && devices.thinPoolDevice == "" {
  1770  		logger.Debug("Pool doesn't exist. Creating it.")
  1771  
  1772  		var (
  1773  			dataFile     *os.File
  1774  			metadataFile *os.File
  1775  		)
  1776  
  1777  		if devices.dataDevice == "" {
  1778  			// Make sure the sparse images exist in <root>/devicemapper/data
  1779  
  1780  			hasData := devices.hasImage("data")
  1781  
  1782  			if !doInit && !hasData {
  1783  				return errors.New("loopback data file not found")
  1784  			}
  1785  
  1786  			if !hasData {
  1787  				createdLoopback = true
  1788  			}
  1789  
  1790  			data, err := devices.ensureImage("data", devices.dataLoopbackSize)
  1791  			if err != nil {
  1792  				logger.Debugf("Error device ensureImage (data): %s", err)
  1793  				return err
  1794  			}
  1795  
  1796  			dataFile, err = loopback.AttachLoopDevice(data)
  1797  			if err != nil {
  1798  				return err
  1799  			}
  1800  			devices.dataLoopFile = data
  1801  			devices.dataDevice = dataFile.Name()
  1802  		} else {
  1803  			dataFile, err = os.OpenFile(devices.dataDevice, os.O_RDWR, 0600)
  1804  			if err != nil {
  1805  				return err
  1806  			}
  1807  		}
  1808  		defer dataFile.Close()
  1809  
  1810  		if devices.metadataDevice == "" {
  1811  			// Make sure the sparse images exist in <root>/devicemapper/metadata
  1812  
  1813  			hasMetadata := devices.hasImage("metadata")
  1814  
  1815  			if !doInit && !hasMetadata {
  1816  				return errors.New("loopback metadata file not found")
  1817  			}
  1818  
  1819  			if !hasMetadata {
  1820  				createdLoopback = true
  1821  			}
  1822  
  1823  			metadata, err := devices.ensureImage("metadata", devices.metaDataLoopbackSize)
  1824  			if err != nil {
  1825  				logger.Debugf("Error device ensureImage (metadata): %s", err)
  1826  				return err
  1827  			}
  1828  
  1829  			metadataFile, err = loopback.AttachLoopDevice(metadata)
  1830  			if err != nil {
  1831  				return err
  1832  			}
  1833  			devices.metadataLoopFile = metadata
  1834  			devices.metadataDevice = metadataFile.Name()
  1835  		} else {
  1836  			metadataFile, err = os.OpenFile(devices.metadataDevice, os.O_RDWR, 0600)
  1837  			if err != nil {
  1838  				return err
  1839  			}
  1840  		}
  1841  		defer metadataFile.Close()
  1842  
  1843  		if err := devicemapper.CreatePool(devices.getPoolName(), dataFile, metadataFile, devices.thinpBlockSize); err != nil {
  1844  			return err
  1845  		}
  1846  		defer func() {
  1847  			if retErr != nil {
  1848  				err = devices.deactivatePool()
  1849  				if err != nil {
  1850  					logger.Warnf("Failed to deactivatePool: %v", err)
  1851  				}
  1852  			}
  1853  		}()
  1854  	}
  1855  
  1856  	// Pool already exists and caller did not pass us a pool. That means
  1857  	// we probably created pool earlier and could not remove it as some
  1858  	// containers were still using it. Detect some of the properties of
  1859  	// pool, like is it using loop devices.
  1860  	if poolExists && devices.thinPoolDevice == "" {
  1861  		if err := devices.loadThinPoolLoopBackInfo(); err != nil {
  1862  			logger.Debugf("Failed to load thin pool loopback device information:%v", err)
  1863  			return err
  1864  		}
  1865  	}
  1866  
  1867  	// If we didn't just create the data or metadata image, we need to
  1868  	// load the transaction id and migrate old metadata
  1869  	if !createdLoopback {
  1870  		if err := devices.initMetaData(); err != nil {
  1871  			return err
  1872  		}
  1873  	}
  1874  
  1875  	if devices.thinPoolDevice == "" {
  1876  		if devices.metadataLoopFile != "" || devices.dataLoopFile != "" {
  1877  			logger.Warn("Usage of loopback devices is strongly discouraged for production use. Please use `--storage-opt dm.thinpooldev` or use `man dockerd` to refer to dm.thinpooldev section.")
  1878  		}
  1879  	}
  1880  
  1881  	// Right now this loads only NextDeviceID. If there is more metadata
  1882  	// down the line, we might have to move it earlier.
  1883  	if err := devices.loadDeviceSetMetaData(); err != nil {
  1884  		return err
  1885  	}
  1886  
  1887  	// Setup the base image
  1888  	if doInit {
  1889  		if err := devices.setupBaseImage(); err != nil {
  1890  			logger.Debugf("Error device setupBaseImage: %s", err)
  1891  			return err
  1892  		}
  1893  	}
  1894  
  1895  	return nil
  1896  }
  1897  
  1898  // AddDevice adds a device and registers in the hash.
  1899  func (devices *DeviceSet) AddDevice(hash, baseHash string, storageOpt map[string]string) error {
  1900  	logrus.WithField("storage-driver", "devicemapper").Debugf("AddDevice START(hash=%s basehash=%s)", hash, baseHash)
  1901  	defer logrus.WithField("storage-driver", "devicemapper").Debugf("AddDevice END(hash=%s basehash=%s)", hash, baseHash)
  1902  
  1903  	// If a deleted device exists, return error.
  1904  	baseInfo, err := devices.lookupDeviceWithLock(baseHash)
  1905  	if err != nil {
  1906  		return err
  1907  	}
  1908  
  1909  	if baseInfo.Deleted {
  1910  		return fmt.Errorf("devmapper: Base device %v has been marked for deferred deletion", baseInfo.Hash)
  1911  	}
  1912  
  1913  	baseInfo.lock.Lock()
  1914  	defer baseInfo.lock.Unlock()
  1915  
  1916  	devices.Lock()
  1917  	defer devices.Unlock()
  1918  
  1919  	// Also include deleted devices in case hash of new device is
  1920  	// same as one of the deleted devices.
  1921  	if info, _ := devices.lookupDevice(hash); info != nil {
  1922  		return fmt.Errorf("devmapper: device %s already exists. Deleted=%v", hash, info.Deleted)
  1923  	}
  1924  
  1925  	size, err := devices.parseStorageOpt(storageOpt)
  1926  	if err != nil {
  1927  		return err
  1928  	}
  1929  
  1930  	if size == 0 {
  1931  		size = baseInfo.Size
  1932  	}
  1933  
  1934  	if size < baseInfo.Size {
  1935  		return fmt.Errorf("devmapper: Container size cannot be smaller than %s", units.HumanSize(float64(baseInfo.Size)))
  1936  	}
  1937  
  1938  	if err := devices.takeSnapshot(hash, baseInfo, size); err != nil {
  1939  		return err
  1940  	}
  1941  
  1942  	// Grow the container rootfs.
  1943  	if size > baseInfo.Size {
  1944  		info, err := devices.lookupDevice(hash)
  1945  		if err != nil {
  1946  			return err
  1947  		}
  1948  
  1949  		if err := devices.growFS(info); err != nil {
  1950  			return err
  1951  		}
  1952  	}
  1953  
  1954  	return nil
  1955  }
  1956  
  1957  func (devices *DeviceSet) parseStorageOpt(storageOpt map[string]string) (uint64, error) {
  1958  
  1959  	// Read size to change the block device size per container.
  1960  	for key, val := range storageOpt {
  1961  		key := strings.ToLower(key)
  1962  		switch key {
  1963  		case "size":
  1964  			size, err := units.RAMInBytes(val)
  1965  			if err != nil {
  1966  				return 0, err
  1967  			}
  1968  			return uint64(size), nil
  1969  		default:
  1970  			return 0, fmt.Errorf("Unknown option %s", key)
  1971  		}
  1972  	}
  1973  
  1974  	return 0, nil
  1975  }
  1976  
  1977  func (devices *DeviceSet) markForDeferredDeletion(info *devInfo) error {
  1978  	// If device is already in deleted state, there is nothing to be done.
  1979  	if info.Deleted {
  1980  		return nil
  1981  	}
  1982  
  1983  	logrus.WithField("storage-driver", "devicemapper").Debugf("Marking device %s for deferred deletion.", info.Hash)
  1984  
  1985  	info.Deleted = true
  1986  
  1987  	// save device metadata to reflect deleted state.
  1988  	if err := devices.saveMetadata(info); err != nil {
  1989  		info.Deleted = false
  1990  		return err
  1991  	}
  1992  
  1993  	devices.nrDeletedDevices++
  1994  	return nil
  1995  }
  1996  
  1997  // Should be called with devices.Lock() held.
  1998  func (devices *DeviceSet) deleteTransaction(info *devInfo, syncDelete bool) error {
  1999  	if err := devices.openTransaction(info.Hash, info.DeviceID); err != nil {
  2000  		logrus.WithField("storage-driver", "devicemapper").Debugf("Error opening transaction hash = %s deviceId = %d", "", info.DeviceID)
  2001  		return err
  2002  	}
  2003  
  2004  	defer devices.closeTransaction()
  2005  
  2006  	err := devicemapper.DeleteDevice(devices.getPoolDevName(), info.DeviceID)
  2007  	if err != nil {
  2008  		// If syncDelete is true, we want to return error. If deferred
  2009  		// deletion is not enabled, we return an error. If error is
  2010  		// something other then EBUSY, return an error.
  2011  		if syncDelete || !devices.deferredDelete || err != devicemapper.ErrBusy {
  2012  			logrus.WithField("storage-driver", "devicemapper").Debugf("Error deleting device: %s", err)
  2013  			return err
  2014  		}
  2015  	}
  2016  
  2017  	if err == nil {
  2018  		if err := devices.unregisterDevice(info.Hash); err != nil {
  2019  			return err
  2020  		}
  2021  		// If device was already in deferred delete state that means
  2022  		// deletion was being tried again later. Reduce the deleted
  2023  		// device count.
  2024  		if info.Deleted {
  2025  			devices.nrDeletedDevices--
  2026  		}
  2027  		devices.markDeviceIDFree(info.DeviceID)
  2028  	} else {
  2029  		if err := devices.markForDeferredDeletion(info); err != nil {
  2030  			return err
  2031  		}
  2032  	}
  2033  
  2034  	return nil
  2035  }
  2036  
  2037  // Issue discard only if device open count is zero.
  2038  func (devices *DeviceSet) issueDiscard(info *devInfo) error {
  2039  	logger := logrus.WithField("storage-driver", "devicemapper")
  2040  	logger.Debugf("issueDiscard START(device: %s).", info.Hash)
  2041  	defer logger.Debugf("issueDiscard END(device: %s).", info.Hash)
  2042  	// This is a workaround for the kernel not discarding block so
  2043  	// on the thin pool when we remove a thinp device, so we do it
  2044  	// manually.
  2045  	// Even if device is deferred deleted, activate it and issue
  2046  	// discards.
  2047  	if err := devices.activateDeviceIfNeeded(info, true); err != nil {
  2048  		return err
  2049  	}
  2050  
  2051  	devinfo, err := devicemapper.GetInfo(info.Name())
  2052  	if err != nil {
  2053  		return err
  2054  	}
  2055  
  2056  	if devinfo.OpenCount != 0 {
  2057  		logger.Debugf("Device: %s is in use. OpenCount=%d. Not issuing discards.", info.Hash, devinfo.OpenCount)
  2058  		return nil
  2059  	}
  2060  
  2061  	if err := devicemapper.BlockDeviceDiscard(info.DevName()); err != nil {
  2062  		logger.Debugf("Error discarding block on device: %s (ignoring)", err)
  2063  	}
  2064  	return nil
  2065  }
  2066  
  2067  // Should be called with devices.Lock() held.
  2068  func (devices *DeviceSet) deleteDevice(info *devInfo, syncDelete bool) error {
  2069  	if devices.doBlkDiscard {
  2070  		devices.issueDiscard(info)
  2071  	}
  2072  
  2073  	// Try to deactivate device in case it is active.
  2074  	// If deferred removal is enabled and deferred deletion is disabled
  2075  	// then make sure device is removed synchronously. There have been
  2076  	// some cases of device being busy for short duration and we would
  2077  	// rather busy wait for device removal to take care of these cases.
  2078  	deferredRemove := devices.deferredRemove
  2079  	if !devices.deferredDelete {
  2080  		deferredRemove = false
  2081  	}
  2082  
  2083  	if err := devices.deactivateDeviceMode(info, deferredRemove); err != nil {
  2084  		logrus.WithField("storage-driver", "devicemapper").Debugf("Error deactivating device: %s", err)
  2085  		return err
  2086  	}
  2087  
  2088  	return devices.deleteTransaction(info, syncDelete)
  2089  }
  2090  
  2091  // DeleteDevice will return success if device has been marked for deferred
  2092  // removal. If one wants to override that and want DeleteDevice() to fail if
  2093  // device was busy and could not be deleted, set syncDelete=true.
  2094  func (devices *DeviceSet) DeleteDevice(hash string, syncDelete bool) error {
  2095  	logrus.WithField("storage-driver", "devicemapper").Debugf("DeleteDevice START(hash=%v syncDelete=%v)", hash, syncDelete)
  2096  	defer logrus.WithField("storage-driver", "devicemapper").Debugf("DeleteDevice END(hash=%v syncDelete=%v)", hash, syncDelete)
  2097  	info, err := devices.lookupDeviceWithLock(hash)
  2098  	if err != nil {
  2099  		return err
  2100  	}
  2101  
  2102  	info.lock.Lock()
  2103  	defer info.lock.Unlock()
  2104  
  2105  	devices.Lock()
  2106  	defer devices.Unlock()
  2107  
  2108  	return devices.deleteDevice(info, syncDelete)
  2109  }
  2110  
  2111  func (devices *DeviceSet) deactivatePool() error {
  2112  	logrus.WithField("storage-driver", "devicemapper").Debug("deactivatePool() START")
  2113  	defer logrus.WithField("storage-driver", "devicemapper").Debug("deactivatePool() END")
  2114  	devname := devices.getPoolDevName()
  2115  
  2116  	devinfo, err := devicemapper.GetInfo(devname)
  2117  	if err != nil {
  2118  		return err
  2119  	}
  2120  
  2121  	if devinfo.Exists == 0 {
  2122  		return nil
  2123  	}
  2124  	if err := devicemapper.RemoveDevice(devname); err != nil {
  2125  		return err
  2126  	}
  2127  
  2128  	if d, err := devicemapper.GetDeps(devname); err == nil {
  2129  		logrus.WithField("storage-driver", "devicemapper").Warnf("device %s still has %d active dependents", devname, d.Count)
  2130  	}
  2131  
  2132  	return nil
  2133  }
  2134  
  2135  func (devices *DeviceSet) deactivateDevice(info *devInfo) error {
  2136  	return devices.deactivateDeviceMode(info, devices.deferredRemove)
  2137  }
  2138  
  2139  func (devices *DeviceSet) deactivateDeviceMode(info *devInfo, deferredRemove bool) error {
  2140  	var err error
  2141  	logrus.WithField("storage-driver", "devicemapper").Debugf("deactivateDevice START(%s)", info.Hash)
  2142  	defer logrus.WithField("storage-driver", "devicemapper").Debugf("deactivateDevice END(%s)", info.Hash)
  2143  
  2144  	devinfo, err := devicemapper.GetInfo(info.Name())
  2145  	if err != nil {
  2146  		return err
  2147  	}
  2148  
  2149  	if devinfo.Exists == 0 {
  2150  		return nil
  2151  	}
  2152  
  2153  	if deferredRemove {
  2154  		err = devicemapper.RemoveDeviceDeferred(info.Name())
  2155  	} else {
  2156  		err = devices.removeDevice(info.Name())
  2157  	}
  2158  
  2159  	// This function's semantics is such that it does not return an
  2160  	// error if device does not exist. So if device went away by
  2161  	// the time we actually tried to remove it, do not return error.
  2162  	if err != devicemapper.ErrEnxio {
  2163  		return err
  2164  	}
  2165  	return nil
  2166  }
  2167  
  2168  // Issues the underlying dm remove operation.
  2169  func (devices *DeviceSet) removeDevice(devname string) error {
  2170  	var err error
  2171  
  2172  	logrus.WithField("storage-driver", "devicemapper").Debugf("removeDevice START(%s)", devname)
  2173  	defer logrus.WithField("storage-driver", "devicemapper").Debugf("removeDevice END(%s)", devname)
  2174  
  2175  	for i := 0; i < 200; i++ {
  2176  		err = devicemapper.RemoveDevice(devname)
  2177  		if err == nil {
  2178  			break
  2179  		}
  2180  		if err != devicemapper.ErrBusy {
  2181  			return err
  2182  		}
  2183  
  2184  		// If we see EBUSY it may be a transient error,
  2185  		// sleep a bit a retry a few times.
  2186  		devices.Unlock()
  2187  		time.Sleep(100 * time.Millisecond)
  2188  		devices.Lock()
  2189  	}
  2190  
  2191  	return err
  2192  }
  2193  
  2194  func (devices *DeviceSet) cancelDeferredRemovalIfNeeded(info *devInfo) error {
  2195  	if !devices.deferredRemove {
  2196  		return nil
  2197  	}
  2198  
  2199  	logrus.WithField("storage-driver", "devicemapper").Debugf("cancelDeferredRemovalIfNeeded START(%s)", info.Name())
  2200  	defer logrus.WithField("storage-driver", "devicemapper").Debugf("cancelDeferredRemovalIfNeeded END(%s)", info.Name())
  2201  
  2202  	devinfo, err := devicemapper.GetInfoWithDeferred(info.Name())
  2203  	if err != nil {
  2204  		return err
  2205  	}
  2206  
  2207  	if devinfo != nil && devinfo.DeferredRemove == 0 {
  2208  		return nil
  2209  	}
  2210  
  2211  	// Cancel deferred remove
  2212  	if err := devices.cancelDeferredRemoval(info); err != nil {
  2213  		// If Error is ErrEnxio. Device is probably already gone. Continue.
  2214  		if err != devicemapper.ErrEnxio {
  2215  			return err
  2216  		}
  2217  	}
  2218  	return nil
  2219  }
  2220  
  2221  func (devices *DeviceSet) cancelDeferredRemoval(info *devInfo) error {
  2222  	logrus.WithField("storage-driver", "devicemapper").Debugf("cancelDeferredRemoval START(%s)", info.Name())
  2223  	defer logrus.WithField("storage-driver", "devicemapper").Debugf("cancelDeferredRemoval END(%s)", info.Name())
  2224  
  2225  	var err error
  2226  
  2227  	// Cancel deferred remove
  2228  	for i := 0; i < 100; i++ {
  2229  		err = devicemapper.CancelDeferredRemove(info.Name())
  2230  		if err != nil {
  2231  			if err == devicemapper.ErrBusy {
  2232  				// If we see EBUSY it may be a transient error,
  2233  				// sleep a bit a retry a few times.
  2234  				devices.Unlock()
  2235  				time.Sleep(100 * time.Millisecond)
  2236  				devices.Lock()
  2237  				continue
  2238  			}
  2239  		}
  2240  		break
  2241  	}
  2242  	return err
  2243  }
  2244  
  2245  func (devices *DeviceSet) unmountAndDeactivateAll(dir string) {
  2246  	logger := logrus.WithField("storage-driver", "devicemapper")
  2247  
  2248  	files, err := ioutil.ReadDir(dir)
  2249  	if err != nil {
  2250  		logger.Warnf("unmountAndDeactivate: %s", err)
  2251  		return
  2252  	}
  2253  
  2254  	for _, d := range files {
  2255  		if !d.IsDir() {
  2256  			continue
  2257  		}
  2258  
  2259  		name := d.Name()
  2260  		fullname := path.Join(dir, name)
  2261  
  2262  		// We use MNT_DETACH here in case it is still busy in some running
  2263  		// container. This means it'll go away from the global scope directly,
  2264  		// and the device will be released when that container dies.
  2265  		if err := unix.Unmount(fullname, unix.MNT_DETACH); err != nil && err != unix.EINVAL {
  2266  			logger.Warnf("Shutdown unmounting %s, error: %s", fullname, err)
  2267  		}
  2268  
  2269  		if devInfo, err := devices.lookupDevice(name); err != nil {
  2270  			logger.Debugf("Shutdown lookup device %s, error: %s", name, err)
  2271  		} else {
  2272  			if err := devices.deactivateDevice(devInfo); err != nil {
  2273  				logger.Debugf("Shutdown deactivate %s, error: %s", devInfo.Hash, err)
  2274  			}
  2275  		}
  2276  	}
  2277  }
  2278  
  2279  // Shutdown shuts down the device by unmounting the root.
  2280  func (devices *DeviceSet) Shutdown(home string) error {
  2281  	logger := logrus.WithField("storage-driver", "devicemapper")
  2282  
  2283  	logger.Debugf("[deviceset %s] Shutdown()", devices.devicePrefix)
  2284  	logger.Debugf("Shutting down DeviceSet: %s", devices.root)
  2285  	defer logger.Debugf("[deviceset %s] Shutdown() END", devices.devicePrefix)
  2286  
  2287  	// Stop deletion worker. This should start delivering new events to
  2288  	// ticker channel. That means no new instance of cleanupDeletedDevice()
  2289  	// will run after this call. If one instance is already running at
  2290  	// the time of the call, it must be holding devices.Lock() and
  2291  	// we will block on this lock till cleanup function exits.
  2292  	devices.deletionWorkerTicker.Stop()
  2293  
  2294  	devices.Lock()
  2295  	// Save DeviceSet Metadata first. Docker kills all threads if they
  2296  	// don't finish in certain time. It is possible that Shutdown()
  2297  	// routine does not finish in time as we loop trying to deactivate
  2298  	// some devices while these are busy. In that case shutdown() routine
  2299  	// will be killed and we will not get a chance to save deviceset
  2300  	// metadata. Hence save this early before trying to deactivate devices.
  2301  	devices.saveDeviceSetMetaData()
  2302  	devices.unmountAndDeactivateAll(path.Join(home, "mnt"))
  2303  	devices.Unlock()
  2304  
  2305  	info, _ := devices.lookupDeviceWithLock("")
  2306  	if info != nil {
  2307  		info.lock.Lock()
  2308  		devices.Lock()
  2309  		if err := devices.deactivateDevice(info); err != nil {
  2310  			logger.Debugf("Shutdown deactivate base , error: %s", err)
  2311  		}
  2312  		devices.Unlock()
  2313  		info.lock.Unlock()
  2314  	}
  2315  
  2316  	devices.Lock()
  2317  	if devices.thinPoolDevice == "" {
  2318  		if err := devices.deactivatePool(); err != nil {
  2319  			logger.Debugf("Shutdown deactivate pool , error: %s", err)
  2320  		}
  2321  	}
  2322  	devices.Unlock()
  2323  
  2324  	return nil
  2325  }
  2326  
  2327  // Recent XFS changes allow changing behavior of filesystem in case of errors.
  2328  // When thin pool gets full and XFS gets ENOSPC error, currently it tries
  2329  // IO infinitely and sometimes it can block the container process
  2330  // and process can't be killWith 0 value, XFS will not retry upon error
  2331  // and instead will shutdown filesystem.
  2332  
  2333  func (devices *DeviceSet) xfsSetNospaceRetries(info *devInfo) error {
  2334  	dmDevicePath, err := os.Readlink(info.DevName())
  2335  	if err != nil {
  2336  		return fmt.Errorf("devmapper: readlink failed for device %v:%v", info.DevName(), err)
  2337  	}
  2338  
  2339  	dmDeviceName := path.Base(dmDevicePath)
  2340  	filePath := "/sys/fs/xfs/" + dmDeviceName + "/error/metadata/ENOSPC/max_retries"
  2341  	maxRetriesFile, err := os.OpenFile(filePath, os.O_WRONLY, 0)
  2342  	if err != nil {
  2343  		return fmt.Errorf("devmapper: user specified daemon option dm.xfs_nospace_max_retries but it does not seem to be supported on this system :%v", err)
  2344  	}
  2345  	defer maxRetriesFile.Close()
  2346  
  2347  	// Set max retries to 0
  2348  	_, err = maxRetriesFile.WriteString(devices.xfsNospaceRetries)
  2349  	if err != nil {
  2350  		return fmt.Errorf("devmapper: Failed to write string %v to file %v:%v", devices.xfsNospaceRetries, filePath, err)
  2351  	}
  2352  	return nil
  2353  }
  2354  
  2355  // MountDevice mounts the device if not already mounted.
  2356  func (devices *DeviceSet) MountDevice(hash, path, mountLabel string) error {
  2357  	info, err := devices.lookupDeviceWithLock(hash)
  2358  	if err != nil {
  2359  		return err
  2360  	}
  2361  
  2362  	if info.Deleted {
  2363  		return fmt.Errorf("devmapper: Can't mount device %v as it has been marked for deferred deletion", info.Hash)
  2364  	}
  2365  
  2366  	info.lock.Lock()
  2367  	defer info.lock.Unlock()
  2368  
  2369  	devices.Lock()
  2370  	defer devices.Unlock()
  2371  
  2372  	if err := devices.activateDeviceIfNeeded(info, false); err != nil {
  2373  		return fmt.Errorf("devmapper: Error activating devmapper device for '%s': %s", hash, err)
  2374  	}
  2375  
  2376  	fstype, err := ProbeFsType(info.DevName())
  2377  	if err != nil {
  2378  		return err
  2379  	}
  2380  
  2381  	options := ""
  2382  
  2383  	if fstype == "xfs" {
  2384  		// XFS needs nouuid or it can't mount filesystems with the same fs
  2385  		options = joinMountOptions(options, "nouuid")
  2386  	}
  2387  
  2388  	options = joinMountOptions(options, devices.mountOptions)
  2389  	options = joinMountOptions(options, label.FormatMountLabel("", mountLabel))
  2390  
  2391  	if err := mount.Mount(info.DevName(), path, fstype, options); err != nil {
  2392  		return errors.Wrapf(err, "Failed to mount; dmesg: %s", string(dmesg.Dmesg(256)))
  2393  	}
  2394  
  2395  	if fstype == "xfs" && devices.xfsNospaceRetries != "" {
  2396  		if err := devices.xfsSetNospaceRetries(info); err != nil {
  2397  			unix.Unmount(path, unix.MNT_DETACH)
  2398  			devices.deactivateDevice(info)
  2399  			return err
  2400  		}
  2401  	}
  2402  
  2403  	return nil
  2404  }
  2405  
  2406  // UnmountDevice unmounts the device and removes it from hash.
  2407  func (devices *DeviceSet) UnmountDevice(hash, mountPath string) error {
  2408  	logger := logrus.WithField("storage-driver", "devicemapper")
  2409  
  2410  	logger.Debugf("UnmountDevice START(hash=%s)", hash)
  2411  	defer logger.Debugf("UnmountDevice END(hash=%s)", hash)
  2412  
  2413  	info, err := devices.lookupDeviceWithLock(hash)
  2414  	if err != nil {
  2415  		return err
  2416  	}
  2417  
  2418  	info.lock.Lock()
  2419  	defer info.lock.Unlock()
  2420  
  2421  	devices.Lock()
  2422  	defer devices.Unlock()
  2423  
  2424  	logger.Debugf("Unmount(%s)", mountPath)
  2425  	if err := unix.Unmount(mountPath, unix.MNT_DETACH); err != nil {
  2426  		return err
  2427  	}
  2428  	logger.Debug("Unmount done")
  2429  
  2430  	// Remove the mountpoint here. Removing the mountpoint (in newer kernels)
  2431  	// will cause all other instances of this mount in other mount namespaces
  2432  	// to be killed (this is an anti-DoS measure that is necessary for things
  2433  	// like devicemapper). This is necessary to avoid cases where a libdm mount
  2434  	// that is present in another namespace will cause subsequent RemoveDevice
  2435  	// operations to fail. We ignore any errors here because this may fail on
  2436  	// older kernels which don't have
  2437  	// torvalds/linux@8ed936b5671bfb33d89bc60bdcc7cf0470ba52fe applied.
  2438  	if err := os.Remove(mountPath); err != nil {
  2439  		logger.Debugf("error doing a remove on unmounted device %s: %v", mountPath, err)
  2440  	}
  2441  
  2442  	return devices.deactivateDevice(info)
  2443  }
  2444  
  2445  // HasDevice returns true if the device metadata exists.
  2446  func (devices *DeviceSet) HasDevice(hash string) bool {
  2447  	info, _ := devices.lookupDeviceWithLock(hash)
  2448  	return info != nil
  2449  }
  2450  
  2451  // List returns a list of device ids.
  2452  func (devices *DeviceSet) List() []string {
  2453  	devices.Lock()
  2454  	defer devices.Unlock()
  2455  
  2456  	ids := make([]string, len(devices.Devices))
  2457  	i := 0
  2458  	for k := range devices.Devices {
  2459  		ids[i] = k
  2460  		i++
  2461  	}
  2462  	return ids
  2463  }
  2464  
  2465  func (devices *DeviceSet) deviceStatus(devName string) (sizeInSectors, mappedSectors, highestMappedSector uint64, err error) {
  2466  	var params string
  2467  	_, sizeInSectors, _, params, err = devicemapper.GetStatus(devName)
  2468  	if err != nil {
  2469  		return
  2470  	}
  2471  	if _, err = fmt.Sscanf(params, "%d %d", &mappedSectors, &highestMappedSector); err == nil {
  2472  		return
  2473  	}
  2474  	return
  2475  }
  2476  
  2477  // GetDeviceStatus provides size, mapped sectors
  2478  func (devices *DeviceSet) GetDeviceStatus(hash string) (*DevStatus, error) {
  2479  	info, err := devices.lookupDeviceWithLock(hash)
  2480  	if err != nil {
  2481  		return nil, err
  2482  	}
  2483  
  2484  	info.lock.Lock()
  2485  	defer info.lock.Unlock()
  2486  
  2487  	devices.Lock()
  2488  	defer devices.Unlock()
  2489  
  2490  	status := &DevStatus{
  2491  		DeviceID:      info.DeviceID,
  2492  		Size:          info.Size,
  2493  		TransactionID: info.TransactionID,
  2494  	}
  2495  
  2496  	if err := devices.activateDeviceIfNeeded(info, false); err != nil {
  2497  		return nil, fmt.Errorf("devmapper: Error activating devmapper device for '%s': %s", hash, err)
  2498  	}
  2499  
  2500  	sizeInSectors, mappedSectors, highestMappedSector, err := devices.deviceStatus(info.DevName())
  2501  
  2502  	if err != nil {
  2503  		return nil, err
  2504  	}
  2505  
  2506  	status.SizeInSectors = sizeInSectors
  2507  	status.MappedSectors = mappedSectors
  2508  	status.HighestMappedSector = highestMappedSector
  2509  
  2510  	return status, nil
  2511  }
  2512  
  2513  func (devices *DeviceSet) poolStatus() (totalSizeInSectors, transactionID, dataUsed, dataTotal, metadataUsed, metadataTotal uint64, err error) {
  2514  	var params string
  2515  	if _, totalSizeInSectors, _, params, err = devicemapper.GetStatus(devices.getPoolName()); err == nil {
  2516  		_, err = fmt.Sscanf(params, "%d %d/%d %d/%d", &transactionID, &metadataUsed, &metadataTotal, &dataUsed, &dataTotal)
  2517  	}
  2518  	return
  2519  }
  2520  
  2521  // DataDevicePath returns the path to the data storage for this deviceset,
  2522  // regardless of loopback or block device
  2523  func (devices *DeviceSet) DataDevicePath() string {
  2524  	return devices.dataDevice
  2525  }
  2526  
  2527  // MetadataDevicePath returns the path to the metadata storage for this deviceset,
  2528  // regardless of loopback or block device
  2529  func (devices *DeviceSet) MetadataDevicePath() string {
  2530  	return devices.metadataDevice
  2531  }
  2532  
  2533  func (devices *DeviceSet) getUnderlyingAvailableSpace(loopFile string) (uint64, error) {
  2534  	buf := new(unix.Statfs_t)
  2535  	if err := unix.Statfs(loopFile, buf); err != nil {
  2536  		logrus.WithField("storage-driver", "devicemapper").Warnf("Couldn't stat loopfile filesystem %v: %v", loopFile, err)
  2537  		return 0, err
  2538  	}
  2539  	return buf.Bfree * uint64(buf.Bsize), nil
  2540  }
  2541  
  2542  func (devices *DeviceSet) isRealFile(loopFile string) (bool, error) {
  2543  	if loopFile != "" {
  2544  		fi, err := os.Stat(loopFile)
  2545  		if err != nil {
  2546  			logrus.WithField("storage-driver", "devicemapper").Warnf("Couldn't stat loopfile %v: %v", loopFile, err)
  2547  			return false, err
  2548  		}
  2549  		return fi.Mode().IsRegular(), nil
  2550  	}
  2551  	return false, nil
  2552  }
  2553  
  2554  // Status returns the current status of this deviceset
  2555  func (devices *DeviceSet) Status() *Status {
  2556  	devices.Lock()
  2557  	defer devices.Unlock()
  2558  
  2559  	status := &Status{}
  2560  
  2561  	status.PoolName = devices.getPoolName()
  2562  	status.DataFile = devices.DataDevicePath()
  2563  	status.DataLoopback = devices.dataLoopFile
  2564  	status.MetadataFile = devices.MetadataDevicePath()
  2565  	status.MetadataLoopback = devices.metadataLoopFile
  2566  	status.UdevSyncSupported = devicemapper.UdevSyncSupported()
  2567  	status.DeferredRemoveEnabled = devices.deferredRemove
  2568  	status.DeferredDeleteEnabled = devices.deferredDelete
  2569  	status.DeferredDeletedDeviceCount = devices.nrDeletedDevices
  2570  	status.BaseDeviceSize = devices.getBaseDeviceSize()
  2571  	status.BaseDeviceFS = devices.getBaseDeviceFS()
  2572  
  2573  	totalSizeInSectors, _, dataUsed, dataTotal, metadataUsed, metadataTotal, err := devices.poolStatus()
  2574  	if err == nil {
  2575  		// Convert from blocks to bytes
  2576  		blockSizeInSectors := totalSizeInSectors / dataTotal
  2577  
  2578  		status.Data.Used = dataUsed * blockSizeInSectors * 512
  2579  		status.Data.Total = dataTotal * blockSizeInSectors * 512
  2580  		status.Data.Available = status.Data.Total - status.Data.Used
  2581  
  2582  		// metadata blocks are always 4k
  2583  		status.Metadata.Used = metadataUsed * 4096
  2584  		status.Metadata.Total = metadataTotal * 4096
  2585  		status.Metadata.Available = status.Metadata.Total - status.Metadata.Used
  2586  
  2587  		status.SectorSize = blockSizeInSectors * 512
  2588  
  2589  		if check, _ := devices.isRealFile(devices.dataLoopFile); check {
  2590  			actualSpace, err := devices.getUnderlyingAvailableSpace(devices.dataLoopFile)
  2591  			if err == nil && actualSpace < status.Data.Available {
  2592  				status.Data.Available = actualSpace
  2593  			}
  2594  		}
  2595  
  2596  		if check, _ := devices.isRealFile(devices.metadataLoopFile); check {
  2597  			actualSpace, err := devices.getUnderlyingAvailableSpace(devices.metadataLoopFile)
  2598  			if err == nil && actualSpace < status.Metadata.Available {
  2599  				status.Metadata.Available = actualSpace
  2600  			}
  2601  		}
  2602  
  2603  		minFreeData := (dataTotal * uint64(devices.minFreeSpacePercent)) / 100
  2604  		status.MinFreeSpace = minFreeData * blockSizeInSectors * 512
  2605  	}
  2606  
  2607  	return status
  2608  }
  2609  
  2610  // Status returns the current status of this deviceset
  2611  func (devices *DeviceSet) exportDeviceMetadata(hash string) (*deviceMetadata, error) {
  2612  	info, err := devices.lookupDeviceWithLock(hash)
  2613  	if err != nil {
  2614  		return nil, err
  2615  	}
  2616  
  2617  	info.lock.Lock()
  2618  	defer info.lock.Unlock()
  2619  
  2620  	metadata := &deviceMetadata{info.DeviceID, info.Size, info.Name()}
  2621  	return metadata, nil
  2622  }
  2623  
  2624  // NewDeviceSet creates the device set based on the options provided.
  2625  func NewDeviceSet(root string, doInit bool, options []string, uidMaps, gidMaps []idtools.IDMap) (*DeviceSet, error) {
  2626  	devicemapper.SetDevDir("/dev")
  2627  
  2628  	devices := &DeviceSet{
  2629  		root:                  root,
  2630  		metaData:              metaData{Devices: make(map[string]*devInfo)},
  2631  		dataLoopbackSize:      defaultDataLoopbackSize,
  2632  		metaDataLoopbackSize:  defaultMetaDataLoopbackSize,
  2633  		baseFsSize:            defaultBaseFsSize,
  2634  		overrideUdevSyncCheck: defaultUdevSyncOverride,
  2635  		doBlkDiscard:          true,
  2636  		thinpBlockSize:        defaultThinpBlockSize,
  2637  		deviceIDMap:           make([]byte, deviceIDMapSz),
  2638  		deletionWorkerTicker:  time.NewTicker(time.Second * 30),
  2639  		uidMaps:               uidMaps,
  2640  		gidMaps:               gidMaps,
  2641  		minFreeSpacePercent:   defaultMinFreeSpacePercent,
  2642  	}
  2643  
  2644  	version, err := devicemapper.GetDriverVersion()
  2645  	if err != nil {
  2646  		// Can't even get driver version, assume not supported
  2647  		return nil, graphdriver.ErrNotSupported
  2648  	}
  2649  
  2650  	if err := determineDriverCapabilities(version); err != nil {
  2651  		return nil, graphdriver.ErrNotSupported
  2652  	}
  2653  
  2654  	if driverDeferredRemovalSupport && devicemapper.LibraryDeferredRemovalSupport {
  2655  		// enable deferred stuff by default
  2656  		enableDeferredDeletion = true
  2657  		enableDeferredRemoval = true
  2658  	}
  2659  
  2660  	foundBlkDiscard := false
  2661  	var lvmSetupConfig directLVMConfig
  2662  	for _, option := range options {
  2663  		key, val, err := parsers.ParseKeyValueOpt(option)
  2664  		if err != nil {
  2665  			return nil, err
  2666  		}
  2667  		key = strings.ToLower(key)
  2668  		switch key {
  2669  		case "dm.basesize":
  2670  			size, err := units.RAMInBytes(val)
  2671  			if err != nil {
  2672  				return nil, err
  2673  			}
  2674  			userBaseSize = true
  2675  			devices.baseFsSize = uint64(size)
  2676  		case "dm.loopdatasize":
  2677  			size, err := units.RAMInBytes(val)
  2678  			if err != nil {
  2679  				return nil, err
  2680  			}
  2681  			devices.dataLoopbackSize = size
  2682  		case "dm.loopmetadatasize":
  2683  			size, err := units.RAMInBytes(val)
  2684  			if err != nil {
  2685  				return nil, err
  2686  			}
  2687  			devices.metaDataLoopbackSize = size
  2688  		case "dm.fs":
  2689  			if val != "ext4" && val != "xfs" {
  2690  				return nil, fmt.Errorf("devmapper: Unsupported filesystem %s", val)
  2691  			}
  2692  			devices.filesystem = val
  2693  		case "dm.mkfsarg":
  2694  			devices.mkfsArgs = append(devices.mkfsArgs, val)
  2695  		case "dm.mountopt":
  2696  			devices.mountOptions = joinMountOptions(devices.mountOptions, val)
  2697  		case "dm.metadatadev":
  2698  			devices.metadataDevice = val
  2699  		case "dm.datadev":
  2700  			devices.dataDevice = val
  2701  		case "dm.thinpooldev":
  2702  			devices.thinPoolDevice = strings.TrimPrefix(val, "/dev/mapper/")
  2703  		case "dm.blkdiscard":
  2704  			foundBlkDiscard = true
  2705  			devices.doBlkDiscard, err = strconv.ParseBool(val)
  2706  			if err != nil {
  2707  				return nil, err
  2708  			}
  2709  		case "dm.blocksize":
  2710  			size, err := units.RAMInBytes(val)
  2711  			if err != nil {
  2712  				return nil, err
  2713  			}
  2714  			// convert to 512b sectors
  2715  			devices.thinpBlockSize = uint32(size) >> 9
  2716  		case "dm.override_udev_sync_check":
  2717  			devices.overrideUdevSyncCheck, err = strconv.ParseBool(val)
  2718  			if err != nil {
  2719  				return nil, err
  2720  			}
  2721  
  2722  		case "dm.use_deferred_removal":
  2723  			enableDeferredRemoval, err = strconv.ParseBool(val)
  2724  			if err != nil {
  2725  				return nil, err
  2726  			}
  2727  
  2728  		case "dm.use_deferred_deletion":
  2729  			enableDeferredDeletion, err = strconv.ParseBool(val)
  2730  			if err != nil {
  2731  				return nil, err
  2732  			}
  2733  
  2734  		case "dm.min_free_space":
  2735  			if !strings.HasSuffix(val, "%") {
  2736  				return nil, fmt.Errorf("devmapper: Option dm.min_free_space requires %% suffix")
  2737  			}
  2738  
  2739  			valstring := strings.TrimSuffix(val, "%")
  2740  			minFreeSpacePercent, err := strconv.ParseUint(valstring, 10, 32)
  2741  			if err != nil {
  2742  				return nil, err
  2743  			}
  2744  
  2745  			if minFreeSpacePercent >= 100 {
  2746  				return nil, fmt.Errorf("devmapper: Invalid value %v for option dm.min_free_space", val)
  2747  			}
  2748  
  2749  			devices.minFreeSpacePercent = uint32(minFreeSpacePercent)
  2750  		case "dm.xfs_nospace_max_retries":
  2751  			_, err := strconv.ParseUint(val, 10, 64)
  2752  			if err != nil {
  2753  				return nil, err
  2754  			}
  2755  			devices.xfsNospaceRetries = val
  2756  		case "dm.directlvm_device":
  2757  			lvmSetupConfig.Device = val
  2758  		case "dm.directlvm_device_force":
  2759  			lvmSetupConfigForce, err = strconv.ParseBool(val)
  2760  			if err != nil {
  2761  				return nil, err
  2762  			}
  2763  		case "dm.thinp_percent":
  2764  			per, err := strconv.ParseUint(strings.TrimSuffix(val, "%"), 10, 32)
  2765  			if err != nil {
  2766  				return nil, errors.Wrapf(err, "could not parse `dm.thinp_percent=%s`", val)
  2767  			}
  2768  			if per >= 100 {
  2769  				return nil, errors.New("dm.thinp_percent must be greater than 0 and less than 100")
  2770  			}
  2771  			lvmSetupConfig.ThinpPercent = per
  2772  		case "dm.thinp_metapercent":
  2773  			per, err := strconv.ParseUint(strings.TrimSuffix(val, "%"), 10, 32)
  2774  			if err != nil {
  2775  				return nil, errors.Wrapf(err, "could not parse `dm.thinp_metapercent=%s`", val)
  2776  			}
  2777  			if per >= 100 {
  2778  				return nil, errors.New("dm.thinp_metapercent must be greater than 0 and less than 100")
  2779  			}
  2780  			lvmSetupConfig.ThinpMetaPercent = per
  2781  		case "dm.thinp_autoextend_percent":
  2782  			per, err := strconv.ParseUint(strings.TrimSuffix(val, "%"), 10, 32)
  2783  			if err != nil {
  2784  				return nil, errors.Wrapf(err, "could not parse `dm.thinp_autoextend_percent=%s`", val)
  2785  			}
  2786  			if per > 100 {
  2787  				return nil, errors.New("dm.thinp_autoextend_percent must be greater than 0 and less than 100")
  2788  			}
  2789  			lvmSetupConfig.AutoExtendPercent = per
  2790  		case "dm.thinp_autoextend_threshold":
  2791  			per, err := strconv.ParseUint(strings.TrimSuffix(val, "%"), 10, 32)
  2792  			if err != nil {
  2793  				return nil, errors.Wrapf(err, "could not parse `dm.thinp_autoextend_threshold=%s`", val)
  2794  			}
  2795  			if per > 100 {
  2796  				return nil, errors.New("dm.thinp_autoextend_threshold must be greater than 0 and less than 100")
  2797  			}
  2798  			lvmSetupConfig.AutoExtendThreshold = per
  2799  		case "dm.libdm_log_level":
  2800  			level, err := strconv.ParseInt(val, 10, 32)
  2801  			if err != nil {
  2802  				return nil, errors.Wrapf(err, "could not parse `dm.libdm_log_level=%s`", val)
  2803  			}
  2804  			if level < devicemapper.LogLevelFatal || level > devicemapper.LogLevelDebug {
  2805  				return nil, errors.Errorf("dm.libdm_log_level must be in range [%d,%d]", devicemapper.LogLevelFatal, devicemapper.LogLevelDebug)
  2806  			}
  2807  			// Register a new logging callback with the specified level.
  2808  			devicemapper.LogInit(devicemapper.DefaultLogger{
  2809  				Level: int(level),
  2810  			})
  2811  		default:
  2812  			return nil, fmt.Errorf("devmapper: Unknown option %s", key)
  2813  		}
  2814  	}
  2815  
  2816  	if err := validateLVMConfig(lvmSetupConfig); err != nil {
  2817  		return nil, err
  2818  	}
  2819  
  2820  	devices.lvmSetupConfig = lvmSetupConfig
  2821  
  2822  	// By default, don't do blk discard hack on raw devices, its rarely useful and is expensive
  2823  	if !foundBlkDiscard && (devices.dataDevice != "" || devices.thinPoolDevice != "") {
  2824  		devices.doBlkDiscard = false
  2825  	}
  2826  
  2827  	if err := devices.initDevmapper(doInit); err != nil {
  2828  		return nil, err
  2829  	}
  2830  
  2831  	return devices, nil
  2832  }