github.com/demonoid81/moby@v0.0.0-20200517203328-62dd8e17c460/daemon/graphdriver/quota/projectquota.go (about)

     1  // +build linux,!exclude_disk_quota,cgo
     2  
     3  //
     4  // projectquota.go - implements XFS project quota controls
     5  // for setting quota limits on a newly created directory.
     6  // It currently supports the legacy XFS specific ioctls.
     7  //
     8  // TODO: use generic quota control ioctl FS_IOC_FS{GET,SET}XATTR
     9  //       for both xfs/ext4 for kernel version >= v4.5
    10  //
    11  
    12  package quota // import "github.com/demonoid81/moby/daemon/graphdriver/quota"
    13  
    14  /*
    15  #include <stdlib.h>
    16  #include <dirent.h>
    17  #include <linux/fs.h>
    18  #include <linux/quota.h>
    19  #include <linux/dqblk_xfs.h>
    20  
    21  #ifndef FS_XFLAG_PROJINHERIT
    22  struct fsxattr {
    23  	__u32		fsx_xflags;
    24  	__u32		fsx_extsize;
    25  	__u32		fsx_nextents;
    26  	__u32		fsx_projid;
    27  	unsigned char	fsx_pad[12];
    28  };
    29  #define FS_XFLAG_PROJINHERIT	0x00000200
    30  #endif
    31  #ifndef FS_IOC_FSGETXATTR
    32  #define FS_IOC_FSGETXATTR		_IOR ('X', 31, struct fsxattr)
    33  #endif
    34  #ifndef FS_IOC_FSSETXATTR
    35  #define FS_IOC_FSSETXATTR		_IOW ('X', 32, struct fsxattr)
    36  #endif
    37  
    38  #ifndef PRJQUOTA
    39  #define PRJQUOTA	2
    40  #endif
    41  #ifndef XFS_PROJ_QUOTA
    42  #define XFS_PROJ_QUOTA	2
    43  #endif
    44  #ifndef Q_XSETPQLIM
    45  #define Q_XSETPQLIM QCMD(Q_XSETQLIM, PRJQUOTA)
    46  #endif
    47  #ifndef Q_XGETPQUOTA
    48  #define Q_XGETPQUOTA QCMD(Q_XGETQUOTA, PRJQUOTA)
    49  #endif
    50  
    51  const int Q_XGETQSTAT_PRJQUOTA = QCMD(Q_XGETQSTAT, PRJQUOTA);
    52  */
    53  import "C"
    54  import (
    55  	"io/ioutil"
    56  	"path"
    57  	"path/filepath"
    58  	"unsafe"
    59  
    60  	rsystem "github.com/opencontainers/runc/libcontainer/system"
    61  	"github.com/pkg/errors"
    62  	"github.com/sirupsen/logrus"
    63  	"golang.org/x/sys/unix"
    64  )
    65  
    66  // NewControl - initialize project quota support.
    67  // Test to make sure that quota can be set on a test dir and find
    68  // the first project id to be used for the next container create.
    69  //
    70  // Returns nil (and error) if project quota is not supported.
    71  //
    72  // First get the project id of the home directory.
    73  // This test will fail if the backing fs is not xfs.
    74  //
    75  // xfs_quota tool can be used to assign a project id to the driver home directory, e.g.:
    76  //    echo 999:/var/lib/docker/overlay2 >> /etc/projects
    77  //    echo docker:999 >> /etc/projid
    78  //    xfs_quota -x -c 'project -s docker' /<xfs mount point>
    79  //
    80  // In that case, the home directory project id will be used as a "start offset"
    81  // and all containers will be assigned larger project ids (e.g. >= 1000).
    82  // This is a way to prevent xfs_quota management from conflicting with docker.
    83  //
    84  // Then try to create a test directory with the next project id and set a quota
    85  // on it. If that works, continue to scan existing containers to map allocated
    86  // project ids.
    87  //
    88  func NewControl(basePath string) (*Control, error) {
    89  	//
    90  	// If we are running in a user namespace quota won't be supported for
    91  	// now since makeBackingFsDev() will try to mknod().
    92  	//
    93  	if rsystem.RunningInUserNS() {
    94  		return nil, ErrQuotaNotSupported
    95  	}
    96  
    97  	//
    98  	// create backing filesystem device node
    99  	//
   100  	backingFsBlockDev, err := makeBackingFsDev(basePath)
   101  	if err != nil {
   102  		return nil, err
   103  	}
   104  
   105  	// check if we can call quotactl with project quotas
   106  	// as a mechanism to determine (early) if we have support
   107  	hasQuotaSupport, err := hasQuotaSupport(backingFsBlockDev)
   108  	if err != nil {
   109  		return nil, err
   110  	}
   111  	if !hasQuotaSupport {
   112  		return nil, ErrQuotaNotSupported
   113  	}
   114  
   115  	//
   116  	// Get project id of parent dir as minimal id to be used by driver
   117  	//
   118  	minProjectID, err := getProjectID(basePath)
   119  	if err != nil {
   120  		return nil, err
   121  	}
   122  	minProjectID++
   123  
   124  	//
   125  	// Test if filesystem supports project quotas by trying to set
   126  	// a quota on the first available project id
   127  	//
   128  	quota := Quota{
   129  		Size: 0,
   130  	}
   131  	if err := setProjectQuota(backingFsBlockDev, minProjectID, quota); err != nil {
   132  		return nil, err
   133  	}
   134  
   135  	q := Control{
   136  		backingFsBlockDev: backingFsBlockDev,
   137  		nextProjectID:     minProjectID + 1,
   138  		quotas:            make(map[string]uint32),
   139  	}
   140  
   141  	//
   142  	// get first project id to be used for next container
   143  	//
   144  	err = q.findNextProjectID(basePath)
   145  	if err != nil {
   146  		return nil, err
   147  	}
   148  
   149  	logrus.Debugf("NewControl(%s): nextProjectID = %d", basePath, q.nextProjectID)
   150  	return &q, nil
   151  }
   152  
   153  // SetQuota - assign a unique project id to directory and set the quota limits
   154  // for that project id
   155  func (q *Control) SetQuota(targetPath string, quota Quota) error {
   156  	q.RLock()
   157  	projectID, ok := q.quotas[targetPath]
   158  	q.RUnlock()
   159  	if !ok {
   160  		q.Lock()
   161  		projectID = q.nextProjectID
   162  
   163  		//
   164  		// assign project id to new container directory
   165  		//
   166  		err := setProjectID(targetPath, projectID)
   167  		if err != nil {
   168  			q.Unlock()
   169  			return err
   170  		}
   171  		q.quotas[targetPath] = projectID
   172  		q.nextProjectID++
   173  		q.Unlock()
   174  	}
   175  
   176  	//
   177  	// set the quota limit for the container's project id
   178  	//
   179  	logrus.Debugf("SetQuota(%s, %d): projectID=%d", targetPath, quota.Size, projectID)
   180  	return setProjectQuota(q.backingFsBlockDev, projectID, quota)
   181  }
   182  
   183  // setProjectQuota - set the quota for project id on xfs block device
   184  func setProjectQuota(backingFsBlockDev string, projectID uint32, quota Quota) error {
   185  	var d C.fs_disk_quota_t
   186  	d.d_version = C.FS_DQUOT_VERSION
   187  	d.d_id = C.__u32(projectID)
   188  	d.d_flags = C.XFS_PROJ_QUOTA
   189  
   190  	d.d_fieldmask = C.FS_DQ_BHARD | C.FS_DQ_BSOFT
   191  	d.d_blk_hardlimit = C.__u64(quota.Size / 512)
   192  	d.d_blk_softlimit = d.d_blk_hardlimit
   193  
   194  	var cs = C.CString(backingFsBlockDev)
   195  	defer C.free(unsafe.Pointer(cs))
   196  
   197  	_, _, errno := unix.Syscall6(unix.SYS_QUOTACTL, C.Q_XSETPQLIM,
   198  		uintptr(unsafe.Pointer(cs)), uintptr(d.d_id),
   199  		uintptr(unsafe.Pointer(&d)), 0, 0)
   200  	if errno != 0 {
   201  		return errors.Wrapf(errno, "failed to set quota limit for projid %d on %s",
   202  			projectID, backingFsBlockDev)
   203  	}
   204  
   205  	return nil
   206  }
   207  
   208  // GetQuota - get the quota limits of a directory that was configured with SetQuota
   209  func (q *Control) GetQuota(targetPath string, quota *Quota) error {
   210  	q.RLock()
   211  	projectID, ok := q.quotas[targetPath]
   212  	q.RUnlock()
   213  	if !ok {
   214  		return errors.Errorf("quota not found for path: %s", targetPath)
   215  	}
   216  
   217  	//
   218  	// get the quota limit for the container's project id
   219  	//
   220  	var d C.fs_disk_quota_t
   221  
   222  	var cs = C.CString(q.backingFsBlockDev)
   223  	defer C.free(unsafe.Pointer(cs))
   224  
   225  	_, _, errno := unix.Syscall6(unix.SYS_QUOTACTL, C.Q_XGETPQUOTA,
   226  		uintptr(unsafe.Pointer(cs)), uintptr(C.__u32(projectID)),
   227  		uintptr(unsafe.Pointer(&d)), 0, 0)
   228  	if errno != 0 {
   229  		return errors.Wrapf(errno, "Failed to get quota limit for projid %d on %s",
   230  			projectID, q.backingFsBlockDev)
   231  	}
   232  	quota.Size = uint64(d.d_blk_hardlimit) * 512
   233  
   234  	return nil
   235  }
   236  
   237  // getProjectID - get the project id of path on xfs
   238  func getProjectID(targetPath string) (uint32, error) {
   239  	dir, err := openDir(targetPath)
   240  	if err != nil {
   241  		return 0, err
   242  	}
   243  	defer closeDir(dir)
   244  
   245  	var fsx C.struct_fsxattr
   246  	_, _, errno := unix.Syscall(unix.SYS_IOCTL, getDirFd(dir), C.FS_IOC_FSGETXATTR,
   247  		uintptr(unsafe.Pointer(&fsx)))
   248  	if errno != 0 {
   249  		return 0, errors.Wrapf(errno, "failed to get projid for %s", targetPath)
   250  	}
   251  
   252  	return uint32(fsx.fsx_projid), nil
   253  }
   254  
   255  // setProjectID - set the project id of path on xfs
   256  func setProjectID(targetPath string, projectID uint32) error {
   257  	dir, err := openDir(targetPath)
   258  	if err != nil {
   259  		return err
   260  	}
   261  	defer closeDir(dir)
   262  
   263  	var fsx C.struct_fsxattr
   264  	_, _, errno := unix.Syscall(unix.SYS_IOCTL, getDirFd(dir), C.FS_IOC_FSGETXATTR,
   265  		uintptr(unsafe.Pointer(&fsx)))
   266  	if errno != 0 {
   267  		return errors.Wrapf(errno, "failed to get projid for %s", targetPath)
   268  	}
   269  	fsx.fsx_projid = C.__u32(projectID)
   270  	fsx.fsx_xflags |= C.FS_XFLAG_PROJINHERIT
   271  	_, _, errno = unix.Syscall(unix.SYS_IOCTL, getDirFd(dir), C.FS_IOC_FSSETXATTR,
   272  		uintptr(unsafe.Pointer(&fsx)))
   273  	if errno != 0 {
   274  		return errors.Wrapf(errno, "failed to set projid for %s", targetPath)
   275  	}
   276  
   277  	return nil
   278  }
   279  
   280  // findNextProjectID - find the next project id to be used for containers
   281  // by scanning driver home directory to find used project ids
   282  func (q *Control) findNextProjectID(home string) error {
   283  	q.Lock()
   284  	defer q.Unlock()
   285  	files, err := ioutil.ReadDir(home)
   286  	if err != nil {
   287  		return errors.Errorf("read directory failed: %s", home)
   288  	}
   289  	for _, file := range files {
   290  		if !file.IsDir() {
   291  			continue
   292  		}
   293  		path := filepath.Join(home, file.Name())
   294  		projid, err := getProjectID(path)
   295  		if err != nil {
   296  			return err
   297  		}
   298  		if projid > 0 {
   299  			q.quotas[path] = projid
   300  		}
   301  		if q.nextProjectID <= projid {
   302  			q.nextProjectID = projid + 1
   303  		}
   304  	}
   305  
   306  	return nil
   307  }
   308  
   309  func free(p *C.char) {
   310  	C.free(unsafe.Pointer(p))
   311  }
   312  
   313  func openDir(path string) (*C.DIR, error) {
   314  	Cpath := C.CString(path)
   315  	defer free(Cpath)
   316  
   317  	dir := C.opendir(Cpath)
   318  	if dir == nil {
   319  		return nil, errors.Errorf("failed to open dir: %s", path)
   320  	}
   321  	return dir, nil
   322  }
   323  
   324  func closeDir(dir *C.DIR) {
   325  	if dir != nil {
   326  		C.closedir(dir)
   327  	}
   328  }
   329  
   330  func getDirFd(dir *C.DIR) uintptr {
   331  	return uintptr(C.dirfd(dir))
   332  }
   333  
   334  // Get the backing block device of the driver home directory
   335  // and create a block device node under the home directory
   336  // to be used by quotactl commands
   337  func makeBackingFsDev(home string) (string, error) {
   338  	var stat unix.Stat_t
   339  	if err := unix.Stat(home, &stat); err != nil {
   340  		return "", err
   341  	}
   342  
   343  	backingFsBlockDev := path.Join(home, "backingFsBlockDev")
   344  	// Re-create just in case someone copied the home directory over to a new device
   345  	unix.Unlink(backingFsBlockDev)
   346  	err := unix.Mknod(backingFsBlockDev, unix.S_IFBLK|0600, int(stat.Dev))
   347  	switch err {
   348  	case nil:
   349  		return backingFsBlockDev, nil
   350  
   351  	case unix.ENOSYS, unix.EPERM:
   352  		return "", ErrQuotaNotSupported
   353  
   354  	default:
   355  		return "", errors.Wrapf(err, "failed to mknod %s", backingFsBlockDev)
   356  	}
   357  }
   358  
   359  func hasQuotaSupport(backingFsBlockDev string) (bool, error) {
   360  	var cs = C.CString(backingFsBlockDev)
   361  	defer free(cs)
   362  	var qstat C.fs_quota_stat_t
   363  
   364  	_, _, errno := unix.Syscall6(unix.SYS_QUOTACTL, uintptr(C.Q_XGETQSTAT_PRJQUOTA), uintptr(unsafe.Pointer(cs)), 0, uintptr(unsafe.Pointer(&qstat)), 0, 0)
   365  	if errno == 0 && qstat.qs_flags&C.FS_QUOTA_PDQ_ENFD > 0 && qstat.qs_flags&C.FS_QUOTA_PDQ_ACCT > 0 {
   366  		return true, nil
   367  	}
   368  
   369  	switch errno {
   370  	// These are the known fatal errors, consider all other errors (ENOTTY, etc.. not supporting quota)
   371  	case unix.EFAULT, unix.ENOENT, unix.ENOTBLK, unix.EPERM:
   372  	default:
   373  		return false, nil
   374  	}
   375  
   376  	return false, errno
   377  }