github.com/lazyboychen7/engine@v17.12.1-ce-rc2+incompatible/daemon/graphdriver/quota/projectquota.go (about)

     1  // +build linux
     2  
     3  //
     4  // projectquota.go - implements XFS project quota controls
     5  // for setting quota limits on a newly created directory.
     6  // It currently supports the legacy XFS specific ioctls.
     7  //
     8  // TODO: use generic quota control ioctl FS_IOC_FS{GET,SET}XATTR
     9  //       for both xfs/ext4 for kernel version >= v4.5
    10  //
    11  
    12  package quota
    13  
    14  /*
    15  #include <stdlib.h>
    16  #include <dirent.h>
    17  #include <linux/fs.h>
    18  #include <linux/quota.h>
    19  #include <linux/dqblk_xfs.h>
    20  
    21  #ifndef FS_XFLAG_PROJINHERIT
    22  struct fsxattr {
    23  	__u32		fsx_xflags;
    24  	__u32		fsx_extsize;
    25  	__u32		fsx_nextents;
    26  	__u32		fsx_projid;
    27  	unsigned char	fsx_pad[12];
    28  };
    29  #define FS_XFLAG_PROJINHERIT	0x00000200
    30  #endif
    31  #ifndef FS_IOC_FSGETXATTR
    32  #define FS_IOC_FSGETXATTR		_IOR ('X', 31, struct fsxattr)
    33  #endif
    34  #ifndef FS_IOC_FSSETXATTR
    35  #define FS_IOC_FSSETXATTR		_IOW ('X', 32, struct fsxattr)
    36  #endif
    37  
    38  #ifndef PRJQUOTA
    39  #define PRJQUOTA	2
    40  #endif
    41  #ifndef XFS_PROJ_QUOTA
    42  #define XFS_PROJ_QUOTA	2
    43  #endif
    44  #ifndef Q_XSETPQLIM
    45  #define Q_XSETPQLIM QCMD(Q_XSETQLIM, PRJQUOTA)
    46  #endif
    47  #ifndef Q_XGETPQUOTA
    48  #define Q_XGETPQUOTA QCMD(Q_XGETQUOTA, PRJQUOTA)
    49  #endif
    50  
    51  const int Q_XGETQSTAT_PRJQUOTA = QCMD(Q_XGETQSTAT, PRJQUOTA);
    52  */
    53  import "C"
    54  import (
    55  	"fmt"
    56  	"io/ioutil"
    57  	"path"
    58  	"path/filepath"
    59  	"unsafe"
    60  
    61  	rsystem "github.com/opencontainers/runc/libcontainer/system"
    62  	"github.com/sirupsen/logrus"
    63  	"golang.org/x/sys/unix"
    64  )
    65  
    66  // Quota limit params - currently we only control blocks hard limit
    67  type Quota struct {
    68  	Size uint64
    69  }
    70  
    71  // Control - Context to be used by storage driver (e.g. overlay)
    72  // who wants to apply project quotas to container dirs
    73  type Control struct {
    74  	backingFsBlockDev string
    75  	nextProjectID     uint32
    76  	quotas            map[string]uint32
    77  }
    78  
    79  // NewControl - initialize project quota support.
    80  // Test to make sure that quota can be set on a test dir and find
    81  // the first project id to be used for the next container create.
    82  //
    83  // Returns nil (and error) if project quota is not supported.
    84  //
    85  // First get the project id of the home directory.
    86  // This test will fail if the backing fs is not xfs.
    87  //
    88  // xfs_quota tool can be used to assign a project id to the driver home directory, e.g.:
    89  //    echo 999:/var/lib/docker/overlay2 >> /etc/projects
    90  //    echo docker:999 >> /etc/projid
    91  //    xfs_quota -x -c 'project -s docker' /<xfs mount point>
    92  //
    93  // In that case, the home directory project id will be used as a "start offset"
    94  // and all containers will be assigned larger project ids (e.g. >= 1000).
    95  // This is a way to prevent xfs_quota management from conflicting with docker.
    96  //
    97  // Then try to create a test directory with the next project id and set a quota
    98  // on it. If that works, continue to scan existing containers to map allocated
    99  // project ids.
   100  //
   101  func NewControl(basePath string) (*Control, error) {
   102  	//
   103  	// If we are running in a user namespace quota won't be supported for
   104  	// now since makeBackingFsDev() will try to mknod().
   105  	//
   106  	if rsystem.RunningInUserNS() {
   107  		return nil, ErrQuotaNotSupported
   108  	}
   109  
   110  	//
   111  	// create backing filesystem device node
   112  	//
   113  	backingFsBlockDev, err := makeBackingFsDev(basePath)
   114  	if err != nil {
   115  		return nil, err
   116  	}
   117  
   118  	// check if we can call quotactl with project quotas
   119  	// as a mechanism to determine (early) if we have support
   120  	hasQuotaSupport, err := hasQuotaSupport(backingFsBlockDev)
   121  	if err != nil {
   122  		return nil, err
   123  	}
   124  	if !hasQuotaSupport {
   125  		return nil, ErrQuotaNotSupported
   126  	}
   127  
   128  	//
   129  	// Get project id of parent dir as minimal id to be used by driver
   130  	//
   131  	minProjectID, err := getProjectID(basePath)
   132  	if err != nil {
   133  		return nil, err
   134  	}
   135  	minProjectID++
   136  
   137  	//
   138  	// Test if filesystem supports project quotas by trying to set
   139  	// a quota on the first available project id
   140  	//
   141  	quota := Quota{
   142  		Size: 0,
   143  	}
   144  	if err := setProjectQuota(backingFsBlockDev, minProjectID, quota); err != nil {
   145  		return nil, err
   146  	}
   147  
   148  	q := Control{
   149  		backingFsBlockDev: backingFsBlockDev,
   150  		nextProjectID:     minProjectID + 1,
   151  		quotas:            make(map[string]uint32),
   152  	}
   153  
   154  	//
   155  	// get first project id to be used for next container
   156  	//
   157  	err = q.findNextProjectID(basePath)
   158  	if err != nil {
   159  		return nil, err
   160  	}
   161  
   162  	logrus.Debugf("NewControl(%s): nextProjectID = %d", basePath, q.nextProjectID)
   163  	return &q, nil
   164  }
   165  
   166  // SetQuota - assign a unique project id to directory and set the quota limits
   167  // for that project id
   168  func (q *Control) SetQuota(targetPath string, quota Quota) error {
   169  
   170  	projectID, ok := q.quotas[targetPath]
   171  	if !ok {
   172  		projectID = q.nextProjectID
   173  
   174  		//
   175  		// assign project id to new container directory
   176  		//
   177  		err := setProjectID(targetPath, projectID)
   178  		if err != nil {
   179  			return err
   180  		}
   181  
   182  		q.quotas[targetPath] = projectID
   183  		q.nextProjectID++
   184  	}
   185  
   186  	//
   187  	// set the quota limit for the container's project id
   188  	//
   189  	logrus.Debugf("SetQuota(%s, %d): projectID=%d", targetPath, quota.Size, projectID)
   190  	return setProjectQuota(q.backingFsBlockDev, projectID, quota)
   191  }
   192  
   193  // setProjectQuota - set the quota for project id on xfs block device
   194  func setProjectQuota(backingFsBlockDev string, projectID uint32, quota Quota) error {
   195  	var d C.fs_disk_quota_t
   196  	d.d_version = C.FS_DQUOT_VERSION
   197  	d.d_id = C.__u32(projectID)
   198  	d.d_flags = C.XFS_PROJ_QUOTA
   199  
   200  	d.d_fieldmask = C.FS_DQ_BHARD | C.FS_DQ_BSOFT
   201  	d.d_blk_hardlimit = C.__u64(quota.Size / 512)
   202  	d.d_blk_softlimit = d.d_blk_hardlimit
   203  
   204  	var cs = C.CString(backingFsBlockDev)
   205  	defer C.free(unsafe.Pointer(cs))
   206  
   207  	_, _, errno := unix.Syscall6(unix.SYS_QUOTACTL, C.Q_XSETPQLIM,
   208  		uintptr(unsafe.Pointer(cs)), uintptr(d.d_id),
   209  		uintptr(unsafe.Pointer(&d)), 0, 0)
   210  	if errno != 0 {
   211  		return fmt.Errorf("Failed to set quota limit for projid %d on %s: %v",
   212  			projectID, backingFsBlockDev, errno.Error())
   213  	}
   214  
   215  	return nil
   216  }
   217  
   218  // GetQuota - get the quota limits of a directory that was configured with SetQuota
   219  func (q *Control) GetQuota(targetPath string, quota *Quota) error {
   220  
   221  	projectID, ok := q.quotas[targetPath]
   222  	if !ok {
   223  		return fmt.Errorf("quota not found for path : %s", targetPath)
   224  	}
   225  
   226  	//
   227  	// get the quota limit for the container's project id
   228  	//
   229  	var d C.fs_disk_quota_t
   230  
   231  	var cs = C.CString(q.backingFsBlockDev)
   232  	defer C.free(unsafe.Pointer(cs))
   233  
   234  	_, _, errno := unix.Syscall6(unix.SYS_QUOTACTL, C.Q_XGETPQUOTA,
   235  		uintptr(unsafe.Pointer(cs)), uintptr(C.__u32(projectID)),
   236  		uintptr(unsafe.Pointer(&d)), 0, 0)
   237  	if errno != 0 {
   238  		return fmt.Errorf("Failed to get quota limit for projid %d on %s: %v",
   239  			projectID, q.backingFsBlockDev, errno.Error())
   240  	}
   241  	quota.Size = uint64(d.d_blk_hardlimit) * 512
   242  
   243  	return nil
   244  }
   245  
   246  // getProjectID - get the project id of path on xfs
   247  func getProjectID(targetPath string) (uint32, error) {
   248  	dir, err := openDir(targetPath)
   249  	if err != nil {
   250  		return 0, err
   251  	}
   252  	defer closeDir(dir)
   253  
   254  	var fsx C.struct_fsxattr
   255  	_, _, errno := unix.Syscall(unix.SYS_IOCTL, getDirFd(dir), C.FS_IOC_FSGETXATTR,
   256  		uintptr(unsafe.Pointer(&fsx)))
   257  	if errno != 0 {
   258  		return 0, fmt.Errorf("Failed to get projid for %s: %v", targetPath, errno.Error())
   259  	}
   260  
   261  	return uint32(fsx.fsx_projid), nil
   262  }
   263  
   264  // setProjectID - set the project id of path on xfs
   265  func setProjectID(targetPath string, projectID uint32) error {
   266  	dir, err := openDir(targetPath)
   267  	if err != nil {
   268  		return err
   269  	}
   270  	defer closeDir(dir)
   271  
   272  	var fsx C.struct_fsxattr
   273  	_, _, errno := unix.Syscall(unix.SYS_IOCTL, getDirFd(dir), C.FS_IOC_FSGETXATTR,
   274  		uintptr(unsafe.Pointer(&fsx)))
   275  	if errno != 0 {
   276  		return fmt.Errorf("Failed to get projid for %s: %v", targetPath, errno.Error())
   277  	}
   278  	fsx.fsx_projid = C.__u32(projectID)
   279  	fsx.fsx_xflags |= C.FS_XFLAG_PROJINHERIT
   280  	_, _, errno = unix.Syscall(unix.SYS_IOCTL, getDirFd(dir), C.FS_IOC_FSSETXATTR,
   281  		uintptr(unsafe.Pointer(&fsx)))
   282  	if errno != 0 {
   283  		return fmt.Errorf("Failed to set projid for %s: %v", targetPath, errno.Error())
   284  	}
   285  
   286  	return nil
   287  }
   288  
   289  // findNextProjectID - find the next project id to be used for containers
   290  // by scanning driver home directory to find used project ids
   291  func (q *Control) findNextProjectID(home string) error {
   292  	files, err := ioutil.ReadDir(home)
   293  	if err != nil {
   294  		return fmt.Errorf("read directory failed : %s", home)
   295  	}
   296  	for _, file := range files {
   297  		if !file.IsDir() {
   298  			continue
   299  		}
   300  		path := filepath.Join(home, file.Name())
   301  		projid, err := getProjectID(path)
   302  		if err != nil {
   303  			return err
   304  		}
   305  		if projid > 0 {
   306  			q.quotas[path] = projid
   307  		}
   308  		if q.nextProjectID <= projid {
   309  			q.nextProjectID = projid + 1
   310  		}
   311  	}
   312  
   313  	return nil
   314  }
   315  
   316  func free(p *C.char) {
   317  	C.free(unsafe.Pointer(p))
   318  }
   319  
   320  func openDir(path string) (*C.DIR, error) {
   321  	Cpath := C.CString(path)
   322  	defer free(Cpath)
   323  
   324  	dir := C.opendir(Cpath)
   325  	if dir == nil {
   326  		return nil, fmt.Errorf("Can't open dir")
   327  	}
   328  	return dir, nil
   329  }
   330  
   331  func closeDir(dir *C.DIR) {
   332  	if dir != nil {
   333  		C.closedir(dir)
   334  	}
   335  }
   336  
   337  func getDirFd(dir *C.DIR) uintptr {
   338  	return uintptr(C.dirfd(dir))
   339  }
   340  
   341  // Get the backing block device of the driver home directory
   342  // and create a block device node under the home directory
   343  // to be used by quotactl commands
   344  func makeBackingFsDev(home string) (string, error) {
   345  	var stat unix.Stat_t
   346  	if err := unix.Stat(home, &stat); err != nil {
   347  		return "", err
   348  	}
   349  
   350  	backingFsBlockDev := path.Join(home, "backingFsBlockDev")
   351  	// Re-create just in case someone copied the home directory over to a new device
   352  	unix.Unlink(backingFsBlockDev)
   353  	err := unix.Mknod(backingFsBlockDev, unix.S_IFBLK|0600, int(stat.Dev))
   354  	switch err {
   355  	case nil:
   356  		return backingFsBlockDev, nil
   357  
   358  	case unix.ENOSYS:
   359  		return "", ErrQuotaNotSupported
   360  
   361  	default:
   362  		return "", fmt.Errorf("Failed to mknod %s: %v", backingFsBlockDev, err)
   363  	}
   364  }
   365  
   366  func hasQuotaSupport(backingFsBlockDev string) (bool, error) {
   367  	var cs = C.CString(backingFsBlockDev)
   368  	defer free(cs)
   369  	var qstat C.fs_quota_stat_t
   370  
   371  	_, _, errno := unix.Syscall6(unix.SYS_QUOTACTL, uintptr(C.Q_XGETQSTAT_PRJQUOTA), uintptr(unsafe.Pointer(cs)), 0, uintptr(unsafe.Pointer(&qstat)), 0, 0)
   372  	if errno == 0 && qstat.qs_flags&C.FS_QUOTA_PDQ_ENFD > 0 && qstat.qs_flags&C.FS_QUOTA_PDQ_ACCT > 0 {
   373  		return true, nil
   374  	}
   375  
   376  	switch errno {
   377  	// These are the known fatal errors, consider all other errors (ENOTTY, etc.. not supporting quota)
   378  	case unix.EFAULT, unix.ENOENT, unix.ENOTBLK, unix.EPERM:
   379  	default:
   380  		return false, nil
   381  	}
   382  
   383  	return false, errno
   384  }