github.com/lazyboychen7/engine@v17.12.1-ce-rc2+incompatible/daemon/graphdriver/quota/projectquota.go (about) 1 // +build linux 2 3 // 4 // projectquota.go - implements XFS project quota controls 5 // for setting quota limits on a newly created directory. 6 // It currently supports the legacy XFS specific ioctls. 7 // 8 // TODO: use generic quota control ioctl FS_IOC_FS{GET,SET}XATTR 9 // for both xfs/ext4 for kernel version >= v4.5 10 // 11 12 package quota 13 14 /* 15 #include <stdlib.h> 16 #include <dirent.h> 17 #include <linux/fs.h> 18 #include <linux/quota.h> 19 #include <linux/dqblk_xfs.h> 20 21 #ifndef FS_XFLAG_PROJINHERIT 22 struct fsxattr { 23 __u32 fsx_xflags; 24 __u32 fsx_extsize; 25 __u32 fsx_nextents; 26 __u32 fsx_projid; 27 unsigned char fsx_pad[12]; 28 }; 29 #define FS_XFLAG_PROJINHERIT 0x00000200 30 #endif 31 #ifndef FS_IOC_FSGETXATTR 32 #define FS_IOC_FSGETXATTR _IOR ('X', 31, struct fsxattr) 33 #endif 34 #ifndef FS_IOC_FSSETXATTR 35 #define FS_IOC_FSSETXATTR _IOW ('X', 32, struct fsxattr) 36 #endif 37 38 #ifndef PRJQUOTA 39 #define PRJQUOTA 2 40 #endif 41 #ifndef XFS_PROJ_QUOTA 42 #define XFS_PROJ_QUOTA 2 43 #endif 44 #ifndef Q_XSETPQLIM 45 #define Q_XSETPQLIM QCMD(Q_XSETQLIM, PRJQUOTA) 46 #endif 47 #ifndef Q_XGETPQUOTA 48 #define Q_XGETPQUOTA QCMD(Q_XGETQUOTA, PRJQUOTA) 49 #endif 50 51 const int Q_XGETQSTAT_PRJQUOTA = QCMD(Q_XGETQSTAT, PRJQUOTA); 52 */ 53 import "C" 54 import ( 55 "fmt" 56 "io/ioutil" 57 "path" 58 "path/filepath" 59 "unsafe" 60 61 rsystem "github.com/opencontainers/runc/libcontainer/system" 62 "github.com/sirupsen/logrus" 63 "golang.org/x/sys/unix" 64 ) 65 66 // Quota limit params - currently we only control blocks hard limit 67 type Quota struct { 68 Size uint64 69 } 70 71 // Control - Context to be used by storage driver (e.g. overlay) 72 // who wants to apply project quotas to container dirs 73 type Control struct { 74 backingFsBlockDev string 75 nextProjectID uint32 76 quotas map[string]uint32 77 } 78 79 // NewControl - initialize project quota support. 80 // Test to make sure that quota can be set on a test dir and find 81 // the first project id to be used for the next container create. 82 // 83 // Returns nil (and error) if project quota is not supported. 84 // 85 // First get the project id of the home directory. 86 // This test will fail if the backing fs is not xfs. 87 // 88 // xfs_quota tool can be used to assign a project id to the driver home directory, e.g.: 89 // echo 999:/var/lib/docker/overlay2 >> /etc/projects 90 // echo docker:999 >> /etc/projid 91 // xfs_quota -x -c 'project -s docker' /<xfs mount point> 92 // 93 // In that case, the home directory project id will be used as a "start offset" 94 // and all containers will be assigned larger project ids (e.g. >= 1000). 95 // This is a way to prevent xfs_quota management from conflicting with docker. 96 // 97 // Then try to create a test directory with the next project id and set a quota 98 // on it. If that works, continue to scan existing containers to map allocated 99 // project ids. 100 // 101 func NewControl(basePath string) (*Control, error) { 102 // 103 // If we are running in a user namespace quota won't be supported for 104 // now since makeBackingFsDev() will try to mknod(). 105 // 106 if rsystem.RunningInUserNS() { 107 return nil, ErrQuotaNotSupported 108 } 109 110 // 111 // create backing filesystem device node 112 // 113 backingFsBlockDev, err := makeBackingFsDev(basePath) 114 if err != nil { 115 return nil, err 116 } 117 118 // check if we can call quotactl with project quotas 119 // as a mechanism to determine (early) if we have support 120 hasQuotaSupport, err := hasQuotaSupport(backingFsBlockDev) 121 if err != nil { 122 return nil, err 123 } 124 if !hasQuotaSupport { 125 return nil, ErrQuotaNotSupported 126 } 127 128 // 129 // Get project id of parent dir as minimal id to be used by driver 130 // 131 minProjectID, err := getProjectID(basePath) 132 if err != nil { 133 return nil, err 134 } 135 minProjectID++ 136 137 // 138 // Test if filesystem supports project quotas by trying to set 139 // a quota on the first available project id 140 // 141 quota := Quota{ 142 Size: 0, 143 } 144 if err := setProjectQuota(backingFsBlockDev, minProjectID, quota); err != nil { 145 return nil, err 146 } 147 148 q := Control{ 149 backingFsBlockDev: backingFsBlockDev, 150 nextProjectID: minProjectID + 1, 151 quotas: make(map[string]uint32), 152 } 153 154 // 155 // get first project id to be used for next container 156 // 157 err = q.findNextProjectID(basePath) 158 if err != nil { 159 return nil, err 160 } 161 162 logrus.Debugf("NewControl(%s): nextProjectID = %d", basePath, q.nextProjectID) 163 return &q, nil 164 } 165 166 // SetQuota - assign a unique project id to directory and set the quota limits 167 // for that project id 168 func (q *Control) SetQuota(targetPath string, quota Quota) error { 169 170 projectID, ok := q.quotas[targetPath] 171 if !ok { 172 projectID = q.nextProjectID 173 174 // 175 // assign project id to new container directory 176 // 177 err := setProjectID(targetPath, projectID) 178 if err != nil { 179 return err 180 } 181 182 q.quotas[targetPath] = projectID 183 q.nextProjectID++ 184 } 185 186 // 187 // set the quota limit for the container's project id 188 // 189 logrus.Debugf("SetQuota(%s, %d): projectID=%d", targetPath, quota.Size, projectID) 190 return setProjectQuota(q.backingFsBlockDev, projectID, quota) 191 } 192 193 // setProjectQuota - set the quota for project id on xfs block device 194 func setProjectQuota(backingFsBlockDev string, projectID uint32, quota Quota) error { 195 var d C.fs_disk_quota_t 196 d.d_version = C.FS_DQUOT_VERSION 197 d.d_id = C.__u32(projectID) 198 d.d_flags = C.XFS_PROJ_QUOTA 199 200 d.d_fieldmask = C.FS_DQ_BHARD | C.FS_DQ_BSOFT 201 d.d_blk_hardlimit = C.__u64(quota.Size / 512) 202 d.d_blk_softlimit = d.d_blk_hardlimit 203 204 var cs = C.CString(backingFsBlockDev) 205 defer C.free(unsafe.Pointer(cs)) 206 207 _, _, errno := unix.Syscall6(unix.SYS_QUOTACTL, C.Q_XSETPQLIM, 208 uintptr(unsafe.Pointer(cs)), uintptr(d.d_id), 209 uintptr(unsafe.Pointer(&d)), 0, 0) 210 if errno != 0 { 211 return fmt.Errorf("Failed to set quota limit for projid %d on %s: %v", 212 projectID, backingFsBlockDev, errno.Error()) 213 } 214 215 return nil 216 } 217 218 // GetQuota - get the quota limits of a directory that was configured with SetQuota 219 func (q *Control) GetQuota(targetPath string, quota *Quota) error { 220 221 projectID, ok := q.quotas[targetPath] 222 if !ok { 223 return fmt.Errorf("quota not found for path : %s", targetPath) 224 } 225 226 // 227 // get the quota limit for the container's project id 228 // 229 var d C.fs_disk_quota_t 230 231 var cs = C.CString(q.backingFsBlockDev) 232 defer C.free(unsafe.Pointer(cs)) 233 234 _, _, errno := unix.Syscall6(unix.SYS_QUOTACTL, C.Q_XGETPQUOTA, 235 uintptr(unsafe.Pointer(cs)), uintptr(C.__u32(projectID)), 236 uintptr(unsafe.Pointer(&d)), 0, 0) 237 if errno != 0 { 238 return fmt.Errorf("Failed to get quota limit for projid %d on %s: %v", 239 projectID, q.backingFsBlockDev, errno.Error()) 240 } 241 quota.Size = uint64(d.d_blk_hardlimit) * 512 242 243 return nil 244 } 245 246 // getProjectID - get the project id of path on xfs 247 func getProjectID(targetPath string) (uint32, error) { 248 dir, err := openDir(targetPath) 249 if err != nil { 250 return 0, err 251 } 252 defer closeDir(dir) 253 254 var fsx C.struct_fsxattr 255 _, _, errno := unix.Syscall(unix.SYS_IOCTL, getDirFd(dir), C.FS_IOC_FSGETXATTR, 256 uintptr(unsafe.Pointer(&fsx))) 257 if errno != 0 { 258 return 0, fmt.Errorf("Failed to get projid for %s: %v", targetPath, errno.Error()) 259 } 260 261 return uint32(fsx.fsx_projid), nil 262 } 263 264 // setProjectID - set the project id of path on xfs 265 func setProjectID(targetPath string, projectID uint32) error { 266 dir, err := openDir(targetPath) 267 if err != nil { 268 return err 269 } 270 defer closeDir(dir) 271 272 var fsx C.struct_fsxattr 273 _, _, errno := unix.Syscall(unix.SYS_IOCTL, getDirFd(dir), C.FS_IOC_FSGETXATTR, 274 uintptr(unsafe.Pointer(&fsx))) 275 if errno != 0 { 276 return fmt.Errorf("Failed to get projid for %s: %v", targetPath, errno.Error()) 277 } 278 fsx.fsx_projid = C.__u32(projectID) 279 fsx.fsx_xflags |= C.FS_XFLAG_PROJINHERIT 280 _, _, errno = unix.Syscall(unix.SYS_IOCTL, getDirFd(dir), C.FS_IOC_FSSETXATTR, 281 uintptr(unsafe.Pointer(&fsx))) 282 if errno != 0 { 283 return fmt.Errorf("Failed to set projid for %s: %v", targetPath, errno.Error()) 284 } 285 286 return nil 287 } 288 289 // findNextProjectID - find the next project id to be used for containers 290 // by scanning driver home directory to find used project ids 291 func (q *Control) findNextProjectID(home string) error { 292 files, err := ioutil.ReadDir(home) 293 if err != nil { 294 return fmt.Errorf("read directory failed : %s", home) 295 } 296 for _, file := range files { 297 if !file.IsDir() { 298 continue 299 } 300 path := filepath.Join(home, file.Name()) 301 projid, err := getProjectID(path) 302 if err != nil { 303 return err 304 } 305 if projid > 0 { 306 q.quotas[path] = projid 307 } 308 if q.nextProjectID <= projid { 309 q.nextProjectID = projid + 1 310 } 311 } 312 313 return nil 314 } 315 316 func free(p *C.char) { 317 C.free(unsafe.Pointer(p)) 318 } 319 320 func openDir(path string) (*C.DIR, error) { 321 Cpath := C.CString(path) 322 defer free(Cpath) 323 324 dir := C.opendir(Cpath) 325 if dir == nil { 326 return nil, fmt.Errorf("Can't open dir") 327 } 328 return dir, nil 329 } 330 331 func closeDir(dir *C.DIR) { 332 if dir != nil { 333 C.closedir(dir) 334 } 335 } 336 337 func getDirFd(dir *C.DIR) uintptr { 338 return uintptr(C.dirfd(dir)) 339 } 340 341 // Get the backing block device of the driver home directory 342 // and create a block device node under the home directory 343 // to be used by quotactl commands 344 func makeBackingFsDev(home string) (string, error) { 345 var stat unix.Stat_t 346 if err := unix.Stat(home, &stat); err != nil { 347 return "", err 348 } 349 350 backingFsBlockDev := path.Join(home, "backingFsBlockDev") 351 // Re-create just in case someone copied the home directory over to a new device 352 unix.Unlink(backingFsBlockDev) 353 err := unix.Mknod(backingFsBlockDev, unix.S_IFBLK|0600, int(stat.Dev)) 354 switch err { 355 case nil: 356 return backingFsBlockDev, nil 357 358 case unix.ENOSYS: 359 return "", ErrQuotaNotSupported 360 361 default: 362 return "", fmt.Errorf("Failed to mknod %s: %v", backingFsBlockDev, err) 363 } 364 } 365 366 func hasQuotaSupport(backingFsBlockDev string) (bool, error) { 367 var cs = C.CString(backingFsBlockDev) 368 defer free(cs) 369 var qstat C.fs_quota_stat_t 370 371 _, _, errno := unix.Syscall6(unix.SYS_QUOTACTL, uintptr(C.Q_XGETQSTAT_PRJQUOTA), uintptr(unsafe.Pointer(cs)), 0, uintptr(unsafe.Pointer(&qstat)), 0, 0) 372 if errno == 0 && qstat.qs_flags&C.FS_QUOTA_PDQ_ENFD > 0 && qstat.qs_flags&C.FS_QUOTA_PDQ_ACCT > 0 { 373 return true, nil 374 } 375 376 switch errno { 377 // These are the known fatal errors, consider all other errors (ENOTTY, etc.. not supporting quota) 378 case unix.EFAULT, unix.ENOENT, unix.ENOTBLK, unix.EPERM: 379 default: 380 return false, nil 381 } 382 383 return false, errno 384 }