github.com/openflowlabs/storage@v1.12.13/drivers/quota/projectquota.go (about) 1 // +build linux,!exclude_disk_quota 2 3 // 4 // projectquota.go - implements XFS project quota controls 5 // for setting quota limits on a newly created directory. 6 // It currently supports the legacy XFS specific ioctls. 7 // 8 // TODO: use generic quota control ioctl FS_IOC_FS{GET,SET}XATTR 9 // for both xfs/ext4 for kernel version >= v4.5 10 // 11 12 package quota 13 14 /* 15 #include <stdlib.h> 16 #include <dirent.h> 17 #include <linux/fs.h> 18 #include <linux/quota.h> 19 #include <linux/dqblk_xfs.h> 20 21 #ifndef FS_XFLAG_PROJINHERIT 22 struct fsxattr { 23 __u32 fsx_xflags; 24 __u32 fsx_extsize; 25 __u32 fsx_nextents; 26 __u32 fsx_projid; 27 unsigned char fsx_pad[12]; 28 }; 29 #define FS_XFLAG_PROJINHERIT 0x00000200 30 #endif 31 #ifndef FS_IOC_FSGETXATTR 32 #define FS_IOC_FSGETXATTR _IOR ('X', 31, struct fsxattr) 33 #endif 34 #ifndef FS_IOC_FSSETXATTR 35 #define FS_IOC_FSSETXATTR _IOW ('X', 32, struct fsxattr) 36 #endif 37 38 #ifndef PRJQUOTA 39 #define PRJQUOTA 2 40 #endif 41 #ifndef XFS_PROJ_QUOTA 42 #define XFS_PROJ_QUOTA 2 43 #endif 44 #ifndef Q_XSETPQLIM 45 #define Q_XSETPQLIM QCMD(Q_XSETQLIM, PRJQUOTA) 46 #endif 47 #ifndef Q_XGETPQUOTA 48 #define Q_XGETPQUOTA QCMD(Q_XGETQUOTA, PRJQUOTA) 49 #endif 50 */ 51 import "C" 52 import ( 53 "fmt" 54 "io/ioutil" 55 "path" 56 "path/filepath" 57 "unsafe" 58 59 "github.com/sirupsen/logrus" 60 "golang.org/x/sys/unix" 61 ) 62 63 // Quota limit params - currently we only control blocks hard limit 64 type Quota struct { 65 Size uint64 66 } 67 68 // Control - Context to be used by storage driver (e.g. overlay) 69 // who wants to apply project quotas to container dirs 70 type Control struct { 71 backingFsBlockDev string 72 nextProjectID uint32 73 quotas map[string]uint32 74 } 75 76 // NewControl - initialize project quota support. 77 // Test to make sure that quota can be set on a test dir and find 78 // the first project id to be used for the next container create. 79 // 80 // Returns nil (and error) if project quota is not supported. 81 // 82 // First get the project id of the home directory. 83 // This test will fail if the backing fs is not xfs. 84 // 85 // xfs_quota tool can be used to assign a project id to the driver home directory, e.g.: 86 // echo 999:/var/lib/containers/storage/overlay >> /etc/projects 87 // echo storage:999 >> /etc/projid 88 // xfs_quota -x -c 'project -s storage' /<xfs mount point> 89 // 90 // In that case, the home directory project id will be used as a "start offset" 91 // and all containers will be assigned larger project ids (e.g. >= 1000). 92 // This is a way to prevent xfs_quota management from conflicting with containers/storage. 93 // 94 // Then try to create a test directory with the next project id and set a quota 95 // on it. If that works, continue to scan existing containers to map allocated 96 // project ids. 97 // 98 func NewControl(basePath string) (*Control, error) { 99 // 100 // Get project id of parent dir as minimal id to be used by driver 101 // 102 minProjectID, err := getProjectID(basePath) 103 if err != nil { 104 return nil, err 105 } 106 minProjectID++ 107 108 // 109 // create backing filesystem device node 110 // 111 backingFsBlockDev, err := makeBackingFsDev(basePath) 112 if err != nil { 113 return nil, err 114 } 115 116 // 117 // Test if filesystem supports project quotas by trying to set 118 // a quota on the first available project id 119 // 120 quota := Quota{ 121 Size: 0, 122 } 123 if err := setProjectQuota(backingFsBlockDev, minProjectID, quota); err != nil { 124 return nil, err 125 } 126 127 q := Control{ 128 backingFsBlockDev: backingFsBlockDev, 129 nextProjectID: minProjectID + 1, 130 quotas: make(map[string]uint32), 131 } 132 133 // 134 // get first project id to be used for next container 135 // 136 err = q.findNextProjectID(basePath) 137 if err != nil { 138 return nil, err 139 } 140 141 logrus.Debugf("NewControl(%s): nextProjectID = %d", basePath, q.nextProjectID) 142 return &q, nil 143 } 144 145 // SetQuota - assign a unique project id to directory and set the quota limits 146 // for that project id 147 func (q *Control) SetQuota(targetPath string, quota Quota) error { 148 149 projectID, ok := q.quotas[targetPath] 150 if !ok { 151 projectID = q.nextProjectID 152 153 // 154 // assign project id to new container directory 155 // 156 err := setProjectID(targetPath, projectID) 157 if err != nil { 158 return err 159 } 160 161 q.quotas[targetPath] = projectID 162 q.nextProjectID++ 163 } 164 165 // 166 // set the quota limit for the container's project id 167 // 168 logrus.Debugf("SetQuota(%s, %d): projectID=%d", targetPath, quota.Size, projectID) 169 return setProjectQuota(q.backingFsBlockDev, projectID, quota) 170 } 171 172 // setProjectQuota - set the quota for project id on xfs block device 173 func setProjectQuota(backingFsBlockDev string, projectID uint32, quota Quota) error { 174 var d C.fs_disk_quota_t 175 d.d_version = C.FS_DQUOT_VERSION 176 d.d_id = C.__u32(projectID) 177 d.d_flags = C.XFS_PROJ_QUOTA 178 179 d.d_fieldmask = C.FS_DQ_BHARD | C.FS_DQ_BSOFT 180 d.d_blk_hardlimit = C.__u64(quota.Size / 512) 181 d.d_blk_softlimit = d.d_blk_hardlimit 182 183 var cs = C.CString(backingFsBlockDev) 184 defer C.free(unsafe.Pointer(cs)) 185 186 _, _, errno := unix.Syscall6(unix.SYS_QUOTACTL, C.Q_XSETPQLIM, 187 uintptr(unsafe.Pointer(cs)), uintptr(d.d_id), 188 uintptr(unsafe.Pointer(&d)), 0, 0) 189 if errno != 0 { 190 return fmt.Errorf("Failed to set quota limit for projid %d on %s: %v", 191 projectID, backingFsBlockDev, errno.Error()) 192 } 193 194 return nil 195 } 196 197 // GetQuota - get the quota limits of a directory that was configured with SetQuota 198 func (q *Control) GetQuota(targetPath string, quota *Quota) error { 199 200 projectID, ok := q.quotas[targetPath] 201 if !ok { 202 return fmt.Errorf("quota not found for path : %s", targetPath) 203 } 204 205 // 206 // get the quota limit for the container's project id 207 // 208 var d C.fs_disk_quota_t 209 210 var cs = C.CString(q.backingFsBlockDev) 211 defer C.free(unsafe.Pointer(cs)) 212 213 _, _, errno := unix.Syscall6(unix.SYS_QUOTACTL, C.Q_XGETPQUOTA, 214 uintptr(unsafe.Pointer(cs)), uintptr(C.__u32(projectID)), 215 uintptr(unsafe.Pointer(&d)), 0, 0) 216 if errno != 0 { 217 return fmt.Errorf("Failed to get quota limit for projid %d on %s: %v", 218 projectID, q.backingFsBlockDev, errno.Error()) 219 } 220 quota.Size = uint64(d.d_blk_hardlimit) * 512 221 222 return nil 223 } 224 225 // getProjectID - get the project id of path on xfs 226 func getProjectID(targetPath string) (uint32, error) { 227 dir, err := openDir(targetPath) 228 if err != nil { 229 return 0, err 230 } 231 defer closeDir(dir) 232 233 var fsx C.struct_fsxattr 234 _, _, errno := unix.Syscall(unix.SYS_IOCTL, getDirFd(dir), C.FS_IOC_FSGETXATTR, 235 uintptr(unsafe.Pointer(&fsx))) 236 if errno != 0 { 237 return 0, fmt.Errorf("Failed to get projid for %s: %v", targetPath, errno.Error()) 238 } 239 240 return uint32(fsx.fsx_projid), nil 241 } 242 243 // setProjectID - set the project id of path on xfs 244 func setProjectID(targetPath string, projectID uint32) error { 245 dir, err := openDir(targetPath) 246 if err != nil { 247 return err 248 } 249 defer closeDir(dir) 250 251 var fsx C.struct_fsxattr 252 _, _, errno := unix.Syscall(unix.SYS_IOCTL, getDirFd(dir), C.FS_IOC_FSGETXATTR, 253 uintptr(unsafe.Pointer(&fsx))) 254 if errno != 0 { 255 return fmt.Errorf("Failed to get projid for %s: %v", targetPath, errno.Error()) 256 } 257 fsx.fsx_projid = C.__u32(projectID) 258 fsx.fsx_xflags |= C.FS_XFLAG_PROJINHERIT 259 _, _, errno = unix.Syscall(unix.SYS_IOCTL, getDirFd(dir), C.FS_IOC_FSSETXATTR, 260 uintptr(unsafe.Pointer(&fsx))) 261 if errno != 0 { 262 return fmt.Errorf("Failed to set projid for %s: %v", targetPath, errno.Error()) 263 } 264 265 return nil 266 } 267 268 // findNextProjectID - find the next project id to be used for containers 269 // by scanning driver home directory to find used project ids 270 func (q *Control) findNextProjectID(home string) error { 271 files, err := ioutil.ReadDir(home) 272 if err != nil { 273 return fmt.Errorf("read directory failed : %s", home) 274 } 275 for _, file := range files { 276 if !file.IsDir() { 277 continue 278 } 279 path := filepath.Join(home, file.Name()) 280 projid, err := getProjectID(path) 281 if err != nil { 282 return err 283 } 284 if projid > 0 { 285 q.quotas[path] = projid 286 } 287 if q.nextProjectID <= projid { 288 q.nextProjectID = projid + 1 289 } 290 } 291 292 return nil 293 } 294 295 func free(p *C.char) { 296 C.free(unsafe.Pointer(p)) 297 } 298 299 func openDir(path string) (*C.DIR, error) { 300 Cpath := C.CString(path) 301 defer free(Cpath) 302 303 dir := C.opendir(Cpath) 304 if dir == nil { 305 return nil, fmt.Errorf("Can't open dir") 306 } 307 return dir, nil 308 } 309 310 func closeDir(dir *C.DIR) { 311 if dir != nil { 312 C.closedir(dir) 313 } 314 } 315 316 func getDirFd(dir *C.DIR) uintptr { 317 return uintptr(C.dirfd(dir)) 318 } 319 320 // Get the backing block device of the driver home directory 321 // and create a block device node under the home directory 322 // to be used by quotactl commands 323 func makeBackingFsDev(home string) (string, error) { 324 var stat unix.Stat_t 325 if err := unix.Stat(home, &stat); err != nil { 326 return "", err 327 } 328 329 backingFsBlockDev := path.Join(home, "backingFsBlockDev") 330 // Re-create just in case someone copied the home directory over to a new device 331 unix.Unlink(backingFsBlockDev) 332 if err := unix.Mknod(backingFsBlockDev, unix.S_IFBLK|0600, int(stat.Dev)); err != nil { 333 return "", fmt.Errorf("Failed to mknod %s: %v", backingFsBlockDev, err) 334 } 335 336 return backingFsBlockDev, nil 337 }