github.com/docker/docker@v299999999.0.0-20200612211812-aaf470eca7b5+incompatible/daemon/graphdriver/btrfs/btrfs.go (about) 1 // +build linux 2 3 package btrfs // import "github.com/docker/docker/daemon/graphdriver/btrfs" 4 5 /* 6 #include <stdlib.h> 7 #include <dirent.h> 8 #include <btrfs/ioctl.h> 9 #include <btrfs/ctree.h> 10 11 static void set_name_btrfs_ioctl_vol_args_v2(struct btrfs_ioctl_vol_args_v2* btrfs_struct, const char* value) { 12 snprintf(btrfs_struct->name, BTRFS_SUBVOL_NAME_MAX, "%s", value); 13 } 14 */ 15 import "C" 16 17 import ( 18 "fmt" 19 "io/ioutil" 20 "math" 21 "os" 22 "path" 23 "path/filepath" 24 "strconv" 25 "strings" 26 "sync" 27 "unsafe" 28 29 "github.com/docker/docker/daemon/graphdriver" 30 "github.com/docker/docker/pkg/containerfs" 31 "github.com/docker/docker/pkg/idtools" 32 "github.com/docker/docker/pkg/parsers" 33 "github.com/docker/docker/pkg/system" 34 units "github.com/docker/go-units" 35 "github.com/moby/sys/mount" 36 "github.com/opencontainers/selinux/go-selinux/label" 37 "github.com/pkg/errors" 38 "github.com/sirupsen/logrus" 39 "golang.org/x/sys/unix" 40 ) 41 42 func init() { 43 graphdriver.Register("btrfs", Init) 44 } 45 46 type btrfsOptions struct { 47 minSpace uint64 48 size uint64 49 } 50 51 // Init returns a new BTRFS driver. 52 // An error is returned if BTRFS is not supported. 53 func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (graphdriver.Driver, error) { 54 55 // Perform feature detection on /var/lib/docker/btrfs if it's an existing directory. 56 // This covers situations where /var/lib/docker/btrfs is a mount, and on a different 57 // filesystem than /var/lib/docker. 58 // If the path does not exist, fall back to using /var/lib/docker for feature detection. 59 testdir := home 60 if _, err := os.Stat(testdir); os.IsNotExist(err) { 61 testdir = filepath.Dir(testdir) 62 } 63 64 fsMagic, err := graphdriver.GetFSMagic(testdir) 65 if err != nil { 66 return nil, err 67 } 68 69 if fsMagic != graphdriver.FsMagicBtrfs { 70 return nil, graphdriver.ErrPrerequisites 71 } 72 73 rootUID, rootGID, err := idtools.GetRootUIDGID(uidMaps, gidMaps) 74 if err != nil { 75 return nil, err 76 } 77 if err := idtools.MkdirAllAndChown(home, 0700, idtools.Identity{UID: rootUID, GID: rootGID}); err != nil { 78 return nil, err 79 } 80 81 opt, userDiskQuota, err := parseOptions(options) 82 if err != nil { 83 return nil, err 84 } 85 86 // For some reason shared mount propagation between a container 87 // and the host does not work for btrfs, and a remedy is to bind 88 // mount graphdriver home to itself (even without changing the 89 // propagation mode). 90 err = mount.MakeMount(home) 91 if err != nil { 92 return nil, errors.Wrapf(err, "failed to make %s a mount", home) 93 } 94 95 driver := &Driver{ 96 home: home, 97 uidMaps: uidMaps, 98 gidMaps: gidMaps, 99 options: opt, 100 } 101 102 if userDiskQuota { 103 if err := driver.subvolEnableQuota(); err != nil { 104 return nil, err 105 } 106 } 107 108 return graphdriver.NewNaiveDiffDriver(driver, uidMaps, gidMaps), nil 109 } 110 111 func parseOptions(opt []string) (btrfsOptions, bool, error) { 112 var options btrfsOptions 113 userDiskQuota := false 114 for _, option := range opt { 115 key, val, err := parsers.ParseKeyValueOpt(option) 116 if err != nil { 117 return options, userDiskQuota, err 118 } 119 key = strings.ToLower(key) 120 switch key { 121 case "btrfs.min_space": 122 minSpace, err := units.RAMInBytes(val) 123 if err != nil { 124 return options, userDiskQuota, err 125 } 126 userDiskQuota = true 127 options.minSpace = uint64(minSpace) 128 default: 129 return options, userDiskQuota, fmt.Errorf("Unknown option %s", key) 130 } 131 } 132 return options, userDiskQuota, nil 133 } 134 135 // Driver contains information about the filesystem mounted. 136 type Driver struct { 137 // root of the file system 138 home string 139 uidMaps []idtools.IDMap 140 gidMaps []idtools.IDMap 141 options btrfsOptions 142 quotaEnabled bool 143 once sync.Once 144 } 145 146 // String prints the name of the driver (btrfs). 147 func (d *Driver) String() string { 148 return "btrfs" 149 } 150 151 // Status returns current driver information in a two dimensional string array. 152 // Output contains "Build Version" and "Library Version" of the btrfs libraries used. 153 // Version information can be used to check compatibility with your kernel. 154 func (d *Driver) Status() [][2]string { 155 status := [][2]string{} 156 if bv := btrfsBuildVersion(); bv != "-" { 157 status = append(status, [2]string{"Build Version", bv}) 158 } 159 if lv := btrfsLibVersion(); lv != -1 { 160 status = append(status, [2]string{"Library Version", fmt.Sprintf("%d", lv)}) 161 } 162 return status 163 } 164 165 // GetMetadata returns empty metadata for this driver. 166 func (d *Driver) GetMetadata(id string) (map[string]string, error) { 167 return nil, nil 168 } 169 170 // Cleanup unmounts the home directory. 171 func (d *Driver) Cleanup() error { 172 err := d.subvolDisableQuota() 173 umountErr := mount.Unmount(d.home) 174 175 // in case we have two errors, prefer the one from disableQuota() 176 if err != nil { 177 return err 178 } 179 180 if umountErr != nil { 181 return umountErr 182 } 183 184 return nil 185 } 186 187 func free(p *C.char) { 188 C.free(unsafe.Pointer(p)) 189 } 190 191 func openDir(path string) (*C.DIR, error) { 192 Cpath := C.CString(path) 193 defer free(Cpath) 194 195 dir := C.opendir(Cpath) 196 if dir == nil { 197 return nil, fmt.Errorf("Can't open dir") 198 } 199 return dir, nil 200 } 201 202 func closeDir(dir *C.DIR) { 203 if dir != nil { 204 C.closedir(dir) 205 } 206 } 207 208 func getDirFd(dir *C.DIR) uintptr { 209 return uintptr(C.dirfd(dir)) 210 } 211 212 func subvolCreate(path, name string) error { 213 dir, err := openDir(path) 214 if err != nil { 215 return err 216 } 217 defer closeDir(dir) 218 219 var args C.struct_btrfs_ioctl_vol_args 220 for i, c := range []byte(name) { 221 args.name[i] = C.char(c) 222 } 223 224 _, _, errno := unix.Syscall(unix.SYS_IOCTL, getDirFd(dir), C.BTRFS_IOC_SUBVOL_CREATE, 225 uintptr(unsafe.Pointer(&args))) 226 if errno != 0 { 227 return fmt.Errorf("Failed to create btrfs subvolume: %v", errno.Error()) 228 } 229 return nil 230 } 231 232 func subvolSnapshot(src, dest, name string) error { 233 srcDir, err := openDir(src) 234 if err != nil { 235 return err 236 } 237 defer closeDir(srcDir) 238 239 destDir, err := openDir(dest) 240 if err != nil { 241 return err 242 } 243 defer closeDir(destDir) 244 245 var args C.struct_btrfs_ioctl_vol_args_v2 246 args.fd = C.__s64(getDirFd(srcDir)) 247 248 var cs = C.CString(name) 249 C.set_name_btrfs_ioctl_vol_args_v2(&args, cs) 250 C.free(unsafe.Pointer(cs)) 251 252 _, _, errno := unix.Syscall(unix.SYS_IOCTL, getDirFd(destDir), C.BTRFS_IOC_SNAP_CREATE_V2, 253 uintptr(unsafe.Pointer(&args))) 254 if errno != 0 { 255 return fmt.Errorf("Failed to create btrfs snapshot: %v", errno.Error()) 256 } 257 return nil 258 } 259 260 func isSubvolume(p string) (bool, error) { 261 var bufStat unix.Stat_t 262 if err := unix.Lstat(p, &bufStat); err != nil { 263 return false, err 264 } 265 266 // return true if it is a btrfs subvolume 267 return bufStat.Ino == C.BTRFS_FIRST_FREE_OBJECTID, nil 268 } 269 270 func subvolDelete(dirpath, name string, quotaEnabled bool) error { 271 dir, err := openDir(dirpath) 272 if err != nil { 273 return err 274 } 275 defer closeDir(dir) 276 fullPath := path.Join(dirpath, name) 277 278 var args C.struct_btrfs_ioctl_vol_args 279 280 // walk the btrfs subvolumes 281 walkSubvolumes := func(p string, f os.FileInfo, err error) error { 282 if err != nil { 283 if os.IsNotExist(err) && p != fullPath { 284 // missing most likely because the path was a subvolume that got removed in the previous iteration 285 // since it's gone anyway, we don't care 286 return nil 287 } 288 return fmt.Errorf("error walking subvolumes: %v", err) 289 } 290 // we want to check children only so skip itself 291 // it will be removed after the filepath walk anyways 292 if f.IsDir() && p != fullPath { 293 sv, err := isSubvolume(p) 294 if err != nil { 295 return fmt.Errorf("Failed to test if %s is a btrfs subvolume: %v", p, err) 296 } 297 if sv { 298 if err := subvolDelete(path.Dir(p), f.Name(), quotaEnabled); err != nil { 299 return fmt.Errorf("Failed to destroy btrfs child subvolume (%s) of parent (%s): %v", p, dirpath, err) 300 } 301 } 302 } 303 return nil 304 } 305 if err := filepath.Walk(path.Join(dirpath, name), walkSubvolumes); err != nil { 306 return fmt.Errorf("Recursively walking subvolumes for %s failed: %v", dirpath, err) 307 } 308 309 if quotaEnabled { 310 if qgroupid, err := subvolLookupQgroup(fullPath); err == nil { 311 var args C.struct_btrfs_ioctl_qgroup_create_args 312 args.qgroupid = C.__u64(qgroupid) 313 314 _, _, errno := unix.Syscall(unix.SYS_IOCTL, getDirFd(dir), C.BTRFS_IOC_QGROUP_CREATE, 315 uintptr(unsafe.Pointer(&args))) 316 if errno != 0 { 317 logrus.WithField("storage-driver", "btrfs").Errorf("Failed to delete btrfs qgroup %v for %s: %v", qgroupid, fullPath, errno.Error()) 318 } 319 } else { 320 logrus.WithField("storage-driver", "btrfs").Errorf("Failed to lookup btrfs qgroup for %s: %v", fullPath, err.Error()) 321 } 322 } 323 324 // all subvolumes have been removed 325 // now remove the one originally passed in 326 for i, c := range []byte(name) { 327 args.name[i] = C.char(c) 328 } 329 _, _, errno := unix.Syscall(unix.SYS_IOCTL, getDirFd(dir), C.BTRFS_IOC_SNAP_DESTROY, 330 uintptr(unsafe.Pointer(&args))) 331 if errno != 0 { 332 return fmt.Errorf("Failed to destroy btrfs snapshot %s for %s: %v", dirpath, name, errno.Error()) 333 } 334 return nil 335 } 336 337 func (d *Driver) updateQuotaStatus() { 338 d.once.Do(func() { 339 if !d.quotaEnabled { 340 // In case quotaEnabled is not set, check qgroup and update quotaEnabled as needed 341 if err := subvolQgroupStatus(d.home); err != nil { 342 // quota is still not enabled 343 return 344 } 345 d.quotaEnabled = true 346 } 347 }) 348 } 349 350 func (d *Driver) subvolEnableQuota() error { 351 d.updateQuotaStatus() 352 353 if d.quotaEnabled { 354 return nil 355 } 356 357 dir, err := openDir(d.home) 358 if err != nil { 359 return err 360 } 361 defer closeDir(dir) 362 363 var args C.struct_btrfs_ioctl_quota_ctl_args 364 args.cmd = C.BTRFS_QUOTA_CTL_ENABLE 365 _, _, errno := unix.Syscall(unix.SYS_IOCTL, getDirFd(dir), C.BTRFS_IOC_QUOTA_CTL, 366 uintptr(unsafe.Pointer(&args))) 367 if errno != 0 { 368 return fmt.Errorf("Failed to enable btrfs quota for %s: %v", dir, errno.Error()) 369 } 370 371 d.quotaEnabled = true 372 373 return nil 374 } 375 376 func (d *Driver) subvolDisableQuota() error { 377 d.updateQuotaStatus() 378 379 if !d.quotaEnabled { 380 return nil 381 } 382 383 dir, err := openDir(d.home) 384 if err != nil { 385 return err 386 } 387 defer closeDir(dir) 388 389 var args C.struct_btrfs_ioctl_quota_ctl_args 390 args.cmd = C.BTRFS_QUOTA_CTL_DISABLE 391 _, _, errno := unix.Syscall(unix.SYS_IOCTL, getDirFd(dir), C.BTRFS_IOC_QUOTA_CTL, 392 uintptr(unsafe.Pointer(&args))) 393 if errno != 0 { 394 return fmt.Errorf("Failed to disable btrfs quota for %s: %v", dir, errno.Error()) 395 } 396 397 d.quotaEnabled = false 398 399 return nil 400 } 401 402 func (d *Driver) subvolRescanQuota() error { 403 d.updateQuotaStatus() 404 405 if !d.quotaEnabled { 406 return nil 407 } 408 409 dir, err := openDir(d.home) 410 if err != nil { 411 return err 412 } 413 defer closeDir(dir) 414 415 var args C.struct_btrfs_ioctl_quota_rescan_args 416 _, _, errno := unix.Syscall(unix.SYS_IOCTL, getDirFd(dir), C.BTRFS_IOC_QUOTA_RESCAN_WAIT, 417 uintptr(unsafe.Pointer(&args))) 418 if errno != 0 { 419 return fmt.Errorf("Failed to rescan btrfs quota for %s: %v", dir, errno.Error()) 420 } 421 422 return nil 423 } 424 425 func subvolLimitQgroup(path string, size uint64) error { 426 dir, err := openDir(path) 427 if err != nil { 428 return err 429 } 430 defer closeDir(dir) 431 432 var args C.struct_btrfs_ioctl_qgroup_limit_args 433 args.lim.max_referenced = C.__u64(size) 434 args.lim.flags = C.BTRFS_QGROUP_LIMIT_MAX_RFER 435 _, _, errno := unix.Syscall(unix.SYS_IOCTL, getDirFd(dir), C.BTRFS_IOC_QGROUP_LIMIT, 436 uintptr(unsafe.Pointer(&args))) 437 if errno != 0 { 438 return fmt.Errorf("Failed to limit qgroup for %s: %v", dir, errno.Error()) 439 } 440 441 return nil 442 } 443 444 // subvolQgroupStatus performs a BTRFS_IOC_TREE_SEARCH on the root path 445 // with search key of BTRFS_QGROUP_STATUS_KEY. 446 // In case qgroup is enabled, the retuned key type will match BTRFS_QGROUP_STATUS_KEY. 447 // For more details please see https://github.com/kdave/btrfs-progs/blob/v4.9/qgroup.c#L1035 448 func subvolQgroupStatus(path string) error { 449 dir, err := openDir(path) 450 if err != nil { 451 return err 452 } 453 defer closeDir(dir) 454 455 var args C.struct_btrfs_ioctl_search_args 456 args.key.tree_id = C.BTRFS_QUOTA_TREE_OBJECTID 457 args.key.min_type = C.BTRFS_QGROUP_STATUS_KEY 458 args.key.max_type = C.BTRFS_QGROUP_STATUS_KEY 459 args.key.max_objectid = C.__u64(math.MaxUint64) 460 args.key.max_offset = C.__u64(math.MaxUint64) 461 args.key.max_transid = C.__u64(math.MaxUint64) 462 args.key.nr_items = 4096 463 464 _, _, errno := unix.Syscall(unix.SYS_IOCTL, getDirFd(dir), C.BTRFS_IOC_TREE_SEARCH, 465 uintptr(unsafe.Pointer(&args))) 466 if errno != 0 { 467 return fmt.Errorf("Failed to search qgroup for %s: %v", path, errno.Error()) 468 } 469 sh := (*C.struct_btrfs_ioctl_search_header)(unsafe.Pointer(&args.buf)) 470 if sh._type != C.BTRFS_QGROUP_STATUS_KEY { 471 return fmt.Errorf("Invalid qgroup search header type for %s: %v", path, sh._type) 472 } 473 return nil 474 } 475 476 func subvolLookupQgroup(path string) (uint64, error) { 477 dir, err := openDir(path) 478 if err != nil { 479 return 0, err 480 } 481 defer closeDir(dir) 482 483 var args C.struct_btrfs_ioctl_ino_lookup_args 484 args.objectid = C.BTRFS_FIRST_FREE_OBJECTID 485 486 _, _, errno := unix.Syscall(unix.SYS_IOCTL, getDirFd(dir), C.BTRFS_IOC_INO_LOOKUP, 487 uintptr(unsafe.Pointer(&args))) 488 if errno != 0 { 489 return 0, fmt.Errorf("Failed to lookup qgroup for %s: %v", dir, errno.Error()) 490 } 491 if args.treeid == 0 { 492 return 0, fmt.Errorf("Invalid qgroup id for %s: 0", dir) 493 } 494 495 return uint64(args.treeid), nil 496 } 497 498 func (d *Driver) subvolumesDir() string { 499 return path.Join(d.home, "subvolumes") 500 } 501 502 func (d *Driver) subvolumesDirID(id string) string { 503 return path.Join(d.subvolumesDir(), id) 504 } 505 506 func (d *Driver) quotasDir() string { 507 return path.Join(d.home, "quotas") 508 } 509 510 func (d *Driver) quotasDirID(id string) string { 511 return path.Join(d.quotasDir(), id) 512 } 513 514 // CreateReadWrite creates a layer that is writable for use as a container 515 // file system. 516 func (d *Driver) CreateReadWrite(id, parent string, opts *graphdriver.CreateOpts) error { 517 return d.Create(id, parent, opts) 518 } 519 520 // Create the filesystem with given id. 521 func (d *Driver) Create(id, parent string, opts *graphdriver.CreateOpts) error { 522 quotas := path.Join(d.home, "quotas") 523 subvolumes := path.Join(d.home, "subvolumes") 524 rootUID, rootGID, err := idtools.GetRootUIDGID(d.uidMaps, d.gidMaps) 525 if err != nil { 526 return err 527 } 528 if err := idtools.MkdirAllAndChown(subvolumes, 0700, idtools.Identity{UID: rootUID, GID: rootGID}); err != nil { 529 return err 530 } 531 if parent == "" { 532 if err := subvolCreate(subvolumes, id); err != nil { 533 return err 534 } 535 } else { 536 parentDir := d.subvolumesDirID(parent) 537 st, err := os.Stat(parentDir) 538 if err != nil { 539 return err 540 } 541 if !st.IsDir() { 542 return fmt.Errorf("%s: not a directory", parentDir) 543 } 544 if err := subvolSnapshot(parentDir, subvolumes, id); err != nil { 545 return err 546 } 547 } 548 549 var storageOpt map[string]string 550 if opts != nil { 551 storageOpt = opts.StorageOpt 552 } 553 554 if _, ok := storageOpt["size"]; ok { 555 driver := &Driver{} 556 if err := d.parseStorageOpt(storageOpt, driver); err != nil { 557 return err 558 } 559 560 if err := d.setStorageSize(path.Join(subvolumes, id), driver); err != nil { 561 return err 562 } 563 if err := idtools.MkdirAllAndChown(quotas, 0700, idtools.Identity{UID: rootUID, GID: rootGID}); err != nil { 564 return err 565 } 566 if err := ioutil.WriteFile(path.Join(quotas, id), []byte(fmt.Sprint(driver.options.size)), 0644); err != nil { 567 return err 568 } 569 } 570 571 // if we have a remapped root (user namespaces enabled), change the created snapshot 572 // dir ownership to match 573 if rootUID != 0 || rootGID != 0 { 574 if err := os.Chown(path.Join(subvolumes, id), rootUID, rootGID); err != nil { 575 return err 576 } 577 } 578 579 mountLabel := "" 580 if opts != nil { 581 mountLabel = opts.MountLabel 582 } 583 584 return label.Relabel(path.Join(subvolumes, id), mountLabel, false) 585 } 586 587 // Parse btrfs storage options 588 func (d *Driver) parseStorageOpt(storageOpt map[string]string, driver *Driver) error { 589 // Read size to change the subvolume disk quota per container 590 for key, val := range storageOpt { 591 key := strings.ToLower(key) 592 switch key { 593 case "size": 594 size, err := units.RAMInBytes(val) 595 if err != nil { 596 return err 597 } 598 driver.options.size = uint64(size) 599 default: 600 return fmt.Errorf("Unknown option %s", key) 601 } 602 } 603 604 return nil 605 } 606 607 // Set btrfs storage size 608 func (d *Driver) setStorageSize(dir string, driver *Driver) error { 609 if driver.options.size == 0 { 610 return fmt.Errorf("btrfs: invalid storage size: %s", units.HumanSize(float64(driver.options.size))) 611 } 612 if d.options.minSpace > 0 && driver.options.size < d.options.minSpace { 613 return fmt.Errorf("btrfs: storage size cannot be less than %s", units.HumanSize(float64(d.options.minSpace))) 614 } 615 if err := d.subvolEnableQuota(); err != nil { 616 return err 617 } 618 return subvolLimitQgroup(dir, driver.options.size) 619 } 620 621 // Remove the filesystem with given id. 622 func (d *Driver) Remove(id string) error { 623 dir := d.subvolumesDirID(id) 624 if _, err := os.Stat(dir); err != nil { 625 return err 626 } 627 quotasDir := d.quotasDirID(id) 628 if _, err := os.Stat(quotasDir); err == nil { 629 if err := os.Remove(quotasDir); err != nil { 630 return err 631 } 632 } else if !os.IsNotExist(err) { 633 return err 634 } 635 636 // Call updateQuotaStatus() to invoke status update 637 d.updateQuotaStatus() 638 639 if err := subvolDelete(d.subvolumesDir(), id, d.quotaEnabled); err != nil { 640 return err 641 } 642 if err := system.EnsureRemoveAll(dir); err != nil { 643 return err 644 } 645 return d.subvolRescanQuota() 646 } 647 648 // Get the requested filesystem id. 649 func (d *Driver) Get(id, mountLabel string) (containerfs.ContainerFS, error) { 650 dir := d.subvolumesDirID(id) 651 st, err := os.Stat(dir) 652 if err != nil { 653 return nil, err 654 } 655 656 if !st.IsDir() { 657 return nil, fmt.Errorf("%s: not a directory", dir) 658 } 659 660 if quota, err := ioutil.ReadFile(d.quotasDirID(id)); err == nil { 661 if size, err := strconv.ParseUint(string(quota), 10, 64); err == nil && size >= d.options.minSpace { 662 if err := d.subvolEnableQuota(); err != nil { 663 return nil, err 664 } 665 if err := subvolLimitQgroup(dir, size); err != nil { 666 return nil, err 667 } 668 } 669 } 670 671 return containerfs.NewLocalContainerFS(dir), nil 672 } 673 674 // Put is not implemented for BTRFS as there is no cleanup required for the id. 675 func (d *Driver) Put(id string) error { 676 // Get() creates no runtime resources (like e.g. mounts) 677 // so this doesn't need to do anything. 678 return nil 679 } 680 681 // Exists checks if the id exists in the filesystem. 682 func (d *Driver) Exists(id string) bool { 683 dir := d.subvolumesDirID(id) 684 _, err := os.Stat(dir) 685 return err == nil 686 }