github.com/zhouyu0/docker-note@v0.0.0-20190722021225-b8d3825084db/daemon/graphdriver/zfs/zfs.go (about) 1 // +build linux freebsd 2 3 package zfs // import "github.com/docker/docker/daemon/graphdriver/zfs" 4 5 import ( 6 "fmt" 7 "os" 8 "os/exec" 9 "path" 10 "strconv" 11 "strings" 12 "sync" 13 "time" 14 15 "github.com/docker/docker/daemon/graphdriver" 16 "github.com/docker/docker/pkg/containerfs" 17 "github.com/docker/docker/pkg/idtools" 18 "github.com/docker/docker/pkg/mount" 19 "github.com/docker/docker/pkg/parsers" 20 "github.com/mistifyio/go-zfs" 21 "github.com/opencontainers/selinux/go-selinux/label" 22 "github.com/pkg/errors" 23 "github.com/sirupsen/logrus" 24 "golang.org/x/sys/unix" 25 ) 26 27 type zfsOptions struct { 28 fsName string 29 mountPath string 30 } 31 32 func init() { 33 graphdriver.Register("zfs", Init) 34 } 35 36 // Logger returns a zfs logger implementation. 37 type Logger struct{} 38 39 // Log wraps log message from ZFS driver with a prefix '[zfs]'. 40 func (*Logger) Log(cmd []string) { 41 logrus.WithField("storage-driver", "zfs").Debugf("[zfs] %s", strings.Join(cmd, " ")) 42 } 43 44 // Init returns a new ZFS driver. 45 // It takes base mount path and an array of options which are represented as key value pairs. 46 // Each option is in the for key=value. 'zfs.fsname' is expected to be a valid key in the options. 47 func Init(base string, opt []string, uidMaps, gidMaps []idtools.IDMap) (graphdriver.Driver, error) { 48 var err error 49 50 logger := logrus.WithField("storage-driver", "zfs") 51 52 if _, err := exec.LookPath("zfs"); err != nil { 53 logger.Debugf("zfs command is not available: %v", err) 54 return nil, graphdriver.ErrPrerequisites 55 } 56 57 file, err := os.OpenFile("/dev/zfs", os.O_RDWR, 600) 58 if err != nil { 59 logger.Debugf("cannot open /dev/zfs: %v", err) 60 return nil, graphdriver.ErrPrerequisites 61 } 62 defer file.Close() 63 64 options, err := parseOptions(opt) 65 if err != nil { 66 return nil, err 67 } 68 options.mountPath = base 69 70 rootdir := path.Dir(base) 71 72 if options.fsName == "" { 73 err = checkRootdirFs(rootdir) 74 if err != nil { 75 return nil, err 76 } 77 } 78 79 if options.fsName == "" { 80 options.fsName, err = lookupZfsDataset(rootdir) 81 if err != nil { 82 return nil, err 83 } 84 } 85 86 zfs.SetLogger(new(Logger)) 87 88 filesystems, err := zfs.Filesystems(options.fsName) 89 if err != nil { 90 return nil, fmt.Errorf("Cannot find root filesystem %s: %v", options.fsName, err) 91 } 92 93 filesystemsCache := make(map[string]bool, len(filesystems)) 94 var rootDataset *zfs.Dataset 95 for _, fs := range filesystems { 96 if fs.Name == options.fsName { 97 rootDataset = fs 98 } 99 filesystemsCache[fs.Name] = true 100 } 101 102 if rootDataset == nil { 103 return nil, fmt.Errorf("BUG: zfs get all -t filesystem -rHp '%s' should contain '%s'", options.fsName, options.fsName) 104 } 105 106 rootUID, rootGID, err := idtools.GetRootUIDGID(uidMaps, gidMaps) 107 if err != nil { 108 return nil, fmt.Errorf("Failed to get root uid/guid: %v", err) 109 } 110 if err := idtools.MkdirAllAndChown(base, 0700, idtools.Identity{UID: rootUID, GID: rootGID}); err != nil { 111 return nil, fmt.Errorf("Failed to create '%s': %v", base, err) 112 } 113 114 d := &Driver{ 115 dataset: rootDataset, 116 options: options, 117 filesystemsCache: filesystemsCache, 118 uidMaps: uidMaps, 119 gidMaps: gidMaps, 120 ctr: graphdriver.NewRefCounter(graphdriver.NewDefaultChecker()), 121 } 122 return graphdriver.NewNaiveDiffDriver(d, uidMaps, gidMaps), nil 123 } 124 125 func parseOptions(opt []string) (zfsOptions, error) { 126 var options zfsOptions 127 options.fsName = "" 128 for _, option := range opt { 129 key, val, err := parsers.ParseKeyValueOpt(option) 130 if err != nil { 131 return options, err 132 } 133 key = strings.ToLower(key) 134 switch key { 135 case "zfs.fsname": 136 options.fsName = val 137 default: 138 return options, fmt.Errorf("Unknown option %s", key) 139 } 140 } 141 return options, nil 142 } 143 144 func lookupZfsDataset(rootdir string) (string, error) { 145 var stat unix.Stat_t 146 if err := unix.Stat(rootdir, &stat); err != nil { 147 return "", fmt.Errorf("Failed to access '%s': %s", rootdir, err) 148 } 149 wantedDev := stat.Dev 150 151 mounts, err := mount.GetMounts(nil) 152 if err != nil { 153 return "", err 154 } 155 for _, m := range mounts { 156 if err := unix.Stat(m.Mountpoint, &stat); err != nil { 157 logrus.WithField("storage-driver", "zfs").Debugf("failed to stat '%s' while scanning for zfs mount: %v", m.Mountpoint, err) 158 continue // may fail on fuse file systems 159 } 160 161 if stat.Dev == wantedDev && m.Fstype == "zfs" { 162 return m.Source, nil 163 } 164 } 165 166 return "", fmt.Errorf("Failed to find zfs dataset mounted on '%s' in /proc/mounts", rootdir) 167 } 168 169 // Driver holds information about the driver, such as zfs dataset, options and cache. 170 type Driver struct { 171 dataset *zfs.Dataset 172 options zfsOptions 173 sync.Mutex // protects filesystem cache against concurrent access 174 filesystemsCache map[string]bool 175 uidMaps []idtools.IDMap 176 gidMaps []idtools.IDMap 177 ctr *graphdriver.RefCounter 178 } 179 180 func (d *Driver) String() string { 181 return "zfs" 182 } 183 184 // Cleanup is called on daemon shutdown, it is a no-op for ZFS. 185 // TODO(@cpuguy83): Walk layer tree and check mounts? 186 func (d *Driver) Cleanup() error { 187 return nil 188 } 189 190 // Status returns information about the ZFS filesystem. It returns a two dimensional array of information 191 // such as pool name, dataset name, disk usage, parent quota and compression used. 192 // Currently it return 'Zpool', 'Zpool Health', 'Parent Dataset', 'Space Used By Parent', 193 // 'Space Available', 'Parent Quota' and 'Compression'. 194 func (d *Driver) Status() [][2]string { 195 parts := strings.Split(d.dataset.Name, "/") 196 pool, err := zfs.GetZpool(parts[0]) 197 198 var poolName, poolHealth string 199 if err == nil { 200 poolName = pool.Name 201 poolHealth = pool.Health 202 } else { 203 poolName = fmt.Sprintf("error while getting pool information %v", err) 204 poolHealth = "not available" 205 } 206 207 quota := "no" 208 if d.dataset.Quota != 0 { 209 quota = strconv.FormatUint(d.dataset.Quota, 10) 210 } 211 212 return [][2]string{ 213 {"Zpool", poolName}, 214 {"Zpool Health", poolHealth}, 215 {"Parent Dataset", d.dataset.Name}, 216 {"Space Used By Parent", strconv.FormatUint(d.dataset.Used, 10)}, 217 {"Space Available", strconv.FormatUint(d.dataset.Avail, 10)}, 218 {"Parent Quota", quota}, 219 {"Compression", d.dataset.Compression}, 220 } 221 } 222 223 // GetMetadata returns image/container metadata related to graph driver 224 func (d *Driver) GetMetadata(id string) (map[string]string, error) { 225 return map[string]string{ 226 "Mountpoint": d.mountPath(id), 227 "Dataset": d.zfsPath(id), 228 }, nil 229 } 230 231 func (d *Driver) cloneFilesystem(name, parentName string) error { 232 snapshotName := fmt.Sprintf("%d", time.Now().Nanosecond()) 233 parentDataset := zfs.Dataset{Name: parentName} 234 snapshot, err := parentDataset.Snapshot(snapshotName /*recursive */, false) 235 if err != nil { 236 return err 237 } 238 239 _, err = snapshot.Clone(name, map[string]string{"mountpoint": "legacy"}) 240 if err == nil { 241 d.Lock() 242 d.filesystemsCache[name] = true 243 d.Unlock() 244 } 245 246 if err != nil { 247 snapshot.Destroy(zfs.DestroyDeferDeletion) 248 return err 249 } 250 return snapshot.Destroy(zfs.DestroyDeferDeletion) 251 } 252 253 func (d *Driver) zfsPath(id string) string { 254 return d.options.fsName + "/" + id 255 } 256 257 func (d *Driver) mountPath(id string) string { 258 return path.Join(d.options.mountPath, "graph", getMountpoint(id)) 259 } 260 261 // CreateReadWrite creates a layer that is writable for use as a container 262 // file system. 263 func (d *Driver) CreateReadWrite(id, parent string, opts *graphdriver.CreateOpts) error { 264 return d.Create(id, parent, opts) 265 } 266 267 // Create prepares the dataset and filesystem for the ZFS driver for the given id under the parent. 268 func (d *Driver) Create(id, parent string, opts *graphdriver.CreateOpts) error { 269 var storageOpt map[string]string 270 if opts != nil { 271 storageOpt = opts.StorageOpt 272 } 273 274 err := d.create(id, parent, storageOpt) 275 if err == nil { 276 return nil 277 } 278 if zfsError, ok := err.(*zfs.Error); ok { 279 if !strings.HasSuffix(zfsError.Stderr, "dataset already exists\n") { 280 return err 281 } 282 // aborted build -> cleanup 283 } else { 284 return err 285 } 286 287 dataset := zfs.Dataset{Name: d.zfsPath(id)} 288 if err := dataset.Destroy(zfs.DestroyRecursiveClones); err != nil { 289 return err 290 } 291 292 // retry 293 return d.create(id, parent, storageOpt) 294 } 295 296 func (d *Driver) create(id, parent string, storageOpt map[string]string) error { 297 name := d.zfsPath(id) 298 quota, err := parseStorageOpt(storageOpt) 299 if err != nil { 300 return err 301 } 302 if parent == "" { 303 mountoptions := map[string]string{"mountpoint": "legacy"} 304 fs, err := zfs.CreateFilesystem(name, mountoptions) 305 if err == nil { 306 err = setQuota(name, quota) 307 if err == nil { 308 d.Lock() 309 d.filesystemsCache[fs.Name] = true 310 d.Unlock() 311 } 312 } 313 return err 314 } 315 err = d.cloneFilesystem(name, d.zfsPath(parent)) 316 if err == nil { 317 err = setQuota(name, quota) 318 } 319 return err 320 } 321 322 func parseStorageOpt(storageOpt map[string]string) (string, error) { 323 // Read size to change the disk quota per container 324 for k, v := range storageOpt { 325 key := strings.ToLower(k) 326 switch key { 327 case "size": 328 return v, nil 329 default: 330 return "0", fmt.Errorf("Unknown option %s", key) 331 } 332 } 333 return "0", nil 334 } 335 336 func setQuota(name string, quota string) error { 337 if quota == "0" { 338 return nil 339 } 340 fs, err := zfs.GetDataset(name) 341 if err != nil { 342 return err 343 } 344 return fs.SetProperty("quota", quota) 345 } 346 347 // Remove deletes the dataset, filesystem and the cache for the given id. 348 func (d *Driver) Remove(id string) error { 349 name := d.zfsPath(id) 350 dataset := zfs.Dataset{Name: name} 351 err := dataset.Destroy(zfs.DestroyRecursive) 352 if err == nil { 353 d.Lock() 354 delete(d.filesystemsCache, name) 355 d.Unlock() 356 } 357 return err 358 } 359 360 // Get returns the mountpoint for the given id after creating the target directories if necessary. 361 func (d *Driver) Get(id, mountLabel string) (_ containerfs.ContainerFS, retErr error) { 362 mountpoint := d.mountPath(id) 363 if count := d.ctr.Increment(mountpoint); count > 1 { 364 return containerfs.NewLocalContainerFS(mountpoint), nil 365 } 366 defer func() { 367 if retErr != nil { 368 if c := d.ctr.Decrement(mountpoint); c <= 0 { 369 if mntErr := unix.Unmount(mountpoint, 0); mntErr != nil { 370 logrus.WithField("storage-driver", "zfs").Errorf("Error unmounting %v: %v", mountpoint, mntErr) 371 } 372 if rmErr := unix.Rmdir(mountpoint); rmErr != nil && !os.IsNotExist(rmErr) { 373 logrus.WithField("storage-driver", "zfs").Debugf("Failed to remove %s: %v", id, rmErr) 374 } 375 376 } 377 } 378 }() 379 380 filesystem := d.zfsPath(id) 381 options := label.FormatMountLabel("", mountLabel) 382 logrus.WithField("storage-driver", "zfs").Debugf(`mount("%s", "%s", "%s")`, filesystem, mountpoint, options) 383 384 rootUID, rootGID, err := idtools.GetRootUIDGID(d.uidMaps, d.gidMaps) 385 if err != nil { 386 return nil, err 387 } 388 // Create the target directories if they don't exist 389 if err := idtools.MkdirAllAndChown(mountpoint, 0755, idtools.Identity{UID: rootUID, GID: rootGID}); err != nil { 390 return nil, err 391 } 392 393 if err := mount.Mount(filesystem, mountpoint, "zfs", options); err != nil { 394 return nil, errors.Wrap(err, "error creating zfs mount") 395 } 396 397 // this could be our first mount after creation of the filesystem, and the root dir may still have root 398 // permissions instead of the remapped root uid:gid (if user namespaces are enabled): 399 if err := os.Chown(mountpoint, rootUID, rootGID); err != nil { 400 return nil, fmt.Errorf("error modifying zfs mountpoint (%s) directory ownership: %v", mountpoint, err) 401 } 402 403 return containerfs.NewLocalContainerFS(mountpoint), nil 404 } 405 406 // Put removes the existing mountpoint for the given id if it exists. 407 func (d *Driver) Put(id string) error { 408 mountpoint := d.mountPath(id) 409 if count := d.ctr.Decrement(mountpoint); count > 0 { 410 return nil 411 } 412 413 logger := logrus.WithField("storage-driver", "zfs") 414 415 logger.Debugf(`unmount("%s")`, mountpoint) 416 417 if err := unix.Unmount(mountpoint, unix.MNT_DETACH); err != nil { 418 logger.Warnf("Failed to unmount %s mount %s: %v", id, mountpoint, err) 419 } 420 if err := unix.Rmdir(mountpoint); err != nil && !os.IsNotExist(err) { 421 logger.Debugf("Failed to remove %s mount point %s: %v", id, mountpoint, err) 422 } 423 424 return nil 425 } 426 427 // Exists checks to see if the cache entry exists for the given id. 428 func (d *Driver) Exists(id string) bool { 429 d.Lock() 430 defer d.Unlock() 431 return d.filesystemsCache[d.zfsPath(id)] 432 }