github.com/sentienttechnologies/studio-go-runner@v0.0.0-20201118202441-6d21f2ced8ee/internal/runner/disk.go (about)

     1  // Copyright 2018-2020 (c) Cognizant Digital Business, Evolutionary AI. All rights reserved. Issued under the Apache 2.0 License.
     2  
     3  package runner
     4  
     5  // This file contains functions and data used to deal with local disk space allocation
     6  
     7  import (
     8  	"sync"
     9  	"syscall"
    10  
    11  	"github.com/dustin/go-humanize"
    12  
    13  	"github.com/go-stack/stack"
    14  	"github.com/jjeffery/kv" // MIT License
    15  )
    16  
    17  type diskTracker struct {
    18  	Device     string   // The local storage device being tracked, if change this will clear our all old allocations and releases will be ignored for the old device
    19  	AllocSpace uint64   // The amount of local storage, in the file system nominated by the user, currently allocated
    20  	MinFree    uint64   // The amount of local storage low water mark, specified by the user, defaults to 10% of physical storage on devices
    21  	InitErr    kv.Error // Any error that might have been recorded during initialization, if set this package may produce unexpected results
    22  
    23  	sync.Mutex
    24  }
    25  
    26  var (
    27  	diskTrack = &diskTracker{}
    28  )
    29  
    30  func initDiskResource(device string) (err kv.Error) {
    31  	_, diskTrack.InitErr = SetDiskLimits(device, 0)
    32  	return diskTrack.InitErr
    33  }
    34  
    35  // GetDiskFree is used to retrieve the amount of available disk
    36  // space we have from a logical perspective with the headroom
    37  // taken out
    38  //
    39  func GetDiskFree() (free uint64) {
    40  	diskTrack.Lock()
    41  	defer diskTrack.Unlock()
    42  
    43  	fs := syscall.Statfs_t{}
    44  	if err := syscall.Statfs(diskTrack.Device, &fs); err != nil {
    45  		return 0
    46  	}
    47  
    48  	hardwareFree := fs.Bfree * uint64(fs.Bsize) // Hardware consumption check
    49  
    50  	// Before we try to do the math with unsigned ints make sure it wont underflow
    51  	// Make sure the hardware free space can handle the logical free space
    52  	if hardwareFree < diskTrack.MinFree {
    53  		return 0
    54  	}
    55  
    56  	highWater := (fs.Bavail * uint64(fs.Bsize)) - diskTrack.MinFree // Space available to user, allows for quotas etc, leave 10% headroom
    57  	if highWater <= diskTrack.AllocSpace {
    58  		return 0
    59  	}
    60  
    61  	return highWater - diskTrack.AllocSpace
    62  }
    63  
    64  // GetPathFree will use the path supplied by the caller as the device context for which
    65  // free space information is returned, this is from a pysical free capacity
    66  // perspective rather than what the application is allowed to allocated
    67  // which has to adjust for minimum amount free.
    68  //
    69  func GetPathFree(path string) (free uint64, err kv.Error) {
    70  	fs := syscall.Statfs_t{}
    71  	if errGo := syscall.Statfs(path, &fs); errGo != nil {
    72  		return 0, kv.Wrap(errGo).With("path", path).With("stack", stack.Trace().TrimRuntime())
    73  	}
    74  
    75  	return fs.Bfree * uint64(fs.Bsize), nil
    76  }
    77  
    78  // SetDiskLimits is used to set a highwater mark for a device as a minimum free quantity
    79  //
    80  func SetDiskLimits(device string, minFree uint64) (avail uint64, err kv.Error) {
    81  
    82  	fs := syscall.Statfs_t{}
    83  	if errGo := syscall.Statfs(device, &fs); err != nil {
    84  		return 0, kv.Wrap(errGo).With("stack", stack.Trace().TrimRuntime())
    85  	}
    86  	blockSize := uint64(fs.Bsize)
    87  
    88  	softMinFree := fs.Blocks / 10 * blockSize // Minimum permitted space is 10 % of the volumes entire capacity
    89  	if minFree > softMinFree {
    90  		softMinFree = minFree
    91  	}
    92  
    93  	diskTrack.Lock()
    94  	defer diskTrack.Unlock()
    95  
    96  	diskTrack.AllocSpace = 0
    97  	diskTrack.MinFree = softMinFree
    98  	diskTrack.Device = device
    99  	diskTrack.InitErr = nil
   100  
   101  	return fs.Bfree*blockSize - diskTrack.MinFree, nil
   102  }
   103  
   104  // AllocDisk will assigned a specified amount of space from a logical bucket for the
   105  // default disk device.  An error is returned if the available amount of disk
   106  // is insufficient
   107  //
   108  func AllocDisk(maxSpace uint64, live bool) (alloc *DiskAllocated, err kv.Error) {
   109  
   110  	avail := GetDiskFree()
   111  
   112  	diskTrack.Lock()
   113  	defer diskTrack.Unlock()
   114  
   115  	if avail < maxSpace {
   116  		return nil, kv.NewError("disk space exhausted").
   117  			With("available", humanize.Bytes(avail), "soft_min_free", humanize.Bytes(diskTrack.MinFree),
   118  				"allocated_already", humanize.Bytes(diskTrack.AllocSpace),
   119  				"device", diskTrack.Device, "maxmimum_space", humanize.Bytes(maxSpace)).
   120  			With("stack", stack.Trace().TrimRuntime())
   121  	}
   122  
   123  	if !live {
   124  		return nil, nil
   125  	}
   126  
   127  	diskTrack.InitErr = nil
   128  	diskTrack.AllocSpace += maxSpace
   129  
   130  	alloc = &DiskAllocated{
   131  		device: diskTrack.Device,
   132  		size:   maxSpace,
   133  	}
   134  
   135  	return alloc, nil
   136  }
   137  
   138  // Release will return assigned disk space back to the free pool of disk space
   139  // for a default disk device.  If the allocation is not recognized then an
   140  // error is returned
   141  //
   142  func (alloc *DiskAllocated) Release() (err kv.Error) {
   143  
   144  	if alloc == nil {
   145  		return kv.NewError("empty allocation supplied for releasing disk storage").With("stack", stack.Trace().TrimRuntime())
   146  	}
   147  
   148  	diskTrack.Lock()
   149  	defer diskTrack.Unlock()
   150  
   151  	if diskTrack.InitErr != nil {
   152  		return diskTrack.InitErr
   153  	}
   154  
   155  	if alloc.device != diskTrack.Device {
   156  		return kv.NewError("allocated space came from untracked local storage").
   157  			With("allocated_size", humanize.Bytes(alloc.size), "device", alloc.device).With("stack", stack.Trace().TrimRuntime())
   158  	}
   159  
   160  	diskTrack.AllocSpace -= alloc.size
   161  
   162  	return nil
   163  }