github.com/sentienttechnologies/studio-go-runner@v0.0.0-20201118202441-6d21f2ced8ee/internal/runner/disk.go (about) 1 // Copyright 2018-2020 (c) Cognizant Digital Business, Evolutionary AI. All rights reserved. Issued under the Apache 2.0 License. 2 3 package runner 4 5 // This file contains functions and data used to deal with local disk space allocation 6 7 import ( 8 "sync" 9 "syscall" 10 11 "github.com/dustin/go-humanize" 12 13 "github.com/go-stack/stack" 14 "github.com/jjeffery/kv" // MIT License 15 ) 16 17 type diskTracker struct { 18 Device string // The local storage device being tracked, if change this will clear our all old allocations and releases will be ignored for the old device 19 AllocSpace uint64 // The amount of local storage, in the file system nominated by the user, currently allocated 20 MinFree uint64 // The amount of local storage low water mark, specified by the user, defaults to 10% of physical storage on devices 21 InitErr kv.Error // Any error that might have been recorded during initialization, if set this package may produce unexpected results 22 23 sync.Mutex 24 } 25 26 var ( 27 diskTrack = &diskTracker{} 28 ) 29 30 func initDiskResource(device string) (err kv.Error) { 31 _, diskTrack.InitErr = SetDiskLimits(device, 0) 32 return diskTrack.InitErr 33 } 34 35 // GetDiskFree is used to retrieve the amount of available disk 36 // space we have from a logical perspective with the headroom 37 // taken out 38 // 39 func GetDiskFree() (free uint64) { 40 diskTrack.Lock() 41 defer diskTrack.Unlock() 42 43 fs := syscall.Statfs_t{} 44 if err := syscall.Statfs(diskTrack.Device, &fs); err != nil { 45 return 0 46 } 47 48 hardwareFree := fs.Bfree * uint64(fs.Bsize) // Hardware consumption check 49 50 // Before we try to do the math with unsigned ints make sure it wont underflow 51 // Make sure the hardware free space can handle the logical free space 52 if hardwareFree < diskTrack.MinFree { 53 return 0 54 } 55 56 highWater := (fs.Bavail * uint64(fs.Bsize)) - diskTrack.MinFree // Space available to user, allows for quotas etc, leave 10% headroom 57 if highWater <= diskTrack.AllocSpace { 58 return 0 59 } 60 61 return highWater - diskTrack.AllocSpace 62 } 63 64 // GetPathFree will use the path supplied by the caller as the device context for which 65 // free space information is returned, this is from a pysical free capacity 66 // perspective rather than what the application is allowed to allocated 67 // which has to adjust for minimum amount free. 68 // 69 func GetPathFree(path string) (free uint64, err kv.Error) { 70 fs := syscall.Statfs_t{} 71 if errGo := syscall.Statfs(path, &fs); errGo != nil { 72 return 0, kv.Wrap(errGo).With("path", path).With("stack", stack.Trace().TrimRuntime()) 73 } 74 75 return fs.Bfree * uint64(fs.Bsize), nil 76 } 77 78 // SetDiskLimits is used to set a highwater mark for a device as a minimum free quantity 79 // 80 func SetDiskLimits(device string, minFree uint64) (avail uint64, err kv.Error) { 81 82 fs := syscall.Statfs_t{} 83 if errGo := syscall.Statfs(device, &fs); err != nil { 84 return 0, kv.Wrap(errGo).With("stack", stack.Trace().TrimRuntime()) 85 } 86 blockSize := uint64(fs.Bsize) 87 88 softMinFree := fs.Blocks / 10 * blockSize // Minimum permitted space is 10 % of the volumes entire capacity 89 if minFree > softMinFree { 90 softMinFree = minFree 91 } 92 93 diskTrack.Lock() 94 defer diskTrack.Unlock() 95 96 diskTrack.AllocSpace = 0 97 diskTrack.MinFree = softMinFree 98 diskTrack.Device = device 99 diskTrack.InitErr = nil 100 101 return fs.Bfree*blockSize - diskTrack.MinFree, nil 102 } 103 104 // AllocDisk will assigned a specified amount of space from a logical bucket for the 105 // default disk device. An error is returned if the available amount of disk 106 // is insufficient 107 // 108 func AllocDisk(maxSpace uint64, live bool) (alloc *DiskAllocated, err kv.Error) { 109 110 avail := GetDiskFree() 111 112 diskTrack.Lock() 113 defer diskTrack.Unlock() 114 115 if avail < maxSpace { 116 return nil, kv.NewError("disk space exhausted"). 117 With("available", humanize.Bytes(avail), "soft_min_free", humanize.Bytes(diskTrack.MinFree), 118 "allocated_already", humanize.Bytes(diskTrack.AllocSpace), 119 "device", diskTrack.Device, "maxmimum_space", humanize.Bytes(maxSpace)). 120 With("stack", stack.Trace().TrimRuntime()) 121 } 122 123 if !live { 124 return nil, nil 125 } 126 127 diskTrack.InitErr = nil 128 diskTrack.AllocSpace += maxSpace 129 130 alloc = &DiskAllocated{ 131 device: diskTrack.Device, 132 size: maxSpace, 133 } 134 135 return alloc, nil 136 } 137 138 // Release will return assigned disk space back to the free pool of disk space 139 // for a default disk device. If the allocation is not recognized then an 140 // error is returned 141 // 142 func (alloc *DiskAllocated) Release() (err kv.Error) { 143 144 if alloc == nil { 145 return kv.NewError("empty allocation supplied for releasing disk storage").With("stack", stack.Trace().TrimRuntime()) 146 } 147 148 diskTrack.Lock() 149 defer diskTrack.Unlock() 150 151 if diskTrack.InitErr != nil { 152 return diskTrack.InitErr 153 } 154 155 if alloc.device != diskTrack.Device { 156 return kv.NewError("allocated space came from untracked local storage"). 157 With("allocated_size", humanize.Bytes(alloc.size), "device", alloc.device).With("stack", stack.Trace().TrimRuntime()) 158 } 159 160 diskTrack.AllocSpace -= alloc.size 161 162 return nil 163 }