github.com/swiftstack/proxyfs@v0.0.0-20201223034610-5434d919416e/fs/config.go (about) 1 package fs 2 3 import ( 4 "container/list" 5 "fmt" 6 "sync" 7 "time" 8 9 "github.com/swiftstack/ProxyFS/bucketstats" 10 "github.com/swiftstack/ProxyFS/conf" 11 "github.com/swiftstack/ProxyFS/headhunter" 12 "github.com/swiftstack/ProxyFS/inode" 13 "github.com/swiftstack/ProxyFS/logger" 14 "github.com/swiftstack/ProxyFS/trackedlock" 15 "github.com/swiftstack/ProxyFS/transitions" 16 ) 17 18 type inFlightFileInodeDataStruct struct { 19 inode.InodeNumber // Indicates the InodeNumber of a fileInode with unflushed 20 volStruct *volumeStruct // Synchronized via volStruct's sync.Mutex 21 control chan bool // Signal with true to flush (and exit), false to simply exit 22 wg sync.WaitGroup // Client can know when done 23 globalsListElement *list.Element // Back-pointer to wrapper used to insert into globals.inFlightFileInodeDataList 24 } 25 26 // inFlightFileInodeDataControlBuffering specifies the inFlightFileInodeDataStruct.control channel buffer size 27 // Note: There are potentially multiple initiators of this signal 28 const inFlightFileInodeDataControlBuffering = 100 29 30 type volumeStruct struct { 31 dataMutex trackedlock.Mutex 32 volumeName string 33 doCheckpointPerFlush bool 34 maxFlushTime time.Duration 35 fileDefragmentChunkSize uint64 36 fileDefragmentChunkDelay time.Duration 37 reportedBlockSize uint64 38 reportedFragmentSize uint64 39 reportedNumBlocks uint64 // Used for Total, Free, and Avail 40 reportedNumInodes uint64 // Used for Total, Free, and Avail 41 FLockMap map[inode.InodeNumber]*list.List 42 inFlightFileInodeDataMap map[inode.InodeNumber]*inFlightFileInodeDataStruct 43 jobRWMutex trackedlock.RWMutex 44 inodeVolumeHandle inode.VolumeHandle 45 headhunterVolumeHandle headhunter.VolumeHandle 46 } 47 48 type tryLockBackoffContextStruct struct { 49 sync.WaitGroup 50 backoffsCompleted uint64 // Note that tryLockBackoffContextStruct{} sets this to zero 51 } 52 53 type globalsStruct struct { 54 trackedlock.Mutex 55 56 tryLockBackoffMin time.Duration 57 tryLockBackoffMax time.Duration 58 tryLockSerializationThreshhold uint64 59 symlinkMax uint16 60 coalesceElementChunkSize uint16 61 62 volumeMap map[string]*volumeStruct // key == volumeStruct.volumeName 63 64 inFlightFileInodeDataList *list.List 65 serializedBackoffList *list.List 66 67 AccessUsec bucketstats.BucketLog2Round 68 CreateUsec bucketstats.BucketLog2Round 69 DestroyUsec bucketstats.BucketLog2Round 70 FlushUsec bucketstats.BucketLog2Round 71 FlockGetUsec bucketstats.BucketLog2Round 72 FlockLockUsec bucketstats.BucketLog2Round 73 FlockUnlockUsec bucketstats.BucketLog2Round 74 GetstatUsec bucketstats.BucketLog2Round 75 GetTypeUsec bucketstats.BucketLog2Round 76 GetXAttrUsec bucketstats.BucketLog2Round 77 IsDirUsec bucketstats.BucketLog2Round 78 IsFileUsec bucketstats.BucketLog2Round 79 IsSymlinkUsec bucketstats.BucketLog2Round 80 LinkUsec bucketstats.BucketLog2Round 81 ListXAttrUsec bucketstats.BucketLog2Round 82 LookupUsec bucketstats.BucketLog2Round 83 LookupPathUsec bucketstats.BucketLog2Round 84 MkdirUsec bucketstats.BucketLog2Round 85 MoveUsec bucketstats.BucketLog2Round 86 RemoveXAttrUsec bucketstats.BucketLog2Round 87 RenameUsec bucketstats.BucketLog2Round 88 ReadUsec bucketstats.BucketLog2Round 89 ReadBytes bucketstats.BucketLog2Round 90 ReaddirUsec bucketstats.BucketLog2Round 91 ReaddirEntries bucketstats.BucketLog2Round 92 ReaddirOneUsec bucketstats.BucketLog2Round 93 ReaddirOnePlusUsec bucketstats.BucketLog2Round 94 ReaddirPlusUsec bucketstats.BucketLog2Round 95 ReaddirPlusBytes bucketstats.BucketLog2Round 96 ReadsymlinkUsec bucketstats.BucketLog2Round 97 ResizeUsec bucketstats.BucketLog2Round 98 RmdirUsec bucketstats.BucketLog2Round 99 SetstatUsec bucketstats.BucketLog2Round 100 SetXAttrUsec bucketstats.BucketLog2Round 101 StatVfsUsec bucketstats.BucketLog2Round 102 SymlinkUsec bucketstats.BucketLog2Round 103 UnlinkUsec bucketstats.BucketLog2Round 104 VolumeNameUsec bucketstats.BucketLog2Round 105 WriteUsec bucketstats.BucketLog2Round 106 WriteBytes bucketstats.BucketLog2Round 107 108 CreateErrors bucketstats.Total 109 DefragmentFileErrors bucketstats.Total 110 DestroyErrors bucketstats.Total 111 FetchExtentMapChunkErrors bucketstats.Total 112 FlushErrors bucketstats.Total 113 FlockOtherErrors bucketstats.Total 114 FlockGetErrors bucketstats.Total 115 FlockLockErrors bucketstats.Total 116 FlockUnlockErrors bucketstats.Total 117 GetstatErrors bucketstats.Total 118 GetTypeErrors bucketstats.Total 119 GetXAttrErrors bucketstats.Total 120 IsDirErrors bucketstats.Total 121 IsFileErrors bucketstats.Total 122 IsSymlinkErrors bucketstats.Total 123 LinkErrors bucketstats.Total 124 ListXAttrErrors bucketstats.Total 125 LookupErrors bucketstats.Total 126 LookupPathErrors bucketstats.Total 127 MkdirErrors bucketstats.Total 128 MoveErrors bucketstats.Total 129 RemoveXAttrErrors bucketstats.Total 130 RenameErrors bucketstats.Total 131 ReadErrors bucketstats.Total 132 ReaddirErrors bucketstats.Total 133 ReaddirOneErrors bucketstats.Total 134 ReaddirOnePlusErrors bucketstats.Total 135 ReaddirPlusErrors bucketstats.Total 136 ReadsymlinkErrors bucketstats.Total 137 ResizeErrors bucketstats.Total 138 RmdirErrors bucketstats.Total 139 SetstatErrors bucketstats.Total 140 SetXAttrErrors bucketstats.Total 141 StatVfsErrors bucketstats.Total 142 SymlinkErrors bucketstats.Total 143 UnlinkErrors bucketstats.Total 144 WriteErrors bucketstats.Total 145 146 DefragmentFileUsec bucketstats.BucketLog2Round 147 FetchExtentMapChunkUsec bucketstats.BucketLog2Round 148 CallInodeToProvisionObjectUsec bucketstats.BucketLog2Round 149 MiddlewareCoalesceUsec bucketstats.BucketLog2Round 150 MiddlewareCoalesceBytes bucketstats.BucketLog2Round 151 MiddlewareDeleteUsec bucketstats.BucketLog2Round 152 MiddlewareGetAccountUsec bucketstats.BucketLog2Round 153 MiddlewareGetContainerUsec bucketstats.BucketLog2Round 154 MiddlewareGetObjectUsec bucketstats.BucketLog2Round 155 MiddlewareGetObjectBytes bucketstats.BucketLog2Round 156 MiddlewareHeadResponseUsec bucketstats.BucketLog2Round 157 MiddlewareMkdirUsec bucketstats.BucketLog2Round 158 MiddlewarePostUsec bucketstats.BucketLog2Round 159 MiddlewarePostBytes bucketstats.BucketLog2Round 160 MiddlewarePutCompleteUsec bucketstats.BucketLog2Round 161 MiddlewarePutCompleteBytes bucketstats.BucketLog2Round 162 MiddlewarePutContainerUsec bucketstats.BucketLog2Round 163 MiddlewarePutContainerBytes bucketstats.BucketLog2Round 164 165 CallInodeToProvisionObjectErrors bucketstats.Total 166 MiddlewareCoalesceErrors bucketstats.Total 167 MiddlewareDeleteErrors bucketstats.Total 168 MiddlewareGetAccountErrors bucketstats.Total 169 MiddlewareGetContainerErrors bucketstats.Total 170 MiddlewareGetObjectErrors bucketstats.Total 171 MiddlewareHeadResponseErrors bucketstats.Total 172 MiddlewareMkdirErrors bucketstats.Total 173 MiddlewarePostErrors bucketstats.Total 174 MiddlewarePutCompleteErrors bucketstats.Total 175 MiddlewarePutContainerErrors bucketstats.Total 176 177 FetchVolumeHandleUsec bucketstats.BucketLog2Round 178 FetchVolumeHandleErrors bucketstats.BucketLog2Round 179 ValidateVolumeUsec bucketstats.BucketLog2Round 180 ScrubVolumeUsec bucketstats.BucketLog2Round 181 ValidateBaseNameUsec bucketstats.BucketLog2Round 182 ValidateBaseNameErrors bucketstats.Total 183 ValidateFullPathUsec bucketstats.BucketLog2Round 184 ValidateFullPathErrors bucketstats.Total 185 AccountNameToVolumeNameUsec bucketstats.BucketLog2Round 186 VolumeNameToActivePeerPrivateIPAddrUsec bucketstats.BucketLog2Round 187 } 188 189 var globals globalsStruct 190 191 func init() { 192 transitions.Register("fs", &globals) 193 } 194 195 func (dummy *globalsStruct) Up(confMap conf.ConfMap) (err error) { 196 globals.tryLockBackoffMin, err = confMap.FetchOptionValueDuration("FSGlobals", "TryLockBackoffMin") 197 if nil != err { 198 globals.tryLockBackoffMin = time.Duration(10 * time.Millisecond) // TODO: Eventually, just return 199 } 200 globals.tryLockBackoffMax, err = confMap.FetchOptionValueDuration("FSGlobals", "TryLockBackoffMax") 201 if nil != err { 202 globals.tryLockBackoffMax = time.Duration(50 * time.Millisecond) // TODO: Eventually, just return 203 } 204 globals.tryLockSerializationThreshhold, err = confMap.FetchOptionValueUint64("FSGlobals", "TryLockSerializationThreshhold") 205 if nil != err { 206 globals.tryLockSerializationThreshhold = 5 // TODO: Eventually, just return 207 } 208 globals.symlinkMax, err = confMap.FetchOptionValueUint16("FSGlobals", "SymlinkMax") 209 if nil != err { 210 globals.symlinkMax = 32 // TODO: Eventually, just return 211 } 212 globals.coalesceElementChunkSize, err = confMap.FetchOptionValueUint16("FSGlobals", "CoalesceElementChunkSize") 213 if nil != err { 214 globals.coalesceElementChunkSize = 16 // TODO: Eventually, just return 215 } 216 217 globals.volumeMap = make(map[string]*volumeStruct) 218 219 globals.inFlightFileInodeDataList = list.New() 220 globals.serializedBackoffList = list.New() 221 222 bucketstats.Register("proxyfs.fs", "", &globals) 223 224 err = nil 225 return nil 226 } 227 228 func (dummy *globalsStruct) VolumeGroupCreated(confMap conf.ConfMap, volumeGroupName string, activePeer string, virtualIPAddr string) (err error) { 229 return nil 230 } 231 func (dummy *globalsStruct) VolumeGroupMoved(confMap conf.ConfMap, volumeGroupName string, activePeer string, virtualIPAddr string) (err error) { 232 return nil 233 } 234 func (dummy *globalsStruct) VolumeGroupDestroyed(confMap conf.ConfMap, volumeGroupName string) (err error) { 235 return nil 236 } 237 func (dummy *globalsStruct) VolumeCreated(confMap conf.ConfMap, volumeName string, volumeGroupName string) (err error) { 238 return nil 239 } 240 func (dummy *globalsStruct) VolumeMoved(confMap conf.ConfMap, volumeName string, volumeGroupName string) (err error) { 241 return nil 242 } 243 func (dummy *globalsStruct) VolumeDestroyed(confMap conf.ConfMap, volumeName string) (err error) { 244 return nil 245 } 246 247 func (dummy *globalsStruct) ServeVolume(confMap conf.ConfMap, volumeName string) (err error) { 248 var ( 249 replayLogFileName string 250 volume *volumeStruct 251 volumeSectionName string 252 ) 253 254 volume = &volumeStruct{ 255 volumeName: volumeName, 256 FLockMap: make(map[inode.InodeNumber]*list.List), 257 inFlightFileInodeDataMap: make(map[inode.InodeNumber]*inFlightFileInodeDataStruct), 258 } 259 260 volumeSectionName = "Volume:" + volumeName 261 262 replayLogFileName, err = confMap.FetchOptionValueString(volumeSectionName, "ReplayLogFileName") 263 if nil == err { 264 volume.doCheckpointPerFlush = ("" == replayLogFileName) 265 } else { 266 volume.doCheckpointPerFlush = true 267 } 268 269 logger.Infof("Checkpoint per Flush for volume %v is %v", volume.volumeName, volume.doCheckpointPerFlush) 270 271 volume.maxFlushTime, err = confMap.FetchOptionValueDuration(volumeSectionName, "MaxFlushTime") 272 if nil != err { 273 return 274 } 275 276 volume.fileDefragmentChunkSize, err = confMap.FetchOptionValueUint64(volumeSectionName, "FileDefragmentChunkSize") 277 if nil != err { 278 volume.fileDefragmentChunkSize = 10485760 // TODO: Eventually, just return 279 } 280 volume.fileDefragmentChunkDelay, err = confMap.FetchOptionValueDuration(volumeSectionName, "FileDefragmentChunkDelay") 281 if nil != err { 282 volume.fileDefragmentChunkDelay = time.Duration(10 * time.Millisecond) // TODO: Eventually, just return 283 } 284 285 volume.reportedBlockSize, err = confMap.FetchOptionValueUint64(volumeSectionName, "ReportedBlockSize") 286 if nil != err { 287 volume.reportedBlockSize = DefaultReportedBlockSize // TODO: Eventually, just return 288 } 289 volume.reportedFragmentSize, err = confMap.FetchOptionValueUint64(volumeSectionName, "ReportedFragmentSize") 290 if nil != err { 291 volume.reportedFragmentSize = DefaultReportedFragmentSize // TODO: Eventually, just return 292 } 293 volume.reportedNumBlocks, err = confMap.FetchOptionValueUint64(volumeSectionName, "ReportedNumBlocks") 294 if nil != err { 295 volume.reportedNumBlocks = DefaultReportedNumBlocks // TODO: Eventually, just return 296 } 297 volume.reportedNumInodes, err = confMap.FetchOptionValueUint64(volumeSectionName, "ReportedNumInodes") 298 if nil != err { 299 volume.reportedNumInodes = DefaultReportedNumInodes // TODO: Eventually, just return 300 } 301 302 volume.inodeVolumeHandle, err = inode.FetchVolumeHandle(volumeName) 303 if nil != err { 304 return 305 } 306 volume.headhunterVolumeHandle, err = headhunter.FetchVolumeHandle(volumeName) 307 if nil != err { 308 return 309 } 310 311 globals.volumeMap[volumeName] = volume 312 313 err = nil 314 return 315 } 316 317 func (dummy *globalsStruct) UnserveVolume(confMap conf.ConfMap, volumeName string) (err error) { 318 var ( 319 ok bool 320 volume *volumeStruct 321 ) 322 323 volume, ok = globals.volumeMap[volumeName] 324 325 if !ok { 326 err = nil 327 return 328 } 329 330 volume.untrackInFlightFileInodeDataAll() 331 332 delete(globals.volumeMap, volumeName) 333 334 err = nil 335 return 336 } 337 338 func (dummy *globalsStruct) VolumeToBeUnserved(confMap conf.ConfMap, volumeName string) (err error) { 339 return nil 340 } 341 func (dummy *globalsStruct) SignaledStart(confMap conf.ConfMap) (err error) { 342 return nil 343 } 344 func (dummy *globalsStruct) SignaledFinish(confMap conf.ConfMap) (err error) { 345 return nil 346 } 347 348 func (dummy *globalsStruct) Down(confMap conf.ConfMap) (err error) { 349 var ( 350 volume *volumeStruct 351 ) 352 353 if 0 != len(globals.volumeMap) { 354 err = fmt.Errorf("fs.Down() called with 0 != len(globals.volumeMap") 355 return 356 } 357 if 0 != globals.inFlightFileInodeDataList.Len() { 358 err = fmt.Errorf("fs.Down() called with 0 != globals.inFlightFileInodeDataList.Len()") 359 return 360 } 361 362 for _, volume = range globals.volumeMap { 363 volume.untrackInFlightFileInodeDataAll() 364 } 365 366 if 0 < globals.inFlightFileInodeDataList.Len() { 367 logger.Fatalf("fs.Down() has completed all un-mount's... but found non-empty globals.inFlightFileInodeDataList") 368 } 369 370 bucketstats.UnRegister("proxyfs.fs", "") 371 372 err = nil 373 return 374 }