github.com/rohankumardubey/proxyfs@v0.0.0-20210108201508-653efa9ab00e/dlm/llm.go (about) 1 package dlm 2 3 import ( 4 "container/list" 5 "errors" 6 "fmt" 7 "sync" 8 "time" 9 10 "github.com/swiftstack/ProxyFS/blunder" 11 "github.com/swiftstack/ProxyFS/trackedlock" 12 ) 13 14 // This struct is used by LLM to track a lock. 15 type localLockTrack struct { 16 trackedlock.Mutex 17 lockId string // lock identity (must be unique) 18 owners uint64 // Count of threads which own lock 19 waiters uint64 // Count of threads which want to own the lock (either shared or exclusive) 20 state lockState 21 exclOwner CallerID 22 listOfOwners []CallerID 23 waitReqQ *list.List // List of requests waiting for lock 24 rwMutexTrack trackedlock.RWMutexTrack // Track the lock to see how long its held 25 } 26 27 var localLockTrackPool = sync.Pool{ 28 New: func() interface{} { 29 var track localLockTrack 30 31 // every localLockTrack should have a waitReqQ 32 track.waitReqQ = list.New() 33 34 return &track 35 }, 36 } 37 38 type localLockRequest struct { 39 requestedState lockState 40 *sync.Cond 41 wakeUp bool 42 LockCallerID CallerID 43 } 44 45 type lockState int 46 47 const ( 48 nilType lockState = iota 49 shared 50 exclusive 51 stale 52 ) 53 54 // NOTE: This is a test-only interface used for unit tests. 55 // 56 // This function assumes that globals.Lock() is held. 57 // TODO - can this be used in more cases without creating entry it if does not exist? 58 func getTrack(lockId string) (track *localLockTrack, ok bool) { 59 track, ok = globals.localLockMap[lockId] 60 if !ok { 61 return track, ok 62 } 63 return track, ok 64 } 65 66 // NOTE: This is a test-only interface used for unit tests. 67 func waitCountWaiters(lockId string, count uint64) { 68 for { 69 globals.Lock() 70 track, ok := getTrack(lockId) 71 72 // If the tracking object has not been created yet, sleep and retry. 73 if !ok { 74 // Sleep 5 milliseconds and test again 75 globals.Unlock() 76 time.Sleep(5 * time.Millisecond) 77 break 78 } 79 80 track.Mutex.Lock() 81 82 globals.Unlock() 83 84 waiters := track.waiters 85 track.Mutex.Unlock() 86 87 if waiters == count { 88 return 89 } else { 90 // Sleep 5 milliseconds and test again 91 time.Sleep(5 * time.Millisecond) 92 } 93 } 94 } 95 96 // NOTE: This is a test-only interface used for unit tests. 97 func waitCountOwners(lockId string, count uint64) { 98 for { 99 globals.Lock() 100 track, ok := getTrack(lockId) 101 102 // If the tracking object has not been created yet, sleep and retry. 103 if !ok { 104 // Sleep 5 milliseconds and test again 105 globals.Unlock() 106 time.Sleep(5 * time.Millisecond) 107 break 108 } 109 110 track.Mutex.Lock() 111 112 globals.Unlock() 113 114 owners := track.owners 115 track.Mutex.Unlock() 116 117 if owners == count { 118 return 119 } else { 120 // Sleep 5 milliseconds and test again 121 time.Sleep(5 * time.Millisecond) 122 } 123 } 124 } 125 126 // This function assumes the mutex is held on the tracker structure 127 func (t *localLockTrack) removeFromListOfOwners(callerID CallerID) { 128 129 // Find Position and delete entry (a map might be more efficient) 130 for i, id := range t.listOfOwners { 131 if id == callerID { 132 lastIdx := len(t.listOfOwners) - 1 133 t.listOfOwners[i] = t.listOfOwners[lastIdx] 134 t.listOfOwners = t.listOfOwners[:lastIdx] 135 return 136 } 137 } 138 139 panic(fmt.Sprintf("Can't find CallerID: %v in list of lock owners!", callerID)) 140 } 141 142 // This function assumes the mutex is held on the tracker structure 143 func callerInListOfOwners(listOfOwners []CallerID, callerID CallerID) (amOwner bool) { 144 // Find Position 145 for _, id := range listOfOwners { 146 if id == callerID { 147 return true 148 } 149 } 150 151 return false 152 } 153 154 func isLockHeld(lockID string, callerID CallerID, lockHeldType LockHeldType) (held bool) { 155 globals.Lock() 156 // NOTE: Not doing a defer globals.Unlock() here since grabbing another lock below. 157 158 track, ok := globals.localLockMap[lockID] 159 if !ok { 160 161 // Lock does not exist in map 162 globals.Unlock() 163 return false 164 } 165 166 track.Mutex.Lock() 167 168 globals.Unlock() 169 170 defer track.Mutex.Unlock() 171 172 switch lockHeldType { 173 case READLOCK: 174 return (track.state == shared) && (callerInListOfOwners(track.listOfOwners, callerID)) 175 case WRITELOCK: 176 return (track.state == exclusive) && (callerInListOfOwners(track.listOfOwners, callerID)) 177 case ANYLOCK: 178 return ((track.state == exclusive) || (track.state == shared)) && (callerInListOfOwners(track.listOfOwners, callerID)) 179 } 180 return false 181 } 182 183 func grantAndSignal(track *localLockTrack, localQRequest *localLockRequest) { 184 track.state = localQRequest.requestedState 185 track.listOfOwners = append(track.listOfOwners, localQRequest.LockCallerID) 186 track.owners++ 187 188 if track.state == exclusive { 189 if track.exclOwner != nil || track.owners != 1 { 190 panic(fmt.Sprintf("granted exclusive lock when (exclOwner != nil || track.owners != 1)! "+ 191 "track lockId %v owners %d waiters %d lockState %v exclOwner %v listOfOwners %v", 192 track.lockId, track.owners, track.waiters, track.state, 193 *track.exclOwner, track.listOfOwners)) 194 } 195 track.exclOwner = localQRequest.LockCallerID 196 } 197 198 localQRequest.wakeUp = true 199 localQRequest.Cond.Broadcast() 200 } 201 202 // Process the waitReqQ and see if any locks can be granted. 203 // 204 // This function assumes that the tracking mutex is held. 205 func processLocalQ(track *localLockTrack) { 206 207 // If nothing on queue then return 208 if track.waitReqQ.Len() == 0 { 209 return 210 } 211 212 // If the lock is already held exclusively then nothing to do. 213 if track.state == exclusive { 214 return 215 } 216 217 // At this point, the lock is either stale or shared 218 // 219 // Loop through Q and see if a request can be granted. If it can then pop it off the Q. 220 for track.waitReqQ.Len() > 0 { 221 elem := track.waitReqQ.Remove(track.waitReqQ.Front()) 222 var localQRequest *localLockRequest 223 var ok bool 224 if localQRequest, ok = elem.(*localLockRequest); !ok { 225 panic("Remove of elem failed!!!") 226 } 227 228 // If the lock is already free and then want it exclusive 229 if (localQRequest.requestedState == exclusive) && (track.state == stale) { 230 grantAndSignal(track, localQRequest) 231 return 232 } 233 234 // If want exclusive and not free, we can't grant so push on front and break from loop. 235 if localQRequest.requestedState == exclusive { 236 track.waitReqQ.PushFront(localQRequest) 237 return 238 } 239 240 // At this point we know the Q entry is shared. Grant it now. 241 grantAndSignal(track, localQRequest) 242 } 243 } 244 245 func (l *RWLockStruct) commonLock(requestedState lockState, try bool) (err error) { 246 247 globals.Lock() 248 track, ok := globals.localLockMap[l.LockID] 249 if !ok { 250 // TODO - handle blocking waiting for lock from DLM 251 252 // Lock does not exist in map, get one 253 track = localLockTrackPool.Get().(*localLockTrack) 254 if track.waitReqQ.Len() != 0 { 255 panic(fmt.Sprintf("localLockTrack object %p from pool does not have empty waitReqQ", 256 track)) 257 } 258 if len(track.listOfOwners) != 0 { 259 panic(fmt.Sprintf("localLockTrack object %p from pool does not have empty ListOfOwners", 260 track)) 261 } 262 track.lockId = l.LockID 263 track.state = stale 264 265 globals.localLockMap[l.LockID] = track 266 267 } 268 269 track.Mutex.Lock() 270 defer track.Mutex.Unlock() 271 272 globals.Unlock() 273 274 // If we are doing a TryWriteLock or TryReadLock, see if we could 275 // grab the lock before putting on queue. 276 if try { 277 if (requestedState == exclusive) && (track.state != stale) { 278 err = errors.New("Lock is busy - try again!") 279 return blunder.AddError(err, blunder.TryAgainError) 280 } else { 281 if track.state == exclusive { 282 err = errors.New("Lock is busy - try again!") 283 return blunder.AddError(err, blunder.TryAgainError) 284 } 285 } 286 } 287 localRequest := localLockRequest{requestedState: requestedState, LockCallerID: l.LockCallerID, wakeUp: false} 288 localRequest.Cond = sync.NewCond(&track.Mutex) 289 track.waitReqQ.PushBack(&localRequest) 290 291 track.waiters++ 292 293 // See if any locks can be granted 294 processLocalQ(track) 295 296 // wakeUp will already be true if processLocalQ() signaled this thread to wakeup. 297 for localRequest.wakeUp == false { 298 localRequest.Cond.Wait() 299 } 300 301 // sanity check request and lock state 302 if localRequest.wakeUp != true { 303 panic(fmt.Sprintf("commonLock(): thread awoke without being signalled; localRequest %v "+ 304 "track lockId %v owners %d waiters %d lockState %v exclOwner %v listOfOwners %v", 305 localRequest, track.lockId, track.owners, track.waiters, track.state, 306 *track.exclOwner, track.listOfOwners)) 307 } 308 if track.state == stale || track.owners == 0 || (track.owners > 1 && track.state != shared) { 309 panic(fmt.Sprintf("commonLock(): lock is in undefined state: localRequest %v "+ 310 "track lockId %v owners %d waiters %d lockState %v exclOwner %v listOfOwners %v", 311 localRequest, track.lockId, track.owners, track.waiters, track.state, 312 *track.exclOwner, track.listOfOwners)) 313 } 314 315 // let trackedlock package track how long we hold the lock 316 if track.state == exclusive { 317 track.rwMutexTrack.LockTrack(track) 318 } else { 319 track.rwMutexTrack.RLockTrack(track) 320 } 321 322 // At this point, we got the lock either by the call to processLocalQ() above 323 // or as a result of processLocalQ() being called from the unlock() path. 324 325 // We decrement waiters here instead of in processLocalQ() so that other threads do not 326 // assume there are no waiters between the time the Cond is signaled and we wakeup this thread. 327 track.waiters-- 328 329 return nil 330 } 331 332 // unlock() releases the lock and signals any waiters that the lock is free. 333 func (l *RWLockStruct) unlock() (err error) { 334 335 // TODO - assert not stale and if shared that count != 0 336 globals.Lock() 337 track, ok := globals.localLockMap[l.LockID] 338 if !ok { 339 panic(fmt.Sprintf("Trying to Unlock() inode: %v and lock not found in localLockMap()!", l.LockID)) 340 } 341 342 track.Mutex.Lock() 343 344 // Remove lock from localLockMap if no other thread using. 345 // 346 // We have track structure for lock. While holding mutex on localLockMap, remove 347 // lock from map if we are the last holder of the lock. 348 // TODO - does this handle revoke case and any others? 349 var deleted = false 350 if (track.owners == 1) && (track.waiters == 0) { 351 deleted = true 352 delete(globals.localLockMap, l.LockID) 353 } 354 355 globals.Unlock() 356 357 // TODO - handle release of lock back to DLM and delete from localLockMap 358 // Set stale and signal any waiters 359 track.owners-- 360 track.removeFromListOfOwners(l.LockCallerID) 361 if track.state == exclusive { 362 if track.owners != 0 || track.exclOwner == nil { 363 panic(fmt.Sprintf("releasing exclusive lock when (exclOwner == nil || track.owners != 0)! "+ 364 "track lockId %v owners %d waiters %d lockState %v exclOwner %v listOfOwners %v", 365 track.lockId, track.owners, track.waiters, track.state, 366 *track.exclOwner, track.listOfOwners)) 367 } 368 track.exclOwner = nil 369 } 370 371 if track.owners == 0 { 372 track.state = stale 373 } else { 374 if track.owners < 0 { 375 panic("track.owners < 0!!!") 376 } 377 } 378 // record the release of the lock 379 track.rwMutexTrack.DLMUnlockTrack(track) 380 381 // See if any locks can be granted 382 processLocalQ(track) 383 384 track.Mutex.Unlock() 385 386 // can't return the 387 if deleted { 388 if track.waitReqQ.Len() != 0 || track.waiters != 0 || track.state != stale { 389 panic(fmt.Sprintf( 390 "localLockTrack object %p retrieved from pool does not have an empty waitReqQ", 391 track.waitReqQ)) 392 } 393 localLockTrackPool.Put(track) 394 } 395 396 // TODO what error is possible? 397 return nil 398 }