github.com/swiftstack/ProxyFS@v0.0.0-20210203235616-4017c267d62f/pfsagentd/lease.go (about)

     1  // Copyright (c) 2015-2021, NVIDIA CORPORATION.
     2  // SPDX-License-Identifier: Apache-2.0
     3  
     4  package main
     5  
     6  import (
     7  	"container/list"
     8  	"encoding/json"
     9  	"time"
    10  
    11  	"github.com/swiftstack/ProxyFS/inode"
    12  	"github.com/swiftstack/ProxyFS/jrpcfs"
    13  )
    14  
    15  func (dummy *globalsStruct) Interrupt(rpcInterruptBuf []byte) {
    16  	var (
    17  		err          error
    18  		fileInode    *fileInodeStruct
    19  		ok           bool
    20  		rpcInterrupt *jrpcfs.RPCInterrupt
    21  	)
    22  
    23  	rpcInterrupt = &jrpcfs.RPCInterrupt{}
    24  
    25  	err = json.Unmarshal(rpcInterruptBuf, rpcInterrupt)
    26  	if nil != err {
    27  		logFatalf("(*globalsStruct).Interrupt() call to json.Unmarshal() failed: %v", err)
    28  	}
    29  
    30  	switch rpcInterrupt.RPCInterruptType {
    31  	case jrpcfs.RPCInterruptTypeUnmount:
    32  		logFatalf("UNSUPPORTED: (*globalsStruct).Interrupt() received jrpcfs.RPCInterruptTypeUnmount")
    33  	case jrpcfs.RPCInterruptTypeDemote:
    34  	case jrpcfs.RPCInterruptTypeRelease:
    35  	default:
    36  		logFatalf("(*globalsStruct).Interrupt() received unknown rpcInterrupt.RPCInterruptType: %v", rpcInterrupt.RPCInterruptType)
    37  	}
    38  
    39  	globals.Lock()
    40  
    41  	fileInode, ok = globals.fileInodeMap[inode.InodeNumber(rpcInterrupt.InodeNumber)]
    42  	if !ok {
    43  		globals.Unlock()
    44  		return
    45  	}
    46  
    47  	switch fileInode.leaseState {
    48  	case fileInodeLeaseStateNone:
    49  		globals.unleasedFileInodeCacheLRU.MoveToBack(fileInode.leaseListElement)
    50  	case fileInodeLeaseStateSharedRequested:
    51  		globals.sharedLeaseFileInodeCacheLRU.MoveToBack(fileInode.leaseListElement)
    52  		fileInode.pendingLeaseInterrupt = &rpcInterrupt.RPCInterruptType
    53  	case fileInodeLeaseStateSharedGranted:
    54  		globals.sharedLeaseFileInodeCacheLRU.MoveToBack(fileInode.leaseListElement)
    55  		if nil == fileInode.lockWaiters {
    56  			// Shared Lease... with nobody currently referencing it
    57  			fileInode.lockWaiters = list.New()
    58  			if jrpcfs.RPCInterruptTypeDemote == rpcInterrupt.RPCInterruptType {
    59  				// We can safely ignore it
    60  			} else { // jrpcfs.RPCInterruptTypeRelease == rpcInterrupt.RPCInterruptType
    61  				fileInode.leaseState = fileInodeLeaseStateSharedReleasing
    62  				_ = globals.sharedLeaseFileInodeCacheLRU.Remove(fileInode.leaseListElement)
    63  				fileInode.leaseListElement = globals.unleasedFileInodeCacheLRU.PushBack(fileInode)
    64  				globals.Unlock()
    65  				// TODO - invalidate cached state (cachedStat and extentMap)
    66  				// TODO - inform ProxyFS that we'd like to Release our Shared Lease
    67  				// TODO - finally service any waiters that have arrived (or delete lockWaiters)
    68  			}
    69  			return
    70  		}
    71  		fileInode.pendingLeaseInterrupt = &rpcInterrupt.RPCInterruptType
    72  	case fileInodeLeaseStateSharedPromoting:
    73  		globals.exclusiveLeaseFileInodeCacheLRU.MoveToBack(fileInode.leaseListElement)
    74  		fileInode.pendingLeaseInterrupt = &rpcInterrupt.RPCInterruptType
    75  	case fileInodeLeaseStateSharedReleasing:
    76  		globals.unleasedFileInodeCacheLRU.MoveToBack(fileInode.leaseListElement)
    77  		fileInode.pendingLeaseInterrupt = &rpcInterrupt.RPCInterruptType
    78  	case fileInodeLeaseStateExclusiveRequested:
    79  		globals.exclusiveLeaseFileInodeCacheLRU.MoveToBack(fileInode.leaseListElement)
    80  		fileInode.pendingLeaseInterrupt = &rpcInterrupt.RPCInterruptType
    81  	case fileInodeLeaseStateExclusiveGranted:
    82  		globals.exclusiveLeaseFileInodeCacheLRU.MoveToBack(fileInode.leaseListElement)
    83  		if nil == fileInode.lockWaiters {
    84  			// Exclusive Lease... with nobody currently referencing it
    85  			if jrpcfs.RPCInterruptTypeDemote == rpcInterrupt.RPCInterruptType {
    86  				fileInode.leaseState = fileInodeLeaseStateExclusiveDemoting
    87  				_ = globals.exclusiveLeaseFileInodeCacheLRU.Remove(fileInode.leaseListElement)
    88  				fileInode.leaseListElement = globals.sharedLeaseFileInodeCacheLRU.PushBack(fileInode)
    89  				globals.Unlock()
    90  				// TODO - flush dirty state (chunkedPutList)
    91  				// TODO - inform ProxyFS that we'd like to Demote our Exclusive Lease
    92  				// TODO - finally service any waiters that have arrived (or delete lockWaiters)
    93  			} else { // jrpcfs.RPCInterruptTypeRelease == rpcInterrupt.RPCInterruptType
    94  				fileInode.leaseState = fileInodeLeaseStateExclusiveReleasing
    95  				_ = globals.exclusiveLeaseFileInodeCacheLRU.Remove(fileInode.leaseListElement)
    96  				fileInode.leaseListElement = globals.unleasedFileInodeCacheLRU.PushBack(fileInode)
    97  				globals.Unlock()
    98  				// TODO - flush dirty state (chunkedPutList)
    99  				// TODO - invalidate cached state (cachedStat and extentMap)
   100  				// TODO - inform ProxyFS that we'd like to Release our Exclusive Lease
   101  				// TODO - finally service any waiters that have arrived (or delete lockWaiters)
   102  			}
   103  			return
   104  		}
   105  		fileInode.pendingLeaseInterrupt = &rpcInterrupt.RPCInterruptType
   106  	case fileInodeLeaseStateExclusiveDemoting:
   107  		globals.sharedLeaseFileInodeCacheLRU.MoveToBack(fileInode.leaseListElement)
   108  		fileInode.pendingLeaseInterrupt = &rpcInterrupt.RPCInterruptType
   109  	case fileInodeLeaseStateExclusiveReleasing:
   110  		globals.unleasedFileInodeCacheLRU.MoveToBack(fileInode.leaseListElement)
   111  		fileInode.pendingLeaseInterrupt = &rpcInterrupt.RPCInterruptType
   112  	default:
   113  		logFatalf("(*globalsStruct).Interrupt() found unknown fileInode.leaseState: %v", fileInode.leaseState)
   114  	}
   115  
   116  	// If fileInode.pendingLeaseInterrupt was set, it'll be serviced in (*fileInodeStruct).unlock()
   117  
   118  	globals.Unlock()
   119  }
   120  
   121  func lockInodeWithSharedLease(inodeNumber inode.InodeNumber) (fileInode *fileInodeStruct) {
   122  	var (
   123  		err                   error
   124  		leaseReply            *jrpcfs.LeaseReply
   125  		leaseRequest          *jrpcfs.LeaseRequest
   126  		leaseRequestEndTime   time.Time
   127  		leaseRequestStartTime time.Time
   128  		ok                    bool
   129  		waitChan              chan struct{}
   130  	)
   131  
   132  	globals.Lock()
   133  
   134  	fileInode, ok = globals.fileInodeMap[inodeNumber]
   135  	if !ok {
   136  		fileInode = &fileInodeStruct{
   137  			InodeNumber:               inodeNumber,
   138  			cachedStat:                nil,
   139  			lockWaiters:               nil,
   140  			leaseState:                fileInodeLeaseStateNone,
   141  			pendingLeaseInterrupt:     nil,
   142  			leaseListElement:          nil,
   143  			extentMap:                 nil,
   144  			chunkedPutList:            list.New(),
   145  			flushInProgress:           false,
   146  			chunkedPutFlushWaiterList: list.New(),
   147  			dirtyListElement:          nil,
   148  		}
   149  
   150  		fileInode.leaseListElement = globals.unleasedFileInodeCacheLRU.PushBack(fileInode)
   151  		globals.fileInodeMap[inodeNumber] = fileInode
   152  	}
   153  
   154  	switch fileInode.leaseState {
   155  	case fileInodeLeaseStateNone:
   156  		if nil == fileInode.lockWaiters {
   157  			// No Lease currently held... so ask for a Shared Lease
   158  
   159  			fileInode.lockWaiters = list.New()
   160  			fileInode.leaseState = fileInodeLeaseStateSharedRequested
   161  			_ = globals.unleasedFileInodeCacheLRU.Remove(fileInode.leaseListElement)
   162  			fileInode.leaseListElement = globals.sharedLeaseFileInodeCacheLRU.PushBack(fileInode)
   163  
   164  			globals.Unlock()
   165  
   166  			leaseRequest = &jrpcfs.LeaseRequest{
   167  				InodeHandle: jrpcfs.InodeHandle{
   168  					MountID:     globals.mountID,
   169  					InodeNumber: int64(inodeNumber),
   170  				},
   171  				LeaseRequestType: jrpcfs.LeaseRequestTypeShared,
   172  			}
   173  			leaseReply = &jrpcfs.LeaseReply{}
   174  
   175  			leaseRequestStartTime = time.Now()
   176  			err = globals.retryRPCClient.Send("RpcLease", leaseRequest, leaseReply)
   177  			if nil != err {
   178  				logFatalf("lockInodeWithSharedLease() unable to obtain Shared Lease [case 1] - retryRPCClient.Send() failed: %v", err)
   179  			}
   180  			leaseRequestEndTime = time.Now()
   181  			globals.stats.LeaseRequests_Shared_Usec.Add(uint64(leaseRequestEndTime.Sub(leaseRequestStartTime) / time.Microsecond))
   182  
   183  			// Record leaseReply.LeaseReplyType
   184  
   185  			globals.Lock()
   186  
   187  			switch leaseReply.LeaseReplyType {
   188  			case jrpcfs.LeaseReplyTypeShared:
   189  				fileInode.leaseState = fileInodeLeaseStateSharedGranted
   190  				globals.sharedLeaseFileInodeCacheLRU.MoveToBack(fileInode.leaseListElement)
   191  			case jrpcfs.LeaseReplyTypeExclusive:
   192  				fileInode.leaseState = fileInodeLeaseStateExclusiveGranted
   193  				_ = globals.sharedLeaseFileInodeCacheLRU.Remove(fileInode.leaseListElement)
   194  				fileInode.leaseListElement = globals.exclusiveLeaseFileInodeCacheLRU.PushBack(fileInode)
   195  			default:
   196  				logFatalf("lockInodeWithSharedLease() unable to obtain Shared Lease [case 2] - LeaseReplyType == %v", leaseReply.LeaseReplyType)
   197  			}
   198  
   199  			// Now that we have either a Shared or Exclusive Lease, we are also the first in line to use it - so just return
   200  
   201  			globals.Unlock()
   202  
   203  			return
   204  		}
   205  
   206  		globals.unleasedFileInodeCacheLRU.MoveToBack(fileInode.leaseListElement)
   207  
   208  		// Somebody else is ahead of us... so fall through
   209  
   210  	case fileInodeLeaseStateSharedRequested:
   211  		globals.sharedLeaseFileInodeCacheLRU.MoveToBack(fileInode.leaseListElement)
   212  
   213  		// Somebody else is ahead of us... so fall through
   214  
   215  	case fileInodeLeaseStateSharedGranted:
   216  		globals.sharedLeaseFileInodeCacheLRU.MoveToBack(fileInode.leaseListElement)
   217  
   218  		if nil == fileInode.lockWaiters {
   219  			// We already have a Shared Lease and we are also the first in line to use it
   220  
   221  			fileInode.lockWaiters = list.New()
   222  
   223  			globals.Unlock()
   224  
   225  			return
   226  		}
   227  
   228  		// Somebody else is ahead of us... so fall through
   229  
   230  	case fileInodeLeaseStateSharedPromoting:
   231  		globals.exclusiveLeaseFileInodeCacheLRU.MoveToBack(fileInode.leaseListElement)
   232  
   233  		// Somebody else is ahead of us... so fall through
   234  
   235  	case fileInodeLeaseStateSharedReleasing:
   236  		globals.unleasedFileInodeCacheLRU.MoveToBack(fileInode.leaseListElement)
   237  
   238  		// Somebody else is ahead of us... so fall through
   239  
   240  	case fileInodeLeaseStateExclusiveRequested:
   241  		globals.exclusiveLeaseFileInodeCacheLRU.MoveToBack(fileInode.leaseListElement)
   242  
   243  		// Somebody else is ahead of us... so fall through
   244  
   245  	case fileInodeLeaseStateExclusiveGranted:
   246  		globals.exclusiveLeaseFileInodeCacheLRU.MoveToBack(fileInode.leaseListElement)
   247  
   248  		if nil == fileInode.lockWaiters {
   249  			// We already have an Exclusive Lease and we are also the first in line to use it
   250  
   251  			fileInode.lockWaiters = list.New()
   252  
   253  			globals.Unlock()
   254  
   255  			return
   256  		}
   257  
   258  		// Somebody else is ahead of us... so fall through
   259  
   260  	case fileInodeLeaseStateExclusiveDemoting:
   261  		globals.sharedLeaseFileInodeCacheLRU.MoveToBack(fileInode.leaseListElement)
   262  
   263  		// Somebody else is ahead of us... so fall through
   264  
   265  	case fileInodeLeaseStateExclusiveReleasing:
   266  		globals.unleasedFileInodeCacheLRU.MoveToBack(fileInode.leaseListElement)
   267  
   268  		// Somebody else is ahead of us... so fall through
   269  
   270  	default:
   271  		logFatalf("lockInodeWithSharedLease() found unknown fileInode.leaseState [case 1]: %v", fileInode.leaseState)
   272  	}
   273  
   274  	// Time to block
   275  
   276  	waitChan = make(chan struct{})
   277  	_ = fileInode.lockWaiters.PushBack(waitChan)
   278  	globals.Unlock()
   279  	_ = <-waitChan
   280  
   281  	// Time to check fileInode.leaseState again [(*fileInodeStruct).unlock() may have changed fileInode.leaseState)
   282  
   283  	globals.Lock()
   284  
   285  	switch fileInode.leaseState {
   286  	case fileInodeLeaseStateNone:
   287  		// No Lease currently held... so ask for a Shared Lease
   288  
   289  		fileInode.leaseState = fileInodeLeaseStateSharedRequested
   290  		_ = globals.unleasedFileInodeCacheLRU.Remove(fileInode.leaseListElement)
   291  		fileInode.leaseListElement = globals.sharedLeaseFileInodeCacheLRU.PushBack(fileInode)
   292  
   293  		globals.Unlock()
   294  
   295  		leaseRequest = &jrpcfs.LeaseRequest{
   296  			InodeHandle: jrpcfs.InodeHandle{
   297  				MountID:     globals.mountID,
   298  				InodeNumber: int64(inodeNumber),
   299  			},
   300  			LeaseRequestType: jrpcfs.LeaseRequestTypeShared,
   301  		}
   302  		leaseReply = &jrpcfs.LeaseReply{}
   303  
   304  		leaseRequestStartTime = time.Now()
   305  		err = globals.retryRPCClient.Send("RpcLease", leaseRequest, leaseReply)
   306  		if nil != err {
   307  			logFatalf("lockInodeWithSharedLease() unable to obtain Shared Lease [case 3] - retryRPCClient.Send() failed: %v", err)
   308  		}
   309  		leaseRequestEndTime = time.Now()
   310  		globals.stats.LeaseRequests_Shared_Usec.Add(uint64(leaseRequestEndTime.Sub(leaseRequestStartTime) / time.Microsecond))
   311  
   312  		// Record leaseReply.LeaseReplyType
   313  
   314  		globals.Lock()
   315  
   316  		switch leaseReply.LeaseReplyType {
   317  		case jrpcfs.LeaseReplyTypeShared:
   318  			fileInode.leaseState = fileInodeLeaseStateSharedGranted
   319  			globals.sharedLeaseFileInodeCacheLRU.MoveToBack(fileInode.leaseListElement)
   320  		case jrpcfs.LeaseReplyTypeExclusive:
   321  			fileInode.leaseState = fileInodeLeaseStateExclusiveGranted
   322  			_ = globals.sharedLeaseFileInodeCacheLRU.Remove(fileInode.leaseListElement)
   323  			fileInode.leaseListElement = globals.exclusiveLeaseFileInodeCacheLRU.PushBack(fileInode)
   324  		default:
   325  			logFatalf("lockInodeWithSharedLease() unable to obtain Shared Lease [case 4] - LeaseReplyType == %v", leaseReply.LeaseReplyType)
   326  		}
   327  
   328  		// Now that we have a Shared or Exclusive Lease, we are also the first in line to use it - so we can grant the Lock
   329  	case fileInodeLeaseStateSharedRequested:
   330  		logFatalf("lockInodeWithSharedLease() found unexpected fileInode.leaseState: fileInodeLeaseStateSharedRequested")
   331  	case fileInodeLeaseStateSharedGranted:
   332  		// Now that we have a Shared Lease, we are also the first in line to use it - so we can grant the Lock
   333  	case fileInodeLeaseStateSharedPromoting:
   334  		logFatalf("lockInodeWithSharedLease() found unexpected fileInode.leaseState: fileInodeLeaseStateSharedPromoting")
   335  	case fileInodeLeaseStateSharedReleasing:
   336  		logFatalf("lockInodeWithSharedLease() found unexpected fileInode.leaseState: fileInodeLeaseStateSharedReleasing")
   337  	case fileInodeLeaseStateExclusiveRequested:
   338  		logFatalf("lockInodeWithSharedLease() found unexpected fileInode.leaseState: fileInodeLeaseStateExclusiveRequested")
   339  	case fileInodeLeaseStateExclusiveGranted:
   340  		// Now that we have an Exclusive Lease, we are also the first in line to use it - so we can grant the Lock
   341  	case fileInodeLeaseStateExclusiveDemoting:
   342  		logFatalf("lockInodeWithSharedLease() found unexpected fileInode.leaseState: fileInodeLeaseStateExclusiveDemoting")
   343  	case fileInodeLeaseStateExclusiveReleasing:
   344  		logFatalf("lockInodeWithSharedLease() found unexpected fileInode.leaseState: fileInodeLeaseStateExclusiveReleasing")
   345  	default:
   346  		logFatalf("lockInodeWithSharedLease() found unknown fileInode.leaseState [case 2]: %v", fileInode.leaseState)
   347  	}
   348  
   349  	globals.Unlock()
   350  
   351  	return
   352  }
   353  
   354  // lockInodeIfExclusiveLeaseGranted is similar to lockInodeWithExclusiveLease() except that
   355  // if an ExclusiveLease is not Granted, it will return nil. Note that callers may also be
   356  // assured any inode state is not cached and dirty in the non-Granted cases. As such, if
   357  // an ExclusiveLease is either being Demoted or Released, lockInodeIfExclusiveLeaseGranted()
   358  // will block until the the Demote or Release has been completed.
   359  //
   360  func lockInodeIfExclusiveLeaseGranted(inodeNumber inode.InodeNumber) (fileInode *fileInodeStruct) {
   361  	var (
   362  		keepLock bool
   363  		ok       bool
   364  		waitChan chan struct{}
   365  	)
   366  
   367  	globals.Lock()
   368  
   369  	fileInode, ok = globals.fileInodeMap[inodeNumber]
   370  	if !ok {
   371  		globals.Unlock()
   372  		fileInode = nil
   373  		return
   374  	}
   375  
   376  	switch fileInode.leaseState {
   377  	case fileInodeLeaseStateNone:
   378  		globals.Unlock()
   379  		fileInode = nil
   380  		return
   381  	case fileInodeLeaseStateSharedRequested:
   382  		globals.sharedLeaseFileInodeCacheLRU.MoveToBack(fileInode.leaseListElement)
   383  		globals.Unlock()
   384  		fileInode = nil
   385  		return
   386  	case fileInodeLeaseStateSharedGranted:
   387  		globals.sharedLeaseFileInodeCacheLRU.MoveToBack(fileInode.leaseListElement)
   388  		globals.Unlock()
   389  		fileInode = nil
   390  		return
   391  	case fileInodeLeaseStateSharedPromoting:
   392  		globals.exclusiveLeaseFileInodeCacheLRU.MoveToBack(fileInode.leaseListElement)
   393  		globals.Unlock()
   394  		fileInode = nil
   395  		return
   396  	case fileInodeLeaseStateSharedReleasing:
   397  		globals.unleasedFileInodeCacheLRU.MoveToBack(fileInode.leaseListElement)
   398  		globals.Unlock()
   399  		fileInode = nil
   400  		return
   401  	case fileInodeLeaseStateExclusiveRequested:
   402  		globals.exclusiveLeaseFileInodeCacheLRU.MoveToBack(fileInode.leaseListElement)
   403  		globals.Unlock()
   404  		fileInode = nil
   405  		return
   406  	case fileInodeLeaseStateExclusiveGranted:
   407  		globals.exclusiveLeaseFileInodeCacheLRU.MoveToBack(fileInode.leaseListElement)
   408  		keepLock = true
   409  	case fileInodeLeaseStateExclusiveDemoting:
   410  		globals.sharedLeaseFileInodeCacheLRU.MoveToBack(fileInode.leaseListElement)
   411  		keepLock = false
   412  	case fileInodeLeaseStateExclusiveReleasing:
   413  		globals.unleasedFileInodeCacheLRU.MoveToBack(fileInode.leaseListElement)
   414  		keepLock = false
   415  	default:
   416  		logFatalf("lockInodeIfExclusiveLeaseGranted() found unknown fileInode.leaseState [case 1]: %v", fileInode.leaseState)
   417  	}
   418  
   419  	if nil == fileInode.lockWaiters {
   420  		fileInode.lockWaiters = list.New()
   421  	} else {
   422  		waitChan = make(chan struct{})
   423  		_ = fileInode.lockWaiters.PushBack(waitChan)
   424  		globals.Unlock()
   425  		_ = <-waitChan
   426  		globals.Lock()
   427  	}
   428  
   429  	if keepLock {
   430  		// Time to check fileInode.leaseState again [(*fileInodeStruct).unlock() may have changed fileInode.leaseState)
   431  
   432  		switch fileInode.leaseState {
   433  		case fileInodeLeaseStateNone:
   434  			globals.Unlock()
   435  			fileInode.unlock(false)
   436  			fileInode = nil
   437  		case fileInodeLeaseStateSharedRequested:
   438  			logFatalf("lockInodeIfExclusiveLeaseGranted() found unexpected fileInode.leaseState: fileInodeLeaseStateSharedRequested")
   439  		case fileInodeLeaseStateSharedGranted:
   440  			globals.Unlock()
   441  			fileInode.unlock(false)
   442  			fileInode = nil
   443  		case fileInodeLeaseStateSharedPromoting:
   444  			logFatalf("lockInodeIfExclusiveLeaseGranted() found unexpected fileInode.leaseState: fileInodeLeaseStateSharedPromoting")
   445  		case fileInodeLeaseStateSharedReleasing:
   446  			logFatalf("lockInodeIfExclusiveLeaseGranted() found unexpected fileInode.leaseState: fileInodeLeaseStateSharedReleasing")
   447  		case fileInodeLeaseStateExclusiveRequested:
   448  			logFatalf("lockInodeIfExclusiveLeaseGranted() found unexpected fileInode.leaseState: fileInodeLeaseStateExclusiveRequested")
   449  		case fileInodeLeaseStateExclusiveGranted:
   450  			// Now that we have an Exclusive Lease, we are also the first in line to use it - so we can grant the Lock
   451  			globals.Unlock()
   452  		case fileInodeLeaseStateExclusiveDemoting:
   453  			logFatalf("lockInodeIfExclusiveLeaseGranted() found unexpected fileInode.leaseState: fileInodeLeaseStateExclusiveDemoting")
   454  		case fileInodeLeaseStateExclusiveReleasing:
   455  			logFatalf("lockInodeIfExclusiveLeaseGranted() found unexpected fileInode.leaseState: fileInodeLeaseStateExclusiveReleasing")
   456  		default:
   457  			logFatalf("lockInodeIfExclusiveLeaseGranted() found unknown fileInode.leaseState [case 2]: %v", fileInode.leaseState)
   458  		}
   459  	} else {
   460  		globals.Unlock()
   461  		fileInode.unlock(false)
   462  		fileInode = nil
   463  	}
   464  
   465  	return
   466  }
   467  
   468  func lockInodeWithExclusiveLease(inodeNumber inode.InodeNumber) (fileInode *fileInodeStruct) {
   469  	var (
   470  		err                   error
   471  		leaseReply            *jrpcfs.LeaseReply
   472  		leaseRequest          *jrpcfs.LeaseRequest
   473  		leaseRequestEndTime   time.Time
   474  		leaseRequestStartTime time.Time
   475  		ok                    bool
   476  		waitChan              chan struct{}
   477  	)
   478  
   479  	globals.Lock()
   480  
   481  	fileInode, ok = globals.fileInodeMap[inodeNumber]
   482  	if !ok {
   483  		fileInode = &fileInodeStruct{
   484  			InodeNumber:               inodeNumber,
   485  			cachedStat:                nil,
   486  			lockWaiters:               nil,
   487  			leaseState:                fileInodeLeaseStateNone,
   488  			pendingLeaseInterrupt:     nil,
   489  			leaseListElement:          nil,
   490  			extentMap:                 nil,
   491  			chunkedPutList:            list.New(),
   492  			flushInProgress:           false,
   493  			chunkedPutFlushWaiterList: list.New(),
   494  			dirtyListElement:          nil,
   495  		}
   496  
   497  		fileInode.leaseListElement = globals.unleasedFileInodeCacheLRU.PushBack(fileInode)
   498  		globals.fileInodeMap[inodeNumber] = fileInode
   499  	}
   500  
   501  	switch fileInode.leaseState {
   502  	case fileInodeLeaseStateNone:
   503  		if nil == fileInode.lockWaiters {
   504  			// No Lease currently held... so ask for an Exclusive Lease
   505  
   506  			fileInode.lockWaiters = list.New()
   507  			fileInode.leaseState = fileInodeLeaseStateExclusiveRequested
   508  			_ = globals.unleasedFileInodeCacheLRU.Remove(fileInode.leaseListElement)
   509  			fileInode.leaseListElement = globals.exclusiveLeaseFileInodeCacheLRU.PushBack(fileInode)
   510  
   511  			globals.Unlock()
   512  
   513  			leaseRequest = &jrpcfs.LeaseRequest{
   514  				InodeHandle: jrpcfs.InodeHandle{
   515  					MountID:     globals.mountID,
   516  					InodeNumber: int64(inodeNumber),
   517  				},
   518  				LeaseRequestType: jrpcfs.LeaseRequestTypeExclusive,
   519  			}
   520  			leaseReply = &jrpcfs.LeaseReply{}
   521  
   522  			leaseRequestStartTime = time.Now()
   523  			err = globals.retryRPCClient.Send("RpcLease", leaseRequest, leaseReply)
   524  			if nil != err {
   525  				logFatalf("lockInodeWithExclusiveLease() unable to obtain Exclusive Lease [case 1] - retryRPCClient.Send() failed: %v", err)
   526  			}
   527  			leaseRequestEndTime = time.Now()
   528  			globals.stats.LeaseRequests_Shared_Usec.Add(uint64(leaseRequestEndTime.Sub(leaseRequestStartTime) / time.Microsecond))
   529  
   530  			// Record leaseReply.LeaseReplyType
   531  
   532  			globals.Lock()
   533  
   534  			switch leaseReply.LeaseReplyType {
   535  			case jrpcfs.LeaseReplyTypeExclusive:
   536  				fileInode.leaseState = fileInodeLeaseStateExclusiveGranted
   537  			default:
   538  				logFatalf("lockInodeWithExclusiveLease() unable to obtain Exclusive Lease [case 2] - LeaseReplyType == %v", leaseReply.LeaseReplyType)
   539  			}
   540  
   541  			// Now that we have an Exclusive Lease, we are also the first in line to use it - so just return
   542  
   543  			globals.Unlock()
   544  
   545  			return
   546  		}
   547  
   548  		globals.unleasedFileInodeCacheLRU.MoveToBack(fileInode.leaseListElement)
   549  
   550  		// Somebody else is ahead of us... so fall through
   551  
   552  	case fileInodeLeaseStateSharedRequested:
   553  		globals.sharedLeaseFileInodeCacheLRU.MoveToBack(fileInode.leaseListElement)
   554  
   555  		// Somebody else is ahead of us... so fall through
   556  
   557  	case fileInodeLeaseStateSharedGranted:
   558  		globals.sharedLeaseFileInodeCacheLRU.MoveToBack(fileInode.leaseListElement)
   559  
   560  		if nil == fileInode.lockWaiters {
   561  			// We already have a Shared Lease and we are also the first in line to use it
   562  			// TODO: We need to promote it first... which could fail !!!
   563  
   564  			fileInode.lockWaiters = list.New()
   565  
   566  			globals.Unlock()
   567  
   568  			return
   569  		}
   570  
   571  		// Somebody else is ahead of us... so fall through
   572  
   573  	case fileInodeLeaseStateSharedPromoting:
   574  		globals.exclusiveLeaseFileInodeCacheLRU.MoveToBack(fileInode.leaseListElement)
   575  
   576  		// Somebody else is ahead of us... so fall through
   577  
   578  	case fileInodeLeaseStateSharedReleasing:
   579  		globals.unleasedFileInodeCacheLRU.MoveToBack(fileInode.leaseListElement)
   580  
   581  		// Somebody else is ahead of us... so fall through
   582  
   583  	case fileInodeLeaseStateExclusiveRequested:
   584  		globals.exclusiveLeaseFileInodeCacheLRU.MoveToBack(fileInode.leaseListElement)
   585  
   586  		// Somebody else is ahead of us... so fall through
   587  
   588  	case fileInodeLeaseStateExclusiveGranted:
   589  		globals.exclusiveLeaseFileInodeCacheLRU.MoveToBack(fileInode.leaseListElement)
   590  
   591  		if nil == fileInode.lockWaiters {
   592  			// We already have an Exclusive Lease and we are also the first in line to use it
   593  
   594  			fileInode.lockWaiters = list.New()
   595  
   596  			globals.Unlock()
   597  
   598  			return
   599  		}
   600  
   601  		// Somebody else is ahead of us... so fall through
   602  
   603  	case fileInodeLeaseStateExclusiveDemoting:
   604  		globals.sharedLeaseFileInodeCacheLRU.MoveToBack(fileInode.leaseListElement)
   605  
   606  		// Somebody else is ahead of us... so fall through
   607  
   608  	case fileInodeLeaseStateExclusiveReleasing:
   609  		globals.unleasedFileInodeCacheLRU.MoveToBack(fileInode.leaseListElement)
   610  
   611  		// Somebody else is ahead of us... so fall through
   612  
   613  	default:
   614  		logFatalf("lockInodeWithExclusiveLease() found unknown fileInode.leaseState [case 1]: %v", fileInode.leaseState)
   615  	}
   616  
   617  	// Time to block
   618  
   619  	waitChan = make(chan struct{})
   620  	_ = fileInode.lockWaiters.PushBack(waitChan)
   621  	globals.Unlock()
   622  	_ = <-waitChan
   623  
   624  	// Time to check fileInode.leaseState again [(*fileInodeStruct).unlock() may have changed fileInode.leaseState)
   625  
   626  	globals.Lock()
   627  
   628  	switch fileInode.leaseState {
   629  	case fileInodeLeaseStateNone:
   630  		// No Lease currently held... so ask for an Exclusive Lease
   631  
   632  		fileInode.leaseState = fileInodeLeaseStateExclusiveRequested
   633  		_ = globals.unleasedFileInodeCacheLRU.Remove(fileInode.leaseListElement)
   634  		fileInode.leaseListElement = globals.exclusiveLeaseFileInodeCacheLRU.PushBack(fileInode)
   635  
   636  		globals.Unlock()
   637  
   638  		leaseRequest = &jrpcfs.LeaseRequest{
   639  			InodeHandle: jrpcfs.InodeHandle{
   640  				MountID:     globals.mountID,
   641  				InodeNumber: int64(inodeNumber),
   642  			},
   643  			LeaseRequestType: jrpcfs.LeaseRequestTypeExclusive,
   644  		}
   645  		leaseReply = &jrpcfs.LeaseReply{}
   646  
   647  		leaseRequestStartTime = time.Now()
   648  		err = globals.retryRPCClient.Send("RpcLease", leaseRequest, leaseReply)
   649  		if nil != err {
   650  			logFatalf("lockInodeWithExclusiveLease() unable to obtain Exclusive Lease [case 3] - retryRPCClient.Send() failed: %v", err)
   651  		}
   652  		leaseRequestEndTime = time.Now()
   653  		globals.stats.LeaseRequests_Shared_Usec.Add(uint64(leaseRequestEndTime.Sub(leaseRequestStartTime) / time.Microsecond))
   654  
   655  		// Record leaseReply.LeaseReplyType
   656  
   657  		globals.Lock()
   658  
   659  		switch leaseReply.LeaseReplyType {
   660  		case jrpcfs.LeaseReplyTypeExclusive:
   661  			fileInode.leaseState = fileInodeLeaseStateExclusiveGranted
   662  		default:
   663  			logFatalf("lockInodeWithExclusiveLease() unable to obtain Exclusive Lease [case 4] - LeaseReplyType == %v", leaseReply.LeaseReplyType)
   664  		}
   665  
   666  		// Now that we have an Exclusive Lease, we are also the first in line to use it - so we can grant the Lock
   667  	case fileInodeLeaseStateSharedRequested:
   668  		logFatalf("lockInodeWithExclusiveLease() found unexpected fileInode.leaseState: fileInodeLeaseStateSharedRequested")
   669  	case fileInodeLeaseStateSharedGranted:
   670  		// Now that we have a Shared Lease, we are also the first in line to use it - so we need to Promote the Lease
   671  
   672  		fileInode.leaseState = fileInodeLeaseStateSharedPromoting
   673  		_ = globals.sharedLeaseFileInodeCacheLRU.Remove(fileInode.leaseListElement)
   674  		fileInode.leaseListElement = globals.exclusiveLeaseFileInodeCacheLRU.PushBack(fileInode)
   675  
   676  		globals.Unlock()
   677  
   678  		leaseRequest = &jrpcfs.LeaseRequest{
   679  			InodeHandle: jrpcfs.InodeHandle{
   680  				MountID:     globals.mountID,
   681  				InodeNumber: int64(inodeNumber),
   682  			},
   683  			LeaseRequestType: jrpcfs.LeaseRequestTypePromote,
   684  		}
   685  		leaseReply = &jrpcfs.LeaseReply{}
   686  
   687  		leaseRequestStartTime = time.Now()
   688  		err = globals.retryRPCClient.Send("RpcLease", leaseRequest, leaseReply)
   689  		if nil != err {
   690  			logFatalf("lockInodeWithExclusiveLease() unable to obtain Exclusive Lease [case 5] - retryRPCClient.Send() failed: %v", err)
   691  		}
   692  		leaseRequestEndTime = time.Now()
   693  		globals.stats.LeaseRequests_Shared_Usec.Add(uint64(leaseRequestEndTime.Sub(leaseRequestStartTime) / time.Microsecond))
   694  
   695  		// Record leaseReply.LeaseReplyType
   696  
   697  		globals.Lock()
   698  
   699  		switch leaseReply.LeaseReplyType {
   700  		case jrpcfs.LeaseReplyTypeExclusive:
   701  			fileInode.leaseState = fileInodeLeaseStateExclusiveGranted
   702  
   703  			// Now that we have an Exclusive Lease, we are also the first in line to use it - so we can grant the Lock
   704  		case jrpcfs.LeaseReplyTypeDenied:
   705  			// We need to Release first... and may have been interrupted to do so already
   706  
   707  			// TODO - invalidate cached state (cachedStat and extentMap)
   708  
   709  			fileInode.pendingLeaseInterrupt = nil
   710  
   711  			fileInode.leaseState = fileInodeLeaseStateSharedReleasing
   712  			_ = globals.exclusiveLeaseFileInodeCacheLRU.Remove(fileInode.leaseListElement)
   713  			fileInode.leaseListElement = globals.unleasedFileInodeCacheLRU.PushBack(fileInode)
   714  
   715  			globals.Unlock()
   716  
   717  			leaseRequest = &jrpcfs.LeaseRequest{
   718  				InodeHandle: jrpcfs.InodeHandle{
   719  					MountID:     globals.mountID,
   720  					InodeNumber: int64(inodeNumber),
   721  				},
   722  				LeaseRequestType: jrpcfs.LeaseRequestTypeRelease,
   723  			}
   724  			leaseReply = &jrpcfs.LeaseReply{}
   725  
   726  			leaseRequestStartTime = time.Now()
   727  			err = globals.retryRPCClient.Send("RpcLease", leaseRequest, leaseReply)
   728  			if nil != err {
   729  				logFatalf("lockInodeWithExclusiveLease() unable to obtain Exclusive Lease [case 6] - retryRPCClient.Send() failed: %v", err)
   730  			}
   731  			leaseRequestEndTime = time.Now()
   732  			globals.stats.LeaseRequests_Shared_Usec.Add(uint64(leaseRequestEndTime.Sub(leaseRequestStartTime) / time.Microsecond))
   733  
   734  			globals.Lock()
   735  
   736  			switch leaseReply.LeaseReplyType {
   737  			case jrpcfs.LeaseReplyTypeReleased:
   738  				// Now we can obtain Exclusive Lease
   739  
   740  				fileInode.leaseState = fileInodeLeaseStateExclusiveRequested
   741  				_ = globals.unleasedFileInodeCacheLRU.Remove(fileInode.leaseListElement)
   742  				fileInode.leaseListElement = globals.exclusiveLeaseFileInodeCacheLRU.PushBack(fileInode)
   743  
   744  				globals.Unlock()
   745  
   746  				leaseRequest = &jrpcfs.LeaseRequest{
   747  					InodeHandle: jrpcfs.InodeHandle{
   748  						MountID:     globals.mountID,
   749  						InodeNumber: int64(inodeNumber),
   750  					},
   751  					LeaseRequestType: jrpcfs.LeaseRequestTypeRelease,
   752  				}
   753  				leaseReply = &jrpcfs.LeaseReply{}
   754  
   755  				leaseRequestStartTime = time.Now()
   756  				err = globals.retryRPCClient.Send("RpcLease", leaseRequest, leaseReply)
   757  				if nil != err {
   758  					logFatalf("lockInodeWithExclusiveLease() unable to obtain Exclusive Lease [case 7] - retryRPCClient.Send() failed: %v", err)
   759  				}
   760  				leaseRequestEndTime = time.Now()
   761  				globals.stats.LeaseRequests_Shared_Usec.Add(uint64(leaseRequestEndTime.Sub(leaseRequestStartTime) / time.Microsecond))
   762  
   763  				globals.Lock()
   764  
   765  				switch leaseReply.LeaseReplyType {
   766  				case jrpcfs.LeaseReplyTypeExclusive:
   767  					fileInode.leaseState = fileInodeLeaseStateExclusiveGranted
   768  				default:
   769  					logFatalf("lockInodeWithExclusiveLease() unable to obtain Exclusive Lease [case 8] - LeaseReplyType == %v", leaseReply.LeaseReplyType)
   770  				}
   771  
   772  				// Now that we have an Exclusive Lease, we are also the first in line to use it - so we can grant the Lock
   773  			default:
   774  				logFatalf("lockInodeWithExclusiveLease() unable to release Shared Lease - LeaseReplyType == %v", leaseReply.LeaseReplyType)
   775  			}
   776  		default:
   777  			logFatalf("lockInodeWithExclusiveLease() unable to obtain Exclusive Lease [case 8] - LeaseReplyType == %v", leaseReply.LeaseReplyType)
   778  		}
   779  	case fileInodeLeaseStateSharedPromoting:
   780  		logFatalf("lockInodeWithExclusiveLease() found unexpected fileInode.leaseState: fileInodeLeaseStateSharedPromoting")
   781  	case fileInodeLeaseStateSharedReleasing:
   782  		logFatalf("lockInodeWithExclusiveLease() found unexpected fileInode.leaseState: fileInodeLeaseStateSharedReleasing")
   783  	case fileInodeLeaseStateExclusiveRequested:
   784  		logFatalf("lockInodeWithExclusiveLease() found unexpected fileInode.leaseState: fileInodeLeaseStateExclusiveRequested")
   785  	case fileInodeLeaseStateExclusiveGranted:
   786  		// Now that we have an Exclusive Lease, we are also the first in line to use it - so we can grant the Lock
   787  	case fileInodeLeaseStateExclusiveDemoting:
   788  		logFatalf("lockInodeWithExclusiveLease() found unexpected fileInode.leaseState: fileInodeLeaseStateExclusiveDemoting")
   789  	case fileInodeLeaseStateExclusiveReleasing:
   790  		logFatalf("lockInodeWithExclusiveLease() found unexpected fileInode.leaseState: fileInodeLeaseStateExclusiveReleasing")
   791  	default:
   792  		logFatalf("lockInodeWithExclusiveLease() found unknown fileInode.leaseState [case 2]: %v", fileInode.leaseState)
   793  	}
   794  
   795  	globals.Unlock()
   796  
   797  	return
   798  }
   799  
   800  func (fileInode *fileInodeStruct) unlock(forceRelease bool) {
   801  	var (
   802  		err                           error
   803  		forcedRPCInterruptTypeRelease jrpcfs.RPCInterruptType
   804  		leaseReply                    *jrpcfs.LeaseReply
   805  		leaseRequest                  *jrpcfs.LeaseRequest
   806  		leaseRequestEndTime           time.Time
   807  		leaseRequestStartTime         time.Time
   808  		lockWaitersListElement        *list.Element
   809  		waitChan                      chan struct{}
   810  	)
   811  
   812  	globals.Lock()
   813  
   814  	if forceRelease {
   815  		// The unlock() call was likely made for a non-existing Inode
   816  		// in which case we might as well clear out both the current
   817  		// Lease as well as the fileInodeStruct (if not being referenced)
   818  
   819  		forcedRPCInterruptTypeRelease = jrpcfs.RPCInterruptTypeRelease
   820  		fileInode.pendingLeaseInterrupt = &forcedRPCInterruptTypeRelease
   821  	}
   822  
   823  	for nil != fileInode.pendingLeaseInterrupt {
   824  		switch *fileInode.pendingLeaseInterrupt {
   825  		case jrpcfs.RPCInterruptTypeUnmount:
   826  			logFatalf("UNSUPPORTED: (*fileInodeStruct).unlock() received jrpcfs.RPCInterruptTypeUnmount")
   827  		case jrpcfs.RPCInterruptTypeDemote:
   828  			fileInode.pendingLeaseInterrupt = nil
   829  
   830  			if fileInodeLeaseStateExclusiveGranted == fileInode.leaseState {
   831  				fileInode.leaseState = fileInodeLeaseStateExclusiveDemoting
   832  				_ = globals.exclusiveLeaseFileInodeCacheLRU.Remove(fileInode.leaseListElement)
   833  				fileInode.leaseListElement = globals.sharedLeaseFileInodeCacheLRU.PushBack(fileInode)
   834  
   835  				globals.Unlock()
   836  
   837  				// TODO - flush dirty state (chunkedPutList)
   838  
   839  				leaseRequest = &jrpcfs.LeaseRequest{
   840  					InodeHandle: jrpcfs.InodeHandle{
   841  						MountID:     globals.mountID,
   842  						InodeNumber: int64(fileInode.InodeNumber),
   843  					},
   844  					LeaseRequestType: jrpcfs.LeaseRequestTypeDemote,
   845  				}
   846  				leaseReply = &jrpcfs.LeaseReply{}
   847  
   848  				leaseRequestStartTime = time.Now()
   849  				err = globals.retryRPCClient.Send("RpcLease", leaseRequest, leaseReply)
   850  				if nil != err {
   851  					logFatalf("(*fileInodeStruct).unlock() unable to Demote Exclusive Lease [case 1] - retryRPCClient.Send() failed: %v", err)
   852  				}
   853  				leaseRequestEndTime = time.Now()
   854  				globals.stats.LeaseRequests_Shared_Usec.Add(uint64(leaseRequestEndTime.Sub(leaseRequestStartTime) / time.Microsecond))
   855  
   856  				// Record leaseReply.LeaseReplyType
   857  
   858  				globals.Lock()
   859  
   860  				switch leaseReply.LeaseReplyType {
   861  				case jrpcfs.LeaseReplyTypeDemoted:
   862  					fileInode.leaseState = fileInodeLeaseStateSharedGranted
   863  				default:
   864  					logFatalf("(*fileInodeStruct).unlock() unable to Demote Exclusive Lease [case 2] - LeaseReplyType == %v", leaseReply.LeaseReplyType)
   865  				}
   866  			}
   867  		case jrpcfs.RPCInterruptTypeRelease:
   868  			fileInode.pendingLeaseInterrupt = nil
   869  
   870  			if fileInodeLeaseStateSharedGranted == fileInode.leaseState {
   871  				fileInode.leaseState = fileInodeLeaseStateSharedReleasing
   872  				_ = globals.sharedLeaseFileInodeCacheLRU.Remove(fileInode.leaseListElement)
   873  				fileInode.leaseListElement = globals.unleasedFileInodeCacheLRU.PushBack(fileInode)
   874  
   875  				globals.Unlock()
   876  
   877  				// TODO - flush dirty state (chunkedPutList)
   878  
   879  				leaseRequest = &jrpcfs.LeaseRequest{
   880  					InodeHandle: jrpcfs.InodeHandle{
   881  						MountID:     globals.mountID,
   882  						InodeNumber: int64(fileInode.InodeNumber),
   883  					},
   884  					LeaseRequestType: jrpcfs.LeaseRequestTypeRelease,
   885  				}
   886  				leaseReply = &jrpcfs.LeaseReply{}
   887  
   888  				leaseRequestStartTime = time.Now()
   889  				err = globals.retryRPCClient.Send("RpcLease", leaseRequest, leaseReply)
   890  				if nil != err {
   891  					logFatalf("(*fileInodeStruct).unlock() unable to Demote Exclusive Lease [case 3] - retryRPCClient.Send() failed: %v", err)
   892  				}
   893  				leaseRequestEndTime = time.Now()
   894  				globals.stats.LeaseRequests_Shared_Usec.Add(uint64(leaseRequestEndTime.Sub(leaseRequestStartTime) / time.Microsecond))
   895  
   896  				// Record leaseReply.LeaseReplyType
   897  
   898  				globals.Lock()
   899  
   900  				switch leaseReply.LeaseReplyType {
   901  				case jrpcfs.LeaseReplyTypeReleased:
   902  					fileInode.leaseState = fileInodeLeaseStateNone
   903  				default:
   904  					logFatalf("(*fileInodeStruct).unlock() unable to Demote Exclusive Lease [case 4] - LeaseReplyType == %v", leaseReply.LeaseReplyType)
   905  				}
   906  			} else if fileInodeLeaseStateExclusiveGranted == fileInode.leaseState {
   907  				fileInode.leaseState = fileInodeLeaseStateExclusiveReleasing
   908  				_ = globals.exclusiveLeaseFileInodeCacheLRU.Remove(fileInode.leaseListElement)
   909  				fileInode.leaseListElement = globals.unleasedFileInodeCacheLRU.PushBack(fileInode)
   910  
   911  				globals.Unlock()
   912  
   913  				// TODO - invalidate cached state (cachedStat and extentMap)
   914  				// TODO - flush dirty state (chunkedPutList)
   915  
   916  				leaseRequest = &jrpcfs.LeaseRequest{
   917  					InodeHandle: jrpcfs.InodeHandle{
   918  						MountID:     globals.mountID,
   919  						InodeNumber: int64(fileInode.InodeNumber),
   920  					},
   921  					LeaseRequestType: jrpcfs.LeaseRequestTypeRelease,
   922  				}
   923  				leaseReply = &jrpcfs.LeaseReply{}
   924  
   925  				leaseRequestStartTime = time.Now()
   926  				err = globals.retryRPCClient.Send("RpcLease", leaseRequest, leaseReply)
   927  				if nil != err {
   928  					logFatalf("(*fileInodeStruct).unlock() unable to Demote Exclusive Lease [case 5] - retryRPCClient.Send() failed: %v", err)
   929  				}
   930  				leaseRequestEndTime = time.Now()
   931  				globals.stats.LeaseRequests_Shared_Usec.Add(uint64(leaseRequestEndTime.Sub(leaseRequestStartTime) / time.Microsecond))
   932  
   933  				// Record leaseReply.LeaseReplyType
   934  
   935  				globals.Lock()
   936  
   937  				switch leaseReply.LeaseReplyType {
   938  				case jrpcfs.LeaseReplyTypeReleased:
   939  					fileInode.leaseState = fileInodeLeaseStateNone
   940  				default:
   941  					logFatalf("(*fileInodeStruct).unlock() unable to Demote Exclusive Lease [case 6] - LeaseReplyType == %v", leaseReply.LeaseReplyType)
   942  				}
   943  			}
   944  		default:
   945  			logFatalf("(*fileInodeStruct).unlock() received unknown rpcInterrupt.RPCInterruptType: %v", *fileInode.pendingLeaseInterrupt)
   946  		}
   947  
   948  		// We need to loop here in case another Interrupt arrived while
   949  		// doing any of the above Demotions/Releases while globals unlocked
   950  	}
   951  
   952  	if 0 == fileInode.lockWaiters.Len() {
   953  		// Nobody was waiting for the fileInode lock
   954  
   955  		if fileInodeLeaseStateNone == fileInode.leaseState {
   956  			// And there is no held Lease... so just destroy it
   957  
   958  			delete(globals.fileInodeMap, fileInode.InodeNumber)
   959  			_ = globals.unleasedFileInodeCacheLRU.Remove(fileInode.leaseListElement)
   960  		} else {
   961  			// We should keep holding the Lease we have... but indicate the lock is available
   962  
   963  			fileInode.lockWaiters = nil
   964  		}
   965  
   966  		globals.Unlock()
   967  		return
   968  	}
   969  
   970  	// At least one caller to lockInodeWith{Shared|Exclusive}Lease() is waiting... so wake them up before exiting
   971  
   972  	lockWaitersListElement = fileInode.lockWaiters.Front()
   973  	fileInode.lockWaiters.Remove(lockWaitersListElement)
   974  
   975  	globals.Unlock()
   976  
   977  	waitChan = lockWaitersListElement.Value.(chan struct{})
   978  
   979  	waitChan <- struct{}{}
   980  }