github.com/tidwall/go@v0.0.0-20170415222209-6694a6888b7d/src/sync/mutex.go (about) 1 // Copyright 2009 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 // Package sync provides basic synchronization primitives such as mutual 6 // exclusion locks. Other than the Once and WaitGroup types, most are intended 7 // for use by low-level library routines. Higher-level synchronization is 8 // better done via channels and communication. 9 // 10 // Values containing the types defined in this package should not be copied. 11 package sync 12 13 import ( 14 "internal/race" 15 "sync/atomic" 16 "unsafe" 17 ) 18 19 func throw(string) // provided by runtime 20 21 // A Mutex is a mutual exclusion lock. 22 // Mutexes can be created as part of other structures; 23 // the zero value for a Mutex is an unlocked mutex. 24 // 25 // A Mutex must not be copied after first use. 26 type Mutex struct { 27 state int32 28 sema uint32 29 } 30 31 // A Locker represents an object that can be locked and unlocked. 32 type Locker interface { 33 Lock() 34 Unlock() 35 } 36 37 const ( 38 mutexLocked = 1 << iota // mutex is locked 39 mutexWoken 40 mutexStarving 41 mutexWaiterShift = iota 42 43 // Mutex fairness. 44 // 45 // Mutex can be in 2 modes of operations: normal and starvation. 46 // In normal mode waiters are queued in FIFO order, but a woken up waiter 47 // does not own the mutex and competes with new arriving goroutines over 48 // the ownership. New arriving goroutines have an advantage -- they are 49 // already running on CPU and there can be lots of them, so a woken up 50 // waiter has good chances of losing. In such case it is queued at front 51 // of the wait queue. If a waiter fails to acquire the mutex for more than 1ms, 52 // it switches mutex to the starvation mode. 53 // 54 // In starvation mode ownership of the mutex is directly handed off from 55 // the unlocking goroutine to the waiter at the front of the queue. 56 // New arriving goroutines don't try to acquire the mutex even if it appears 57 // to be unlocked, and don't try to spin. Instead they queue themselves at 58 // the tail of the wait queue. 59 // 60 // If a waiter receives ownership of the mutex and sees that either 61 // (1) it is the last waiter in the queue, or (2) it waited for less than 1 ms, 62 // it switches mutex back to normal operation mode. 63 // 64 // Normal mode has considerably better performance as a goroutine can acquire 65 // a mutex several times in a row even if there are blocked waiters. 66 // Starvation mode is important to prevent pathological cases of tail latency. 67 starvationThresholdNs = 1e6 68 ) 69 70 // Lock locks m. 71 // If the lock is already in use, the calling goroutine 72 // blocks until the mutex is available. 73 func (m *Mutex) Lock() { 74 // Fast path: grab unlocked mutex. 75 if atomic.CompareAndSwapInt32(&m.state, 0, mutexLocked) { 76 if race.Enabled { 77 race.Acquire(unsafe.Pointer(m)) 78 } 79 return 80 } 81 82 var waitStartTime int64 83 starving := false 84 awoke := false 85 iter := 0 86 old := m.state 87 for { 88 // Don't spin in starvation mode, ownership is handed off to waiters 89 // so we won't be able to acquire the mutex anyway. 90 if old&(mutexLocked|mutexStarving) == mutexLocked && runtime_canSpin(iter) { 91 // Active spinning makes sense. 92 // Try to set mutexWoken flag to inform Unlock 93 // to not wake other blocked goroutines. 94 if !awoke && old&mutexWoken == 0 && old>>mutexWaiterShift != 0 && 95 atomic.CompareAndSwapInt32(&m.state, old, old|mutexWoken) { 96 awoke = true 97 } 98 runtime_doSpin() 99 iter++ 100 old = m.state 101 continue 102 } 103 new := old 104 // Don't try to acquire starving mutex, new arriving goroutines must queue. 105 if old&mutexStarving == 0 { 106 new |= mutexLocked 107 } 108 if old&(mutexLocked|mutexStarving) != 0 { 109 new += 1 << mutexWaiterShift 110 } 111 // The current goroutine switches mutex to starvation mode. 112 // But if the mutex is currently unlocked, don't do the switch. 113 // Unlock expects that starving mutex has waiters, which will not 114 // be true in this case. 115 if starving && old&mutexLocked != 0 { 116 new |= mutexStarving 117 } 118 if awoke { 119 // The goroutine has been woken from sleep, 120 // so we need to reset the flag in either case. 121 if new&mutexWoken == 0 { 122 panic("sync: inconsistent mutex state") 123 } 124 new &^= mutexWoken 125 } 126 if atomic.CompareAndSwapInt32(&m.state, old, new) { 127 if old&(mutexLocked|mutexStarving) == 0 { 128 break // locked the mutex with CAS 129 } 130 // If we were already waiting before, queue at the front of the queue. 131 queueLifo := waitStartTime != 0 132 if waitStartTime == 0 { 133 waitStartTime = runtime_nanotime() 134 } 135 runtime_SemacquireMutex(&m.sema, queueLifo) 136 starving = starving || runtime_nanotime()-waitStartTime > starvationThresholdNs 137 old = m.state 138 if old&mutexStarving != 0 { 139 // If this goroutine was woken and mutex is in starvation mode, 140 // ownership was handed off to us but mutex is in somewhat 141 // inconsistent state: mutexLocked is not set and we are still 142 // accounted as waiter. Fix that. 143 if old&(mutexLocked|mutexWoken) != 0 || old>>mutexWaiterShift == 0 { 144 panic("sync: inconsistent mutex state") 145 } 146 delta := int32(mutexLocked - 1<<mutexWaiterShift) 147 if !starving || old>>mutexWaiterShift == 1 { 148 // Exit starvation mode. 149 // Critical to do it here and consider wait time. 150 // Starvation mode is so inefficient, that two goroutines 151 // can go lock-step infinitely once they switch mutex 152 // to starvation mode. 153 delta -= mutexStarving 154 } 155 atomic.AddInt32(&m.state, delta) 156 break 157 } 158 awoke = true 159 iter = 0 160 } else { 161 old = m.state 162 } 163 } 164 165 if race.Enabled { 166 race.Acquire(unsafe.Pointer(m)) 167 } 168 } 169 170 // Unlock unlocks m. 171 // It is a run-time error if m is not locked on entry to Unlock. 172 // 173 // A locked Mutex is not associated with a particular goroutine. 174 // It is allowed for one goroutine to lock a Mutex and then 175 // arrange for another goroutine to unlock it. 176 func (m *Mutex) Unlock() { 177 if race.Enabled { 178 _ = m.state 179 race.Release(unsafe.Pointer(m)) 180 } 181 182 // Fast path: drop lock bit. 183 new := atomic.AddInt32(&m.state, -mutexLocked) 184 if (new+mutexLocked)&mutexLocked == 0 { 185 panic("sync: unlock of unlocked mutex") 186 } 187 if new&mutexStarving == 0 { 188 old := new 189 for { 190 // If there are no waiters or a goroutine has already 191 // been woken or grabbed the lock, no need to wake anyone. 192 // In starvation mode ownership is directly handed off from unlocking 193 // goroutine to the next waiter. We are not part of this chain, 194 // since we did not observe mutexStarving when we unlocked the mutex above. 195 // So get off the way. 196 if old>>mutexWaiterShift == 0 || old&(mutexLocked|mutexWoken|mutexStarving) != 0 { 197 return 198 } 199 // Grab the right to wake someone. 200 new = (old - 1<<mutexWaiterShift) | mutexWoken 201 if atomic.CompareAndSwapInt32(&m.state, old, new) { 202 runtime_Semrelease(&m.sema, false) 203 return 204 } 205 old = m.state 206 } 207 } else { 208 // Starving mode: handoff mutex ownership to the next waiter. 209 // Note: mutexLocked is not set, the waiter will set it after wakeup. 210 // But mutex is still considered locked if mutexStarving is set, 211 // so new coming goroutines won't acquire it. 212 runtime_Semrelease(&m.sema, true) 213 } 214 }