github.com/AESNooper/go/src@v0.0.0-20220218095104-b56a4ab1bbbb/runtime/lockrank_on.go (about) 1 // Copyright 2020 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 //go:build goexperiment.staticlockranking 6 7 package runtime 8 9 import ( 10 "runtime/internal/atomic" 11 "unsafe" 12 ) 13 14 // worldIsStopped is accessed atomically to track world-stops. 1 == world 15 // stopped. 16 var worldIsStopped uint32 17 18 // lockRankStruct is embedded in mutex 19 type lockRankStruct struct { 20 // static lock ranking of the lock 21 rank lockRank 22 // pad field to make sure lockRankStruct is a multiple of 8 bytes, even on 23 // 32-bit systems. 24 pad int 25 } 26 27 func lockInit(l *mutex, rank lockRank) { 28 l.rank = rank 29 } 30 31 func getLockRank(l *mutex) lockRank { 32 return l.rank 33 } 34 35 // lockWithRank is like lock(l), but allows the caller to specify a lock rank 36 // when acquiring a non-static lock. 37 // 38 // Note that we need to be careful about stack splits: 39 // 40 // This function is not nosplit, thus it may split at function entry. This may 41 // introduce a new edge in the lock order, but it is no different from any 42 // other (nosplit) call before this call (including the call to lock() itself). 43 // 44 // However, we switch to the systemstack to record the lock held to ensure that 45 // we record an accurate lock ordering. e.g., without systemstack, a stack 46 // split on entry to lock2() would record stack split locks as taken after l, 47 // even though l is not actually locked yet. 48 func lockWithRank(l *mutex, rank lockRank) { 49 if l == &debuglock || l == &paniclk { 50 // debuglock is only used for println/printlock(). Don't do lock 51 // rank recording for it, since print/println are used when 52 // printing out a lock ordering problem below. 53 // 54 // paniclk is only used for fatal throw/panic. Don't do lock 55 // ranking recording for it, since we throw after reporting a 56 // lock ordering problem. Additionally, paniclk may be taken 57 // after effectively any lock (anywhere we might panic), which 58 // the partial order doesn't cover. 59 lock2(l) 60 return 61 } 62 if rank == 0 { 63 rank = lockRankLeafRank 64 } 65 gp := getg() 66 // Log the new class. 67 systemstack(func() { 68 i := gp.m.locksHeldLen 69 if i >= len(gp.m.locksHeld) { 70 throw("too many locks held concurrently for rank checking") 71 } 72 gp.m.locksHeld[i].rank = rank 73 gp.m.locksHeld[i].lockAddr = uintptr(unsafe.Pointer(l)) 74 gp.m.locksHeldLen++ 75 76 // i is the index of the lock being acquired 77 if i > 0 { 78 checkRanks(gp, gp.m.locksHeld[i-1].rank, rank) 79 } 80 lock2(l) 81 }) 82 } 83 84 // nosplit to ensure it can be called in as many contexts as possible. 85 //go:nosplit 86 func printHeldLocks(gp *g) { 87 if gp.m.locksHeldLen == 0 { 88 println("<none>") 89 return 90 } 91 92 for j, held := range gp.m.locksHeld[:gp.m.locksHeldLen] { 93 println(j, ":", held.rank.String(), held.rank, unsafe.Pointer(gp.m.locksHeld[j].lockAddr)) 94 } 95 } 96 97 // acquireLockRank acquires a rank which is not associated with a mutex lock 98 // 99 // This function may be called in nosplit context and thus must be nosplit. 100 //go:nosplit 101 func acquireLockRank(rank lockRank) { 102 gp := getg() 103 // Log the new class. See comment on lockWithRank. 104 systemstack(func() { 105 i := gp.m.locksHeldLen 106 if i >= len(gp.m.locksHeld) { 107 throw("too many locks held concurrently for rank checking") 108 } 109 gp.m.locksHeld[i].rank = rank 110 gp.m.locksHeld[i].lockAddr = 0 111 gp.m.locksHeldLen++ 112 113 // i is the index of the lock being acquired 114 if i > 0 { 115 checkRanks(gp, gp.m.locksHeld[i-1].rank, rank) 116 } 117 }) 118 } 119 120 // checkRanks checks if goroutine g, which has mostly recently acquired a lock 121 // with rank 'prevRank', can now acquire a lock with rank 'rank'. 122 // 123 //go:systemstack 124 func checkRanks(gp *g, prevRank, rank lockRank) { 125 rankOK := false 126 if rank < prevRank { 127 // If rank < prevRank, then we definitely have a rank error 128 rankOK = false 129 } else if rank == lockRankLeafRank { 130 // If new lock is a leaf lock, then the preceding lock can 131 // be anything except another leaf lock. 132 rankOK = prevRank < lockRankLeafRank 133 } else { 134 // We've now verified the total lock ranking, but we 135 // also enforce the partial ordering specified by 136 // lockPartialOrder as well. Two locks with the same rank 137 // can only be acquired at the same time if explicitly 138 // listed in the lockPartialOrder table. 139 list := lockPartialOrder[rank] 140 for _, entry := range list { 141 if entry == prevRank { 142 rankOK = true 143 break 144 } 145 } 146 } 147 if !rankOK { 148 printlock() 149 println(gp.m.procid, " ======") 150 printHeldLocks(gp) 151 throw("lock ordering problem") 152 } 153 } 154 155 // See comment on lockWithRank regarding stack splitting. 156 func unlockWithRank(l *mutex) { 157 if l == &debuglock || l == &paniclk { 158 // See comment at beginning of lockWithRank. 159 unlock2(l) 160 return 161 } 162 gp := getg() 163 systemstack(func() { 164 found := false 165 for i := gp.m.locksHeldLen - 1; i >= 0; i-- { 166 if gp.m.locksHeld[i].lockAddr == uintptr(unsafe.Pointer(l)) { 167 found = true 168 copy(gp.m.locksHeld[i:gp.m.locksHeldLen-1], gp.m.locksHeld[i+1:gp.m.locksHeldLen]) 169 gp.m.locksHeldLen-- 170 break 171 } 172 } 173 if !found { 174 println(gp.m.procid, ":", l.rank.String(), l.rank, l) 175 throw("unlock without matching lock acquire") 176 } 177 unlock2(l) 178 }) 179 } 180 181 // releaseLockRank releases a rank which is not associated with a mutex lock 182 // 183 // This function may be called in nosplit context and thus must be nosplit. 184 //go:nosplit 185 func releaseLockRank(rank lockRank) { 186 gp := getg() 187 systemstack(func() { 188 found := false 189 for i := gp.m.locksHeldLen - 1; i >= 0; i-- { 190 if gp.m.locksHeld[i].rank == rank && gp.m.locksHeld[i].lockAddr == 0 { 191 found = true 192 copy(gp.m.locksHeld[i:gp.m.locksHeldLen-1], gp.m.locksHeld[i+1:gp.m.locksHeldLen]) 193 gp.m.locksHeldLen-- 194 break 195 } 196 } 197 if !found { 198 println(gp.m.procid, ":", rank.String(), rank) 199 throw("lockRank release without matching lockRank acquire") 200 } 201 }) 202 } 203 204 // See comment on lockWithRank regarding stack splitting. 205 func lockWithRankMayAcquire(l *mutex, rank lockRank) { 206 gp := getg() 207 if gp.m.locksHeldLen == 0 { 208 // No possibility of lock ordering problem if no other locks held 209 return 210 } 211 212 systemstack(func() { 213 i := gp.m.locksHeldLen 214 if i >= len(gp.m.locksHeld) { 215 throw("too many locks held concurrently for rank checking") 216 } 217 // Temporarily add this lock to the locksHeld list, so 218 // checkRanks() will print out list, including this lock, if there 219 // is a lock ordering problem. 220 gp.m.locksHeld[i].rank = rank 221 gp.m.locksHeld[i].lockAddr = uintptr(unsafe.Pointer(l)) 222 gp.m.locksHeldLen++ 223 checkRanks(gp, gp.m.locksHeld[i-1].rank, rank) 224 gp.m.locksHeldLen-- 225 }) 226 } 227 228 // nosplit to ensure it can be called in as many contexts as possible. 229 //go:nosplit 230 func checkLockHeld(gp *g, l *mutex) bool { 231 for i := gp.m.locksHeldLen - 1; i >= 0; i-- { 232 if gp.m.locksHeld[i].lockAddr == uintptr(unsafe.Pointer(l)) { 233 return true 234 } 235 } 236 return false 237 } 238 239 // assertLockHeld throws if l is not held by the caller. 240 // 241 // nosplit to ensure it can be called in as many contexts as possible. 242 //go:nosplit 243 func assertLockHeld(l *mutex) { 244 gp := getg() 245 246 held := checkLockHeld(gp, l) 247 if held { 248 return 249 } 250 251 // Crash from system stack to avoid splits that may cause 252 // additional issues. 253 systemstack(func() { 254 printlock() 255 print("caller requires lock ", l, " (rank ", l.rank.String(), "), holding:\n") 256 printHeldLocks(gp) 257 throw("not holding required lock!") 258 }) 259 } 260 261 // assertRankHeld throws if a mutex with rank r is not held by the caller. 262 // 263 // This is less precise than assertLockHeld, but can be used in places where a 264 // pointer to the exact mutex is not available. 265 // 266 // nosplit to ensure it can be called in as many contexts as possible. 267 //go:nosplit 268 func assertRankHeld(r lockRank) { 269 gp := getg() 270 271 for i := gp.m.locksHeldLen - 1; i >= 0; i-- { 272 if gp.m.locksHeld[i].rank == r { 273 return 274 } 275 } 276 277 // Crash from system stack to avoid splits that may cause 278 // additional issues. 279 systemstack(func() { 280 printlock() 281 print("caller requires lock with rank ", r.String(), "), holding:\n") 282 printHeldLocks(gp) 283 throw("not holding required lock!") 284 }) 285 } 286 287 // worldStopped notes that the world is stopped. 288 // 289 // Caller must hold worldsema. 290 // 291 // nosplit to ensure it can be called in as many contexts as possible. 292 //go:nosplit 293 func worldStopped() { 294 if stopped := atomic.Xadd(&worldIsStopped, 1); stopped != 1 { 295 systemstack(func() { 296 print("world stop count=", stopped, "\n") 297 throw("recursive world stop") 298 }) 299 } 300 } 301 302 // worldStarted that the world is starting. 303 // 304 // Caller must hold worldsema. 305 // 306 // nosplit to ensure it can be called in as many contexts as possible. 307 //go:nosplit 308 func worldStarted() { 309 if stopped := atomic.Xadd(&worldIsStopped, -1); stopped != 0 { 310 systemstack(func() { 311 print("world stop count=", stopped, "\n") 312 throw("released non-stopped world stop") 313 }) 314 } 315 } 316 317 // nosplit to ensure it can be called in as many contexts as possible. 318 //go:nosplit 319 func checkWorldStopped() bool { 320 stopped := atomic.Load(&worldIsStopped) 321 if stopped > 1 { 322 systemstack(func() { 323 print("inconsistent world stop count=", stopped, "\n") 324 throw("inconsistent world stop count") 325 }) 326 } 327 328 return stopped == 1 329 } 330 331 // assertWorldStopped throws if the world is not stopped. It does not check 332 // which M stopped the world. 333 // 334 // nosplit to ensure it can be called in as many contexts as possible. 335 //go:nosplit 336 func assertWorldStopped() { 337 if checkWorldStopped() { 338 return 339 } 340 341 throw("world not stopped") 342 } 343 344 // assertWorldStoppedOrLockHeld throws if the world is not stopped and the 345 // passed lock is not held. 346 // 347 // nosplit to ensure it can be called in as many contexts as possible. 348 //go:nosplit 349 func assertWorldStoppedOrLockHeld(l *mutex) { 350 if checkWorldStopped() { 351 return 352 } 353 354 gp := getg() 355 held := checkLockHeld(gp, l) 356 if held { 357 return 358 } 359 360 // Crash from system stack to avoid splits that may cause 361 // additional issues. 362 systemstack(func() { 363 printlock() 364 print("caller requires world stop or lock ", l, " (rank ", l.rank.String(), "), holding:\n") 365 println("<no world stop>") 366 printHeldLocks(gp) 367 throw("no world stop or required lock!") 368 }) 369 }