github.com/comwrg/go/src@v0.0.0-20220319063731-c238d0440370/runtime/lockrank_on.go (about) 1 // Copyright 2020 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 //go:build goexperiment.staticlockranking 6 // +build goexperiment.staticlockranking 7 8 package runtime 9 10 import ( 11 "runtime/internal/atomic" 12 "unsafe" 13 ) 14 15 // worldIsStopped is accessed atomically to track world-stops. 1 == world 16 // stopped. 17 var worldIsStopped uint32 18 19 // lockRankStruct is embedded in mutex 20 type lockRankStruct struct { 21 // static lock ranking of the lock 22 rank lockRank 23 // pad field to make sure lockRankStruct is a multiple of 8 bytes, even on 24 // 32-bit systems. 25 pad int 26 } 27 28 func lockInit(l *mutex, rank lockRank) { 29 l.rank = rank 30 } 31 32 func getLockRank(l *mutex) lockRank { 33 return l.rank 34 } 35 36 // lockWithRank is like lock(l), but allows the caller to specify a lock rank 37 // when acquiring a non-static lock. 38 // 39 // Note that we need to be careful about stack splits: 40 // 41 // This function is not nosplit, thus it may split at function entry. This may 42 // introduce a new edge in the lock order, but it is no different from any 43 // other (nosplit) call before this call (including the call to lock() itself). 44 // 45 // However, we switch to the systemstack to record the lock held to ensure that 46 // we record an accurate lock ordering. e.g., without systemstack, a stack 47 // split on entry to lock2() would record stack split locks as taken after l, 48 // even though l is not actually locked yet. 49 func lockWithRank(l *mutex, rank lockRank) { 50 if l == &debuglock || l == &paniclk { 51 // debuglock is only used for println/printlock(). Don't do lock 52 // rank recording for it, since print/println are used when 53 // printing out a lock ordering problem below. 54 // 55 // paniclk is only used for fatal throw/panic. Don't do lock 56 // ranking recording for it, since we throw after reporting a 57 // lock ordering problem. Additionally, paniclk may be taken 58 // after effectively any lock (anywhere we might panic), which 59 // the partial order doesn't cover. 60 lock2(l) 61 return 62 } 63 if rank == 0 { 64 rank = lockRankLeafRank 65 } 66 gp := getg() 67 // Log the new class. 68 systemstack(func() { 69 i := gp.m.locksHeldLen 70 if i >= len(gp.m.locksHeld) { 71 throw("too many locks held concurrently for rank checking") 72 } 73 gp.m.locksHeld[i].rank = rank 74 gp.m.locksHeld[i].lockAddr = uintptr(unsafe.Pointer(l)) 75 gp.m.locksHeldLen++ 76 77 // i is the index of the lock being acquired 78 if i > 0 { 79 checkRanks(gp, gp.m.locksHeld[i-1].rank, rank) 80 } 81 lock2(l) 82 }) 83 } 84 85 // nosplit to ensure it can be called in as many contexts as possible. 86 //go:nosplit 87 func printHeldLocks(gp *g) { 88 if gp.m.locksHeldLen == 0 { 89 println("<none>") 90 return 91 } 92 93 for j, held := range gp.m.locksHeld[:gp.m.locksHeldLen] { 94 println(j, ":", held.rank.String(), held.rank, unsafe.Pointer(gp.m.locksHeld[j].lockAddr)) 95 } 96 } 97 98 // acquireLockRank acquires a rank which is not associated with a mutex lock 99 // 100 // This function may be called in nosplit context and thus must be nosplit. 101 //go:nosplit 102 func acquireLockRank(rank lockRank) { 103 gp := getg() 104 // Log the new class. See comment on lockWithRank. 105 systemstack(func() { 106 i := gp.m.locksHeldLen 107 if i >= len(gp.m.locksHeld) { 108 throw("too many locks held concurrently for rank checking") 109 } 110 gp.m.locksHeld[i].rank = rank 111 gp.m.locksHeld[i].lockAddr = 0 112 gp.m.locksHeldLen++ 113 114 // i is the index of the lock being acquired 115 if i > 0 { 116 checkRanks(gp, gp.m.locksHeld[i-1].rank, rank) 117 } 118 }) 119 } 120 121 // checkRanks checks if goroutine g, which has mostly recently acquired a lock 122 // with rank 'prevRank', can now acquire a lock with rank 'rank'. 123 // 124 //go:systemstack 125 func checkRanks(gp *g, prevRank, rank lockRank) { 126 rankOK := false 127 if rank < prevRank { 128 // If rank < prevRank, then we definitely have a rank error 129 rankOK = false 130 } else if rank == lockRankLeafRank { 131 // If new lock is a leaf lock, then the preceding lock can 132 // be anything except another leaf lock. 133 rankOK = prevRank < lockRankLeafRank 134 } else { 135 // We've now verified the total lock ranking, but we 136 // also enforce the partial ordering specified by 137 // lockPartialOrder as well. Two locks with the same rank 138 // can only be acquired at the same time if explicitly 139 // listed in the lockPartialOrder table. 140 list := lockPartialOrder[rank] 141 for _, entry := range list { 142 if entry == prevRank { 143 rankOK = true 144 break 145 } 146 } 147 } 148 if !rankOK { 149 printlock() 150 println(gp.m.procid, " ======") 151 printHeldLocks(gp) 152 throw("lock ordering problem") 153 } 154 } 155 156 // See comment on lockWithRank regarding stack splitting. 157 func unlockWithRank(l *mutex) { 158 if l == &debuglock || l == &paniclk { 159 // See comment at beginning of lockWithRank. 160 unlock2(l) 161 return 162 } 163 gp := getg() 164 systemstack(func() { 165 found := false 166 for i := gp.m.locksHeldLen - 1; i >= 0; i-- { 167 if gp.m.locksHeld[i].lockAddr == uintptr(unsafe.Pointer(l)) { 168 found = true 169 copy(gp.m.locksHeld[i:gp.m.locksHeldLen-1], gp.m.locksHeld[i+1:gp.m.locksHeldLen]) 170 gp.m.locksHeldLen-- 171 break 172 } 173 } 174 if !found { 175 println(gp.m.procid, ":", l.rank.String(), l.rank, l) 176 throw("unlock without matching lock acquire") 177 } 178 unlock2(l) 179 }) 180 } 181 182 // releaseLockRank releases a rank which is not associated with a mutex lock 183 // 184 // This function may be called in nosplit context and thus must be nosplit. 185 //go:nosplit 186 func releaseLockRank(rank lockRank) { 187 gp := getg() 188 systemstack(func() { 189 found := false 190 for i := gp.m.locksHeldLen - 1; i >= 0; i-- { 191 if gp.m.locksHeld[i].rank == rank && gp.m.locksHeld[i].lockAddr == 0 { 192 found = true 193 copy(gp.m.locksHeld[i:gp.m.locksHeldLen-1], gp.m.locksHeld[i+1:gp.m.locksHeldLen]) 194 gp.m.locksHeldLen-- 195 break 196 } 197 } 198 if !found { 199 println(gp.m.procid, ":", rank.String(), rank) 200 throw("lockRank release without matching lockRank acquire") 201 } 202 }) 203 } 204 205 // See comment on lockWithRank regarding stack splitting. 206 func lockWithRankMayAcquire(l *mutex, rank lockRank) { 207 gp := getg() 208 if gp.m.locksHeldLen == 0 { 209 // No possibility of lock ordering problem if no other locks held 210 return 211 } 212 213 systemstack(func() { 214 i := gp.m.locksHeldLen 215 if i >= len(gp.m.locksHeld) { 216 throw("too many locks held concurrently for rank checking") 217 } 218 // Temporarily add this lock to the locksHeld list, so 219 // checkRanks() will print out list, including this lock, if there 220 // is a lock ordering problem. 221 gp.m.locksHeld[i].rank = rank 222 gp.m.locksHeld[i].lockAddr = uintptr(unsafe.Pointer(l)) 223 gp.m.locksHeldLen++ 224 checkRanks(gp, gp.m.locksHeld[i-1].rank, rank) 225 gp.m.locksHeldLen-- 226 }) 227 } 228 229 // nosplit to ensure it can be called in as many contexts as possible. 230 //go:nosplit 231 func checkLockHeld(gp *g, l *mutex) bool { 232 for i := gp.m.locksHeldLen - 1; i >= 0; i-- { 233 if gp.m.locksHeld[i].lockAddr == uintptr(unsafe.Pointer(l)) { 234 return true 235 } 236 } 237 return false 238 } 239 240 // assertLockHeld throws if l is not held by the caller. 241 // 242 // nosplit to ensure it can be called in as many contexts as possible. 243 //go:nosplit 244 func assertLockHeld(l *mutex) { 245 gp := getg() 246 247 held := checkLockHeld(gp, l) 248 if held { 249 return 250 } 251 252 // Crash from system stack to avoid splits that may cause 253 // additional issues. 254 systemstack(func() { 255 printlock() 256 print("caller requires lock ", l, " (rank ", l.rank.String(), "), holding:\n") 257 printHeldLocks(gp) 258 throw("not holding required lock!") 259 }) 260 } 261 262 // assertRankHeld throws if a mutex with rank r is not held by the caller. 263 // 264 // This is less precise than assertLockHeld, but can be used in places where a 265 // pointer to the exact mutex is not available. 266 // 267 // nosplit to ensure it can be called in as many contexts as possible. 268 //go:nosplit 269 func assertRankHeld(r lockRank) { 270 gp := getg() 271 272 for i := gp.m.locksHeldLen - 1; i >= 0; i-- { 273 if gp.m.locksHeld[i].rank == r { 274 return 275 } 276 } 277 278 // Crash from system stack to avoid splits that may cause 279 // additional issues. 280 systemstack(func() { 281 printlock() 282 print("caller requires lock with rank ", r.String(), "), holding:\n") 283 printHeldLocks(gp) 284 throw("not holding required lock!") 285 }) 286 } 287 288 // worldStopped notes that the world is stopped. 289 // 290 // Caller must hold worldsema. 291 // 292 // nosplit to ensure it can be called in as many contexts as possible. 293 //go:nosplit 294 func worldStopped() { 295 if stopped := atomic.Xadd(&worldIsStopped, 1); stopped != 1 { 296 systemstack(func() { 297 print("world stop count=", stopped, "\n") 298 throw("recursive world stop") 299 }) 300 } 301 } 302 303 // worldStarted that the world is starting. 304 // 305 // Caller must hold worldsema. 306 // 307 // nosplit to ensure it can be called in as many contexts as possible. 308 //go:nosplit 309 func worldStarted() { 310 if stopped := atomic.Xadd(&worldIsStopped, -1); stopped != 0 { 311 systemstack(func() { 312 print("world stop count=", stopped, "\n") 313 throw("released non-stopped world stop") 314 }) 315 } 316 } 317 318 // nosplit to ensure it can be called in as many contexts as possible. 319 //go:nosplit 320 func checkWorldStopped() bool { 321 stopped := atomic.Load(&worldIsStopped) 322 if stopped > 1 { 323 systemstack(func() { 324 print("inconsistent world stop count=", stopped, "\n") 325 throw("inconsistent world stop count") 326 }) 327 } 328 329 return stopped == 1 330 } 331 332 // assertWorldStopped throws if the world is not stopped. It does not check 333 // which M stopped the world. 334 // 335 // nosplit to ensure it can be called in as many contexts as possible. 336 //go:nosplit 337 func assertWorldStopped() { 338 if checkWorldStopped() { 339 return 340 } 341 342 throw("world not stopped") 343 } 344 345 // assertWorldStoppedOrLockHeld throws if the world is not stopped and the 346 // passed lock is not held. 347 // 348 // nosplit to ensure it can be called in as many contexts as possible. 349 //go:nosplit 350 func assertWorldStoppedOrLockHeld(l *mutex) { 351 if checkWorldStopped() { 352 return 353 } 354 355 gp := getg() 356 held := checkLockHeld(gp, l) 357 if held { 358 return 359 } 360 361 // Crash from system stack to avoid splits that may cause 362 // additional issues. 363 systemstack(func() { 364 printlock() 365 print("caller requires world stop or lock ", l, " (rank ", l.rank.String(), "), holding:\n") 366 println("<no world stop>") 367 printHeldLocks(gp) 368 throw("no world stop or required lock!") 369 }) 370 }