github.com/MangoDowner/go-gm@v0.0.0-20180818020936-8baa2bd4408c/src/runtime/mbarrier.go (about) 1 // Copyright 2015 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 // Garbage collector: write barriers. 6 // 7 // For the concurrent garbage collector, the Go compiler implements 8 // updates to pointer-valued fields that may be in heap objects by 9 // emitting calls to write barriers. This file contains the actual write barrier 10 // implementation, gcmarkwb_m, and the various wrappers called by the 11 // compiler to implement pointer assignment, slice assignment, 12 // typed memmove, and so on. 13 14 package runtime 15 16 import ( 17 "runtime/internal/sys" 18 "unsafe" 19 ) 20 21 // gcmarkwb_m is the mark-phase write barrier, the only barrier we have. 22 // The rest of this file exists only to make calls to this function. 23 // 24 // This is a hybrid barrier that combines a Yuasa-style deletion 25 // barrier—which shades the object whose reference is being 26 // overwritten—with Dijkstra insertion barrier—which shades the object 27 // whose reference is being written. The insertion part of the barrier 28 // is necessary while the calling goroutine's stack is grey. In 29 // pseudocode, the barrier is: 30 // 31 // writePointer(slot, ptr): 32 // shade(*slot) 33 // if current stack is grey: 34 // shade(ptr) 35 // *slot = ptr 36 // 37 // slot is the destination in Go code. 38 // ptr is the value that goes into the slot in Go code. 39 // 40 // Shade indicates that it has seen a white pointer by adding the referent 41 // to wbuf as well as marking it. 42 // 43 // The two shades and the condition work together to prevent a mutator 44 // from hiding an object from the garbage collector: 45 // 46 // 1. shade(*slot) prevents a mutator from hiding an object by moving 47 // the sole pointer to it from the heap to its stack. If it attempts 48 // to unlink an object from the heap, this will shade it. 49 // 50 // 2. shade(ptr) prevents a mutator from hiding an object by moving 51 // the sole pointer to it from its stack into a black object in the 52 // heap. If it attempts to install the pointer into a black object, 53 // this will shade it. 54 // 55 // 3. Once a goroutine's stack is black, the shade(ptr) becomes 56 // unnecessary. shade(ptr) prevents hiding an object by moving it from 57 // the stack to the heap, but this requires first having a pointer 58 // hidden on the stack. Immediately after a stack is scanned, it only 59 // points to shaded objects, so it's not hiding anything, and the 60 // shade(*slot) prevents it from hiding any other pointers on its 61 // stack. 62 // 63 // For a detailed description of this barrier and proof of 64 // correctness, see https://github.com/golang/proposal/blob/master/design/17503-eliminate-rescan.md 65 // 66 // 67 // 68 // Dealing with memory ordering: 69 // 70 // Both the Yuasa and Dijkstra barriers can be made conditional on the 71 // color of the object containing the slot. We chose not to make these 72 // conditional because the cost of ensuring that the object holding 73 // the slot doesn't concurrently change color without the mutator 74 // noticing seems prohibitive. 75 // 76 // Consider the following example where the mutator writes into 77 // a slot and then loads the slot's mark bit while the GC thread 78 // writes to the slot's mark bit and then as part of scanning reads 79 // the slot. 80 // 81 // Initially both [slot] and [slotmark] are 0 (nil) 82 // Mutator thread GC thread 83 // st [slot], ptr st [slotmark], 1 84 // 85 // ld r1, [slotmark] ld r2, [slot] 86 // 87 // Without an expensive memory barrier between the st and the ld, the final 88 // result on most HW (including 386/amd64) can be r1==r2==0. This is a classic 89 // example of what can happen when loads are allowed to be reordered with older 90 // stores (avoiding such reorderings lies at the heart of the classic 91 // Peterson/Dekker algorithms for mutual exclusion). Rather than require memory 92 // barriers, which will slow down both the mutator and the GC, we always grey 93 // the ptr object regardless of the slot's color. 94 // 95 // Another place where we intentionally omit memory barriers is when 96 // accessing mheap_.arena_used to check if a pointer points into the 97 // heap. On relaxed memory machines, it's possible for a mutator to 98 // extend the size of the heap by updating arena_used, allocate an 99 // object from this new region, and publish a pointer to that object, 100 // but for tracing running on another processor to observe the pointer 101 // but use the old value of arena_used. In this case, tracing will not 102 // mark the object, even though it's reachable. However, the mutator 103 // is guaranteed to execute a write barrier when it publishes the 104 // pointer, so it will take care of marking the object. A general 105 // consequence of this is that the garbage collector may cache the 106 // value of mheap_.arena_used. (See issue #9984.) 107 // 108 // 109 // Stack writes: 110 // 111 // The compiler omits write barriers for writes to the current frame, 112 // but if a stack pointer has been passed down the call stack, the 113 // compiler will generate a write barrier for writes through that 114 // pointer (because it doesn't know it's not a heap pointer). 115 // 116 // One might be tempted to ignore the write barrier if slot points 117 // into to the stack. Don't do it! Mark termination only re-scans 118 // frames that have potentially been active since the concurrent scan, 119 // so it depends on write barriers to track changes to pointers in 120 // stack frames that have not been active. 121 // 122 // 123 // Global writes: 124 // 125 // The Go garbage collector requires write barriers when heap pointers 126 // are stored in globals. Many garbage collectors ignore writes to 127 // globals and instead pick up global -> heap pointers during 128 // termination. This increases pause time, so we instead rely on write 129 // barriers for writes to globals so that we don't have to rescan 130 // global during mark termination. 131 // 132 // 133 // Publication ordering: 134 // 135 // The write barrier is *pre-publication*, meaning that the write 136 // barrier happens prior to the *slot = ptr write that may make ptr 137 // reachable by some goroutine that currently cannot reach it. 138 // 139 // 140 //go:nowritebarrierrec 141 //go:systemstack 142 func gcmarkwb_m(slot *uintptr, ptr uintptr) { 143 if writeBarrier.needed { 144 // Note: This turns bad pointer writes into bad 145 // pointer reads, which could be confusing. We avoid 146 // reading from obviously bad pointers, which should 147 // take care of the vast majority of these. We could 148 // patch this up in the signal handler, or use XCHG to 149 // combine the read and the write. Checking inheap is 150 // insufficient since we need to track changes to 151 // roots outside the heap. 152 // 153 // Note: profbuf.go omits a barrier during signal handler 154 // profile logging; that's safe only because this deletion barrier exists. 155 // If we remove the deletion barrier, we'll have to work out 156 // a new way to handle the profile logging. 157 if slot1 := uintptr(unsafe.Pointer(slot)); slot1 >= minPhysPageSize { 158 if optr := *slot; optr != 0 { 159 shade(optr) 160 } 161 } 162 // TODO: Make this conditional on the caller's stack color. 163 if ptr != 0 && inheap(ptr) { 164 shade(ptr) 165 } 166 } 167 } 168 169 // writebarrierptr_prewrite1 invokes a write barrier for *dst = src 170 // prior to the write happening. 171 // 172 // Write barrier calls must not happen during critical GC and scheduler 173 // related operations. In particular there are times when the GC assumes 174 // that the world is stopped but scheduler related code is still being 175 // executed, dealing with syscalls, dealing with putting gs on runnable 176 // queues and so forth. This code cannot execute write barriers because 177 // the GC might drop them on the floor. Stopping the world involves removing 178 // the p associated with an m. We use the fact that m.p == nil to indicate 179 // that we are in one these critical section and throw if the write is of 180 // a pointer to a heap object. 181 //go:nosplit 182 func writebarrierptr_prewrite1(dst *uintptr, src uintptr) { 183 mp := acquirem() 184 if mp.inwb || mp.dying > 0 { 185 releasem(mp) 186 return 187 } 188 systemstack(func() { 189 if mp.p == 0 && memstats.enablegc && !mp.inwb && inheap(src) { 190 throw("writebarrierptr_prewrite1 called with mp.p == nil") 191 } 192 mp.inwb = true 193 gcmarkwb_m(dst, src) 194 }) 195 mp.inwb = false 196 releasem(mp) 197 } 198 199 // NOTE: Really dst *unsafe.Pointer, src unsafe.Pointer, 200 // but if we do that, Go inserts a write barrier on *dst = src. 201 //go:nosplit 202 func writebarrierptr(dst *uintptr, src uintptr) { 203 if writeBarrier.cgo { 204 cgoCheckWriteBarrier(dst, src) 205 } 206 if !writeBarrier.needed { 207 *dst = src 208 return 209 } 210 if src != 0 && src < minPhysPageSize { 211 systemstack(func() { 212 print("runtime: writebarrierptr *", dst, " = ", hex(src), "\n") 213 throw("bad pointer in write barrier") 214 }) 215 } 216 writebarrierptr_prewrite1(dst, src) 217 *dst = src 218 } 219 220 // writebarrierptr_prewrite is like writebarrierptr, but the store 221 // will be performed by the caller after this call. The caller must 222 // not allow preemption between this call and the write. 223 // 224 //go:nosplit 225 func writebarrierptr_prewrite(dst *uintptr, src uintptr) { 226 if writeBarrier.cgo { 227 cgoCheckWriteBarrier(dst, src) 228 } 229 if !writeBarrier.needed { 230 return 231 } 232 if src != 0 && src < minPhysPageSize { 233 systemstack(func() { throw("bad pointer in write barrier") }) 234 } 235 writebarrierptr_prewrite1(dst, src) 236 } 237 238 // typedmemmove copies a value of type t to dst from src. 239 // Must be nosplit, see #16026. 240 //go:nosplit 241 func typedmemmove(typ *_type, dst, src unsafe.Pointer) { 242 if typ.kind&kindNoPointers == 0 { 243 bulkBarrierPreWrite(uintptr(dst), uintptr(src), typ.size) 244 } 245 // There's a race here: if some other goroutine can write to 246 // src, it may change some pointer in src after we've 247 // performed the write barrier but before we perform the 248 // memory copy. This safe because the write performed by that 249 // other goroutine must also be accompanied by a write 250 // barrier, so at worst we've unnecessarily greyed the old 251 // pointer that was in src. 252 memmove(dst, src, typ.size) 253 if writeBarrier.cgo { 254 cgoCheckMemmove(typ, dst, src, 0, typ.size) 255 } 256 } 257 258 //go:linkname reflect_typedmemmove reflect.typedmemmove 259 func reflect_typedmemmove(typ *_type, dst, src unsafe.Pointer) { 260 if raceenabled { 261 raceWriteObjectPC(typ, dst, getcallerpc(unsafe.Pointer(&typ)), funcPC(reflect_typedmemmove)) 262 raceReadObjectPC(typ, src, getcallerpc(unsafe.Pointer(&typ)), funcPC(reflect_typedmemmove)) 263 } 264 if msanenabled { 265 msanwrite(dst, typ.size) 266 msanread(src, typ.size) 267 } 268 typedmemmove(typ, dst, src) 269 } 270 271 // typedmemmovepartial is like typedmemmove but assumes that 272 // dst and src point off bytes into the value and only copies size bytes. 273 //go:linkname reflect_typedmemmovepartial reflect.typedmemmovepartial 274 func reflect_typedmemmovepartial(typ *_type, dst, src unsafe.Pointer, off, size uintptr) { 275 if writeBarrier.needed && typ.kind&kindNoPointers == 0 && size >= sys.PtrSize { 276 // Pointer-align start address for bulk barrier. 277 adst, asrc, asize := dst, src, size 278 if frag := -off & (sys.PtrSize - 1); frag != 0 { 279 adst = add(dst, frag) 280 asrc = add(src, frag) 281 asize -= frag 282 } 283 bulkBarrierPreWrite(uintptr(adst), uintptr(asrc), asize&^(sys.PtrSize-1)) 284 } 285 286 memmove(dst, src, size) 287 if writeBarrier.cgo { 288 cgoCheckMemmove(typ, dst, src, off, size) 289 } 290 } 291 292 // reflectcallmove is invoked by reflectcall to copy the return values 293 // out of the stack and into the heap, invoking the necessary write 294 // barriers. dst, src, and size describe the return value area to 295 // copy. typ describes the entire frame (not just the return values). 296 // typ may be nil, which indicates write barriers are not needed. 297 // 298 // It must be nosplit and must only call nosplit functions because the 299 // stack map of reflectcall is wrong. 300 // 301 //go:nosplit 302 func reflectcallmove(typ *_type, dst, src unsafe.Pointer, size uintptr) { 303 if writeBarrier.needed && typ != nil && typ.kind&kindNoPointers == 0 && size >= sys.PtrSize { 304 bulkBarrierPreWrite(uintptr(dst), uintptr(src), size) 305 } 306 memmove(dst, src, size) 307 } 308 309 //go:nosplit 310 func typedslicecopy(typ *_type, dst, src slice) int { 311 // TODO(rsc): If typedslicecopy becomes faster than calling 312 // typedmemmove repeatedly, consider using during func growslice. 313 n := dst.len 314 if n > src.len { 315 n = src.len 316 } 317 if n == 0 { 318 return 0 319 } 320 dstp := dst.array 321 srcp := src.array 322 323 if raceenabled { 324 callerpc := getcallerpc(unsafe.Pointer(&typ)) 325 pc := funcPC(slicecopy) 326 racewriterangepc(dstp, uintptr(n)*typ.size, callerpc, pc) 327 racereadrangepc(srcp, uintptr(n)*typ.size, callerpc, pc) 328 } 329 if msanenabled { 330 msanwrite(dstp, uintptr(n)*typ.size) 331 msanread(srcp, uintptr(n)*typ.size) 332 } 333 334 if writeBarrier.cgo { 335 cgoCheckSliceCopy(typ, dst, src, n) 336 } 337 338 // Note: No point in checking typ.kind&kindNoPointers here: 339 // compiler only emits calls to typedslicecopy for types with pointers, 340 // and growslice and reflect_typedslicecopy check for pointers 341 // before calling typedslicecopy. 342 if !writeBarrier.needed { 343 memmove(dstp, srcp, uintptr(n)*typ.size) 344 return n 345 } 346 347 systemstack(func() { 348 if uintptr(srcp) < uintptr(dstp) && uintptr(srcp)+uintptr(n)*typ.size > uintptr(dstp) { 349 // Overlap with src before dst. 350 // Copy backward, being careful not to move dstp/srcp 351 // out of the array they point into. 352 dstp = add(dstp, uintptr(n-1)*typ.size) 353 srcp = add(srcp, uintptr(n-1)*typ.size) 354 i := 0 355 for { 356 typedmemmove(typ, dstp, srcp) 357 if i++; i >= n { 358 break 359 } 360 dstp = add(dstp, -typ.size) 361 srcp = add(srcp, -typ.size) 362 } 363 } else { 364 // Copy forward, being careful not to move dstp/srcp 365 // out of the array they point into. 366 i := 0 367 for { 368 typedmemmove(typ, dstp, srcp) 369 if i++; i >= n { 370 break 371 } 372 dstp = add(dstp, typ.size) 373 srcp = add(srcp, typ.size) 374 } 375 } 376 }) 377 return n 378 } 379 380 //go:linkname reflect_typedslicecopy reflect.typedslicecopy 381 func reflect_typedslicecopy(elemType *_type, dst, src slice) int { 382 if elemType.kind&kindNoPointers != 0 { 383 n := dst.len 384 if n > src.len { 385 n = src.len 386 } 387 if n == 0 { 388 return 0 389 } 390 391 size := uintptr(n) * elemType.size 392 if raceenabled { 393 callerpc := getcallerpc(unsafe.Pointer(&elemType)) 394 pc := funcPC(reflect_typedslicecopy) 395 racewriterangepc(dst.array, size, callerpc, pc) 396 racereadrangepc(src.array, size, callerpc, pc) 397 } 398 if msanenabled { 399 msanwrite(dst.array, size) 400 msanread(src.array, size) 401 } 402 403 memmove(dst.array, src.array, size) 404 return n 405 } 406 return typedslicecopy(elemType, dst, src) 407 } 408 409 // typedmemclr clears the typed memory at ptr with type typ. The 410 // memory at ptr must already be initialized (and hence in type-safe 411 // state). If the memory is being initialized for the first time, see 412 // memclrNoHeapPointers. 413 // 414 // If the caller knows that typ has pointers, it can alternatively 415 // call memclrHasPointers. 416 // 417 //go:nosplit 418 func typedmemclr(typ *_type, ptr unsafe.Pointer) { 419 if typ.kind&kindNoPointers == 0 { 420 bulkBarrierPreWrite(uintptr(ptr), 0, typ.size) 421 } 422 memclrNoHeapPointers(ptr, typ.size) 423 } 424 425 // memclrHasPointers clears n bytes of typed memory starting at ptr. 426 // The caller must ensure that the type of the object at ptr has 427 // pointers, usually by checking typ.kind&kindNoPointers. However, ptr 428 // does not have to point to the start of the allocation. 429 // 430 //go:nosplit 431 func memclrHasPointers(ptr unsafe.Pointer, n uintptr) { 432 bulkBarrierPreWrite(uintptr(ptr), 0, n) 433 memclrNoHeapPointers(ptr, n) 434 }