github.com/m10x/go/src@v0.0.0-20220112094212-ba61592315da/runtime/mgcwork.go (about) 1 // Copyright 2009 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package runtime 6 7 import ( 8 "internal/goarch" 9 "runtime/internal/atomic" 10 "unsafe" 11 ) 12 13 const ( 14 _WorkbufSize = 2048 // in bytes; larger values result in less contention 15 16 // workbufAlloc is the number of bytes to allocate at a time 17 // for new workbufs. This must be a multiple of pageSize and 18 // should be a multiple of _WorkbufSize. 19 // 20 // Larger values reduce workbuf allocation overhead. Smaller 21 // values reduce heap fragmentation. 22 workbufAlloc = 32 << 10 23 ) 24 25 func init() { 26 if workbufAlloc%pageSize != 0 || workbufAlloc%_WorkbufSize != 0 { 27 throw("bad workbufAlloc") 28 } 29 } 30 31 // Garbage collector work pool abstraction. 32 // 33 // This implements a producer/consumer model for pointers to grey 34 // objects. A grey object is one that is marked and on a work 35 // queue. A black object is marked and not on a work queue. 36 // 37 // Write barriers, root discovery, stack scanning, and object scanning 38 // produce pointers to grey objects. Scanning consumes pointers to 39 // grey objects, thus blackening them, and then scans them, 40 // potentially producing new pointers to grey objects. 41 42 // A gcWork provides the interface to produce and consume work for the 43 // garbage collector. 44 // 45 // A gcWork can be used on the stack as follows: 46 // 47 // (preemption must be disabled) 48 // gcw := &getg().m.p.ptr().gcw 49 // .. call gcw.put() to produce and gcw.tryGet() to consume .. 50 // 51 // It's important that any use of gcWork during the mark phase prevent 52 // the garbage collector from transitioning to mark termination since 53 // gcWork may locally hold GC work buffers. This can be done by 54 // disabling preemption (systemstack or acquirem). 55 type gcWork struct { 56 // wbuf1 and wbuf2 are the primary and secondary work buffers. 57 // 58 // This can be thought of as a stack of both work buffers' 59 // pointers concatenated. When we pop the last pointer, we 60 // shift the stack up by one work buffer by bringing in a new 61 // full buffer and discarding an empty one. When we fill both 62 // buffers, we shift the stack down by one work buffer by 63 // bringing in a new empty buffer and discarding a full one. 64 // This way we have one buffer's worth of hysteresis, which 65 // amortizes the cost of getting or putting a work buffer over 66 // at least one buffer of work and reduces contention on the 67 // global work lists. 68 // 69 // wbuf1 is always the buffer we're currently pushing to and 70 // popping from and wbuf2 is the buffer that will be discarded 71 // next. 72 // 73 // Invariant: Both wbuf1 and wbuf2 are nil or neither are. 74 wbuf1, wbuf2 *workbuf 75 76 // Bytes marked (blackened) on this gcWork. This is aggregated 77 // into work.bytesMarked by dispose. 78 bytesMarked uint64 79 80 // Heap scan work performed on this gcWork. This is aggregated into 81 // gcController by dispose and may also be flushed by callers. 82 // Other types of scan work are flushed immediately. 83 heapScanWork int64 84 85 // flushedWork indicates that a non-empty work buffer was 86 // flushed to the global work list since the last gcMarkDone 87 // termination check. Specifically, this indicates that this 88 // gcWork may have communicated work to another gcWork. 89 flushedWork bool 90 } 91 92 // Most of the methods of gcWork are go:nowritebarrierrec because the 93 // write barrier itself can invoke gcWork methods but the methods are 94 // not generally re-entrant. Hence, if a gcWork method invoked the 95 // write barrier while the gcWork was in an inconsistent state, and 96 // the write barrier in turn invoked a gcWork method, it could 97 // permanently corrupt the gcWork. 98 99 func (w *gcWork) init() { 100 w.wbuf1 = getempty() 101 wbuf2 := trygetfull() 102 if wbuf2 == nil { 103 wbuf2 = getempty() 104 } 105 w.wbuf2 = wbuf2 106 } 107 108 // put enqueues a pointer for the garbage collector to trace. 109 // obj must point to the beginning of a heap object or an oblet. 110 //go:nowritebarrierrec 111 func (w *gcWork) put(obj uintptr) { 112 flushed := false 113 wbuf := w.wbuf1 114 // Record that this may acquire the wbufSpans or heap lock to 115 // allocate a workbuf. 116 lockWithRankMayAcquire(&work.wbufSpans.lock, lockRankWbufSpans) 117 lockWithRankMayAcquire(&mheap_.lock, lockRankMheap) 118 if wbuf == nil { 119 w.init() 120 wbuf = w.wbuf1 121 // wbuf is empty at this point. 122 } else if wbuf.nobj == len(wbuf.obj) { 123 w.wbuf1, w.wbuf2 = w.wbuf2, w.wbuf1 124 wbuf = w.wbuf1 125 if wbuf.nobj == len(wbuf.obj) { 126 putfull(wbuf) 127 w.flushedWork = true 128 wbuf = getempty() 129 w.wbuf1 = wbuf 130 flushed = true 131 } 132 } 133 134 wbuf.obj[wbuf.nobj] = obj 135 wbuf.nobj++ 136 137 // If we put a buffer on full, let the GC controller know so 138 // it can encourage more workers to run. We delay this until 139 // the end of put so that w is in a consistent state, since 140 // enlistWorker may itself manipulate w. 141 if flushed && gcphase == _GCmark { 142 gcController.enlistWorker() 143 } 144 } 145 146 // putFast does a put and reports whether it can be done quickly 147 // otherwise it returns false and the caller needs to call put. 148 //go:nowritebarrierrec 149 func (w *gcWork) putFast(obj uintptr) bool { 150 wbuf := w.wbuf1 151 if wbuf == nil { 152 return false 153 } else if wbuf.nobj == len(wbuf.obj) { 154 return false 155 } 156 157 wbuf.obj[wbuf.nobj] = obj 158 wbuf.nobj++ 159 return true 160 } 161 162 // putBatch performs a put on every pointer in obj. See put for 163 // constraints on these pointers. 164 // 165 //go:nowritebarrierrec 166 func (w *gcWork) putBatch(obj []uintptr) { 167 if len(obj) == 0 { 168 return 169 } 170 171 flushed := false 172 wbuf := w.wbuf1 173 if wbuf == nil { 174 w.init() 175 wbuf = w.wbuf1 176 } 177 178 for len(obj) > 0 { 179 for wbuf.nobj == len(wbuf.obj) { 180 putfull(wbuf) 181 w.flushedWork = true 182 w.wbuf1, w.wbuf2 = w.wbuf2, getempty() 183 wbuf = w.wbuf1 184 flushed = true 185 } 186 n := copy(wbuf.obj[wbuf.nobj:], obj) 187 wbuf.nobj += n 188 obj = obj[n:] 189 } 190 191 if flushed && gcphase == _GCmark { 192 gcController.enlistWorker() 193 } 194 } 195 196 // tryGet dequeues a pointer for the garbage collector to trace. 197 // 198 // If there are no pointers remaining in this gcWork or in the global 199 // queue, tryGet returns 0. Note that there may still be pointers in 200 // other gcWork instances or other caches. 201 //go:nowritebarrierrec 202 func (w *gcWork) tryGet() uintptr { 203 wbuf := w.wbuf1 204 if wbuf == nil { 205 w.init() 206 wbuf = w.wbuf1 207 // wbuf is empty at this point. 208 } 209 if wbuf.nobj == 0 { 210 w.wbuf1, w.wbuf2 = w.wbuf2, w.wbuf1 211 wbuf = w.wbuf1 212 if wbuf.nobj == 0 { 213 owbuf := wbuf 214 wbuf = trygetfull() 215 if wbuf == nil { 216 return 0 217 } 218 putempty(owbuf) 219 w.wbuf1 = wbuf 220 } 221 } 222 223 wbuf.nobj-- 224 return wbuf.obj[wbuf.nobj] 225 } 226 227 // tryGetFast dequeues a pointer for the garbage collector to trace 228 // if one is readily available. Otherwise it returns 0 and 229 // the caller is expected to call tryGet(). 230 //go:nowritebarrierrec 231 func (w *gcWork) tryGetFast() uintptr { 232 wbuf := w.wbuf1 233 if wbuf == nil { 234 return 0 235 } 236 if wbuf.nobj == 0 { 237 return 0 238 } 239 240 wbuf.nobj-- 241 return wbuf.obj[wbuf.nobj] 242 } 243 244 // dispose returns any cached pointers to the global queue. 245 // The buffers are being put on the full queue so that the 246 // write barriers will not simply reacquire them before the 247 // GC can inspect them. This helps reduce the mutator's 248 // ability to hide pointers during the concurrent mark phase. 249 // 250 //go:nowritebarrierrec 251 func (w *gcWork) dispose() { 252 if wbuf := w.wbuf1; wbuf != nil { 253 if wbuf.nobj == 0 { 254 putempty(wbuf) 255 } else { 256 putfull(wbuf) 257 w.flushedWork = true 258 } 259 w.wbuf1 = nil 260 261 wbuf = w.wbuf2 262 if wbuf.nobj == 0 { 263 putempty(wbuf) 264 } else { 265 putfull(wbuf) 266 w.flushedWork = true 267 } 268 w.wbuf2 = nil 269 } 270 if w.bytesMarked != 0 { 271 // dispose happens relatively infrequently. If this 272 // atomic becomes a problem, we should first try to 273 // dispose less and if necessary aggregate in a per-P 274 // counter. 275 atomic.Xadd64(&work.bytesMarked, int64(w.bytesMarked)) 276 w.bytesMarked = 0 277 } 278 if w.heapScanWork != 0 { 279 gcController.heapScanWork.Add(w.heapScanWork) 280 w.heapScanWork = 0 281 } 282 } 283 284 // balance moves some work that's cached in this gcWork back on the 285 // global queue. 286 //go:nowritebarrierrec 287 func (w *gcWork) balance() { 288 if w.wbuf1 == nil { 289 return 290 } 291 if wbuf := w.wbuf2; wbuf.nobj != 0 { 292 putfull(wbuf) 293 w.flushedWork = true 294 w.wbuf2 = getempty() 295 } else if wbuf := w.wbuf1; wbuf.nobj > 4 { 296 w.wbuf1 = handoff(wbuf) 297 w.flushedWork = true // handoff did putfull 298 } else { 299 return 300 } 301 // We flushed a buffer to the full list, so wake a worker. 302 if gcphase == _GCmark { 303 gcController.enlistWorker() 304 } 305 } 306 307 // empty reports whether w has no mark work available. 308 //go:nowritebarrierrec 309 func (w *gcWork) empty() bool { 310 return w.wbuf1 == nil || (w.wbuf1.nobj == 0 && w.wbuf2.nobj == 0) 311 } 312 313 // Internally, the GC work pool is kept in arrays in work buffers. 314 // The gcWork interface caches a work buffer until full (or empty) to 315 // avoid contending on the global work buffer lists. 316 317 type workbufhdr struct { 318 node lfnode // must be first 319 nobj int 320 } 321 322 //go:notinheap 323 type workbuf struct { 324 workbufhdr 325 // account for the above fields 326 obj [(_WorkbufSize - unsafe.Sizeof(workbufhdr{})) / goarch.PtrSize]uintptr 327 } 328 329 // workbuf factory routines. These funcs are used to manage the 330 // workbufs. 331 // If the GC asks for some work these are the only routines that 332 // make wbufs available to the GC. 333 334 func (b *workbuf) checknonempty() { 335 if b.nobj == 0 { 336 throw("workbuf is empty") 337 } 338 } 339 340 func (b *workbuf) checkempty() { 341 if b.nobj != 0 { 342 throw("workbuf is not empty") 343 } 344 } 345 346 // getempty pops an empty work buffer off the work.empty list, 347 // allocating new buffers if none are available. 348 //go:nowritebarrier 349 func getempty() *workbuf { 350 var b *workbuf 351 if work.empty != 0 { 352 b = (*workbuf)(work.empty.pop()) 353 if b != nil { 354 b.checkempty() 355 } 356 } 357 // Record that this may acquire the wbufSpans or heap lock to 358 // allocate a workbuf. 359 lockWithRankMayAcquire(&work.wbufSpans.lock, lockRankWbufSpans) 360 lockWithRankMayAcquire(&mheap_.lock, lockRankMheap) 361 if b == nil { 362 // Allocate more workbufs. 363 var s *mspan 364 if work.wbufSpans.free.first != nil { 365 lock(&work.wbufSpans.lock) 366 s = work.wbufSpans.free.first 367 if s != nil { 368 work.wbufSpans.free.remove(s) 369 work.wbufSpans.busy.insert(s) 370 } 371 unlock(&work.wbufSpans.lock) 372 } 373 if s == nil { 374 systemstack(func() { 375 s = mheap_.allocManual(workbufAlloc/pageSize, spanAllocWorkBuf) 376 }) 377 if s == nil { 378 throw("out of memory") 379 } 380 // Record the new span in the busy list. 381 lock(&work.wbufSpans.lock) 382 work.wbufSpans.busy.insert(s) 383 unlock(&work.wbufSpans.lock) 384 } 385 // Slice up the span into new workbufs. Return one and 386 // put the rest on the empty list. 387 for i := uintptr(0); i+_WorkbufSize <= workbufAlloc; i += _WorkbufSize { 388 newb := (*workbuf)(unsafe.Pointer(s.base() + i)) 389 newb.nobj = 0 390 lfnodeValidate(&newb.node) 391 if i == 0 { 392 b = newb 393 } else { 394 putempty(newb) 395 } 396 } 397 } 398 return b 399 } 400 401 // putempty puts a workbuf onto the work.empty list. 402 // Upon entry this goroutine owns b. The lfstack.push relinquishes ownership. 403 //go:nowritebarrier 404 func putempty(b *workbuf) { 405 b.checkempty() 406 work.empty.push(&b.node) 407 } 408 409 // putfull puts the workbuf on the work.full list for the GC. 410 // putfull accepts partially full buffers so the GC can avoid competing 411 // with the mutators for ownership of partially full buffers. 412 //go:nowritebarrier 413 func putfull(b *workbuf) { 414 b.checknonempty() 415 work.full.push(&b.node) 416 } 417 418 // trygetfull tries to get a full or partially empty workbuffer. 419 // If one is not immediately available return nil 420 //go:nowritebarrier 421 func trygetfull() *workbuf { 422 b := (*workbuf)(work.full.pop()) 423 if b != nil { 424 b.checknonempty() 425 return b 426 } 427 return b 428 } 429 430 //go:nowritebarrier 431 func handoff(b *workbuf) *workbuf { 432 // Make new buffer with half of b's pointers. 433 b1 := getempty() 434 n := b.nobj / 2 435 b.nobj -= n 436 b1.nobj = n 437 memmove(unsafe.Pointer(&b1.obj[0]), unsafe.Pointer(&b.obj[b.nobj]), uintptr(n)*unsafe.Sizeof(b1.obj[0])) 438 439 // Put b on full list - let first half of b get stolen. 440 putfull(b) 441 return b1 442 } 443 444 // prepareFreeWorkbufs moves busy workbuf spans to free list so they 445 // can be freed to the heap. This must only be called when all 446 // workbufs are on the empty list. 447 func prepareFreeWorkbufs() { 448 lock(&work.wbufSpans.lock) 449 if work.full != 0 { 450 throw("cannot free workbufs when work.full != 0") 451 } 452 // Since all workbufs are on the empty list, we don't care 453 // which ones are in which spans. We can wipe the entire empty 454 // list and move all workbuf spans to the free list. 455 work.empty = 0 456 work.wbufSpans.free.takeAll(&work.wbufSpans.busy) 457 unlock(&work.wbufSpans.lock) 458 } 459 460 // freeSomeWbufs frees some workbufs back to the heap and returns 461 // true if it should be called again to free more. 462 func freeSomeWbufs(preemptible bool) bool { 463 const batchSize = 64 // ~1–2 µs per span. 464 lock(&work.wbufSpans.lock) 465 if gcphase != _GCoff || work.wbufSpans.free.isEmpty() { 466 unlock(&work.wbufSpans.lock) 467 return false 468 } 469 systemstack(func() { 470 gp := getg().m.curg 471 for i := 0; i < batchSize && !(preemptible && gp.preempt); i++ { 472 span := work.wbufSpans.free.first 473 if span == nil { 474 break 475 } 476 work.wbufSpans.free.remove(span) 477 mheap_.freeManual(span, spanAllocWorkBuf) 478 } 479 }) 480 more := !work.wbufSpans.free.isEmpty() 481 unlock(&work.wbufSpans.lock) 482 return more 483 }