golang.org/toolchain@v0.0.1-go1.9rc2.windows-amd64/src/runtime/mgcwork.go (about) 1 // Copyright 2009 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package runtime 6 7 import ( 8 "runtime/internal/atomic" 9 "runtime/internal/sys" 10 "unsafe" 11 ) 12 13 const ( 14 _WorkbufSize = 2048 // in bytes; larger values result in less contention 15 16 // workbufAlloc is the number of bytes to allocate at a time 17 // for new workbufs. This must be a multiple of pageSize and 18 // should be a multiple of _WorkbufSize. 19 // 20 // Larger values reduce workbuf allocation overhead. Smaller 21 // values reduce heap fragmentation. 22 workbufAlloc = 32 << 10 23 ) 24 25 func init() { 26 if workbufAlloc%pageSize != 0 || workbufAlloc%_WorkbufSize != 0 { 27 throw("bad workbufAlloc") 28 } 29 } 30 31 // Garbage collector work pool abstraction. 32 // 33 // This implements a producer/consumer model for pointers to grey 34 // objects. A grey object is one that is marked and on a work 35 // queue. A black object is marked and not on a work queue. 36 // 37 // Write barriers, root discovery, stack scanning, and object scanning 38 // produce pointers to grey objects. Scanning consumes pointers to 39 // grey objects, thus blackening them, and then scans them, 40 // potentially producing new pointers to grey objects. 41 42 // A gcWork provides the interface to produce and consume work for the 43 // garbage collector. 44 // 45 // A gcWork can be used on the stack as follows: 46 // 47 // (preemption must be disabled) 48 // gcw := &getg().m.p.ptr().gcw 49 // .. call gcw.put() to produce and gcw.get() to consume .. 50 // if gcBlackenPromptly { 51 // gcw.dispose() 52 // } 53 // 54 // It's important that any use of gcWork during the mark phase prevent 55 // the garbage collector from transitioning to mark termination since 56 // gcWork may locally hold GC work buffers. This can be done by 57 // disabling preemption (systemstack or acquirem). 58 type gcWork struct { 59 // wbuf1 and wbuf2 are the primary and secondary work buffers. 60 // 61 // This can be thought of as a stack of both work buffers' 62 // pointers concatenated. When we pop the last pointer, we 63 // shift the stack up by one work buffer by bringing in a new 64 // full buffer and discarding an empty one. When we fill both 65 // buffers, we shift the stack down by one work buffer by 66 // bringing in a new empty buffer and discarding a full one. 67 // This way we have one buffer's worth of hysteresis, which 68 // amortizes the cost of getting or putting a work buffer over 69 // at least one buffer of work and reduces contention on the 70 // global work lists. 71 // 72 // wbuf1 is always the buffer we're currently pushing to and 73 // popping from and wbuf2 is the buffer that will be discarded 74 // next. 75 // 76 // Invariant: Both wbuf1 and wbuf2 are nil or neither are. 77 wbuf1, wbuf2 *workbuf 78 79 // Bytes marked (blackened) on this gcWork. This is aggregated 80 // into work.bytesMarked by dispose. 81 bytesMarked uint64 82 83 // Scan work performed on this gcWork. This is aggregated into 84 // gcController by dispose and may also be flushed by callers. 85 scanWork int64 86 } 87 88 func (w *gcWork) init() { 89 w.wbuf1 = getempty() 90 wbuf2 := trygetfull() 91 if wbuf2 == nil { 92 wbuf2 = getempty() 93 } 94 w.wbuf2 = wbuf2 95 } 96 97 // put enqueues a pointer for the garbage collector to trace. 98 // obj must point to the beginning of a heap object or an oblet. 99 //go:nowritebarrier 100 func (w *gcWork) put(obj uintptr) { 101 flushed := false 102 wbuf := w.wbuf1 103 if wbuf == nil { 104 w.init() 105 wbuf = w.wbuf1 106 // wbuf is empty at this point. 107 } else if wbuf.nobj == len(wbuf.obj) { 108 w.wbuf1, w.wbuf2 = w.wbuf2, w.wbuf1 109 wbuf = w.wbuf1 110 if wbuf.nobj == len(wbuf.obj) { 111 putfull(wbuf) 112 wbuf = getempty() 113 w.wbuf1 = wbuf 114 flushed = true 115 } 116 } 117 118 wbuf.obj[wbuf.nobj] = obj 119 wbuf.nobj++ 120 121 // If we put a buffer on full, let the GC controller know so 122 // it can encourage more workers to run. We delay this until 123 // the end of put so that w is in a consistent state, since 124 // enlistWorker may itself manipulate w. 125 if flushed && gcphase == _GCmark { 126 gcController.enlistWorker() 127 } 128 } 129 130 // putFast does a put and returns true if it can be done quickly 131 // otherwise it returns false and the caller needs to call put. 132 //go:nowritebarrier 133 func (w *gcWork) putFast(obj uintptr) bool { 134 wbuf := w.wbuf1 135 if wbuf == nil { 136 return false 137 } else if wbuf.nobj == len(wbuf.obj) { 138 return false 139 } 140 141 wbuf.obj[wbuf.nobj] = obj 142 wbuf.nobj++ 143 return true 144 } 145 146 // tryGet dequeues a pointer for the garbage collector to trace. 147 // 148 // If there are no pointers remaining in this gcWork or in the global 149 // queue, tryGet returns 0. Note that there may still be pointers in 150 // other gcWork instances or other caches. 151 //go:nowritebarrier 152 func (w *gcWork) tryGet() uintptr { 153 wbuf := w.wbuf1 154 if wbuf == nil { 155 w.init() 156 wbuf = w.wbuf1 157 // wbuf is empty at this point. 158 } 159 if wbuf.nobj == 0 { 160 w.wbuf1, w.wbuf2 = w.wbuf2, w.wbuf1 161 wbuf = w.wbuf1 162 if wbuf.nobj == 0 { 163 owbuf := wbuf 164 wbuf = trygetfull() 165 if wbuf == nil { 166 return 0 167 } 168 putempty(owbuf) 169 w.wbuf1 = wbuf 170 } 171 } 172 173 wbuf.nobj-- 174 return wbuf.obj[wbuf.nobj] 175 } 176 177 // tryGetFast dequeues a pointer for the garbage collector to trace 178 // if one is readily available. Otherwise it returns 0 and 179 // the caller is expected to call tryGet(). 180 //go:nowritebarrier 181 func (w *gcWork) tryGetFast() uintptr { 182 wbuf := w.wbuf1 183 if wbuf == nil { 184 return 0 185 } 186 if wbuf.nobj == 0 { 187 return 0 188 } 189 190 wbuf.nobj-- 191 return wbuf.obj[wbuf.nobj] 192 } 193 194 // get dequeues a pointer for the garbage collector to trace, blocking 195 // if necessary to ensure all pointers from all queues and caches have 196 // been retrieved. get returns 0 if there are no pointers remaining. 197 //go:nowritebarrier 198 func (w *gcWork) get() uintptr { 199 wbuf := w.wbuf1 200 if wbuf == nil { 201 w.init() 202 wbuf = w.wbuf1 203 // wbuf is empty at this point. 204 } 205 if wbuf.nobj == 0 { 206 w.wbuf1, w.wbuf2 = w.wbuf2, w.wbuf1 207 wbuf = w.wbuf1 208 if wbuf.nobj == 0 { 209 owbuf := wbuf 210 wbuf = getfull() 211 if wbuf == nil { 212 return 0 213 } 214 putempty(owbuf) 215 w.wbuf1 = wbuf 216 } 217 } 218 219 // TODO: This might be a good place to add prefetch code 220 221 wbuf.nobj-- 222 return wbuf.obj[wbuf.nobj] 223 } 224 225 // dispose returns any cached pointers to the global queue. 226 // The buffers are being put on the full queue so that the 227 // write barriers will not simply reacquire them before the 228 // GC can inspect them. This helps reduce the mutator's 229 // ability to hide pointers during the concurrent mark phase. 230 // 231 //go:nowritebarrier 232 func (w *gcWork) dispose() { 233 if wbuf := w.wbuf1; wbuf != nil { 234 if wbuf.nobj == 0 { 235 putempty(wbuf) 236 } else { 237 putfull(wbuf) 238 } 239 w.wbuf1 = nil 240 241 wbuf = w.wbuf2 242 if wbuf.nobj == 0 { 243 putempty(wbuf) 244 } else { 245 putfull(wbuf) 246 } 247 w.wbuf2 = nil 248 } 249 if w.bytesMarked != 0 { 250 // dispose happens relatively infrequently. If this 251 // atomic becomes a problem, we should first try to 252 // dispose less and if necessary aggregate in a per-P 253 // counter. 254 atomic.Xadd64(&work.bytesMarked, int64(w.bytesMarked)) 255 w.bytesMarked = 0 256 } 257 if w.scanWork != 0 { 258 atomic.Xaddint64(&gcController.scanWork, w.scanWork) 259 w.scanWork = 0 260 } 261 } 262 263 // balance moves some work that's cached in this gcWork back on the 264 // global queue. 265 //go:nowritebarrier 266 func (w *gcWork) balance() { 267 if w.wbuf1 == nil { 268 return 269 } 270 if wbuf := w.wbuf2; wbuf.nobj != 0 { 271 putfull(wbuf) 272 w.wbuf2 = getempty() 273 } else if wbuf := w.wbuf1; wbuf.nobj > 4 { 274 w.wbuf1 = handoff(wbuf) 275 } else { 276 return 277 } 278 // We flushed a buffer to the full list, so wake a worker. 279 if gcphase == _GCmark { 280 gcController.enlistWorker() 281 } 282 } 283 284 // empty returns true if w has no mark work available. 285 //go:nowritebarrier 286 func (w *gcWork) empty() bool { 287 return w.wbuf1 == nil || (w.wbuf1.nobj == 0 && w.wbuf2.nobj == 0) 288 } 289 290 // Internally, the GC work pool is kept in arrays in work buffers. 291 // The gcWork interface caches a work buffer until full (or empty) to 292 // avoid contending on the global work buffer lists. 293 294 type workbufhdr struct { 295 node lfnode // must be first 296 nobj int 297 } 298 299 //go:notinheap 300 type workbuf struct { 301 workbufhdr 302 // account for the above fields 303 obj [(_WorkbufSize - unsafe.Sizeof(workbufhdr{})) / sys.PtrSize]uintptr 304 } 305 306 // workbuf factory routines. These funcs are used to manage the 307 // workbufs. 308 // If the GC asks for some work these are the only routines that 309 // make wbufs available to the GC. 310 311 func (b *workbuf) checknonempty() { 312 if b.nobj == 0 { 313 throw("workbuf is empty") 314 } 315 } 316 317 func (b *workbuf) checkempty() { 318 if b.nobj != 0 { 319 throw("workbuf is not empty") 320 } 321 } 322 323 // getempty pops an empty work buffer off the work.empty list, 324 // allocating new buffers if none are available. 325 //go:nowritebarrier 326 func getempty() *workbuf { 327 var b *workbuf 328 if work.empty != 0 { 329 b = (*workbuf)(work.empty.pop()) 330 if b != nil { 331 b.checkempty() 332 } 333 } 334 if b == nil { 335 // Allocate more workbufs. 336 var s *mspan 337 if work.wbufSpans.free.first != nil { 338 lock(&work.wbufSpans.lock) 339 s = work.wbufSpans.free.first 340 if s != nil { 341 work.wbufSpans.free.remove(s) 342 work.wbufSpans.busy.insert(s) 343 } 344 unlock(&work.wbufSpans.lock) 345 } 346 if s == nil { 347 systemstack(func() { 348 s = mheap_.allocManual(workbufAlloc/pageSize, &memstats.gc_sys) 349 }) 350 if s == nil { 351 throw("out of memory") 352 } 353 // Record the new span in the busy list. 354 lock(&work.wbufSpans.lock) 355 work.wbufSpans.busy.insert(s) 356 unlock(&work.wbufSpans.lock) 357 } 358 // Slice up the span into new workbufs. Return one and 359 // put the rest on the empty list. 360 for i := uintptr(0); i+_WorkbufSize <= workbufAlloc; i += _WorkbufSize { 361 newb := (*workbuf)(unsafe.Pointer(s.base() + i)) 362 newb.nobj = 0 363 if i == 0 { 364 b = newb 365 } else { 366 putempty(newb) 367 } 368 } 369 } 370 return b 371 } 372 373 // putempty puts a workbuf onto the work.empty list. 374 // Upon entry this go routine owns b. The lfstack.push relinquishes ownership. 375 //go:nowritebarrier 376 func putempty(b *workbuf) { 377 b.checkempty() 378 work.empty.push(&b.node) 379 } 380 381 // putfull puts the workbuf on the work.full list for the GC. 382 // putfull accepts partially full buffers so the GC can avoid competing 383 // with the mutators for ownership of partially full buffers. 384 //go:nowritebarrier 385 func putfull(b *workbuf) { 386 b.checknonempty() 387 work.full.push(&b.node) 388 } 389 390 // trygetfull tries to get a full or partially empty workbuffer. 391 // If one is not immediately available return nil 392 //go:nowritebarrier 393 func trygetfull() *workbuf { 394 b := (*workbuf)(work.full.pop()) 395 if b != nil { 396 b.checknonempty() 397 return b 398 } 399 return b 400 } 401 402 // Get a full work buffer off the work.full list. 403 // If nothing is available wait until all the other gc helpers have 404 // finished and then return nil. 405 // getfull acts as a barrier for work.nproc helpers. As long as one 406 // gchelper is actively marking objects it 407 // may create a workbuffer that the other helpers can work on. 408 // The for loop either exits when a work buffer is found 409 // or when _all_ of the work.nproc GC helpers are in the loop 410 // looking for work and thus not capable of creating new work. 411 // This is in fact the termination condition for the STW mark 412 // phase. 413 //go:nowritebarrier 414 func getfull() *workbuf { 415 b := (*workbuf)(work.full.pop()) 416 if b != nil { 417 b.checknonempty() 418 return b 419 } 420 421 incnwait := atomic.Xadd(&work.nwait, +1) 422 if incnwait > work.nproc { 423 println("runtime: work.nwait=", incnwait, "work.nproc=", work.nproc) 424 throw("work.nwait > work.nproc") 425 } 426 for i := 0; ; i++ { 427 if work.full != 0 { 428 decnwait := atomic.Xadd(&work.nwait, -1) 429 if decnwait == work.nproc { 430 println("runtime: work.nwait=", decnwait, "work.nproc=", work.nproc) 431 throw("work.nwait > work.nproc") 432 } 433 b = (*workbuf)(work.full.pop()) 434 if b != nil { 435 b.checknonempty() 436 return b 437 } 438 incnwait := atomic.Xadd(&work.nwait, +1) 439 if incnwait > work.nproc { 440 println("runtime: work.nwait=", incnwait, "work.nproc=", work.nproc) 441 throw("work.nwait > work.nproc") 442 } 443 } 444 if work.nwait == work.nproc && work.markrootNext >= work.markrootJobs { 445 return nil 446 } 447 if i < 10 { 448 procyield(20) 449 } else if i < 20 { 450 osyield() 451 } else { 452 usleep(100) 453 } 454 } 455 } 456 457 //go:nowritebarrier 458 func handoff(b *workbuf) *workbuf { 459 // Make new buffer with half of b's pointers. 460 b1 := getempty() 461 n := b.nobj / 2 462 b.nobj -= n 463 b1.nobj = n 464 memmove(unsafe.Pointer(&b1.obj[0]), unsafe.Pointer(&b.obj[b.nobj]), uintptr(n)*unsafe.Sizeof(b1.obj[0])) 465 466 // Put b on full list - let first half of b get stolen. 467 putfull(b) 468 return b1 469 } 470 471 // prepareFreeWorkbufs moves busy workbuf spans to free list so they 472 // can be freed to the heap. This must only be called when all 473 // workbufs are on the empty list. 474 func prepareFreeWorkbufs() { 475 lock(&work.wbufSpans.lock) 476 if work.full != 0 { 477 throw("cannot free workbufs when work.full != 0") 478 } 479 // Since all workbufs are on the empty list, we don't care 480 // which ones are in which spans. We can wipe the entire empty 481 // list and move all workbuf spans to the free list. 482 work.empty = 0 483 work.wbufSpans.free.takeAll(&work.wbufSpans.busy) 484 unlock(&work.wbufSpans.lock) 485 } 486 487 // freeSomeWbufs frees some workbufs back to the heap and returns 488 // true if it should be called again to free more. 489 func freeSomeWbufs(preemptible bool) bool { 490 const batchSize = 64 // ~1–2 µs per span. 491 lock(&work.wbufSpans.lock) 492 if gcphase != _GCoff || work.wbufSpans.free.isEmpty() { 493 unlock(&work.wbufSpans.lock) 494 return false 495 } 496 systemstack(func() { 497 gp := getg().m.curg 498 for i := 0; i < batchSize && !(preemptible && gp.preempt); i++ { 499 span := work.wbufSpans.free.first 500 if span == nil { 501 break 502 } 503 work.wbufSpans.free.remove(span) 504 mheap_.freeManual(span, &memstats.gc_sys) 505 } 506 }) 507 more := !work.wbufSpans.free.isEmpty() 508 unlock(&work.wbufSpans.lock) 509 return more 510 }