github.com/miolini/go@v0.0.0-20160405192216-fca68c8cb408/src/runtime/mgcwork.go (about) 1 // Copyright 2009 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package runtime 6 7 import ( 8 "runtime/internal/atomic" 9 "runtime/internal/sys" 10 "unsafe" 11 ) 12 13 const ( 14 _WorkbufSize = 2048 // in bytes; larger values result in less contention 15 ) 16 17 // Garbage collector work pool abstraction. 18 // 19 // This implements a producer/consumer model for pointers to grey 20 // objects. A grey object is one that is marked and on a work 21 // queue. A black object is marked and not on a work queue. 22 // 23 // Write barriers, root discovery, stack scanning, and object scanning 24 // produce pointers to grey objects. Scanning consumes pointers to 25 // grey objects, thus blackening them, and then scans them, 26 // potentially producing new pointers to grey objects. 27 28 // A wbufptr holds a workbuf*, but protects it from write barriers. 29 // workbufs never live on the heap, so write barriers are unnecessary. 30 // Write barriers on workbuf pointers may also be dangerous in the GC. 31 type wbufptr uintptr 32 33 func wbufptrOf(w *workbuf) wbufptr { 34 return wbufptr(unsafe.Pointer(w)) 35 } 36 37 func (wp wbufptr) ptr() *workbuf { 38 return (*workbuf)(unsafe.Pointer(wp)) 39 } 40 41 // A gcWork provides the interface to produce and consume work for the 42 // garbage collector. 43 // 44 // A gcWork can be used on the stack as follows: 45 // 46 // (preemption must be disabled) 47 // gcw := &getg().m.p.ptr().gcw 48 // .. call gcw.put() to produce and gcw.get() to consume .. 49 // if gcBlackenPromptly { 50 // gcw.dispose() 51 // } 52 // 53 // It's important that any use of gcWork during the mark phase prevent 54 // the garbage collector from transitioning to mark termination since 55 // gcWork may locally hold GC work buffers. This can be done by 56 // disabling preemption (systemstack or acquirem). 57 type gcWork struct { 58 // wbuf1 and wbuf2 are the primary and secondary work buffers. 59 // 60 // This can be thought of as a stack of both work buffers' 61 // pointers concatenated. When we pop the last pointer, we 62 // shift the stack up by one work buffer by bringing in a new 63 // full buffer and discarding an empty one. When we fill both 64 // buffers, we shift the stack down by one work buffer by 65 // bringing in a new empty buffer and discarding a full one. 66 // This way we have one buffer's worth of hysteresis, which 67 // amortizes the cost of getting or putting a work buffer over 68 // at least one buffer of work and reduces contention on the 69 // global work lists. 70 // 71 // wbuf1 is always the buffer we're currently pushing to and 72 // popping from and wbuf2 is the buffer that will be discarded 73 // next. 74 // 75 // Invariant: Both wbuf1 and wbuf2 are nil or neither are. 76 wbuf1, wbuf2 wbufptr 77 78 // Bytes marked (blackened) on this gcWork. This is aggregated 79 // into work.bytesMarked by dispose. 80 bytesMarked uint64 81 82 // Scan work performed on this gcWork. This is aggregated into 83 // gcController by dispose and may also be flushed by callers. 84 scanWork int64 85 } 86 87 func (w *gcWork) init() { 88 w.wbuf1 = wbufptrOf(getempty()) 89 wbuf2 := trygetfull() 90 if wbuf2 == nil { 91 wbuf2 = getempty() 92 } 93 w.wbuf2 = wbufptrOf(wbuf2) 94 } 95 96 // put enqueues a pointer for the garbage collector to trace. 97 // obj must point to the beginning of a heap object. 98 //go:nowritebarrier 99 func (w *gcWork) put(obj uintptr) { 100 wbuf := w.wbuf1.ptr() 101 if wbuf == nil { 102 w.init() 103 wbuf = w.wbuf1.ptr() 104 // wbuf is empty at this point. 105 } else if wbuf.nobj == len(wbuf.obj) { 106 w.wbuf1, w.wbuf2 = w.wbuf2, w.wbuf1 107 wbuf = w.wbuf1.ptr() 108 if wbuf.nobj == len(wbuf.obj) { 109 putfull(wbuf) 110 wbuf = getempty() 111 w.wbuf1 = wbufptrOf(wbuf) 112 } 113 } 114 115 wbuf.obj[wbuf.nobj] = obj 116 wbuf.nobj++ 117 } 118 119 // tryGet dequeues a pointer for the garbage collector to trace. 120 // 121 // If there are no pointers remaining in this gcWork or in the global 122 // queue, tryGet returns 0. Note that there may still be pointers in 123 // other gcWork instances or other caches. 124 //go:nowritebarrier 125 func (w *gcWork) tryGet() uintptr { 126 wbuf := w.wbuf1.ptr() 127 if wbuf == nil { 128 w.init() 129 wbuf = w.wbuf1.ptr() 130 // wbuf is empty at this point. 131 } 132 if wbuf.nobj == 0 { 133 w.wbuf1, w.wbuf2 = w.wbuf2, w.wbuf1 134 wbuf = w.wbuf1.ptr() 135 if wbuf.nobj == 0 { 136 owbuf := wbuf 137 wbuf = trygetfull() 138 if wbuf == nil { 139 return 0 140 } 141 putempty(owbuf) 142 w.wbuf1 = wbufptrOf(wbuf) 143 } 144 } 145 146 wbuf.nobj-- 147 return wbuf.obj[wbuf.nobj] 148 } 149 150 // get dequeues a pointer for the garbage collector to trace, blocking 151 // if necessary to ensure all pointers from all queues and caches have 152 // been retrieved. get returns 0 if there are no pointers remaining. 153 //go:nowritebarrier 154 func (w *gcWork) get() uintptr { 155 wbuf := w.wbuf1.ptr() 156 if wbuf == nil { 157 w.init() 158 wbuf = w.wbuf1.ptr() 159 // wbuf is empty at this point. 160 } 161 if wbuf.nobj == 0 { 162 w.wbuf1, w.wbuf2 = w.wbuf2, w.wbuf1 163 wbuf = w.wbuf1.ptr() 164 if wbuf.nobj == 0 { 165 owbuf := wbuf 166 wbuf = getfull() 167 if wbuf == nil { 168 return 0 169 } 170 putempty(owbuf) 171 w.wbuf1 = wbufptrOf(wbuf) 172 } 173 } 174 175 // TODO: This might be a good place to add prefetch code 176 177 wbuf.nobj-- 178 return wbuf.obj[wbuf.nobj] 179 } 180 181 // dispose returns any cached pointers to the global queue. 182 // The buffers are being put on the full queue so that the 183 // write barriers will not simply reacquire them before the 184 // GC can inspect them. This helps reduce the mutator's 185 // ability to hide pointers during the concurrent mark phase. 186 // 187 //go:nowritebarrier 188 func (w *gcWork) dispose() { 189 if wbuf := w.wbuf1.ptr(); wbuf != nil { 190 if wbuf.nobj == 0 { 191 putempty(wbuf) 192 } else { 193 putfull(wbuf) 194 } 195 w.wbuf1 = 0 196 197 wbuf = w.wbuf2.ptr() 198 if wbuf.nobj == 0 { 199 putempty(wbuf) 200 } else { 201 putfull(wbuf) 202 } 203 w.wbuf2 = 0 204 } 205 if w.bytesMarked != 0 { 206 // dispose happens relatively infrequently. If this 207 // atomic becomes a problem, we should first try to 208 // dispose less and if necessary aggregate in a per-P 209 // counter. 210 atomic.Xadd64(&work.bytesMarked, int64(w.bytesMarked)) 211 w.bytesMarked = 0 212 } 213 if w.scanWork != 0 { 214 atomic.Xaddint64(&gcController.scanWork, w.scanWork) 215 w.scanWork = 0 216 } 217 } 218 219 // balance moves some work that's cached in this gcWork back on the 220 // global queue. 221 //go:nowritebarrier 222 func (w *gcWork) balance() { 223 if w.wbuf1 == 0 { 224 return 225 } 226 if wbuf := w.wbuf2.ptr(); wbuf.nobj != 0 { 227 putfull(wbuf) 228 w.wbuf2 = wbufptrOf(getempty()) 229 } else if wbuf := w.wbuf1.ptr(); wbuf.nobj > 4 { 230 w.wbuf1 = wbufptrOf(handoff(wbuf)) 231 } 232 } 233 234 // empty returns true if w has no mark work available. 235 //go:nowritebarrier 236 func (w *gcWork) empty() bool { 237 return w.wbuf1 == 0 || (w.wbuf1.ptr().nobj == 0 && w.wbuf2.ptr().nobj == 0) 238 } 239 240 // Internally, the GC work pool is kept in arrays in work buffers. 241 // The gcWork interface caches a work buffer until full (or empty) to 242 // avoid contending on the global work buffer lists. 243 244 type workbufhdr struct { 245 node lfnode // must be first 246 nobj int 247 } 248 249 type workbuf struct { 250 workbufhdr 251 // account for the above fields 252 obj [(_WorkbufSize - unsafe.Sizeof(workbufhdr{})) / sys.PtrSize]uintptr 253 } 254 255 // workbuf factory routines. These funcs are used to manage the 256 // workbufs. 257 // If the GC asks for some work these are the only routines that 258 // make wbufs available to the GC. 259 260 func (b *workbuf) checknonempty() { 261 if b.nobj == 0 { 262 throw("workbuf is empty") 263 } 264 } 265 266 func (b *workbuf) checkempty() { 267 if b.nobj != 0 { 268 throw("workbuf is not empty") 269 } 270 } 271 272 // getempty pops an empty work buffer off the work.empty list, 273 // allocating new buffers if none are available. 274 //go:nowritebarrier 275 func getempty() *workbuf { 276 var b *workbuf 277 if work.empty != 0 { 278 b = (*workbuf)(lfstackpop(&work.empty)) 279 if b != nil { 280 b.checkempty() 281 } 282 } 283 if b == nil { 284 b = (*workbuf)(persistentalloc(unsafe.Sizeof(*b), sys.CacheLineSize, &memstats.gc_sys)) 285 } 286 return b 287 } 288 289 // putempty puts a workbuf onto the work.empty list. 290 // Upon entry this go routine owns b. The lfstackpush relinquishes ownership. 291 //go:nowritebarrier 292 func putempty(b *workbuf) { 293 b.checkempty() 294 lfstackpush(&work.empty, &b.node) 295 } 296 297 // putfull puts the workbuf on the work.full list for the GC. 298 // putfull accepts partially full buffers so the GC can avoid competing 299 // with the mutators for ownership of partially full buffers. 300 //go:nowritebarrier 301 func putfull(b *workbuf) { 302 b.checknonempty() 303 lfstackpush(&work.full, &b.node) 304 305 // We just made more work available. Let the GC controller 306 // know so it can encourage more workers to run. 307 if gcphase == _GCmark { 308 gcController.enlistWorker() 309 } 310 } 311 312 // trygetfull tries to get a full or partially empty workbuffer. 313 // If one is not immediately available return nil 314 //go:nowritebarrier 315 func trygetfull() *workbuf { 316 b := (*workbuf)(lfstackpop(&work.full)) 317 if b != nil { 318 b.checknonempty() 319 return b 320 } 321 return b 322 } 323 324 // Get a full work buffer off the work.full list. 325 // If nothing is available wait until all the other gc helpers have 326 // finished and then return nil. 327 // getfull acts as a barrier for work.nproc helpers. As long as one 328 // gchelper is actively marking objects it 329 // may create a workbuffer that the other helpers can work on. 330 // The for loop either exits when a work buffer is found 331 // or when _all_ of the work.nproc GC helpers are in the loop 332 // looking for work and thus not capable of creating new work. 333 // This is in fact the termination condition for the STW mark 334 // phase. 335 //go:nowritebarrier 336 func getfull() *workbuf { 337 b := (*workbuf)(lfstackpop(&work.full)) 338 if b != nil { 339 b.checknonempty() 340 return b 341 } 342 343 incnwait := atomic.Xadd(&work.nwait, +1) 344 if incnwait > work.nproc { 345 println("runtime: work.nwait=", incnwait, "work.nproc=", work.nproc) 346 throw("work.nwait > work.nproc") 347 } 348 for i := 0; ; i++ { 349 if work.full != 0 { 350 decnwait := atomic.Xadd(&work.nwait, -1) 351 if decnwait == work.nproc { 352 println("runtime: work.nwait=", decnwait, "work.nproc=", work.nproc) 353 throw("work.nwait > work.nproc") 354 } 355 b = (*workbuf)(lfstackpop(&work.full)) 356 if b != nil { 357 b.checknonempty() 358 return b 359 } 360 incnwait := atomic.Xadd(&work.nwait, +1) 361 if incnwait > work.nproc { 362 println("runtime: work.nwait=", incnwait, "work.nproc=", work.nproc) 363 throw("work.nwait > work.nproc") 364 } 365 } 366 if work.nwait == work.nproc && work.markrootNext >= work.markrootJobs { 367 return nil 368 } 369 _g_ := getg() 370 if i < 10 { 371 _g_.m.gcstats.nprocyield++ 372 procyield(20) 373 } else if i < 20 { 374 _g_.m.gcstats.nosyield++ 375 osyield() 376 } else { 377 _g_.m.gcstats.nsleep++ 378 usleep(100) 379 } 380 } 381 } 382 383 //go:nowritebarrier 384 func handoff(b *workbuf) *workbuf { 385 // Make new buffer with half of b's pointers. 386 b1 := getempty() 387 n := b.nobj / 2 388 b.nobj -= n 389 b1.nobj = n 390 memmove(unsafe.Pointer(&b1.obj[0]), unsafe.Pointer(&b.obj[b.nobj]), uintptr(n)*unsafe.Sizeof(b1.obj[0])) 391 _g_ := getg() 392 _g_.m.gcstats.nhandoff++ 393 _g_.m.gcstats.nhandoffcnt += uint64(n) 394 395 // Put b on full list - let first half of b get stolen. 396 putfull(b) 397 return b1 398 }