github.com/letsencrypt/go@v0.0.0-20160714163537-4054769a31f6/src/runtime/mgcwork.go (about) 1 // Copyright 2009 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package runtime 6 7 import ( 8 "runtime/internal/atomic" 9 "runtime/internal/sys" 10 "unsafe" 11 ) 12 13 const ( 14 _WorkbufSize = 2048 // in bytes; larger values result in less contention 15 ) 16 17 // Garbage collector work pool abstraction. 18 // 19 // This implements a producer/consumer model for pointers to grey 20 // objects. A grey object is one that is marked and on a work 21 // queue. A black object is marked and not on a work queue. 22 // 23 // Write barriers, root discovery, stack scanning, and object scanning 24 // produce pointers to grey objects. Scanning consumes pointers to 25 // grey objects, thus blackening them, and then scans them, 26 // potentially producing new pointers to grey objects. 27 28 // A wbufptr holds a workbuf*, but protects it from write barriers. 29 // workbufs never live on the heap, so write barriers are unnecessary. 30 // Write barriers on workbuf pointers may also be dangerous in the GC. 31 type wbufptr uintptr 32 33 func wbufptrOf(w *workbuf) wbufptr { 34 return wbufptr(unsafe.Pointer(w)) 35 } 36 37 func (wp wbufptr) ptr() *workbuf { 38 return (*workbuf)(unsafe.Pointer(wp)) 39 } 40 41 // A gcWork provides the interface to produce and consume work for the 42 // garbage collector. 43 // 44 // A gcWork can be used on the stack as follows: 45 // 46 // (preemption must be disabled) 47 // gcw := &getg().m.p.ptr().gcw 48 // .. call gcw.put() to produce and gcw.get() to consume .. 49 // if gcBlackenPromptly { 50 // gcw.dispose() 51 // } 52 // 53 // It's important that any use of gcWork during the mark phase prevent 54 // the garbage collector from transitioning to mark termination since 55 // gcWork may locally hold GC work buffers. This can be done by 56 // disabling preemption (systemstack or acquirem). 57 type gcWork struct { 58 // wbuf1 and wbuf2 are the primary and secondary work buffers. 59 // 60 // This can be thought of as a stack of both work buffers' 61 // pointers concatenated. When we pop the last pointer, we 62 // shift the stack up by one work buffer by bringing in a new 63 // full buffer and discarding an empty one. When we fill both 64 // buffers, we shift the stack down by one work buffer by 65 // bringing in a new empty buffer and discarding a full one. 66 // This way we have one buffer's worth of hysteresis, which 67 // amortizes the cost of getting or putting a work buffer over 68 // at least one buffer of work and reduces contention on the 69 // global work lists. 70 // 71 // wbuf1 is always the buffer we're currently pushing to and 72 // popping from and wbuf2 is the buffer that will be discarded 73 // next. 74 // 75 // Invariant: Both wbuf1 and wbuf2 are nil or neither are. 76 wbuf1, wbuf2 wbufptr 77 78 // Bytes marked (blackened) on this gcWork. This is aggregated 79 // into work.bytesMarked by dispose. 80 bytesMarked uint64 81 82 // Scan work performed on this gcWork. This is aggregated into 83 // gcController by dispose and may also be flushed by callers. 84 scanWork int64 85 } 86 87 func (w *gcWork) init() { 88 w.wbuf1 = wbufptrOf(getempty()) 89 wbuf2 := trygetfull() 90 if wbuf2 == nil { 91 wbuf2 = getempty() 92 } 93 w.wbuf2 = wbufptrOf(wbuf2) 94 } 95 96 // put enqueues a pointer for the garbage collector to trace. 97 // obj must point to the beginning of a heap object. 98 //go:nowritebarrier 99 func (w *gcWork) put(obj uintptr) { 100 wbuf := w.wbuf1.ptr() 101 if wbuf == nil { 102 w.init() 103 wbuf = w.wbuf1.ptr() 104 // wbuf is empty at this point. 105 } else if wbuf.nobj == len(wbuf.obj) { 106 w.wbuf1, w.wbuf2 = w.wbuf2, w.wbuf1 107 wbuf = w.wbuf1.ptr() 108 if wbuf.nobj == len(wbuf.obj) { 109 putfull(wbuf) 110 wbuf = getempty() 111 w.wbuf1 = wbufptrOf(wbuf) 112 } 113 } 114 115 wbuf.obj[wbuf.nobj] = obj 116 wbuf.nobj++ 117 } 118 119 // putFast does a put and returns true if it can be done quickly 120 // otherwise it returns false and the caller needs to call put. 121 //go:nowritebarrier 122 func (w *gcWork) putFast(obj uintptr) bool { 123 wbuf := w.wbuf1.ptr() 124 if wbuf == nil { 125 return false 126 } else if wbuf.nobj == len(wbuf.obj) { 127 return false 128 } 129 130 wbuf.obj[wbuf.nobj] = obj 131 wbuf.nobj++ 132 return true 133 } 134 135 // tryGet dequeues a pointer for the garbage collector to trace. 136 // 137 // If there are no pointers remaining in this gcWork or in the global 138 // queue, tryGet returns 0. Note that there may still be pointers in 139 // other gcWork instances or other caches. 140 //go:nowritebarrier 141 func (w *gcWork) tryGet() uintptr { 142 wbuf := w.wbuf1.ptr() 143 if wbuf == nil { 144 w.init() 145 wbuf = w.wbuf1.ptr() 146 // wbuf is empty at this point. 147 } 148 if wbuf.nobj == 0 { 149 w.wbuf1, w.wbuf2 = w.wbuf2, w.wbuf1 150 wbuf = w.wbuf1.ptr() 151 if wbuf.nobj == 0 { 152 owbuf := wbuf 153 wbuf = trygetfull() 154 if wbuf == nil { 155 return 0 156 } 157 putempty(owbuf) 158 w.wbuf1 = wbufptrOf(wbuf) 159 } 160 } 161 162 wbuf.nobj-- 163 return wbuf.obj[wbuf.nobj] 164 } 165 166 // tryGetFast dequeues a pointer for the garbage collector to trace 167 // if one is readily available. Otherwise it returns 0 and 168 // the caller is expected to call tryGet(). 169 //go:nowritebarrier 170 func (w *gcWork) tryGetFast() uintptr { 171 wbuf := w.wbuf1.ptr() 172 if wbuf == nil { 173 return 0 174 } 175 if wbuf.nobj == 0 { 176 return 0 177 } 178 179 wbuf.nobj-- 180 return wbuf.obj[wbuf.nobj] 181 } 182 183 // get dequeues a pointer for the garbage collector to trace, blocking 184 // if necessary to ensure all pointers from all queues and caches have 185 // been retrieved. get returns 0 if there are no pointers remaining. 186 //go:nowritebarrier 187 func (w *gcWork) get() uintptr { 188 wbuf := w.wbuf1.ptr() 189 if wbuf == nil { 190 w.init() 191 wbuf = w.wbuf1.ptr() 192 // wbuf is empty at this point. 193 } 194 if wbuf.nobj == 0 { 195 w.wbuf1, w.wbuf2 = w.wbuf2, w.wbuf1 196 wbuf = w.wbuf1.ptr() 197 if wbuf.nobj == 0 { 198 owbuf := wbuf 199 wbuf = getfull() 200 if wbuf == nil { 201 return 0 202 } 203 putempty(owbuf) 204 w.wbuf1 = wbufptrOf(wbuf) 205 } 206 } 207 208 // TODO: This might be a good place to add prefetch code 209 210 wbuf.nobj-- 211 return wbuf.obj[wbuf.nobj] 212 } 213 214 // dispose returns any cached pointers to the global queue. 215 // The buffers are being put on the full queue so that the 216 // write barriers will not simply reacquire them before the 217 // GC can inspect them. This helps reduce the mutator's 218 // ability to hide pointers during the concurrent mark phase. 219 // 220 //go:nowritebarrier 221 func (w *gcWork) dispose() { 222 if wbuf := w.wbuf1.ptr(); wbuf != nil { 223 if wbuf.nobj == 0 { 224 putempty(wbuf) 225 } else { 226 putfull(wbuf) 227 } 228 w.wbuf1 = 0 229 230 wbuf = w.wbuf2.ptr() 231 if wbuf.nobj == 0 { 232 putempty(wbuf) 233 } else { 234 putfull(wbuf) 235 } 236 w.wbuf2 = 0 237 } 238 if w.bytesMarked != 0 { 239 // dispose happens relatively infrequently. If this 240 // atomic becomes a problem, we should first try to 241 // dispose less and if necessary aggregate in a per-P 242 // counter. 243 atomic.Xadd64(&work.bytesMarked, int64(w.bytesMarked)) 244 w.bytesMarked = 0 245 } 246 if w.scanWork != 0 { 247 atomic.Xaddint64(&gcController.scanWork, w.scanWork) 248 w.scanWork = 0 249 } 250 } 251 252 // balance moves some work that's cached in this gcWork back on the 253 // global queue. 254 //go:nowritebarrier 255 func (w *gcWork) balance() { 256 if w.wbuf1 == 0 { 257 return 258 } 259 if wbuf := w.wbuf2.ptr(); wbuf.nobj != 0 { 260 putfull(wbuf) 261 w.wbuf2 = wbufptrOf(getempty()) 262 } else if wbuf := w.wbuf1.ptr(); wbuf.nobj > 4 { 263 w.wbuf1 = wbufptrOf(handoff(wbuf)) 264 } 265 } 266 267 // empty returns true if w has no mark work available. 268 //go:nowritebarrier 269 func (w *gcWork) empty() bool { 270 return w.wbuf1 == 0 || (w.wbuf1.ptr().nobj == 0 && w.wbuf2.ptr().nobj == 0) 271 } 272 273 // Internally, the GC work pool is kept in arrays in work buffers. 274 // The gcWork interface caches a work buffer until full (or empty) to 275 // avoid contending on the global work buffer lists. 276 277 type workbufhdr struct { 278 node lfnode // must be first 279 nobj int 280 } 281 282 type workbuf struct { 283 workbufhdr 284 // account for the above fields 285 obj [(_WorkbufSize - unsafe.Sizeof(workbufhdr{})) / sys.PtrSize]uintptr 286 } 287 288 // workbuf factory routines. These funcs are used to manage the 289 // workbufs. 290 // If the GC asks for some work these are the only routines that 291 // make wbufs available to the GC. 292 293 func (b *workbuf) checknonempty() { 294 if b.nobj == 0 { 295 throw("workbuf is empty") 296 } 297 } 298 299 func (b *workbuf) checkempty() { 300 if b.nobj != 0 { 301 throw("workbuf is not empty") 302 } 303 } 304 305 // getempty pops an empty work buffer off the work.empty list, 306 // allocating new buffers if none are available. 307 //go:nowritebarrier 308 func getempty() *workbuf { 309 var b *workbuf 310 if work.empty != 0 { 311 b = (*workbuf)(lfstackpop(&work.empty)) 312 if b != nil { 313 b.checkempty() 314 } 315 } 316 if b == nil { 317 b = (*workbuf)(persistentalloc(unsafe.Sizeof(*b), sys.CacheLineSize, &memstats.gc_sys)) 318 } 319 return b 320 } 321 322 // putempty puts a workbuf onto the work.empty list. 323 // Upon entry this go routine owns b. The lfstackpush relinquishes ownership. 324 //go:nowritebarrier 325 func putempty(b *workbuf) { 326 b.checkempty() 327 lfstackpush(&work.empty, &b.node) 328 } 329 330 // putfull puts the workbuf on the work.full list for the GC. 331 // putfull accepts partially full buffers so the GC can avoid competing 332 // with the mutators for ownership of partially full buffers. 333 //go:nowritebarrier 334 func putfull(b *workbuf) { 335 b.checknonempty() 336 lfstackpush(&work.full, &b.node) 337 338 // We just made more work available. Let the GC controller 339 // know so it can encourage more workers to run. 340 if gcphase == _GCmark { 341 gcController.enlistWorker() 342 } 343 } 344 345 // trygetfull tries to get a full or partially empty workbuffer. 346 // If one is not immediately available return nil 347 //go:nowritebarrier 348 func trygetfull() *workbuf { 349 b := (*workbuf)(lfstackpop(&work.full)) 350 if b != nil { 351 b.checknonempty() 352 return b 353 } 354 return b 355 } 356 357 // Get a full work buffer off the work.full list. 358 // If nothing is available wait until all the other gc helpers have 359 // finished and then return nil. 360 // getfull acts as a barrier for work.nproc helpers. As long as one 361 // gchelper is actively marking objects it 362 // may create a workbuffer that the other helpers can work on. 363 // The for loop either exits when a work buffer is found 364 // or when _all_ of the work.nproc GC helpers are in the loop 365 // looking for work and thus not capable of creating new work. 366 // This is in fact the termination condition for the STW mark 367 // phase. 368 //go:nowritebarrier 369 func getfull() *workbuf { 370 b := (*workbuf)(lfstackpop(&work.full)) 371 if b != nil { 372 b.checknonempty() 373 return b 374 } 375 376 incnwait := atomic.Xadd(&work.nwait, +1) 377 if incnwait > work.nproc { 378 println("runtime: work.nwait=", incnwait, "work.nproc=", work.nproc) 379 throw("work.nwait > work.nproc") 380 } 381 for i := 0; ; i++ { 382 if work.full != 0 { 383 decnwait := atomic.Xadd(&work.nwait, -1) 384 if decnwait == work.nproc { 385 println("runtime: work.nwait=", decnwait, "work.nproc=", work.nproc) 386 throw("work.nwait > work.nproc") 387 } 388 b = (*workbuf)(lfstackpop(&work.full)) 389 if b != nil { 390 b.checknonempty() 391 return b 392 } 393 incnwait := atomic.Xadd(&work.nwait, +1) 394 if incnwait > work.nproc { 395 println("runtime: work.nwait=", incnwait, "work.nproc=", work.nproc) 396 throw("work.nwait > work.nproc") 397 } 398 } 399 if work.nwait == work.nproc && work.markrootNext >= work.markrootJobs { 400 return nil 401 } 402 _g_ := getg() 403 if i < 10 { 404 _g_.m.gcstats.nprocyield++ 405 procyield(20) 406 } else if i < 20 { 407 _g_.m.gcstats.nosyield++ 408 osyield() 409 } else { 410 _g_.m.gcstats.nsleep++ 411 usleep(100) 412 } 413 } 414 } 415 416 //go:nowritebarrier 417 func handoff(b *workbuf) *workbuf { 418 // Make new buffer with half of b's pointers. 419 b1 := getempty() 420 n := b.nobj / 2 421 b.nobj -= n 422 b1.nobj = n 423 memmove(unsafe.Pointer(&b1.obj[0]), unsafe.Pointer(&b.obj[b.nobj]), uintptr(n)*unsafe.Sizeof(b1.obj[0])) 424 _g_ := getg() 425 _g_.m.gcstats.nhandoff++ 426 _g_.m.gcstats.nhandoffcnt += uint64(n) 427 428 // Put b on full list - let first half of b get stolen. 429 putfull(b) 430 return b1 431 }