github.com/4ad/go@v0.0.0-20161219182952-69a12818b605/src/runtime/mgcsweep.go (about) 1 // Copyright 2009 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 // Garbage collector: sweeping 6 7 package runtime 8 9 import ( 10 "runtime/internal/atomic" 11 "unsafe" 12 ) 13 14 var sweep sweepdata 15 16 // State of background sweep. 17 type sweepdata struct { 18 lock mutex 19 g *g 20 parked bool 21 started bool 22 23 spanidx uint32 // background sweeper position 24 25 nbgsweep uint32 26 npausesweep uint32 27 } 28 29 //go:nowritebarrier 30 func finishsweep_m(stw bool) { 31 // Sweeping must be complete before marking commences, so 32 // sweep any unswept spans. If this is a concurrent GC, there 33 // shouldn't be any spans left to sweep, so this should finish 34 // instantly. If GC was forced before the concurrent sweep 35 // finished, there may be spans to sweep. 36 for sweepone() != ^uintptr(0) { 37 sweep.npausesweep++ 38 } 39 40 // There may be some other spans being swept concurrently that 41 // we need to wait for. If finishsweep_m is done with the world stopped 42 // this is not required because the STW must have waited for sweeps. 43 // 44 // TODO(austin): As of this writing, we always pass true for stw. 45 // Consider removing this code. 46 if !stw { 47 sg := mheap_.sweepgen 48 for _, s := range work.spans { 49 if s.sweepgen != sg && s.state == _MSpanInUse { 50 s.ensureSwept() 51 } 52 } 53 } 54 nextMarkBitArenaEpoch() 55 } 56 57 func bgsweep(c chan int) { 58 sweep.g = getg() 59 60 lock(&sweep.lock) 61 sweep.parked = true 62 c <- 1 63 goparkunlock(&sweep.lock, "GC sweep wait", traceEvGoBlock, 1) 64 65 for { 66 for gosweepone() != ^uintptr(0) { 67 sweep.nbgsweep++ 68 Gosched() 69 } 70 lock(&sweep.lock) 71 if !gosweepdone() { 72 // This can happen if a GC runs between 73 // gosweepone returning ^0 above 74 // and the lock being acquired. 75 unlock(&sweep.lock) 76 continue 77 } 78 sweep.parked = true 79 goparkunlock(&sweep.lock, "GC sweep wait", traceEvGoBlock, 1) 80 } 81 } 82 83 // sweeps one span 84 // returns number of pages returned to heap, or ^uintptr(0) if there is nothing to sweep 85 //go:nowritebarrier 86 func sweepone() uintptr { 87 _g_ := getg() 88 89 // increment locks to ensure that the goroutine is not preempted 90 // in the middle of sweep thus leaving the span in an inconsistent state for next GC 91 _g_.m.locks++ 92 sg := mheap_.sweepgen 93 for { 94 idx := atomic.Xadd(&sweep.spanidx, 1) - 1 95 if idx >= uint32(len(work.spans)) { 96 mheap_.sweepdone = 1 97 _g_.m.locks-- 98 if debug.gcpacertrace > 0 && idx == uint32(len(work.spans)) { 99 print("pacer: sweep done at heap size ", memstats.heap_live>>20, "MB; allocated ", mheap_.spanBytesAlloc>>20, "MB of spans; swept ", mheap_.pagesSwept, " pages at ", mheap_.sweepPagesPerByte, " pages/byte\n") 100 } 101 return ^uintptr(0) 102 } 103 s := work.spans[idx] 104 if s.state != mSpanInUse { 105 s.sweepgen = sg 106 continue 107 } 108 if s.sweepgen != sg-2 || !atomic.Cas(&s.sweepgen, sg-2, sg-1) { 109 continue 110 } 111 npages := s.npages 112 if !s.sweep(false) { 113 npages = 0 114 } 115 _g_.m.locks-- 116 return npages 117 } 118 } 119 120 //go:nowritebarrier 121 func gosweepone() uintptr { 122 var ret uintptr 123 systemstack(func() { 124 ret = sweepone() 125 }) 126 return ret 127 } 128 129 //go:nowritebarrier 130 func gosweepdone() bool { 131 return mheap_.sweepdone != 0 132 } 133 134 // Returns only when span s has been swept. 135 //go:nowritebarrier 136 func (s *mspan) ensureSwept() { 137 // Caller must disable preemption. 138 // Otherwise when this function returns the span can become unswept again 139 // (if GC is triggered on another goroutine). 140 _g_ := getg() 141 if _g_.m.locks == 0 && _g_.m.mallocing == 0 && _g_ != _g_.m.g0 { 142 throw("MSpan_EnsureSwept: m is not locked") 143 } 144 145 sg := mheap_.sweepgen 146 if atomic.Load(&s.sweepgen) == sg { 147 return 148 } 149 // The caller must be sure that the span is a MSpanInUse span. 150 if atomic.Cas(&s.sweepgen, sg-2, sg-1) { 151 s.sweep(false) 152 return 153 } 154 // unfortunate condition, and we don't have efficient means to wait 155 for atomic.Load(&s.sweepgen) != sg { 156 osyield() 157 } 158 } 159 160 // Sweep frees or collects finalizers for blocks not marked in the mark phase. 161 // It clears the mark bits in preparation for the next GC round. 162 // Returns true if the span was returned to heap. 163 // If preserve=true, don't return it to heap nor relink in MCentral lists; 164 // caller takes care of it. 165 //TODO go:nowritebarrier 166 func (s *mspan) sweep(preserve bool) bool { 167 // It's critical that we enter this function with preemption disabled, 168 // GC must not start while we are in the middle of this function. 169 _g_ := getg() 170 if _g_.m.locks == 0 && _g_.m.mallocing == 0 && _g_ != _g_.m.g0 { 171 throw("MSpan_Sweep: m is not locked") 172 } 173 sweepgen := mheap_.sweepgen 174 if s.state != mSpanInUse || s.sweepgen != sweepgen-1 { 175 print("MSpan_Sweep: state=", s.state, " sweepgen=", s.sweepgen, " mheap.sweepgen=", sweepgen, "\n") 176 throw("MSpan_Sweep: bad span state") 177 } 178 179 if trace.enabled { 180 traceGCSweepStart() 181 } 182 183 atomic.Xadd64(&mheap_.pagesSwept, int64(s.npages)) 184 185 cl := s.sizeclass 186 size := s.elemsize 187 res := false 188 nfree := 0 189 190 c := _g_.m.mcache 191 freeToHeap := false 192 193 // The allocBits indicate which unmarked objects don't need to be 194 // processed since they were free at the end of the last GC cycle 195 // and were not allocated since then. 196 // If the allocBits index is >= s.freeindex and the bit 197 // is not marked then the object remains unallocated 198 // since the last GC. 199 // This situation is analogous to being on a freelist. 200 201 // Unlink & free special records for any objects we're about to free. 202 // Two complications here: 203 // 1. An object can have both finalizer and profile special records. 204 // In such case we need to queue finalizer for execution, 205 // mark the object as live and preserve the profile special. 206 // 2. A tiny object can have several finalizers setup for different offsets. 207 // If such object is not marked, we need to queue all finalizers at once. 208 // Both 1 and 2 are possible at the same time. 209 specialp := &s.specials 210 special := *specialp 211 for special != nil { 212 // A finalizer can be set for an inner byte of an object, find object beginning. 213 objIndex := uintptr(special.offset) / size 214 p := s.base() + objIndex*size 215 mbits := s.markBitsForIndex(objIndex) 216 if !mbits.isMarked() { 217 // This object is not marked and has at least one special record. 218 // Pass 1: see if it has at least one finalizer. 219 hasFin := false 220 endOffset := p - s.base() + size 221 for tmp := special; tmp != nil && uintptr(tmp.offset) < endOffset; tmp = tmp.next { 222 if tmp.kind == _KindSpecialFinalizer { 223 // Stop freeing of object if it has a finalizer. 224 mbits.setMarkedNonAtomic() 225 hasFin = true 226 break 227 } 228 } 229 // Pass 2: queue all finalizers _or_ handle profile record. 230 for special != nil && uintptr(special.offset) < endOffset { 231 // Find the exact byte for which the special was setup 232 // (as opposed to object beginning). 233 p := s.base() + uintptr(special.offset) 234 if special.kind == _KindSpecialFinalizer || !hasFin { 235 // Splice out special record. 236 y := special 237 special = special.next 238 *specialp = special 239 freespecial(y, unsafe.Pointer(p), size) 240 } else { 241 // This is profile record, but the object has finalizers (so kept alive). 242 // Keep special record. 243 specialp = &special.next 244 special = *specialp 245 } 246 } 247 } else { 248 // object is still live: keep special record 249 specialp = &special.next 250 special = *specialp 251 } 252 } 253 254 if debug.allocfreetrace != 0 || raceenabled || msanenabled { 255 // Find all newly freed objects. This doesn't have to 256 // efficient; allocfreetrace has massive overhead. 257 mbits := s.markBitsForBase() 258 abits := s.allocBitsForIndex(0) 259 for i := uintptr(0); i < s.nelems; i++ { 260 if !mbits.isMarked() && (abits.index < s.freeindex || abits.isMarked()) { 261 x := s.base() + i*s.elemsize 262 if debug.allocfreetrace != 0 { 263 tracefree(unsafe.Pointer(x), size) 264 } 265 if raceenabled { 266 racefree(unsafe.Pointer(x), size) 267 } 268 if msanenabled { 269 msanfree(unsafe.Pointer(x), size) 270 } 271 } 272 mbits.advance() 273 abits.advance() 274 } 275 } 276 277 // Count the number of free objects in this span. 278 nfree = s.countFree() 279 if cl == 0 && nfree != 0 { 280 s.needzero = 1 281 freeToHeap = true 282 } 283 nalloc := uint16(s.nelems) - uint16(nfree) 284 nfreed := s.allocCount - nalloc 285 if nalloc > s.allocCount { 286 print("runtime: nelems=", s.nelems, " nfree=", nfree, " nalloc=", nalloc, " previous allocCount=", s.allocCount, " nfreed=", nfreed, "\n") 287 throw("sweep increased allocation count") 288 } 289 290 s.allocCount = nalloc 291 wasempty := s.nextFreeIndex() == s.nelems 292 s.freeindex = 0 // reset allocation index to start of span. 293 294 // gcmarkBits becomes the allocBits. 295 // get a fresh cleared gcmarkBits in preparation for next GC 296 s.allocBits = s.gcmarkBits 297 s.gcmarkBits = newMarkBits(s.nelems) 298 299 // Initialize alloc bits cache. 300 s.refillAllocCache(0) 301 302 // We need to set s.sweepgen = h.sweepgen only when all blocks are swept, 303 // because of the potential for a concurrent free/SetFinalizer. 304 // But we need to set it before we make the span available for allocation 305 // (return it to heap or mcentral), because allocation code assumes that a 306 // span is already swept if available for allocation. 307 if freeToHeap || nfreed == 0 { 308 // The span must be in our exclusive ownership until we update sweepgen, 309 // check for potential races. 310 if s.state != mSpanInUse || s.sweepgen != sweepgen-1 { 311 print("MSpan_Sweep: state=", s.state, " sweepgen=", s.sweepgen, " mheap.sweepgen=", sweepgen, "\n") 312 throw("MSpan_Sweep: bad span state after sweep") 313 } 314 // Serialization point. 315 // At this point the mark bits are cleared and allocation ready 316 // to go so release the span. 317 atomic.Store(&s.sweepgen, sweepgen) 318 } 319 320 if nfreed > 0 && cl != 0 { 321 c.local_nsmallfree[cl] += uintptr(nfreed) 322 res = mheap_.central[cl].mcentral.freeSpan(s, preserve, wasempty) 323 // MCentral_FreeSpan updates sweepgen 324 } else if freeToHeap { 325 // Free large span to heap 326 327 // NOTE(rsc,dvyukov): The original implementation of efence 328 // in CL 22060046 used SysFree instead of SysFault, so that 329 // the operating system would eventually give the memory 330 // back to us again, so that an efence program could run 331 // longer without running out of memory. Unfortunately, 332 // calling SysFree here without any kind of adjustment of the 333 // heap data structures means that when the memory does 334 // come back to us, we have the wrong metadata for it, either in 335 // the MSpan structures or in the garbage collection bitmap. 336 // Using SysFault here means that the program will run out of 337 // memory fairly quickly in efence mode, but at least it won't 338 // have mysterious crashes due to confused memory reuse. 339 // It should be possible to switch back to SysFree if we also 340 // implement and then call some kind of MHeap_DeleteSpan. 341 if debug.efence > 0 { 342 s.limit = 0 // prevent mlookup from finding this span 343 sysFault(unsafe.Pointer(s.base()), size) 344 } else { 345 mheap_.freeSpan(s, 1) 346 } 347 c.local_nlargefree++ 348 c.local_largefree += size 349 res = true 350 } 351 if trace.enabled { 352 traceGCSweepDone() 353 } 354 return res 355 } 356 357 // deductSweepCredit deducts sweep credit for allocating a span of 358 // size spanBytes. This must be performed *before* the span is 359 // allocated to ensure the system has enough credit. If necessary, it 360 // performs sweeping to prevent going in to debt. If the caller will 361 // also sweep pages (e.g., for a large allocation), it can pass a 362 // non-zero callerSweepPages to leave that many pages unswept. 363 // 364 // deductSweepCredit makes a worst-case assumption that all spanBytes 365 // bytes of the ultimately allocated span will be available for object 366 // allocation. The caller should call reimburseSweepCredit if that 367 // turns out not to be the case once the span is allocated. 368 // 369 // deductSweepCredit is the core of the "proportional sweep" system. 370 // It uses statistics gathered by the garbage collector to perform 371 // enough sweeping so that all pages are swept during the concurrent 372 // sweep phase between GC cycles. 373 // 374 // mheap_ must NOT be locked. 375 func deductSweepCredit(spanBytes uintptr, callerSweepPages uintptr) { 376 if mheap_.sweepPagesPerByte == 0 { 377 // Proportional sweep is done or disabled. 378 return 379 } 380 381 // Account for this span allocation. 382 spanBytesAlloc := atomic.Xadd64(&mheap_.spanBytesAlloc, int64(spanBytes)) 383 384 // Fix debt if necessary. 385 pagesOwed := int64(mheap_.sweepPagesPerByte * float64(spanBytesAlloc)) 386 for pagesOwed-int64(atomic.Load64(&mheap_.pagesSwept)) > int64(callerSweepPages) { 387 if gosweepone() == ^uintptr(0) { 388 mheap_.sweepPagesPerByte = 0 389 break 390 } 391 } 392 } 393 394 // reimburseSweepCredit records that unusableBytes bytes of a 395 // just-allocated span are not available for object allocation. This 396 // offsets the worst-case charge performed by deductSweepCredit. 397 func reimburseSweepCredit(unusableBytes uintptr) { 398 if mheap_.sweepPagesPerByte == 0 { 399 // Nobody cares about the credit. Avoid the atomic. 400 return 401 } 402 if int64(atomic.Xadd64(&mheap_.spanBytesAlloc, -int64(unusableBytes))) < 0 { 403 throw("spanBytesAlloc underflow") 404 } 405 }