github.com/c0deoo1/golang1.5@v0.0.0-20220525150107-c87c805d4593/src/runtime/mbitmap.go (about) 1 // Copyright 2009 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 // Garbage collector: type and heap bitmaps. 6 // 7 // Stack, data, and bss bitmaps 8 // 9 // Stack frames and global variables in the data and bss sections are described 10 // by 1-bit bitmaps in which 0 means uninteresting and 1 means live pointer 11 // to be visited during GC. The bits in each byte are consumed starting with 12 // the low bit: 1<<0, 1<<1, and so on. 13 // 14 // Heap bitmap 15 // 16 // The allocated heap comes from a subset of the memory in the range [start, used), 17 // where start == mheap_.arena_start and used == mheap_.arena_used. 18 // The heap bitmap comprises 2 bits for each pointer-sized word in that range, 19 // stored in bytes indexed backward in memory from start. 20 // That is, the byte at address start-1 holds the 2-bit entries for the four words 21 // start through start+3*ptrSize, the byte at start-2 holds the entries for 22 // start+4*ptrSize through start+7*ptrSize, and so on. 23 // 24 // In each 2-bit entry, the lower bit holds the same information as in the 1-bit 25 // bitmaps: 0 means uninteresting and 1 means live pointer to be visited during GC. 26 // The meaning of the high bit depends on the position of the word being described 27 // in its allocated object. In the first word, the high bit is the GC ``marked'' bit. 28 // In the second word, the high bit is the GC ``checkmarked'' bit (see below). 29 // In the third and later words, the high bit indicates that the object is still 30 // being described. In these words, if a bit pair with a high bit 0 is encountered, 31 // the low bit can also be assumed to be 0, and the object description is over. 32 // This 00 is called the ``dead'' encoding: it signals that the rest of the words 33 // in the object are uninteresting to the garbage collector. 34 // 35 // The 2-bit entries are split when written into the byte, so that the top half 36 // of the byte contains 4 mark bits and the bottom half contains 4 pointer bits. 37 // This form allows a copy from the 1-bit to the 4-bit form to keep the 38 // pointer bits contiguous, instead of having to space them out. 39 // 40 // The code makes use of the fact that the zero value for a heap bitmap 41 // has no live pointer bit set and is (depending on position), not marked, 42 // not checkmarked, and is the dead encoding. 43 // These properties must be preserved when modifying the encoding. 44 // 45 // Checkmarks 46 // 47 // In a concurrent garbage collector, one worries about failing to mark 48 // a live object due to mutations without write barriers or bugs in the 49 // collector implementation. As a sanity check, the GC has a 'checkmark' 50 // mode that retraverses the object graph with the world stopped, to make 51 // sure that everything that should be marked is marked. 52 // In checkmark mode, in the heap bitmap, the high bit of the 2-bit entry 53 // for the second word of the object holds the checkmark bit. 54 // When not in checkmark mode, this bit is set to 1. 55 // 56 // The smallest possible allocation is 8 bytes. On a 32-bit machine, that 57 // means every allocated object has two words, so there is room for the 58 // checkmark bit. On a 64-bit machine, however, the 8-byte allocation is 59 // just one word, so the second bit pair is not available for encoding the 60 // checkmark. However, because non-pointer allocations are combined 61 // into larger 16-byte (maxTinySize) allocations, a plain 8-byte allocation 62 // must be a pointer, so the type bit in the first word is not actually needed. 63 // It is still used in general, except in checkmark the type bit is repurposed 64 // as the checkmark bit and then reinitialized (to 1) as the type bit when 65 // finished. 66 67 package runtime 68 69 import "unsafe" 70 71 const ( 72 bitPointer = 1 << 0 73 bitMarked = 1 << 4 74 75 heapBitsShift = 1 // shift offset between successive bitPointer or bitMarked entries 76 heapBitmapScale = ptrSize * (8 / 2) // number of data bytes described by one heap bitmap byte 77 78 // all mark/pointer bits in a byte 79 bitMarkedAll = bitMarked | bitMarked<<heapBitsShift | bitMarked<<(2*heapBitsShift) | bitMarked<<(3*heapBitsShift) 80 bitPointerAll = bitPointer | bitPointer<<heapBitsShift | bitPointer<<(2*heapBitsShift) | bitPointer<<(3*heapBitsShift) 81 ) 82 83 // addb returns the byte pointer p+n. 84 //go:nowritebarrier 85 func addb(p *byte, n uintptr) *byte { 86 // Note: wrote out full expression instead of calling add(p, n) 87 // to reduce the number of temporaries generated by the 88 // compiler for this trivial expression during inlining. 89 return (*byte)(unsafe.Pointer(uintptr(unsafe.Pointer(p)) + n)) 90 } 91 92 // subtractb returns the byte pointer p-n. 93 //go:nowritebarrier 94 func subtractb(p *byte, n uintptr) *byte { 95 // Note: wrote out full expression instead of calling add(p, -n) 96 // to reduce the number of temporaries generated by the 97 // compiler for this trivial expression during inlining. 98 return (*byte)(unsafe.Pointer(uintptr(unsafe.Pointer(p)) - n)) 99 } 100 101 // add1 returns the byte pointer p+1. 102 //go:nowritebarrier 103 func add1(p *byte) *byte { 104 // Note: wrote out full expression instead of calling addb(p, 1) 105 // to reduce the number of temporaries generated by the 106 // compiler for this trivial expression during inlining. 107 return (*byte)(unsafe.Pointer(uintptr(unsafe.Pointer(p)) + 1)) 108 } 109 110 // subtract1 returns the byte pointer p-1. 111 //go:nowritebarrier 112 func subtract1(p *byte) *byte { 113 // Note: wrote out full expression instead of calling subtractb(p, 1) 114 // to reduce the number of temporaries generated by the 115 // compiler for this trivial expression during inlining. 116 return (*byte)(unsafe.Pointer(uintptr(unsafe.Pointer(p)) - 1)) 117 } 118 119 // mHeap_MapBits is called each time arena_used is extended. 120 // It maps any additional bitmap memory needed for the new arena memory. 121 // It must be called with the expected new value of arena_used, 122 // *before* h.arena_used has been updated. 123 // Waiting to update arena_used until after the memory has been mapped 124 // avoids faults when other threads try access the bitmap immediately 125 // after observing the change to arena_used. 126 // 127 //go:nowritebarrier 128 func mHeap_MapBits(h *mheap, arena_used uintptr) { 129 // Caller has added extra mappings to the arena. 130 // Add extra mappings of bitmap words as needed. 131 // We allocate extra bitmap pieces in chunks of bitmapChunk. 132 const bitmapChunk = 8192 133 134 n := (arena_used - mheap_.arena_start) / heapBitmapScale 135 n = round(n, bitmapChunk) 136 n = round(n, _PhysPageSize) 137 if h.bitmap_mapped >= n { 138 return 139 } 140 141 sysMap(unsafe.Pointer(h.arena_start-n), n-h.bitmap_mapped, h.arena_reserved, &memstats.gc_sys) 142 h.bitmap_mapped = n 143 } 144 145 // heapBits provides access to the bitmap bits for a single heap word. 146 // The methods on heapBits take value receivers so that the compiler 147 // can more easily inline calls to those methods and registerize the 148 // struct fields independently. 149 type heapBits struct { 150 bitp *uint8 151 shift uint32 152 } 153 154 // heapBitsForAddr returns the heapBits for the address addr. 155 // The caller must have already checked that addr is in the range [mheap_.arena_start, mheap_.arena_used). 156 // 157 // nosplit because it is used during write barriers and must not be preempted. 158 //go:nosplit 159 func heapBitsForAddr(addr uintptr) heapBits { 160 // 2 bits per work, 4 pairs per byte, and a mask is hard coded. 161 off := (addr - mheap_.arena_start) / ptrSize 162 return heapBits{(*uint8)(unsafe.Pointer(mheap_.arena_start - off/4 - 1)), uint32(off & 3)} 163 } 164 165 // heapBitsForSpan returns the heapBits for the span base address base. 166 func heapBitsForSpan(base uintptr) (hbits heapBits) { 167 if base < mheap_.arena_start || base >= mheap_.arena_used { 168 throw("heapBitsForSpan: base out of range") 169 } 170 hbits = heapBitsForAddr(base) 171 if hbits.shift != 0 { 172 throw("heapBitsForSpan: unaligned start") 173 } 174 return hbits 175 } 176 177 // heapBitsForObject returns the base address for the heap object 178 // containing the address p, along with the heapBits for base. 179 // If p does not point into a heap object, 180 // return base == 0 181 // otherwise return the base of the object. 182 func heapBitsForObject(p uintptr) (base uintptr, hbits heapBits, s *mspan) { 183 arenaStart := mheap_.arena_start 184 if p < arenaStart || p >= mheap_.arena_used { 185 return 186 } 187 off := p - arenaStart 188 idx := off >> _PageShift 189 // p points into the heap, but possibly to the middle of an object. 190 // Consult the span table to find the block beginning. 191 k := p >> _PageShift 192 s = h_spans[idx] 193 if s == nil || pageID(k) < s.start || p >= s.limit || s.state != mSpanInUse { 194 if s == nil || s.state == _MSpanStack { 195 // If s is nil, the virtual address has never been part of the heap. 196 // This pointer may be to some mmap'd region, so we allow it. 197 // Pointers into stacks are also ok, the runtime manages these explicitly. 198 return 199 } 200 201 // The following ensures that we are rigorous about what data 202 // structures hold valid pointers. 203 // TODO(rsc): Check if this still happens. 204 if debug.invalidptr != 0 { 205 // Still happens sometimes. We don't know why. 206 printlock() 207 print("runtime:objectstart Span weird: p=", hex(p), " k=", hex(k)) 208 if s == nil { 209 print(" s=nil\n") 210 } else { 211 print(" s.start=", hex(s.start<<_PageShift), " s.limit=", hex(s.limit), " s.state=", s.state, "\n") 212 } 213 printunlock() 214 throw("objectstart: bad pointer in unexpected span") 215 } 216 return 217 } 218 // If this span holds object of a power of 2 size, just mask off the bits to 219 // the interior of the object. Otherwise use the size to get the base. 220 if s.baseMask != 0 { 221 // optimize for power of 2 sized objects. 222 base = s.base() 223 base = base + (p-base)&s.baseMask 224 // base = p & s.baseMask is faster for small spans, 225 // but doesn't work for large spans. 226 // Overall, it's faster to use the more general computation above. 227 } else { 228 base = s.base() 229 if p-base >= s.elemsize { 230 // n := (p - base) / s.elemsize, using division by multiplication 231 n := uintptr(uint64(p-base) >> s.divShift * uint64(s.divMul) >> s.divShift2) 232 base += n * s.elemsize 233 } 234 } 235 // Now that we know the actual base, compute heapBits to return to caller. 236 hbits = heapBitsForAddr(base) 237 return 238 } 239 240 // prefetch the bits. 241 func (h heapBits) prefetch() { 242 prefetchnta(uintptr(unsafe.Pointer((h.bitp)))) 243 } 244 245 // next returns the heapBits describing the next pointer-sized word in memory. 246 // That is, if h describes address p, h.next() describes p+ptrSize. 247 // Note that next does not modify h. The caller must record the result. 248 func (h heapBits) next() heapBits { 249 if h.shift < 3*heapBitsShift { 250 return heapBits{h.bitp, h.shift + heapBitsShift} 251 } 252 return heapBits{subtract1(h.bitp), 0} 253 } 254 255 // forward returns the heapBits describing n pointer-sized words ahead of h in memory. 256 // That is, if h describes address p, h.forward(n) describes p+n*ptrSize. 257 // h.forward(1) is equivalent to h.next(), just slower. 258 // Note that forward does not modify h. The caller must record the result. 259 // bits returns the heap bits for the current word. 260 func (h heapBits) forward(n uintptr) heapBits { 261 n += uintptr(h.shift) / heapBitsShift 262 return heapBits{subtractb(h.bitp, n/4), uint32(n%4) * heapBitsShift} 263 } 264 265 // The caller can test isMarked and isPointer by &-ing with bitMarked and bitPointer. 266 // The result includes in its higher bits the bits for subsequent words 267 // described by the same bitmap byte. 268 func (h heapBits) bits() uint32 { 269 return uint32(*h.bitp) >> h.shift 270 } 271 272 // isMarked reports whether the heap bits have the marked bit set. 273 // h must describe the initial word of the object. 274 func (h heapBits) isMarked() bool { 275 return *h.bitp&(bitMarked<<h.shift) != 0 276 } 277 278 // setMarked sets the marked bit in the heap bits, atomically. 279 // h must describe the initial word of the object. 280 func (h heapBits) setMarked() { 281 // Each byte of GC bitmap holds info for four words. 282 // Might be racing with other updates, so use atomic update always. 283 // We used to be clever here and use a non-atomic update in certain 284 // cases, but it's not worth the risk. 285 atomicor8(h.bitp, bitMarked<<h.shift) 286 } 287 288 // setMarkedNonAtomic sets the marked bit in the heap bits, non-atomically. 289 // h must describe the initial word of the object. 290 func (h heapBits) setMarkedNonAtomic() { 291 *h.bitp |= bitMarked << h.shift 292 } 293 294 // isPointer reports whether the heap bits describe a pointer word. 295 // h must describe the initial word of the object. 296 func (h heapBits) isPointer() bool { 297 return (*h.bitp>>h.shift)&bitPointer != 0 298 } 299 300 // hasPointers reports whether the given object has any pointers. 301 // It must be told how large the object at h is, so that it does not read too 302 // far into the bitmap. 303 // h must describe the initial word of the object. 304 func (h heapBits) hasPointers(size uintptr) bool { 305 if size == ptrSize { // 1-word objects are always pointers 306 return true 307 } 308 // Otherwise, at least a 2-word object, and at least 2-word aligned, 309 // so h.shift is either 0 or 4, so we know we can get the bits for the 310 // first two words out of *h.bitp. 311 // If either of the first two words is a pointer, not pointer free. 312 b := uint32(*h.bitp >> h.shift) 313 if b&(bitPointer|bitPointer<<heapBitsShift) != 0 { 314 return true 315 } 316 if size == 2*ptrSize { 317 return false 318 } 319 // At least a 4-word object. Check scan bit (aka marked bit) in third word. 320 if h.shift == 0 { 321 return b&(bitMarked<<(2*heapBitsShift)) != 0 322 } 323 return uint32(*subtract1(h.bitp))&bitMarked != 0 324 } 325 326 // isCheckmarked reports whether the heap bits have the checkmarked bit set. 327 // It must be told how large the object at h is, because the encoding of the 328 // checkmark bit varies by size. 329 // h must describe the initial word of the object. 330 func (h heapBits) isCheckmarked(size uintptr) bool { 331 if size == ptrSize { 332 return (*h.bitp>>h.shift)&bitPointer != 0 333 } 334 // All multiword objects are 2-word aligned, 335 // so we know that the initial word's 2-bit pair 336 // and the second word's 2-bit pair are in the 337 // same heap bitmap byte, *h.bitp. 338 return (*h.bitp>>(heapBitsShift+h.shift))&bitMarked != 0 339 } 340 341 // setCheckmarked sets the checkmarked bit. 342 // It must be told how large the object at h is, because the encoding of the 343 // checkmark bit varies by size. 344 // h must describe the initial word of the object. 345 func (h heapBits) setCheckmarked(size uintptr) { 346 if size == ptrSize { 347 atomicor8(h.bitp, bitPointer<<h.shift) 348 return 349 } 350 atomicor8(h.bitp, bitMarked<<(heapBitsShift+h.shift)) 351 } 352 353 // heapBitsBulkBarrier executes writebarrierptr_nostore 354 // for every pointer slot in the memory range [p, p+size), 355 // using the heap bitmap to locate those pointer slots. 356 // This executes the write barriers necessary after a memmove. 357 // Both p and size must be pointer-aligned. 358 // The range [p, p+size) must lie within a single allocation. 359 // 360 // Callers should call heapBitsBulkBarrier immediately after 361 // calling memmove(p, src, size). This function is marked nosplit 362 // to avoid being preempted; the GC must not stop the goroutine 363 // between the memmove and the execution of the barriers. 364 // 365 // The heap bitmap is not maintained for allocations containing 366 // no pointers at all; any caller of heapBitsBulkBarrier must first 367 // make sure the underlying allocation contains pointers, usually 368 // by checking typ.kind&kindNoPointers. 369 // 370 //go:nosplit 371 func heapBitsBulkBarrier(p, size uintptr) { 372 if (p|size)&(ptrSize-1) != 0 { 373 throw("heapBitsBulkBarrier: unaligned arguments") 374 } 375 if !writeBarrierEnabled { 376 return 377 } 378 if !inheap(p) { 379 // If p is on the stack and in a higher frame than the 380 // caller, we either need to execute write barriers on 381 // it (which is what happens for normal stack writes 382 // through pointers to higher frames), or we need to 383 // force the mark termination stack scan to scan the 384 // frame containing p. 385 // 386 // Executing write barriers on p is complicated in the 387 // general case because we either need to unwind the 388 // stack to get the stack map, or we need the type's 389 // bitmap, which may be a GC program. 390 // 391 // Hence, we opt for forcing the re-scan to scan the 392 // frame containing p, which we can do by simply 393 // unwinding the stack barriers between the current SP 394 // and p's frame. 395 gp := getg().m.curg 396 if gp != nil && gp.stack.lo <= p && p < gp.stack.hi { 397 // Run on the system stack to give it more 398 // stack space. 399 systemstack(func() { 400 gcUnwindBarriers(gp, p) 401 }) 402 } 403 return 404 } 405 406 h := heapBitsForAddr(p) 407 for i := uintptr(0); i < size; i += ptrSize { 408 if h.isPointer() { 409 x := (*uintptr)(unsafe.Pointer(p + i)) 410 writebarrierptr_nostore(x, *x) 411 } 412 h = h.next() 413 } 414 } 415 416 // typeBitsBulkBarrier executes writebarrierptr_nostore 417 // for every pointer slot in the memory range [p, p+size), 418 // using the type bitmap to locate those pointer slots. 419 // The type typ must correspond exactly to [p, p+size). 420 // This executes the write barriers necessary after a copy. 421 // Both p and size must be pointer-aligned. 422 // The type typ must have a plain bitmap, not a GC program. 423 // The only use of this function is in channel sends, and the 424 // 64 kB channel element limit takes care of this for us. 425 // 426 // Must not be preempted because it typically runs right after memmove, 427 // and the GC must not complete between those two. 428 // 429 //go:nosplit 430 func typeBitsBulkBarrier(typ *_type, p, size uintptr) { 431 if typ == nil { 432 throw("runtime: typeBitsBulkBarrier without type") 433 } 434 if typ.size != size { 435 println("runtime: typeBitsBulkBarrier with type ", *typ._string, " of size ", typ.size, " but memory size", size) 436 throw("runtime: invalid typeBitsBulkBarrier") 437 } 438 if typ.kind&kindGCProg != 0 { 439 println("runtime: typeBitsBulkBarrier with type ", *typ._string, " with GC prog") 440 throw("runtime: invalid typeBitsBulkBarrier") 441 } 442 if !writeBarrierEnabled { 443 return 444 } 445 ptrmask := typ.gcdata 446 var bits uint32 447 for i := uintptr(0); i < typ.ptrdata; i += ptrSize { 448 if i&(ptrSize*8-1) == 0 { 449 bits = uint32(*ptrmask) 450 ptrmask = addb(ptrmask, 1) 451 } else { 452 bits = bits >> 1 453 } 454 if bits&1 != 0 { 455 x := (*uintptr)(unsafe.Pointer(p + i)) 456 writebarrierptr_nostore(x, *x) 457 } 458 } 459 } 460 461 // The methods operating on spans all require that h has been returned 462 // by heapBitsForSpan and that size, n, total are the span layout description 463 // returned by the mspan's layout method. 464 // If total > size*n, it means that there is extra leftover memory in the span, 465 // usually due to rounding. 466 // 467 // TODO(rsc): Perhaps introduce a different heapBitsSpan type. 468 469 // initSpan initializes the heap bitmap for a span. 470 func (h heapBits) initSpan(size, n, total uintptr) { 471 if total%heapBitmapScale != 0 { 472 throw("initSpan: unaligned length") 473 } 474 nbyte := total / heapBitmapScale 475 if ptrSize == 8 && size == ptrSize { 476 end := h.bitp 477 bitp := subtractb(end, nbyte-1) 478 for { 479 *bitp = bitPointerAll 480 if bitp == end { 481 break 482 } 483 bitp = add1(bitp) 484 } 485 return 486 } 487 memclr(unsafe.Pointer(subtractb(h.bitp, nbyte-1)), nbyte) 488 } 489 490 // initCheckmarkSpan initializes a span for being checkmarked. 491 // It clears the checkmark bits, which are set to 1 in normal operation. 492 func (h heapBits) initCheckmarkSpan(size, n, total uintptr) { 493 // The ptrSize == 8 is a compile-time constant false on 32-bit and eliminates this code entirely. 494 if ptrSize == 8 && size == ptrSize { 495 // Checkmark bit is type bit, bottom bit of every 2-bit entry. 496 // Only possible on 64-bit system, since minimum size is 8. 497 // Must clear type bit (checkmark bit) of every word. 498 // The type bit is the lower of every two-bit pair. 499 bitp := h.bitp 500 for i := uintptr(0); i < n; i += 4 { 501 *bitp &^= bitPointerAll 502 bitp = subtract1(bitp) 503 } 504 return 505 } 506 for i := uintptr(0); i < n; i++ { 507 *h.bitp &^= bitMarked << (heapBitsShift + h.shift) 508 h = h.forward(size / ptrSize) 509 } 510 } 511 512 // clearCheckmarkSpan undoes all the checkmarking in a span. 513 // The actual checkmark bits are ignored, so the only work to do 514 // is to fix the pointer bits. (Pointer bits are ignored by scanobject 515 // but consulted by typedmemmove.) 516 func (h heapBits) clearCheckmarkSpan(size, n, total uintptr) { 517 // The ptrSize == 8 is a compile-time constant false on 32-bit and eliminates this code entirely. 518 if ptrSize == 8 && size == ptrSize { 519 // Checkmark bit is type bit, bottom bit of every 2-bit entry. 520 // Only possible on 64-bit system, since minimum size is 8. 521 // Must clear type bit (checkmark bit) of every word. 522 // The type bit is the lower of every two-bit pair. 523 bitp := h.bitp 524 for i := uintptr(0); i < n; i += 4 { 525 *bitp |= bitPointerAll 526 bitp = subtract1(bitp) 527 } 528 } 529 } 530 531 // heapBitsSweepSpan coordinates the sweeping of a span by reading 532 // and updating the corresponding heap bitmap entries. 533 // For each free object in the span, heapBitsSweepSpan sets the type 534 // bits for the first two words (or one for single-word objects) to typeDead 535 // and then calls f(p), where p is the object's base address. 536 // f is expected to add the object to a free list. 537 // For non-free objects, heapBitsSweepSpan turns off the marked bit. 538 func heapBitsSweepSpan(base, size, n uintptr, f func(uintptr)) { 539 h := heapBitsForSpan(base) 540 switch { 541 default: 542 throw("heapBitsSweepSpan") 543 case ptrSize == 8 && size == ptrSize: 544 // Consider mark bits in all four 2-bit entries of each bitmap byte. 545 bitp := h.bitp 546 for i := uintptr(0); i < n; i += 4 { 547 x := uint32(*bitp) 548 // Note that unlike the other size cases, we leave the pointer bits set here. 549 // These are initialized during initSpan when the span is created and left 550 // in place the whole time the span is used for pointer-sized objects. 551 // That lets heapBitsSetType avoid an atomic update to set the pointer bit 552 // during allocation. 553 if x&bitMarked != 0 { 554 x &^= bitMarked 555 } else { 556 f(base + i*ptrSize) 557 } 558 if x&(bitMarked<<heapBitsShift) != 0 { 559 x &^= bitMarked << heapBitsShift 560 } else { 561 f(base + (i+1)*ptrSize) 562 } 563 if x&(bitMarked<<(2*heapBitsShift)) != 0 { 564 x &^= bitMarked << (2 * heapBitsShift) 565 } else { 566 f(base + (i+2)*ptrSize) 567 } 568 if x&(bitMarked<<(3*heapBitsShift)) != 0 { 569 x &^= bitMarked << (3 * heapBitsShift) 570 } else { 571 f(base + (i+3)*ptrSize) 572 } 573 *bitp = uint8(x) 574 bitp = subtract1(bitp) 575 } 576 577 case size%(4*ptrSize) == 0: 578 // Mark bit is in first word of each object. 579 // Each object starts at bit 0 of a heap bitmap byte. 580 bitp := h.bitp 581 step := size / heapBitmapScale 582 for i := uintptr(0); i < n; i++ { 583 x := uint32(*bitp) 584 if x&bitMarked != 0 { 585 x &^= bitMarked 586 } else { 587 x = 0 588 f(base + i*size) 589 } 590 *bitp = uint8(x) 591 bitp = subtractb(bitp, step) 592 } 593 594 case size%(4*ptrSize) == 2*ptrSize: 595 // Mark bit is in first word of each object, 596 // but every other object starts halfway through a heap bitmap byte. 597 // Unroll loop 2x to handle alternating shift count and step size. 598 bitp := h.bitp 599 step := size / heapBitmapScale 600 var i uintptr 601 for i = uintptr(0); i < n; i += 2 { 602 x := uint32(*bitp) 603 if x&bitMarked != 0 { 604 x &^= bitMarked 605 } else { 606 x &^= bitMarked | bitPointer | (bitMarked|bitPointer)<<heapBitsShift 607 f(base + i*size) 608 if size > 2*ptrSize { 609 x = 0 610 } 611 } 612 *bitp = uint8(x) 613 if i+1 >= n { 614 break 615 } 616 bitp = subtractb(bitp, step) 617 x = uint32(*bitp) 618 if x&(bitMarked<<(2*heapBitsShift)) != 0 { 619 x &^= bitMarked << (2 * heapBitsShift) 620 } else { 621 x &^= (bitMarked|bitPointer)<<(2*heapBitsShift) | (bitMarked|bitPointer)<<(3*heapBitsShift) 622 f(base + (i+1)*size) 623 if size > 2*ptrSize { 624 *subtract1(bitp) = 0 625 } 626 } 627 *bitp = uint8(x) 628 bitp = subtractb(bitp, step+1) 629 } 630 } 631 } 632 633 // heapBitsSetType records that the new allocation [x, x+size) 634 // holds in [x, x+dataSize) one or more values of type typ. 635 // (The number of values is given by dataSize / typ.size.) 636 // If dataSize < size, the fragment [x+dataSize, x+size) is 637 // recorded as non-pointer data. 638 // It is known that the type has pointers somewhere; 639 // malloc does not call heapBitsSetType when there are no pointers, 640 // because all free objects are marked as noscan during 641 // heapBitsSweepSpan. 642 // There can only be one allocation from a given span active at a time, 643 // so this code is not racing with other instances of itself, 644 // and we don't allocate from a span until it has been swept, 645 // so this code is not racing with heapBitsSweepSpan. 646 // It is, however, racing with the concurrent GC mark phase, 647 // which can be setting the mark bit in the leading 2-bit entry 648 // of an allocated block. The block we are modifying is not quite 649 // allocated yet, so the GC marker is not racing with updates to x's bits, 650 // but if the start or end of x shares a bitmap byte with an adjacent 651 // object, the GC marker is racing with updates to those object's mark bits. 652 func heapBitsSetType(x, size, dataSize uintptr, typ *_type) { 653 const doubleCheck = false // slow but helpful; enable to test modifications to this code 654 655 // dataSize is always size rounded up to the next malloc size class, 656 // except in the case of allocating a defer block, in which case 657 // size is sizeof(_defer{}) (at least 6 words) and dataSize may be 658 // arbitrarily larger. 659 // 660 // The checks for size == ptrSize and size == 2*ptrSize can therefore 661 // assume that dataSize == size without checking it explicitly. 662 663 if ptrSize == 8 && size == ptrSize { 664 // It's one word and it has pointers, it must be a pointer. 665 // In general we'd need an atomic update here if the 666 // concurrent GC were marking objects in this span, 667 // because each bitmap byte describes 3 other objects 668 // in addition to the one being allocated. 669 // However, since all allocated one-word objects are pointers 670 // (non-pointers are aggregated into tinySize allocations), 671 // initSpan sets the pointer bits for us. Nothing to do here. 672 if doubleCheck { 673 h := heapBitsForAddr(x) 674 if !h.isPointer() { 675 throw("heapBitsSetType: pointer bit missing") 676 } 677 } 678 return 679 } 680 681 h := heapBitsForAddr(x) 682 ptrmask := typ.gcdata // start of 1-bit pointer mask (or GC program, handled below) 683 684 // Heap bitmap bits for 2-word object are only 4 bits, 685 // so also shared with objects next to it; use atomic updates. 686 // This is called out as a special case primarily for 32-bit systems, 687 // so that on 32-bit systems the code below can assume all objects 688 // are 4-word aligned (because they're all 16-byte aligned). 689 if size == 2*ptrSize { 690 if typ.size == ptrSize { 691 // We're allocating a block big enough to hold two pointers. 692 // On 64-bit, that means the actual object must be two pointers, 693 // or else we'd have used the one-pointer-sized block. 694 // On 32-bit, however, this is the 8-byte block, the smallest one. 695 // So it could be that we're allocating one pointer and this was 696 // just the smallest block available. Distinguish by checking dataSize. 697 // (In general the number of instances of typ being allocated is 698 // dataSize/typ.size.) 699 if ptrSize == 4 && dataSize == ptrSize { 700 // 1 pointer. 701 if gcphase == _GCoff { 702 *h.bitp |= bitPointer << h.shift 703 } else { 704 atomicor8(h.bitp, bitPointer<<h.shift) 705 } 706 } else { 707 // 2-element slice of pointer. 708 if gcphase == _GCoff { 709 *h.bitp |= (bitPointer | bitPointer<<heapBitsShift) << h.shift 710 } else { 711 atomicor8(h.bitp, (bitPointer|bitPointer<<heapBitsShift)<<h.shift) 712 } 713 } 714 return 715 } 716 // Otherwise typ.size must be 2*ptrSize, and typ.kind&kindGCProg == 0. 717 if doubleCheck { 718 if typ.size != 2*ptrSize || typ.kind&kindGCProg != 0 { 719 print("runtime: heapBitsSetType size=", size, " but typ.size=", typ.size, " gcprog=", typ.kind&kindGCProg != 0, "\n") 720 throw("heapBitsSetType") 721 } 722 } 723 b := uint32(*ptrmask) 724 hb := b & 3 725 if gcphase == _GCoff { 726 *h.bitp |= uint8(hb << h.shift) 727 } else { 728 atomicor8(h.bitp, uint8(hb<<h.shift)) 729 } 730 return 731 } 732 733 // Copy from 1-bit ptrmask into 2-bit bitmap. 734 // The basic approach is to use a single uintptr as a bit buffer, 735 // alternating between reloading the buffer and writing bitmap bytes. 736 // In general, one load can supply two bitmap byte writes. 737 // This is a lot of lines of code, but it compiles into relatively few 738 // machine instructions. 739 740 var ( 741 // Ptrmask input. 742 p *byte // last ptrmask byte read 743 b uintptr // ptrmask bits already loaded 744 nb uintptr // number of bits in b at next read 745 endp *byte // final ptrmask byte to read (then repeat) 746 endnb uintptr // number of valid bits in *endp 747 pbits uintptr // alternate source of bits 748 749 // Heap bitmap output. 750 w uintptr // words processed 751 nw uintptr // number of words to process 752 hbitp *byte // next heap bitmap byte to write 753 hb uintptr // bits being prepared for *hbitp 754 ) 755 756 hbitp = h.bitp 757 758 // Handle GC program. Delayed until this part of the code 759 // so that we can use the same double-checking mechanism 760 // as the 1-bit case. Nothing above could have encountered 761 // GC programs: the cases were all too small. 762 if typ.kind&kindGCProg != 0 { 763 heapBitsSetTypeGCProg(h, typ.ptrdata, typ.size, dataSize, size, addb(typ.gcdata, 4)) 764 if doubleCheck { 765 // Double-check the heap bits written by GC program 766 // by running the GC program to create a 1-bit pointer mask 767 // and then jumping to the double-check code below. 768 // This doesn't catch bugs shared between the 1-bit and 4-bit 769 // GC program execution, but it does catch mistakes specific 770 // to just one of those and bugs in heapBitsSetTypeGCProg's 771 // implementation of arrays. 772 lock(&debugPtrmask.lock) 773 if debugPtrmask.data == nil { 774 debugPtrmask.data = (*byte)(persistentalloc(1<<20, 1, &memstats.other_sys)) 775 } 776 ptrmask = debugPtrmask.data 777 runGCProg(addb(typ.gcdata, 4), nil, ptrmask, 1) 778 goto Phase4 779 } 780 return 781 } 782 783 // Note about sizes: 784 // 785 // typ.size is the number of words in the object, 786 // and typ.ptrdata is the number of words in the prefix 787 // of the object that contains pointers. That is, the final 788 // typ.size - typ.ptrdata words contain no pointers. 789 // This allows optimization of a common pattern where 790 // an object has a small header followed by a large scalar 791 // buffer. If we know the pointers are over, we don't have 792 // to scan the buffer's heap bitmap at all. 793 // The 1-bit ptrmasks are sized to contain only bits for 794 // the typ.ptrdata prefix, zero padded out to a full byte 795 // of bitmap. This code sets nw (below) so that heap bitmap 796 // bits are only written for the typ.ptrdata prefix; if there is 797 // more room in the allocated object, the next heap bitmap 798 // entry is a 00, indicating that there are no more pointers 799 // to scan. So only the ptrmask for the ptrdata bytes is needed. 800 // 801 // Replicated copies are not as nice: if there is an array of 802 // objects with scalar tails, all but the last tail does have to 803 // be initialized, because there is no way to say "skip forward". 804 // However, because of the possibility of a repeated type with 805 // size not a multiple of 4 pointers (one heap bitmap byte), 806 // the code already must handle the last ptrmask byte specially 807 // by treating it as containing only the bits for endnb pointers, 808 // where endnb <= 4. We represent large scalar tails that must 809 // be expanded in the replication by setting endnb larger than 4. 810 // This will have the effect of reading many bits out of b, 811 // but once the real bits are shifted out, b will supply as many 812 // zero bits as we try to read, which is exactly what we need. 813 814 p = ptrmask 815 if typ.size < dataSize { 816 // Filling in bits for an array of typ. 817 // Set up for repetition of ptrmask during main loop. 818 // Note that ptrmask describes only a prefix of 819 const maxBits = ptrSize*8 - 7 820 if typ.ptrdata/ptrSize <= maxBits { 821 // Entire ptrmask fits in uintptr with room for a byte fragment. 822 // Load into pbits and never read from ptrmask again. 823 // This is especially important when the ptrmask has 824 // fewer than 8 bits in it; otherwise the reload in the middle 825 // of the Phase 2 loop would itself need to loop to gather 826 // at least 8 bits. 827 828 // Accumulate ptrmask into b. 829 // ptrmask is sized to describe only typ.ptrdata, but we record 830 // it as describing typ.size bytes, since all the high bits are zero. 831 nb = typ.ptrdata / ptrSize 832 for i := uintptr(0); i < nb; i += 8 { 833 b |= uintptr(*p) << i 834 p = add1(p) 835 } 836 nb = typ.size / ptrSize 837 838 // Replicate ptrmask to fill entire pbits uintptr. 839 // Doubling and truncating is fewer steps than 840 // iterating by nb each time. (nb could be 1.) 841 // Since we loaded typ.ptrdata/ptrSize bits 842 // but are pretending to have typ.size/ptrSize, 843 // there might be no replication necessary/possible. 844 pbits = b 845 endnb = nb 846 if nb+nb <= maxBits { 847 for endnb <= ptrSize*8 { 848 pbits |= pbits << endnb 849 endnb += endnb 850 } 851 // Truncate to a multiple of original ptrmask. 852 endnb = maxBits / nb * nb 853 pbits &= 1<<endnb - 1 854 b = pbits 855 nb = endnb 856 } 857 858 // Clear p and endp as sentinel for using pbits. 859 // Checked during Phase 2 loop. 860 p = nil 861 endp = nil 862 } else { 863 // Ptrmask is larger. Read it multiple times. 864 n := (typ.ptrdata/ptrSize+7)/8 - 1 865 endp = addb(ptrmask, n) 866 endnb = typ.size/ptrSize - n*8 867 } 868 } 869 if p != nil { 870 b = uintptr(*p) 871 p = add1(p) 872 nb = 8 873 } 874 875 if typ.size == dataSize { 876 // Single entry: can stop once we reach the non-pointer data. 877 nw = typ.ptrdata / ptrSize 878 } else { 879 // Repeated instances of typ in an array. 880 // Have to process first N-1 entries in full, but can stop 881 // once we reach the non-pointer data in the final entry. 882 nw = ((dataSize/typ.size-1)*typ.size + typ.ptrdata) / ptrSize 883 } 884 if nw == 0 { 885 // No pointers! Caller was supposed to check. 886 println("runtime: invalid type ", *typ._string) 887 throw("heapBitsSetType: called with non-pointer type") 888 return 889 } 890 if nw < 2 { 891 // Must write at least 2 words, because the "no scan" 892 // encoding doesn't take effect until the third word. 893 nw = 2 894 } 895 896 // Phase 1: Special case for leading byte (shift==0) or half-byte (shift==4). 897 // The leading byte is special because it contains the bits for words 0 and 1, 898 // which do not have the marked bits set. 899 // The leading half-byte is special because it's a half a byte and must be 900 // manipulated atomically. 901 switch { 902 default: 903 throw("heapBitsSetType: unexpected shift") 904 905 case h.shift == 0: 906 // Ptrmask and heap bitmap are aligned. 907 // Handle first byte of bitmap specially. 908 // The first byte we write out contains the first two words of the object. 909 // In those words, the mark bits are mark and checkmark, respectively, 910 // and must not be set. In all following words, we want to set the mark bit 911 // as a signal that the object continues to the next 2-bit entry in the bitmap. 912 hb = b & bitPointerAll 913 hb |= bitMarked<<(2*heapBitsShift) | bitMarked<<(3*heapBitsShift) 914 if w += 4; w >= nw { 915 goto Phase3 916 } 917 *hbitp = uint8(hb) 918 hbitp = subtract1(hbitp) 919 b >>= 4 920 nb -= 4 921 922 case ptrSize == 8 && h.shift == 2: 923 // Ptrmask and heap bitmap are misaligned. 924 // The bits for the first two words are in a byte shared with another object 925 // and must be updated atomically. 926 // NOTE(rsc): The atomic here may not be necessary. 927 // We took care of 1-word and 2-word objects above, 928 // so this is at least a 6-word object, so our start bits 929 // are shared only with the type bits of another object, 930 // not with its mark bit. Since there is only one allocation 931 // from a given span at a time, we should be able to set 932 // these bits non-atomically. Not worth the risk right now. 933 hb = (b & 3) << (2 * heapBitsShift) 934 b >>= 2 935 nb -= 2 936 // Note: no bitMarker in hb because the first two words don't get markers from us. 937 if gcphase == _GCoff { 938 *hbitp |= uint8(hb) 939 } else { 940 atomicor8(hbitp, uint8(hb)) 941 } 942 hbitp = subtract1(hbitp) 943 if w += 2; w >= nw { 944 // We know that there is more data, because we handled 2-word objects above. 945 // This must be at least a 6-word object. If we're out of pointer words, 946 // mark no scan in next bitmap byte and finish. 947 hb = 0 948 w += 4 949 goto Phase3 950 } 951 } 952 953 // Phase 2: Full bytes in bitmap, up to but not including write to last byte (full or partial) in bitmap. 954 // The loop computes the bits for that last write but does not execute the write; 955 // it leaves the bits in hb for processing by phase 3. 956 // To avoid repeated adjustment of nb, we subtract out the 4 bits we're going to 957 // use in the first half of the loop right now, and then we only adjust nb explicitly 958 // if the 8 bits used by each iteration isn't balanced by 8 bits loaded mid-loop. 959 nb -= 4 960 for { 961 // Emit bitmap byte. 962 // b has at least nb+4 bits, with one exception: 963 // if w+4 >= nw, then b has only nw-w bits, 964 // but we'll stop at the break and then truncate 965 // appropriately in Phase 3. 966 hb = b & bitPointerAll 967 hb |= bitMarkedAll 968 if w += 4; w >= nw { 969 break 970 } 971 *hbitp = uint8(hb) 972 hbitp = subtract1(hbitp) 973 b >>= 4 974 975 // Load more bits. b has nb right now. 976 if p != endp { 977 // Fast path: keep reading from ptrmask. 978 // nb unmodified: we just loaded 8 bits, 979 // and the next iteration will consume 8 bits, 980 // leaving us with the same nb the next time we're here. 981 if nb < 8 { 982 b |= uintptr(*p) << nb 983 p = add1(p) 984 } else { 985 // Reduce the number of bits in b. 986 // This is important if we skipped 987 // over a scalar tail, since nb could 988 // be larger than the bit width of b. 989 nb -= 8 990 } 991 } else if p == nil { 992 // Almost as fast path: track bit count and refill from pbits. 993 // For short repetitions. 994 if nb < 8 { 995 b |= pbits << nb 996 nb += endnb 997 } 998 nb -= 8 // for next iteration 999 } else { 1000 // Slow path: reached end of ptrmask. 1001 // Process final partial byte and rewind to start. 1002 b |= uintptr(*p) << nb 1003 nb += endnb 1004 if nb < 8 { 1005 b |= uintptr(*ptrmask) << nb 1006 p = add1(ptrmask) 1007 } else { 1008 nb -= 8 1009 p = ptrmask 1010 } 1011 } 1012 1013 // Emit bitmap byte. 1014 hb = b & bitPointerAll 1015 hb |= bitMarkedAll 1016 if w += 4; w >= nw { 1017 break 1018 } 1019 *hbitp = uint8(hb) 1020 hbitp = subtract1(hbitp) 1021 b >>= 4 1022 } 1023 1024 Phase3: 1025 // Phase 3: Write last byte or partial byte and zero the rest of the bitmap entries. 1026 if w > nw { 1027 // Counting the 4 entries in hb not yet written to memory, 1028 // there are more entries than possible pointer slots. 1029 // Discard the excess entries (can't be more than 3). 1030 mask := uintptr(1)<<(4-(w-nw)) - 1 1031 hb &= mask | mask<<4 // apply mask to both pointer bits and mark bits 1032 } 1033 1034 // Change nw from counting possibly-pointer words to total words in allocation. 1035 nw = size / ptrSize 1036 1037 // Write whole bitmap bytes. 1038 // The first is hb, the rest are zero. 1039 if w <= nw { 1040 *hbitp = uint8(hb) 1041 hbitp = subtract1(hbitp) 1042 hb = 0 // for possible final half-byte below 1043 for w += 4; w <= nw; w += 4 { 1044 *hbitp = 0 1045 hbitp = subtract1(hbitp) 1046 } 1047 } 1048 1049 // Write final partial bitmap byte if any. 1050 // We know w > nw, or else we'd still be in the loop above. 1051 // It can be bigger only due to the 4 entries in hb that it counts. 1052 // If w == nw+4 then there's nothing left to do: we wrote all nw entries 1053 // and can discard the 4 sitting in hb. 1054 // But if w == nw+2, we need to write first two in hb. 1055 // The byte is shared with the next object so we may need an atomic. 1056 if w == nw+2 { 1057 if gcphase == _GCoff { 1058 *hbitp = *hbitp&^(bitPointer|bitMarked|(bitPointer|bitMarked)<<heapBitsShift) | uint8(hb) 1059 } else { 1060 atomicand8(hbitp, ^uint8(bitPointer|bitMarked|(bitPointer|bitMarked)<<heapBitsShift)) 1061 atomicor8(hbitp, uint8(hb)) 1062 } 1063 } 1064 1065 Phase4: 1066 // Phase 4: all done, but perhaps double check. 1067 if doubleCheck { 1068 end := heapBitsForAddr(x + size) 1069 if typ.kind&kindGCProg == 0 && (hbitp != end.bitp || (w == nw+2) != (end.shift == 2)) { 1070 println("ended at wrong bitmap byte for", *typ._string, "x", dataSize/typ.size) 1071 print("typ.size=", typ.size, " typ.ptrdata=", typ.ptrdata, " dataSize=", dataSize, " size=", size, "\n") 1072 print("w=", w, " nw=", nw, " b=", hex(b), " nb=", nb, " hb=", hex(hb), "\n") 1073 h0 := heapBitsForAddr(x) 1074 print("initial bits h0.bitp=", h0.bitp, " h0.shift=", h0.shift, "\n") 1075 print("ended at hbitp=", hbitp, " but next starts at bitp=", end.bitp, " shift=", end.shift, "\n") 1076 throw("bad heapBitsSetType") 1077 } 1078 1079 // Double-check that bits to be written were written correctly. 1080 // Does not check that other bits were not written, unfortunately. 1081 h := heapBitsForAddr(x) 1082 nptr := typ.ptrdata / ptrSize 1083 ndata := typ.size / ptrSize 1084 count := dataSize / typ.size 1085 totalptr := ((count-1)*typ.size + typ.ptrdata) / ptrSize 1086 for i := uintptr(0); i < size/ptrSize; i++ { 1087 j := i % ndata 1088 var have, want uint8 1089 have = (*h.bitp >> h.shift) & (bitPointer | bitMarked) 1090 if i >= totalptr { 1091 want = 0 // deadmarker 1092 if typ.kind&kindGCProg != 0 && i < (totalptr+3)/4*4 { 1093 want = bitMarked 1094 } 1095 } else { 1096 if j < nptr && (*addb(ptrmask, j/8)>>(j%8))&1 != 0 { 1097 want |= bitPointer 1098 } 1099 if i >= 2 { 1100 want |= bitMarked 1101 } else { 1102 have &^= bitMarked 1103 } 1104 } 1105 if have != want { 1106 println("mismatch writing bits for", *typ._string, "x", dataSize/typ.size) 1107 print("typ.size=", typ.size, " typ.ptrdata=", typ.ptrdata, " dataSize=", dataSize, " size=", size, "\n") 1108 print("kindGCProg=", typ.kind&kindGCProg != 0, "\n") 1109 print("w=", w, " nw=", nw, " b=", hex(b), " nb=", nb, " hb=", hex(hb), "\n") 1110 h0 := heapBitsForAddr(x) 1111 print("initial bits h0.bitp=", h0.bitp, " h0.shift=", h0.shift, "\n") 1112 print("current bits h.bitp=", h.bitp, " h.shift=", h.shift, " *h.bitp=", hex(*h.bitp), "\n") 1113 print("ptrmask=", ptrmask, " p=", p, " endp=", endp, " endnb=", endnb, " pbits=", hex(pbits), " b=", hex(b), " nb=", nb, "\n") 1114 println("at word", i, "offset", i*ptrSize, "have", have, "want", want) 1115 if typ.kind&kindGCProg != 0 { 1116 println("GC program:") 1117 dumpGCProg(addb(typ.gcdata, 4)) 1118 } 1119 throw("bad heapBitsSetType") 1120 } 1121 h = h.next() 1122 } 1123 if ptrmask == debugPtrmask.data { 1124 unlock(&debugPtrmask.lock) 1125 } 1126 } 1127 } 1128 1129 var debugPtrmask struct { 1130 lock mutex 1131 data *byte 1132 } 1133 1134 // heapBitsSetTypeGCProg implements heapBitsSetType using a GC program. 1135 // progSize is the size of the memory described by the program. 1136 // elemSize is the size of the element that the GC program describes (a prefix of). 1137 // dataSize is the total size of the intended data, a multiple of elemSize. 1138 // allocSize is the total size of the allocated memory. 1139 // 1140 // GC programs are only used for large allocations. 1141 // heapBitsSetType requires that allocSize is a multiple of 4 words, 1142 // so that the relevant bitmap bytes are not shared with surrounding 1143 // objects and need not be accessed with atomic instructions. 1144 func heapBitsSetTypeGCProg(h heapBits, progSize, elemSize, dataSize, allocSize uintptr, prog *byte) { 1145 if ptrSize == 8 && allocSize%(4*ptrSize) != 0 { 1146 // Alignment will be wrong. 1147 throw("heapBitsSetTypeGCProg: small allocation") 1148 } 1149 var totalBits uintptr 1150 if elemSize == dataSize { 1151 totalBits = runGCProg(prog, nil, h.bitp, 2) 1152 if totalBits*ptrSize != progSize { 1153 println("runtime: heapBitsSetTypeGCProg: total bits", totalBits, "but progSize", progSize) 1154 throw("heapBitsSetTypeGCProg: unexpected bit count") 1155 } 1156 } else { 1157 count := dataSize / elemSize 1158 1159 // Piece together program trailer to run after prog that does: 1160 // literal(0) 1161 // repeat(1, elemSize-progSize-1) // zeros to fill element size 1162 // repeat(elemSize, count-1) // repeat that element for count 1163 // This zero-pads the data remaining in the first element and then 1164 // repeats that first element to fill the array. 1165 var trailer [40]byte // 3 varints (max 10 each) + some bytes 1166 i := 0 1167 if n := elemSize/ptrSize - progSize/ptrSize; n > 0 { 1168 // literal(0) 1169 trailer[i] = 0x01 1170 i++ 1171 trailer[i] = 0 1172 i++ 1173 if n > 1 { 1174 // repeat(1, n-1) 1175 trailer[i] = 0x81 1176 i++ 1177 n-- 1178 for ; n >= 0x80; n >>= 7 { 1179 trailer[i] = byte(n | 0x80) 1180 i++ 1181 } 1182 trailer[i] = byte(n) 1183 i++ 1184 } 1185 } 1186 // repeat(elemSize/ptrSize, count-1) 1187 trailer[i] = 0x80 1188 i++ 1189 n := elemSize / ptrSize 1190 for ; n >= 0x80; n >>= 7 { 1191 trailer[i] = byte(n | 0x80) 1192 i++ 1193 } 1194 trailer[i] = byte(n) 1195 i++ 1196 n = count - 1 1197 for ; n >= 0x80; n >>= 7 { 1198 trailer[i] = byte(n | 0x80) 1199 i++ 1200 } 1201 trailer[i] = byte(n) 1202 i++ 1203 trailer[i] = 0 1204 i++ 1205 1206 runGCProg(prog, &trailer[0], h.bitp, 2) 1207 1208 // Even though we filled in the full array just now, 1209 // record that we only filled in up to the ptrdata of the 1210 // last element. This will cause the code below to 1211 // memclr the dead section of the final array element, 1212 // so that scanobject can stop early in the final element. 1213 totalBits = (elemSize*(count-1) + progSize) / ptrSize 1214 } 1215 endProg := unsafe.Pointer(subtractb(h.bitp, (totalBits+3)/4)) 1216 endAlloc := unsafe.Pointer(subtractb(h.bitp, allocSize/heapBitmapScale)) 1217 memclr(add(endAlloc, 1), uintptr(endProg)-uintptr(endAlloc)) 1218 } 1219 1220 // progToPointerMask returns the 1-bit pointer mask output by the GC program prog. 1221 // size the size of the region described by prog, in bytes. 1222 // The resulting bitvector will have no more than size/ptrSize bits. 1223 func progToPointerMask(prog *byte, size uintptr) bitvector { 1224 n := (size/ptrSize + 7) / 8 1225 x := (*[1 << 30]byte)(persistentalloc(n+1, 1, &memstats.buckhash_sys))[:n+1] 1226 x[len(x)-1] = 0xa1 // overflow check sentinel 1227 n = runGCProg(prog, nil, &x[0], 1) 1228 if x[len(x)-1] != 0xa1 { 1229 throw("progToPointerMask: overflow") 1230 } 1231 return bitvector{int32(n), &x[0]} 1232 } 1233 1234 // Packed GC pointer bitmaps, aka GC programs. 1235 // 1236 // For large types containing arrays, the type information has a 1237 // natural repetition that can be encoded to save space in the 1238 // binary and in the memory representation of the type information. 1239 // 1240 // The encoding is a simple Lempel-Ziv style bytecode machine 1241 // with the following instructions: 1242 // 1243 // 00000000: stop 1244 // 0nnnnnnn: emit n bits copied from the next (n+7)/8 bytes 1245 // 10000000 n c: repeat the previous n bits c times; n, c are varints 1246 // 1nnnnnnn c: repeat the previous n bits c times; c is a varint 1247 1248 // runGCProg executes the GC program prog, and then trailer if non-nil, 1249 // writing to dst with entries of the given size. 1250 // If size == 1, dst is a 1-bit pointer mask laid out moving forward from dst. 1251 // If size == 2, dst is the 2-bit heap bitmap, and writes move backward 1252 // starting at dst (because the heap bitmap does). In this case, the caller guarantees 1253 // that only whole bytes in dst need to be written. 1254 // 1255 // runGCProg returns the number of 1- or 2-bit entries written to memory. 1256 func runGCProg(prog, trailer, dst *byte, size int) uintptr { 1257 dstStart := dst 1258 1259 // Bits waiting to be written to memory. 1260 var bits uintptr 1261 var nbits uintptr 1262 1263 p := prog 1264 Run: 1265 for { 1266 // Flush accumulated full bytes. 1267 // The rest of the loop assumes that nbits <= 7. 1268 for ; nbits >= 8; nbits -= 8 { 1269 if size == 1 { 1270 *dst = uint8(bits) 1271 dst = add1(dst) 1272 bits >>= 8 1273 } else { 1274 v := bits&bitPointerAll | bitMarkedAll 1275 *dst = uint8(v) 1276 dst = subtract1(dst) 1277 bits >>= 4 1278 v = bits&bitPointerAll | bitMarkedAll 1279 *dst = uint8(v) 1280 dst = subtract1(dst) 1281 bits >>= 4 1282 } 1283 } 1284 1285 // Process one instruction. 1286 inst := uintptr(*p) 1287 p = add1(p) 1288 n := inst & 0x7F 1289 if inst&0x80 == 0 { 1290 // Literal bits; n == 0 means end of program. 1291 if n == 0 { 1292 // Program is over; continue in trailer if present. 1293 if trailer != nil { 1294 //println("trailer") 1295 p = trailer 1296 trailer = nil 1297 continue 1298 } 1299 //println("done") 1300 break Run 1301 } 1302 //println("lit", n, dst) 1303 nbyte := n / 8 1304 for i := uintptr(0); i < nbyte; i++ { 1305 bits |= uintptr(*p) << nbits 1306 p = add1(p) 1307 if size == 1 { 1308 *dst = uint8(bits) 1309 dst = add1(dst) 1310 bits >>= 8 1311 } else { 1312 v := bits&0xf | bitMarkedAll 1313 *dst = uint8(v) 1314 dst = subtract1(dst) 1315 bits >>= 4 1316 v = bits&0xf | bitMarkedAll 1317 *dst = uint8(v) 1318 dst = subtract1(dst) 1319 bits >>= 4 1320 } 1321 } 1322 if n %= 8; n > 0 { 1323 bits |= uintptr(*p) << nbits 1324 p = add1(p) 1325 nbits += n 1326 } 1327 continue Run 1328 } 1329 1330 // Repeat. If n == 0, it is encoded in a varint in the next bytes. 1331 if n == 0 { 1332 for off := uint(0); ; off += 7 { 1333 x := uintptr(*p) 1334 p = add1(p) 1335 n |= (x & 0x7F) << off 1336 if x&0x80 == 0 { 1337 break 1338 } 1339 } 1340 } 1341 1342 // Count is encoded in a varint in the next bytes. 1343 c := uintptr(0) 1344 for off := uint(0); ; off += 7 { 1345 x := uintptr(*p) 1346 p = add1(p) 1347 c |= (x & 0x7F) << off 1348 if x&0x80 == 0 { 1349 break 1350 } 1351 } 1352 c *= n // now total number of bits to copy 1353 1354 // If the number of bits being repeated is small, load them 1355 // into a register and use that register for the entire loop 1356 // instead of repeatedly reading from memory. 1357 // Handling fewer than 8 bits here makes the general loop simpler. 1358 // The cutoff is ptrSize*8 - 7 to guarantee that when we add 1359 // the pattern to a bit buffer holding at most 7 bits (a partial byte) 1360 // it will not overflow. 1361 src := dst 1362 const maxBits = ptrSize*8 - 7 1363 if n <= maxBits { 1364 // Start with bits in output buffer. 1365 pattern := bits 1366 npattern := nbits 1367 1368 // If we need more bits, fetch them from memory. 1369 if size == 1 { 1370 src = subtract1(src) 1371 for npattern < n { 1372 pattern <<= 8 1373 pattern |= uintptr(*src) 1374 src = subtract1(src) 1375 npattern += 8 1376 } 1377 } else { 1378 src = add1(src) 1379 for npattern < n { 1380 pattern <<= 4 1381 pattern |= uintptr(*src) & 0xf 1382 src = add1(src) 1383 npattern += 4 1384 } 1385 } 1386 1387 // We started with the whole bit output buffer, 1388 // and then we loaded bits from whole bytes. 1389 // Either way, we might now have too many instead of too few. 1390 // Discard the extra. 1391 if npattern > n { 1392 pattern >>= npattern - n 1393 npattern = n 1394 } 1395 1396 // Replicate pattern to at most maxBits. 1397 if npattern == 1 { 1398 // One bit being repeated. 1399 // If the bit is 1, make the pattern all 1s. 1400 // If the bit is 0, the pattern is already all 0s, 1401 // but we can claim that the number of bits 1402 // in the word is equal to the number we need (c), 1403 // because right shift of bits will zero fill. 1404 if pattern == 1 { 1405 pattern = 1<<maxBits - 1 1406 npattern = maxBits 1407 } else { 1408 npattern = c 1409 } 1410 } else { 1411 b := pattern 1412 nb := npattern 1413 if nb+nb <= maxBits { 1414 // Double pattern until the whole uintptr is filled. 1415 for nb <= ptrSize*8 { 1416 b |= b << nb 1417 nb += nb 1418 } 1419 // Trim away incomplete copy of original pattern in high bits. 1420 // TODO(rsc): Replace with table lookup or loop on systems without divide? 1421 nb = maxBits / npattern * npattern 1422 b &= 1<<nb - 1 1423 pattern = b 1424 npattern = nb 1425 } 1426 } 1427 1428 // Add pattern to bit buffer and flush bit buffer, c/npattern times. 1429 // Since pattern contains >8 bits, there will be full bytes to flush 1430 // on each iteration. 1431 for ; c >= npattern; c -= npattern { 1432 bits |= pattern << nbits 1433 nbits += npattern 1434 if size == 1 { 1435 for nbits >= 8 { 1436 *dst = uint8(bits) 1437 dst = add1(dst) 1438 bits >>= 8 1439 nbits -= 8 1440 } 1441 } else { 1442 for nbits >= 4 { 1443 *dst = uint8(bits&0xf | bitMarkedAll) 1444 dst = subtract1(dst) 1445 bits >>= 4 1446 nbits -= 4 1447 } 1448 } 1449 } 1450 1451 // Add final fragment to bit buffer. 1452 if c > 0 { 1453 pattern &= 1<<c - 1 1454 bits |= pattern << nbits 1455 nbits += c 1456 } 1457 continue Run 1458 } 1459 1460 // Repeat; n too large to fit in a register. 1461 // Since nbits <= 7, we know the first few bytes of repeated data 1462 // are already written to memory. 1463 off := n - nbits // n > nbits because n > maxBits and nbits <= 7 1464 if size == 1 { 1465 // Leading src fragment. 1466 src = subtractb(src, (off+7)/8) 1467 if frag := off & 7; frag != 0 { 1468 bits |= uintptr(*src) >> (8 - frag) << nbits 1469 src = add1(src) 1470 nbits += frag 1471 c -= frag 1472 } 1473 // Main loop: load one byte, write another. 1474 // The bits are rotating through the bit buffer. 1475 for i := c / 8; i > 0; i-- { 1476 bits |= uintptr(*src) << nbits 1477 src = add1(src) 1478 *dst = uint8(bits) 1479 dst = add1(dst) 1480 bits >>= 8 1481 } 1482 // Final src fragment. 1483 if c %= 8; c > 0 { 1484 bits |= (uintptr(*src) & (1<<c - 1)) << nbits 1485 nbits += c 1486 } 1487 } else { 1488 // Leading src fragment. 1489 src = addb(src, (off+3)/4) 1490 if frag := off & 3; frag != 0 { 1491 bits |= (uintptr(*src) & 0xf) >> (4 - frag) << nbits 1492 src = subtract1(src) 1493 nbits += frag 1494 c -= frag 1495 } 1496 // Main loop: load one byte, write another. 1497 // The bits are rotating through the bit buffer. 1498 for i := c / 4; i > 0; i-- { 1499 bits |= (uintptr(*src) & 0xf) << nbits 1500 src = subtract1(src) 1501 *dst = uint8(bits&0xf | bitMarkedAll) 1502 dst = subtract1(dst) 1503 bits >>= 4 1504 } 1505 // Final src fragment. 1506 if c %= 4; c > 0 { 1507 bits |= (uintptr(*src) & (1<<c - 1)) << nbits 1508 nbits += c 1509 } 1510 } 1511 } 1512 1513 // Write any final bits out, using full-byte writes, even for the final byte. 1514 var totalBits uintptr 1515 if size == 1 { 1516 totalBits = (uintptr(unsafe.Pointer(dst))-uintptr(unsafe.Pointer(dstStart)))*8 + nbits 1517 nbits += -nbits & 7 1518 for ; nbits > 0; nbits -= 8 { 1519 *dst = uint8(bits) 1520 dst = add1(dst) 1521 bits >>= 8 1522 } 1523 } else { 1524 totalBits = (uintptr(unsafe.Pointer(dstStart))-uintptr(unsafe.Pointer(dst)))*4 + nbits 1525 nbits += -nbits & 3 1526 for ; nbits > 0; nbits -= 4 { 1527 v := bits&0xf | bitMarkedAll 1528 *dst = uint8(v) 1529 dst = subtract1(dst) 1530 bits >>= 4 1531 } 1532 // Clear the mark bits in the first two entries. 1533 // They are the actual mark and checkmark bits, 1534 // not non-dead markers. It simplified the code 1535 // above to set the marker in every bit written and 1536 // then clear these two as a special case at the end. 1537 *dstStart &^= bitMarked | bitMarked<<heapBitsShift 1538 } 1539 return totalBits 1540 } 1541 1542 func dumpGCProg(p *byte) { 1543 nptr := 0 1544 for { 1545 x := *p 1546 p = add1(p) 1547 if x == 0 { 1548 print("\t", nptr, " end\n") 1549 break 1550 } 1551 if x&0x80 == 0 { 1552 print("\t", nptr, " lit ", x, ":") 1553 n := int(x+7) / 8 1554 for i := 0; i < n; i++ { 1555 print(" ", hex(*p)) 1556 p = add1(p) 1557 } 1558 print("\n") 1559 nptr += int(x) 1560 } else { 1561 nbit := int(x &^ 0x80) 1562 if nbit == 0 { 1563 for nb := uint(0); ; nb += 7 { 1564 x := *p 1565 p = add1(p) 1566 nbit |= int(x&0x7f) << nb 1567 if x&0x80 == 0 { 1568 break 1569 } 1570 } 1571 } 1572 count := 0 1573 for nb := uint(0); ; nb += 7 { 1574 x := *p 1575 p = add1(p) 1576 count |= int(x&0x7f) << nb 1577 if x&0x80 == 0 { 1578 break 1579 } 1580 } 1581 print("\t", nptr, " repeat ", nbit, " × ", count, "\n") 1582 nptr += nbit * count 1583 } 1584 } 1585 } 1586 1587 // Testing. 1588 1589 func getgcmaskcb(frame *stkframe, ctxt unsafe.Pointer) bool { 1590 target := (*stkframe)(ctxt) 1591 if frame.sp <= target.sp && target.sp < frame.varp { 1592 *target = *frame 1593 return false 1594 } 1595 return true 1596 } 1597 1598 // gcbits returns the GC type info for x, for testing. 1599 // The result is the bitmap entries (0 or 1), one entry per byte. 1600 //go:linkname reflect_gcbits reflect.gcbits 1601 func reflect_gcbits(x interface{}) []byte { 1602 ret := getgcmask(x) 1603 typ := (*ptrtype)(unsafe.Pointer((*eface)(unsafe.Pointer(&x))._type)).elem 1604 nptr := typ.ptrdata / ptrSize 1605 for uintptr(len(ret)) > nptr && ret[len(ret)-1] == 0 { 1606 ret = ret[:len(ret)-1] 1607 } 1608 return ret 1609 } 1610 1611 // Returns GC type info for object p for testing. 1612 func getgcmask(ep interface{}) (mask []byte) { 1613 e := *(*eface)(unsafe.Pointer(&ep)) 1614 p := e.data 1615 t := e._type 1616 // data or bss 1617 for datap := &firstmoduledata; datap != nil; datap = datap.next { 1618 // data 1619 if datap.data <= uintptr(p) && uintptr(p) < datap.edata { 1620 bitmap := datap.gcdatamask.bytedata 1621 n := (*ptrtype)(unsafe.Pointer(t)).elem.size 1622 mask = make([]byte, n/ptrSize) 1623 for i := uintptr(0); i < n; i += ptrSize { 1624 off := (uintptr(p) + i - datap.data) / ptrSize 1625 mask[i/ptrSize] = (*addb(bitmap, off/8) >> (off % 8)) & 1 1626 } 1627 return 1628 } 1629 1630 // bss 1631 if datap.bss <= uintptr(p) && uintptr(p) < datap.ebss { 1632 bitmap := datap.gcbssmask.bytedata 1633 n := (*ptrtype)(unsafe.Pointer(t)).elem.size 1634 mask = make([]byte, n/ptrSize) 1635 for i := uintptr(0); i < n; i += ptrSize { 1636 off := (uintptr(p) + i - datap.bss) / ptrSize 1637 mask[i/ptrSize] = (*addb(bitmap, off/8) >> (off % 8)) & 1 1638 } 1639 return 1640 } 1641 } 1642 1643 // heap 1644 var n uintptr 1645 var base uintptr 1646 if mlookup(uintptr(p), &base, &n, nil) != 0 { 1647 mask = make([]byte, n/ptrSize) 1648 for i := uintptr(0); i < n; i += ptrSize { 1649 hbits := heapBitsForAddr(base + i) 1650 if hbits.isPointer() { 1651 mask[i/ptrSize] = 1 1652 } 1653 if i >= 2*ptrSize && !hbits.isMarked() { 1654 mask = mask[:i/ptrSize] 1655 break 1656 } 1657 } 1658 return 1659 } 1660 1661 // stack 1662 if _g_ := getg(); _g_.m.curg.stack.lo <= uintptr(p) && uintptr(p) < _g_.m.curg.stack.hi { 1663 var frame stkframe 1664 frame.sp = uintptr(p) 1665 _g_ := getg() 1666 gentraceback(_g_.m.curg.sched.pc, _g_.m.curg.sched.sp, 0, _g_.m.curg, 0, nil, 1000, getgcmaskcb, noescape(unsafe.Pointer(&frame)), 0) 1667 if frame.fn != nil { 1668 f := frame.fn 1669 targetpc := frame.continpc 1670 if targetpc == 0 { 1671 return 1672 } 1673 if targetpc != f.entry { 1674 targetpc-- 1675 } 1676 pcdata := pcdatavalue(f, _PCDATA_StackMapIndex, targetpc) 1677 if pcdata == -1 { 1678 return 1679 } 1680 stkmap := (*stackmap)(funcdata(f, _FUNCDATA_LocalsPointerMaps)) 1681 if stkmap == nil || stkmap.n <= 0 { 1682 return 1683 } 1684 bv := stackmapdata(stkmap, pcdata) 1685 size := uintptr(bv.n) * ptrSize 1686 n := (*ptrtype)(unsafe.Pointer(t)).elem.size 1687 mask = make([]byte, n/ptrSize) 1688 for i := uintptr(0); i < n; i += ptrSize { 1689 bitmap := bv.bytedata 1690 off := (uintptr(p) + i - frame.varp + size) / ptrSize 1691 mask[i/ptrSize] = (*addb(bitmap, off/8) >> (off % 8)) & 1 1692 } 1693 } 1694 return 1695 } 1696 1697 // otherwise, not something the GC knows about. 1698 // possibly read-only data, like malloc(0). 1699 // must not have pointers 1700 return 1701 }