github.com/akaros/go-akaros@v0.0.0-20181004170632-85005d477eab/src/runtime/malloc.go (about) 1 // Copyright 2014 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package runtime 6 7 import ( 8 "unsafe" 9 ) 10 11 const ( 12 debugMalloc = false 13 14 flagNoScan = _FlagNoScan 15 flagNoZero = _FlagNoZero 16 17 maxTinySize = _TinySize 18 tinySizeClass = _TinySizeClass 19 maxSmallSize = _MaxSmallSize 20 21 pageShift = _PageShift 22 pageSize = _PageSize 23 pageMask = _PageMask 24 25 bitsPerPointer = _BitsPerPointer 26 bitsMask = _BitsMask 27 pointersPerByte = _PointersPerByte 28 maxGCMask = _MaxGCMask 29 bitsDead = _BitsDead 30 bitsPointer = _BitsPointer 31 32 mSpanInUse = _MSpanInUse 33 34 concurrentSweep = _ConcurrentSweep != 0 35 ) 36 37 // Page number (address>>pageShift) 38 type pageID uintptr 39 40 // base address for all 0-byte allocations 41 var zerobase uintptr 42 43 // Allocate an object of size bytes. 44 // Small objects are allocated from the per-P cache's free lists. 45 // Large objects (> 32 kB) are allocated straight from the heap. 46 func mallocgc(size uintptr, typ *_type, flags uint32) unsafe.Pointer { 47 if size == 0 { 48 return unsafe.Pointer(&zerobase) 49 } 50 size0 := size 51 52 if flags&flagNoScan == 0 && typ == nil { 53 gothrow("malloc missing type") 54 } 55 56 // This function must be atomic wrt GC, but for performance reasons 57 // we don't acquirem/releasem on fast path. The code below does not have 58 // split stack checks, so it can't be preempted by GC. 59 // Functions like roundup/add are inlined. And onM/racemalloc are nosplit. 60 // If debugMalloc = true, these assumptions are checked below. 61 if debugMalloc { 62 mp := acquirem() 63 if mp.mallocing != 0 { 64 gothrow("malloc deadlock") 65 } 66 mp.mallocing = 1 67 if mp.curg != nil { 68 mp.curg.stackguard0 = ^uintptr(0xfff) | 0xbad 69 } 70 } 71 72 c := gomcache() 73 var s *mspan 74 var x unsafe.Pointer 75 if size <= maxSmallSize { 76 if flags&flagNoScan != 0 && size < maxTinySize { 77 // Tiny allocator. 78 // 79 // Tiny allocator combines several tiny allocation requests 80 // into a single memory block. The resulting memory block 81 // is freed when all subobjects are unreachable. The subobjects 82 // must be FlagNoScan (don't have pointers), this ensures that 83 // the amount of potentially wasted memory is bounded. 84 // 85 // Size of the memory block used for combining (maxTinySize) is tunable. 86 // Current setting is 16 bytes, which relates to 2x worst case memory 87 // wastage (when all but one subobjects are unreachable). 88 // 8 bytes would result in no wastage at all, but provides less 89 // opportunities for combining. 90 // 32 bytes provides more opportunities for combining, 91 // but can lead to 4x worst case wastage. 92 // The best case winning is 8x regardless of block size. 93 // 94 // Objects obtained from tiny allocator must not be freed explicitly. 95 // So when an object will be freed explicitly, we ensure that 96 // its size >= maxTinySize. 97 // 98 // SetFinalizer has a special case for objects potentially coming 99 // from tiny allocator, it such case it allows to set finalizers 100 // for an inner byte of a memory block. 101 // 102 // The main targets of tiny allocator are small strings and 103 // standalone escaping variables. On a json benchmark 104 // the allocator reduces number of allocations by ~12% and 105 // reduces heap size by ~20%. 106 tinysize := uintptr(c.tinysize) 107 if size <= tinysize { 108 tiny := unsafe.Pointer(c.tiny) 109 // Align tiny pointer for required (conservative) alignment. 110 if size&7 == 0 { 111 tiny = roundup(tiny, 8) 112 } else if size&3 == 0 { 113 tiny = roundup(tiny, 4) 114 } else if size&1 == 0 { 115 tiny = roundup(tiny, 2) 116 } 117 size1 := size + (uintptr(tiny) - uintptr(unsafe.Pointer(c.tiny))) 118 if size1 <= tinysize { 119 // The object fits into existing tiny block. 120 x = tiny 121 c.tiny = (*byte)(add(x, size)) 122 c.tinysize -= uintptr(size1) 123 c.local_tinyallocs++ 124 if debugMalloc { 125 mp := acquirem() 126 if mp.mallocing == 0 { 127 gothrow("bad malloc") 128 } 129 mp.mallocing = 0 130 if mp.curg != nil { 131 mp.curg.stackguard0 = mp.curg.stack.lo + _StackGuard 132 } 133 // Note: one releasem for the acquirem just above. 134 // The other for the acquirem at start of malloc. 135 releasem(mp) 136 releasem(mp) 137 } 138 return x 139 } 140 } 141 // Allocate a new maxTinySize block. 142 s = c.alloc[tinySizeClass] 143 v := s.freelist 144 if v == nil { 145 mp := acquirem() 146 mp.scalararg[0] = tinySizeClass 147 onM(mcacheRefill_m) 148 releasem(mp) 149 s = c.alloc[tinySizeClass] 150 v = s.freelist 151 } 152 s.freelist = v.next 153 s.ref++ 154 //TODO: prefetch v.next 155 x = unsafe.Pointer(v) 156 (*[2]uint64)(x)[0] = 0 157 (*[2]uint64)(x)[1] = 0 158 // See if we need to replace the existing tiny block with the new one 159 // based on amount of remaining free space. 160 if maxTinySize-size > tinysize { 161 c.tiny = (*byte)(add(x, size)) 162 c.tinysize = uintptr(maxTinySize - size) 163 } 164 size = maxTinySize 165 } else { 166 var sizeclass int8 167 if size <= 1024-8 { 168 sizeclass = size_to_class8[(size+7)>>3] 169 } else { 170 sizeclass = size_to_class128[(size-1024+127)>>7] 171 } 172 size = uintptr(class_to_size[sizeclass]) 173 s = c.alloc[sizeclass] 174 v := s.freelist 175 if v == nil { 176 mp := acquirem() 177 mp.scalararg[0] = uintptr(sizeclass) 178 onM(mcacheRefill_m) 179 releasem(mp) 180 s = c.alloc[sizeclass] 181 v = s.freelist 182 } 183 s.freelist = v.next 184 s.ref++ 185 //TODO: prefetch 186 x = unsafe.Pointer(v) 187 if flags&flagNoZero == 0 { 188 v.next = nil 189 if size > 2*ptrSize && ((*[2]uintptr)(x))[1] != 0 { 190 memclr(unsafe.Pointer(v), size) 191 } 192 } 193 } 194 c.local_cachealloc += intptr(size) 195 } else { 196 mp := acquirem() 197 mp.scalararg[0] = uintptr(size) 198 mp.scalararg[1] = uintptr(flags) 199 onM(largeAlloc_m) 200 s = (*mspan)(mp.ptrarg[0]) 201 mp.ptrarg[0] = nil 202 releasem(mp) 203 x = unsafe.Pointer(uintptr(s.start << pageShift)) 204 size = uintptr(s.elemsize) 205 } 206 207 if flags&flagNoScan != 0 { 208 // All objects are pre-marked as noscan. 209 goto marked 210 } 211 212 // If allocating a defer+arg block, now that we've picked a malloc size 213 // large enough to hold everything, cut the "asked for" size down to 214 // just the defer header, so that the GC bitmap will record the arg block 215 // as containing nothing at all (as if it were unused space at the end of 216 // a malloc block caused by size rounding). 217 // The defer arg areas are scanned as part of scanstack. 218 if typ == deferType { 219 size0 = unsafe.Sizeof(_defer{}) 220 } 221 222 // From here till marked label marking the object as allocated 223 // and storing type info in the GC bitmap. 224 { 225 arena_start := uintptr(unsafe.Pointer(mheap_.arena_start)) 226 off := (uintptr(x) - arena_start) / ptrSize 227 xbits := (*uint8)(unsafe.Pointer(arena_start - off/wordsPerBitmapByte - 1)) 228 shift := (off % wordsPerBitmapByte) * gcBits 229 if debugMalloc && ((*xbits>>shift)&(bitMask|bitPtrMask)) != bitBoundary { 230 println("runtime: bits =", (*xbits>>shift)&(bitMask|bitPtrMask)) 231 gothrow("bad bits in markallocated") 232 } 233 234 var ti, te uintptr 235 var ptrmask *uint8 236 if size == ptrSize { 237 // It's one word and it has pointers, it must be a pointer. 238 *xbits |= (bitsPointer << 2) << shift 239 goto marked 240 } 241 if typ.kind&kindGCProg != 0 { 242 nptr := (uintptr(typ.size) + ptrSize - 1) / ptrSize 243 masksize := nptr 244 if masksize%2 != 0 { 245 masksize *= 2 // repeated 246 } 247 masksize = masksize * pointersPerByte / 8 // 4 bits per word 248 masksize++ // unroll flag in the beginning 249 if masksize > maxGCMask && typ.gc[1] != 0 { 250 // If the mask is too large, unroll the program directly 251 // into the GC bitmap. It's 7 times slower than copying 252 // from the pre-unrolled mask, but saves 1/16 of type size 253 // memory for the mask. 254 mp := acquirem() 255 mp.ptrarg[0] = x 256 mp.ptrarg[1] = unsafe.Pointer(typ) 257 mp.scalararg[0] = uintptr(size) 258 mp.scalararg[1] = uintptr(size0) 259 onM(unrollgcproginplace_m) 260 releasem(mp) 261 goto marked 262 } 263 ptrmask = (*uint8)(unsafe.Pointer(uintptr(typ.gc[0]))) 264 // Check whether the program is already unrolled. 265 if uintptr(atomicloadp(unsafe.Pointer(ptrmask)))&0xff == 0 { 266 mp := acquirem() 267 mp.ptrarg[0] = unsafe.Pointer(typ) 268 onM(unrollgcprog_m) 269 releasem(mp) 270 } 271 ptrmask = (*uint8)(add(unsafe.Pointer(ptrmask), 1)) // skip the unroll flag byte 272 } else { 273 ptrmask = (*uint8)(unsafe.Pointer(typ.gc[0])) // pointer to unrolled mask 274 } 275 if size == 2*ptrSize { 276 *xbits = *ptrmask | bitBoundary 277 goto marked 278 } 279 te = uintptr(typ.size) / ptrSize 280 // If the type occupies odd number of words, its mask is repeated. 281 if te%2 == 0 { 282 te /= 2 283 } 284 // Copy pointer bitmask into the bitmap. 285 for i := uintptr(0); i < size0; i += 2 * ptrSize { 286 v := *(*uint8)(add(unsafe.Pointer(ptrmask), ti)) 287 ti++ 288 if ti == te { 289 ti = 0 290 } 291 if i == 0 { 292 v |= bitBoundary 293 } 294 if i+ptrSize == size0 { 295 v &^= uint8(bitPtrMask << 4) 296 } 297 298 *xbits = v 299 xbits = (*byte)(add(unsafe.Pointer(xbits), ^uintptr(0))) 300 } 301 if size0%(2*ptrSize) == 0 && size0 < size { 302 // Mark the word after last object's word as bitsDead. 303 *xbits = bitsDead << 2 304 } 305 } 306 marked: 307 if raceenabled { 308 racemalloc(x, size) 309 } 310 311 if debugMalloc { 312 mp := acquirem() 313 if mp.mallocing == 0 { 314 gothrow("bad malloc") 315 } 316 mp.mallocing = 0 317 if mp.curg != nil { 318 mp.curg.stackguard0 = mp.curg.stack.lo + _StackGuard 319 } 320 // Note: one releasem for the acquirem just above. 321 // The other for the acquirem at start of malloc. 322 releasem(mp) 323 releasem(mp) 324 } 325 326 if debug.allocfreetrace != 0 { 327 tracealloc(x, size, typ) 328 } 329 330 if rate := MemProfileRate; rate > 0 { 331 if size < uintptr(rate) && int32(size) < c.next_sample { 332 c.next_sample -= int32(size) 333 } else { 334 mp := acquirem() 335 profilealloc(mp, x, size) 336 releasem(mp) 337 } 338 } 339 340 if memstats.heap_alloc >= memstats.next_gc { 341 gogc(0) 342 } 343 344 return x 345 } 346 347 // implementation of new builtin 348 func newobject(typ *_type) unsafe.Pointer { 349 flags := uint32(0) 350 if typ.kind&kindNoPointers != 0 { 351 flags |= flagNoScan 352 } 353 return mallocgc(uintptr(typ.size), typ, flags) 354 } 355 356 // implementation of make builtin for slices 357 func newarray(typ *_type, n uintptr) unsafe.Pointer { 358 flags := uint32(0) 359 if typ.kind&kindNoPointers != 0 { 360 flags |= flagNoScan 361 } 362 if int(n) < 0 || (typ.size > 0 && n > maxmem/uintptr(typ.size)) { 363 panic("runtime: allocation size out of range") 364 } 365 return mallocgc(uintptr(typ.size)*n, typ, flags) 366 } 367 368 // rawmem returns a chunk of pointerless memory. It is 369 // not zeroed. 370 func rawmem(size uintptr) unsafe.Pointer { 371 return mallocgc(size, nil, flagNoScan|flagNoZero) 372 } 373 374 // round size up to next size class 375 func goroundupsize(size uintptr) uintptr { 376 if size < maxSmallSize { 377 if size <= 1024-8 { 378 return uintptr(class_to_size[size_to_class8[(size+7)>>3]]) 379 } 380 return uintptr(class_to_size[size_to_class128[(size-1024+127)>>7]]) 381 } 382 if size+pageSize < size { 383 return size 384 } 385 return (size + pageSize - 1) &^ pageMask 386 } 387 388 func profilealloc(mp *m, x unsafe.Pointer, size uintptr) { 389 c := mp.mcache 390 rate := MemProfileRate 391 if size < uintptr(rate) { 392 // pick next profile time 393 // If you change this, also change allocmcache. 394 if rate > 0x3fffffff { // make 2*rate not overflow 395 rate = 0x3fffffff 396 } 397 next := int32(fastrand1()) % (2 * int32(rate)) 398 // Subtract the "remainder" of the current allocation. 399 // Otherwise objects that are close in size to sampling rate 400 // will be under-sampled, because we consistently discard this remainder. 401 next -= (int32(size) - c.next_sample) 402 if next < 0 { 403 next = 0 404 } 405 c.next_sample = next 406 } 407 408 mProf_Malloc(x, size) 409 } 410 411 // force = 1 - do GC regardless of current heap usage 412 // force = 2 - go GC and eager sweep 413 func gogc(force int32) { 414 // The gc is turned off (via enablegc) until the bootstrap has completed. 415 // Also, malloc gets called in the guts of a number of libraries that might be 416 // holding locks. To avoid deadlocks during stoptheworld, don't bother 417 // trying to run gc while holding a lock. The next mallocgc without a lock 418 // will do the gc instead. 419 mp := acquirem() 420 if gp := getg(); gp == mp.g0 || mp.locks > 1 || !memstats.enablegc || panicking != 0 || gcpercent < 0 { 421 releasem(mp) 422 return 423 } 424 releasem(mp) 425 mp = nil 426 427 semacquire(&worldsema, false) 428 429 if force == 0 && memstats.heap_alloc < memstats.next_gc { 430 // typically threads which lost the race to grab 431 // worldsema exit here when gc is done. 432 semrelease(&worldsema) 433 return 434 } 435 436 // Ok, we're doing it! Stop everybody else 437 startTime := nanotime() 438 mp = acquirem() 439 mp.gcing = 1 440 releasem(mp) 441 onM(stoptheworld) 442 if mp != acquirem() { 443 gothrow("gogc: rescheduled") 444 } 445 446 clearpools() 447 448 // Run gc on the g0 stack. We do this so that the g stack 449 // we're currently running on will no longer change. Cuts 450 // the root set down a bit (g0 stacks are not scanned, and 451 // we don't need to scan gc's internal state). We also 452 // need to switch to g0 so we can shrink the stack. 453 n := 1 454 if debug.gctrace > 1 { 455 n = 2 456 } 457 for i := 0; i < n; i++ { 458 if i > 0 { 459 startTime = nanotime() 460 } 461 // switch to g0, call gc, then switch back 462 mp.scalararg[0] = uintptr(uint32(startTime)) // low 32 bits 463 mp.scalararg[1] = uintptr(startTime >> 32) // high 32 bits 464 if force >= 2 { 465 mp.scalararg[2] = 1 // eagersweep 466 } else { 467 mp.scalararg[2] = 0 468 } 469 onM(gc_m) 470 } 471 472 // all done 473 mp.gcing = 0 474 semrelease(&worldsema) 475 onM(starttheworld) 476 releasem(mp) 477 mp = nil 478 479 // now that gc is done, kick off finalizer thread if needed 480 if !concurrentSweep { 481 // give the queued finalizers, if any, a chance to run 482 Gosched() 483 } 484 } 485 486 // GC runs a garbage collection. 487 func GC() { 488 gogc(2) 489 } 490 491 // linker-provided 492 var noptrdata struct{} 493 var enoptrdata struct{} 494 var noptrbss struct{} 495 var enoptrbss struct{} 496 497 // SetFinalizer sets the finalizer associated with x to f. 498 // When the garbage collector finds an unreachable block 499 // with an associated finalizer, it clears the association and runs 500 // f(x) in a separate goroutine. This makes x reachable again, but 501 // now without an associated finalizer. Assuming that SetFinalizer 502 // is not called again, the next time the garbage collector sees 503 // that x is unreachable, it will free x. 504 // 505 // SetFinalizer(x, nil) clears any finalizer associated with x. 506 // 507 // The argument x must be a pointer to an object allocated by 508 // calling new or by taking the address of a composite literal. 509 // The argument f must be a function that takes a single argument 510 // to which x's type can be assigned, and can have arbitrary ignored return 511 // values. If either of these is not true, SetFinalizer aborts the 512 // program. 513 // 514 // Finalizers are run in dependency order: if A points at B, both have 515 // finalizers, and they are otherwise unreachable, only the finalizer 516 // for A runs; once A is freed, the finalizer for B can run. 517 // If a cyclic structure includes a block with a finalizer, that 518 // cycle is not guaranteed to be garbage collected and the finalizer 519 // is not guaranteed to run, because there is no ordering that 520 // respects the dependencies. 521 // 522 // The finalizer for x is scheduled to run at some arbitrary time after 523 // x becomes unreachable. 524 // There is no guarantee that finalizers will run before a program exits, 525 // so typically they are useful only for releasing non-memory resources 526 // associated with an object during a long-running program. 527 // For example, an os.File object could use a finalizer to close the 528 // associated operating system file descriptor when a program discards 529 // an os.File without calling Close, but it would be a mistake 530 // to depend on a finalizer to flush an in-memory I/O buffer such as a 531 // bufio.Writer, because the buffer would not be flushed at program exit. 532 // 533 // It is not guaranteed that a finalizer will run if the size of *x is 534 // zero bytes. 535 // 536 // It is not guaranteed that a finalizer will run for objects allocated 537 // in initializers for package-level variables. Such objects may be 538 // linker-allocated, not heap-allocated. 539 // 540 // A single goroutine runs all finalizers for a program, sequentially. 541 // If a finalizer must run for a long time, it should do so by starting 542 // a new goroutine. 543 func SetFinalizer(obj interface{}, finalizer interface{}) { 544 e := (*eface)(unsafe.Pointer(&obj)) 545 etyp := e._type 546 if etyp == nil { 547 gothrow("runtime.SetFinalizer: first argument is nil") 548 } 549 if etyp.kind&kindMask != kindPtr { 550 gothrow("runtime.SetFinalizer: first argument is " + *etyp._string + ", not pointer") 551 } 552 ot := (*ptrtype)(unsafe.Pointer(etyp)) 553 if ot.elem == nil { 554 gothrow("nil elem type!") 555 } 556 557 // find the containing object 558 _, base, _ := findObject(e.data) 559 560 if base == nil { 561 // 0-length objects are okay. 562 if e.data == unsafe.Pointer(&zerobase) { 563 return 564 } 565 566 // Global initializers might be linker-allocated. 567 // var Foo = &Object{} 568 // func main() { 569 // runtime.SetFinalizer(Foo, nil) 570 // } 571 // The relevant segments are: noptrdata, data, bss, noptrbss. 572 // We cannot assume they are in any order or even contiguous, 573 // due to external linking. 574 if uintptr(unsafe.Pointer(&noptrdata)) <= uintptr(e.data) && uintptr(e.data) < uintptr(unsafe.Pointer(&enoptrdata)) || 575 uintptr(unsafe.Pointer(&data)) <= uintptr(e.data) && uintptr(e.data) < uintptr(unsafe.Pointer(&edata)) || 576 uintptr(unsafe.Pointer(&bss)) <= uintptr(e.data) && uintptr(e.data) < uintptr(unsafe.Pointer(&ebss)) || 577 uintptr(unsafe.Pointer(&noptrbss)) <= uintptr(e.data) && uintptr(e.data) < uintptr(unsafe.Pointer(&enoptrbss)) { 578 return 579 } 580 gothrow("runtime.SetFinalizer: pointer not in allocated block") 581 } 582 583 if e.data != base { 584 // As an implementation detail we allow to set finalizers for an inner byte 585 // of an object if it could come from tiny alloc (see mallocgc for details). 586 if ot.elem == nil || ot.elem.kind&kindNoPointers == 0 || ot.elem.size >= maxTinySize { 587 gothrow("runtime.SetFinalizer: pointer not at beginning of allocated block") 588 } 589 } 590 591 f := (*eface)(unsafe.Pointer(&finalizer)) 592 ftyp := f._type 593 if ftyp == nil { 594 // switch to M stack and remove finalizer 595 mp := acquirem() 596 mp.ptrarg[0] = e.data 597 onM(removeFinalizer_m) 598 releasem(mp) 599 return 600 } 601 602 if ftyp.kind&kindMask != kindFunc { 603 gothrow("runtime.SetFinalizer: second argument is " + *ftyp._string + ", not a function") 604 } 605 ft := (*functype)(unsafe.Pointer(ftyp)) 606 ins := *(*[]*_type)(unsafe.Pointer(&ft.in)) 607 if ft.dotdotdot || len(ins) != 1 { 608 gothrow("runtime.SetFinalizer: cannot pass " + *etyp._string + " to finalizer " + *ftyp._string) 609 } 610 fint := ins[0] 611 switch { 612 case fint == etyp: 613 // ok - same type 614 goto okarg 615 case fint.kind&kindMask == kindPtr: 616 if (fint.x == nil || fint.x.name == nil || etyp.x == nil || etyp.x.name == nil) && (*ptrtype)(unsafe.Pointer(fint)).elem == ot.elem { 617 // ok - not same type, but both pointers, 618 // one or the other is unnamed, and same element type, so assignable. 619 goto okarg 620 } 621 case fint.kind&kindMask == kindInterface: 622 ityp := (*interfacetype)(unsafe.Pointer(fint)) 623 if len(ityp.mhdr) == 0 { 624 // ok - satisfies empty interface 625 goto okarg 626 } 627 if _, ok := assertE2I2(ityp, obj); ok { 628 goto okarg 629 } 630 } 631 gothrow("runtime.SetFinalizer: cannot pass " + *etyp._string + " to finalizer " + *ftyp._string) 632 okarg: 633 // compute size needed for return parameters 634 nret := uintptr(0) 635 for _, t := range *(*[]*_type)(unsafe.Pointer(&ft.out)) { 636 nret = round(nret, uintptr(t.align)) + uintptr(t.size) 637 } 638 nret = round(nret, ptrSize) 639 640 // make sure we have a finalizer goroutine 641 createfing() 642 643 // switch to M stack to add finalizer record 644 mp := acquirem() 645 mp.ptrarg[0] = f.data 646 mp.ptrarg[1] = e.data 647 mp.scalararg[0] = nret 648 mp.ptrarg[2] = unsafe.Pointer(fint) 649 mp.ptrarg[3] = unsafe.Pointer(ot) 650 onM(setFinalizer_m) 651 if mp.scalararg[0] != 1 { 652 gothrow("runtime.SetFinalizer: finalizer already set") 653 } 654 releasem(mp) 655 } 656 657 // round n up to a multiple of a. a must be a power of 2. 658 func round(n, a uintptr) uintptr { 659 return (n + a - 1) &^ (a - 1) 660 } 661 662 // Look up pointer v in heap. Return the span containing the object, 663 // the start of the object, and the size of the object. If the object 664 // does not exist, return nil, nil, 0. 665 func findObject(v unsafe.Pointer) (s *mspan, x unsafe.Pointer, n uintptr) { 666 c := gomcache() 667 c.local_nlookup++ 668 if ptrSize == 4 && c.local_nlookup >= 1<<30 { 669 // purge cache stats to prevent overflow 670 lock(&mheap_.lock) 671 purgecachedstats(c) 672 unlock(&mheap_.lock) 673 } 674 675 // find span 676 arena_start := uintptr(unsafe.Pointer(mheap_.arena_start)) 677 arena_used := uintptr(unsafe.Pointer(mheap_.arena_used)) 678 if uintptr(v) < arena_start || uintptr(v) >= arena_used { 679 return 680 } 681 p := uintptr(v) >> pageShift 682 q := p - arena_start>>pageShift 683 s = *(**mspan)(add(unsafe.Pointer(mheap_.spans), q*ptrSize)) 684 if s == nil { 685 return 686 } 687 x = unsafe.Pointer(uintptr(s.start) << pageShift) 688 689 if uintptr(v) < uintptr(x) || uintptr(v) >= uintptr(unsafe.Pointer(s.limit)) || s.state != mSpanInUse { 690 s = nil 691 x = nil 692 return 693 } 694 695 n = uintptr(s.elemsize) 696 if s.sizeclass != 0 { 697 x = add(x, (uintptr(v)-uintptr(x))/n*n) 698 } 699 return 700 } 701 702 var fingCreate uint32 703 704 func createfing() { 705 // start the finalizer goroutine exactly once 706 if fingCreate == 0 && cas(&fingCreate, 0, 1) { 707 go runfinq() 708 } 709 } 710 711 // This is the goroutine that runs all of the finalizers 712 func runfinq() { 713 var ( 714 frame unsafe.Pointer 715 framecap uintptr 716 ) 717 718 for { 719 lock(&finlock) 720 fb := finq 721 finq = nil 722 if fb == nil { 723 gp := getg() 724 fing = gp 725 fingwait = true 726 gp.issystem = true 727 goparkunlock(&finlock, "finalizer wait") 728 gp.issystem = false 729 continue 730 } 731 unlock(&finlock) 732 if raceenabled { 733 racefingo() 734 } 735 for fb != nil { 736 for i := int32(0); i < fb.cnt; i++ { 737 f := (*finalizer)(add(unsafe.Pointer(&fb.fin), uintptr(i)*unsafe.Sizeof(finalizer{}))) 738 739 framesz := unsafe.Sizeof((interface{})(nil)) + uintptr(f.nret) 740 if framecap < framesz { 741 // The frame does not contain pointers interesting for GC, 742 // all not yet finalized objects are stored in finq. 743 // If we do not mark it as FlagNoScan, 744 // the last finalized object is not collected. 745 frame = mallocgc(framesz, nil, flagNoScan) 746 framecap = framesz 747 } 748 749 if f.fint == nil { 750 gothrow("missing type in runfinq") 751 } 752 switch f.fint.kind & kindMask { 753 case kindPtr: 754 // direct use of pointer 755 *(*unsafe.Pointer)(frame) = f.arg 756 case kindInterface: 757 ityp := (*interfacetype)(unsafe.Pointer(f.fint)) 758 // set up with empty interface 759 (*eface)(frame)._type = &f.ot.typ 760 (*eface)(frame).data = f.arg 761 if len(ityp.mhdr) != 0 { 762 // convert to interface with methods 763 // this conversion is guaranteed to succeed - we checked in SetFinalizer 764 *(*fInterface)(frame) = assertE2I(ityp, *(*interface{})(frame)) 765 } 766 default: 767 gothrow("bad kind in runfinq") 768 } 769 reflectcall(unsafe.Pointer(f.fn), frame, uint32(framesz), uint32(framesz)) 770 771 // drop finalizer queue references to finalized object 772 f.fn = nil 773 f.arg = nil 774 f.ot = nil 775 } 776 fb.cnt = 0 777 next := fb.next 778 lock(&finlock) 779 fb.next = finc 780 finc = fb 781 unlock(&finlock) 782 fb = next 783 } 784 } 785 } 786 787 var persistent struct { 788 lock mutex 789 pos unsafe.Pointer 790 end unsafe.Pointer 791 } 792 793 // Wrapper around sysAlloc that can allocate small chunks. 794 // There is no associated free operation. 795 // Intended for things like function/type/debug-related persistent data. 796 // If align is 0, uses default align (currently 8). 797 func persistentalloc(size, align uintptr, stat *uint64) unsafe.Pointer { 798 const ( 799 chunk = 256 << 10 800 maxBlock = 64 << 10 // VM reservation granularity is 64K on windows 801 ) 802 803 if align != 0 { 804 if align&(align-1) != 0 { 805 gothrow("persistentalloc: align is not a power of 2") 806 } 807 if align > _PageSize { 808 gothrow("persistentalloc: align is too large") 809 } 810 } else { 811 align = 8 812 } 813 814 if size >= maxBlock { 815 return sysAlloc(size, stat) 816 } 817 818 lock(&persistent.lock) 819 persistent.pos = roundup(persistent.pos, align) 820 if uintptr(persistent.pos)+size > uintptr(persistent.end) { 821 persistent.pos = sysAlloc(chunk, &memstats.other_sys) 822 if persistent.pos == nil { 823 unlock(&persistent.lock) 824 gothrow("runtime: cannot allocate memory") 825 } 826 persistent.end = add(persistent.pos, chunk) 827 } 828 p := persistent.pos 829 persistent.pos = add(persistent.pos, size) 830 unlock(&persistent.lock) 831 832 if stat != &memstats.other_sys { 833 xadd64(stat, int64(size)) 834 xadd64(&memstats.other_sys, -int64(size)) 835 } 836 return p 837 }