github.com/reiver/go@v0.0.0-20150109200633-1d0c7792f172/src/runtime/malloc.go (about)

     1  // Copyright 2014 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package runtime
     6  
     7  import "unsafe"
     8  
     9  const (
    10  	debugMalloc = false
    11  
    12  	flagNoScan = _FlagNoScan
    13  	flagNoZero = _FlagNoZero
    14  
    15  	maxTinySize   = _TinySize
    16  	tinySizeClass = _TinySizeClass
    17  	maxSmallSize  = _MaxSmallSize
    18  
    19  	pageShift = _PageShift
    20  	pageSize  = _PageSize
    21  	pageMask  = _PageMask
    22  
    23  	bitsPerPointer  = _BitsPerPointer
    24  	bitsMask        = _BitsMask
    25  	pointersPerByte = _PointersPerByte
    26  	maxGCMask       = _MaxGCMask
    27  	bitsDead        = _BitsDead
    28  	bitsPointer     = _BitsPointer
    29  	bitsScalar      = _BitsScalar
    30  
    31  	mSpanInUse = _MSpanInUse
    32  
    33  	concurrentSweep = _ConcurrentSweep
    34  )
    35  
    36  // Page number (address>>pageShift)
    37  type pageID uintptr
    38  
    39  // base address for all 0-byte allocations
    40  var zerobase uintptr
    41  
    42  // Determine whether to initiate a GC.
    43  // Currently the primitive heuristic we use will start a new
    44  // concurrent GC when approximately half the available space
    45  // made available by the last GC cycle has been used.
    46  // If the GC is already working no need to trigger another one.
    47  // This should establish a feedback loop where if the GC does not
    48  // have sufficient time to complete then more memory will be
    49  // requested from the OS increasing heap size thus allow future
    50  // GCs more time to complete.
    51  // memstat.heap_alloc and memstat.next_gc reads have benign races
    52  // A false negative simple does not start a GC, a false positive
    53  // will start a GC needlessly. Neither have correctness issues.
    54  func shouldtriggergc() bool {
    55  	return memstats.heap_alloc+memstats.heap_alloc*3/4 >= memstats.next_gc && atomicloaduint(&bggc.working) == 0
    56  }
    57  
    58  // Allocate an object of size bytes.
    59  // Small objects are allocated from the per-P cache's free lists.
    60  // Large objects (> 32 kB) are allocated straight from the heap.
    61  func mallocgc(size uintptr, typ *_type, flags uint32) unsafe.Pointer {
    62  	shouldhelpgc := false
    63  	if size == 0 {
    64  		return unsafe.Pointer(&zerobase)
    65  	}
    66  	size0 := size
    67  
    68  	if flags&flagNoScan == 0 && typ == nil {
    69  		throw("malloc missing type")
    70  	}
    71  
    72  	// This function must be atomic wrt GC, but for performance reasons
    73  	// we don't acquirem/releasem on fast path. The code below does not have
    74  	// split stack checks, so it can't be preempted by GC.
    75  	// Functions like roundup/add are inlined. And systemstack/racemalloc are nosplit.
    76  	// If debugMalloc = true, these assumptions are checked below.
    77  	if debugMalloc {
    78  		mp := acquirem()
    79  		if mp.mallocing != 0 {
    80  			throw("malloc deadlock")
    81  		}
    82  		mp.mallocing = 1
    83  		if mp.curg != nil {
    84  			mp.curg.stackguard0 = ^uintptr(0xfff) | 0xbad
    85  		}
    86  	}
    87  
    88  	c := gomcache()
    89  	var s *mspan
    90  	var x unsafe.Pointer
    91  	if size <= maxSmallSize {
    92  		if flags&flagNoScan != 0 && size < maxTinySize {
    93  			// Tiny allocator.
    94  			//
    95  			// Tiny allocator combines several tiny allocation requests
    96  			// into a single memory block. The resulting memory block
    97  			// is freed when all subobjects are unreachable. The subobjects
    98  			// must be FlagNoScan (don't have pointers), this ensures that
    99  			// the amount of potentially wasted memory is bounded.
   100  			//
   101  			// Size of the memory block used for combining (maxTinySize) is tunable.
   102  			// Current setting is 16 bytes, which relates to 2x worst case memory
   103  			// wastage (when all but one subobjects are unreachable).
   104  			// 8 bytes would result in no wastage at all, but provides less
   105  			// opportunities for combining.
   106  			// 32 bytes provides more opportunities for combining,
   107  			// but can lead to 4x worst case wastage.
   108  			// The best case winning is 8x regardless of block size.
   109  			//
   110  			// Objects obtained from tiny allocator must not be freed explicitly.
   111  			// So when an object will be freed explicitly, we ensure that
   112  			// its size >= maxTinySize.
   113  			//
   114  			// SetFinalizer has a special case for objects potentially coming
   115  			// from tiny allocator, it such case it allows to set finalizers
   116  			// for an inner byte of a memory block.
   117  			//
   118  			// The main targets of tiny allocator are small strings and
   119  			// standalone escaping variables. On a json benchmark
   120  			// the allocator reduces number of allocations by ~12% and
   121  			// reduces heap size by ~20%.
   122  			tinysize := uintptr(c.tinysize)
   123  			if size <= tinysize {
   124  				tiny := unsafe.Pointer(c.tiny)
   125  				// Align tiny pointer for required (conservative) alignment.
   126  				if size&7 == 0 {
   127  					tiny = roundup(tiny, 8)
   128  				} else if size&3 == 0 {
   129  					tiny = roundup(tiny, 4)
   130  				} else if size&1 == 0 {
   131  					tiny = roundup(tiny, 2)
   132  				}
   133  				size1 := size + (uintptr(tiny) - uintptr(unsafe.Pointer(c.tiny)))
   134  				if size1 <= tinysize {
   135  					// The object fits into existing tiny block.
   136  					x = tiny
   137  					c.tiny = (*byte)(add(x, size))
   138  					c.tinysize -= uintptr(size1)
   139  					c.local_tinyallocs++
   140  					if debugMalloc {
   141  						mp := acquirem()
   142  						if mp.mallocing == 0 {
   143  							throw("bad malloc")
   144  						}
   145  						mp.mallocing = 0
   146  						if mp.curg != nil {
   147  							mp.curg.stackguard0 = mp.curg.stack.lo + _StackGuard
   148  						}
   149  						// Note: one releasem for the acquirem just above.
   150  						// The other for the acquirem at start of malloc.
   151  						releasem(mp)
   152  						releasem(mp)
   153  					}
   154  					return x
   155  				}
   156  			}
   157  			// Allocate a new maxTinySize block.
   158  			s = c.alloc[tinySizeClass]
   159  			v := s.freelist
   160  			if v.ptr() == nil {
   161  				systemstack(func() {
   162  					mCache_Refill(c, tinySizeClass)
   163  				})
   164  				shouldhelpgc = true
   165  				s = c.alloc[tinySizeClass]
   166  				v = s.freelist
   167  			}
   168  			s.freelist = v.ptr().next
   169  			s.ref++
   170  			//TODO: prefetch v.next
   171  			x = unsafe.Pointer(v)
   172  			(*[2]uint64)(x)[0] = 0
   173  			(*[2]uint64)(x)[1] = 0
   174  			// See if we need to replace the existing tiny block with the new one
   175  			// based on amount of remaining free space.
   176  			if maxTinySize-size > tinysize {
   177  				c.tiny = (*byte)(add(x, size))
   178  				c.tinysize = uintptr(maxTinySize - size)
   179  			}
   180  			size = maxTinySize
   181  		} else {
   182  			var sizeclass int8
   183  			if size <= 1024-8 {
   184  				sizeclass = size_to_class8[(size+7)>>3]
   185  			} else {
   186  				sizeclass = size_to_class128[(size-1024+127)>>7]
   187  			}
   188  			size = uintptr(class_to_size[sizeclass])
   189  			s = c.alloc[sizeclass]
   190  			v := s.freelist
   191  			if v.ptr() == nil {
   192  				systemstack(func() {
   193  					mCache_Refill(c, int32(sizeclass))
   194  				})
   195  				shouldhelpgc = true
   196  				s = c.alloc[sizeclass]
   197  				v = s.freelist
   198  			}
   199  			s.freelist = v.ptr().next
   200  			s.ref++
   201  			//TODO: prefetch
   202  			x = unsafe.Pointer(v)
   203  			if flags&flagNoZero == 0 {
   204  				v.ptr().next = 0
   205  				if size > 2*ptrSize && ((*[2]uintptr)(x))[1] != 0 {
   206  					memclr(unsafe.Pointer(v), size)
   207  				}
   208  			}
   209  		}
   210  		c.local_cachealloc += intptr(size)
   211  	} else {
   212  		var s *mspan
   213  		shouldhelpgc = true
   214  		systemstack(func() {
   215  			s = largeAlloc(size, uint32(flags))
   216  		})
   217  		x = unsafe.Pointer(uintptr(s.start << pageShift))
   218  		size = uintptr(s.elemsize)
   219  	}
   220  
   221  	if flags&flagNoScan != 0 {
   222  		// All objects are pre-marked as noscan.
   223  		goto marked
   224  	}
   225  
   226  	// If allocating a defer+arg block, now that we've picked a malloc size
   227  	// large enough to hold everything, cut the "asked for" size down to
   228  	// just the defer header, so that the GC bitmap will record the arg block
   229  	// as containing nothing at all (as if it were unused space at the end of
   230  	// a malloc block caused by size rounding).
   231  	// The defer arg areas are scanned as part of scanstack.
   232  	if typ == deferType {
   233  		size0 = unsafe.Sizeof(_defer{})
   234  	}
   235  
   236  	// From here till marked label marking the object as allocated
   237  	// and storing type info in the GC bitmap.
   238  	{
   239  		arena_start := uintptr(unsafe.Pointer(mheap_.arena_start))
   240  		off := (uintptr(x) - arena_start) / ptrSize
   241  		xbits := (*uint8)(unsafe.Pointer(arena_start - off/wordsPerBitmapByte - 1))
   242  		shift := (off % wordsPerBitmapByte) * gcBits
   243  		if debugMalloc && ((*xbits>>shift)&(bitMask|bitPtrMask)) != bitBoundary {
   244  			println("runtime: bits =", (*xbits>>shift)&(bitMask|bitPtrMask))
   245  			throw("bad bits in markallocated")
   246  		}
   247  
   248  		var ti, te uintptr
   249  		var ptrmask *uint8
   250  		if size == ptrSize {
   251  			// It's one word and it has pointers, it must be a pointer.
   252  			*xbits |= (bitsPointer << 2) << shift
   253  			goto marked
   254  		}
   255  		if typ.kind&kindGCProg != 0 {
   256  			nptr := (uintptr(typ.size) + ptrSize - 1) / ptrSize
   257  			masksize := nptr
   258  			if masksize%2 != 0 {
   259  				masksize *= 2 // repeated
   260  			}
   261  			masksize = masksize * pointersPerByte / 8 // 4 bits per word
   262  			masksize++                                // unroll flag in the beginning
   263  			if masksize > maxGCMask && typ.gc[1] != 0 {
   264  				// write barriers have not been updated to deal with this case yet.
   265  				throw("maxGCMask too small for now")
   266  				// If the mask is too large, unroll the program directly
   267  				// into the GC bitmap. It's 7 times slower than copying
   268  				// from the pre-unrolled mask, but saves 1/16 of type size
   269  				// memory for the mask.
   270  				systemstack(func() {
   271  					unrollgcproginplace_m(x, typ, size, size0)
   272  				})
   273  				goto marked
   274  			}
   275  			ptrmask = (*uint8)(unsafe.Pointer(uintptr(typ.gc[0])))
   276  			// Check whether the program is already unrolled
   277  			// by checking if the unroll flag byte is set
   278  			maskword := uintptr(atomicloadp(unsafe.Pointer(ptrmask)))
   279  			if *(*uint8)(unsafe.Pointer(&maskword)) == 0 {
   280  				systemstack(func() {
   281  					unrollgcprog_m(typ)
   282  				})
   283  			}
   284  			ptrmask = (*uint8)(add(unsafe.Pointer(ptrmask), 1)) // skip the unroll flag byte
   285  		} else {
   286  			ptrmask = (*uint8)(unsafe.Pointer(typ.gc[0])) // pointer to unrolled mask
   287  		}
   288  		if size == 2*ptrSize {
   289  			*xbits = *ptrmask | bitBoundary
   290  			goto marked
   291  		}
   292  		te = uintptr(typ.size) / ptrSize
   293  		// If the type occupies odd number of words, its mask is repeated.
   294  		if te%2 == 0 {
   295  			te /= 2
   296  		}
   297  		// Copy pointer bitmask into the bitmap.
   298  		for i := uintptr(0); i < size0; i += 2 * ptrSize {
   299  			v := *(*uint8)(add(unsafe.Pointer(ptrmask), ti))
   300  			ti++
   301  			if ti == te {
   302  				ti = 0
   303  			}
   304  			if i == 0 {
   305  				v |= bitBoundary
   306  			}
   307  			if i+ptrSize == size0 {
   308  				v &^= uint8(bitPtrMask << 4)
   309  			}
   310  
   311  			*xbits = v
   312  			xbits = (*byte)(add(unsafe.Pointer(xbits), ^uintptr(0)))
   313  		}
   314  		if size0%(2*ptrSize) == 0 && size0 < size {
   315  			// Mark the word after last object's word as bitsDead.
   316  			*xbits = bitsDead << 2
   317  		}
   318  	}
   319  marked:
   320  
   321  	// GCmarkterminate allocates black
   322  	// All slots hold nil so no scanning is needed.
   323  	// This may be racing with GC so do it atomically if there can be
   324  	// a race marking the bit.
   325  	if gcphase == _GCmarktermination {
   326  		systemstack(func() {
   327  			gcmarknewobject_m(uintptr(x))
   328  		})
   329  	}
   330  
   331  	if mheap_.shadow_enabled {
   332  		clearshadow(uintptr(x), size)
   333  	}
   334  
   335  	if raceenabled {
   336  		racemalloc(x, size)
   337  	}
   338  
   339  	if debugMalloc {
   340  		mp := acquirem()
   341  		if mp.mallocing == 0 {
   342  			throw("bad malloc")
   343  		}
   344  		mp.mallocing = 0
   345  		if mp.curg != nil {
   346  			mp.curg.stackguard0 = mp.curg.stack.lo + _StackGuard
   347  		}
   348  		// Note: one releasem for the acquirem just above.
   349  		// The other for the acquirem at start of malloc.
   350  		releasem(mp)
   351  		releasem(mp)
   352  	}
   353  
   354  	if debug.allocfreetrace != 0 {
   355  		tracealloc(x, size, typ)
   356  	}
   357  
   358  	if rate := MemProfileRate; rate > 0 {
   359  		if size < uintptr(rate) && int32(size) < c.next_sample {
   360  			c.next_sample -= int32(size)
   361  		} else {
   362  			mp := acquirem()
   363  			profilealloc(mp, x, size)
   364  			releasem(mp)
   365  		}
   366  	}
   367  
   368  	if shouldtriggergc() {
   369  		gogc(0)
   370  	} else if shouldhelpgc && atomicloaduint(&bggc.working) == 1 {
   371  		// bggc.lock not taken since race on bggc.working is benign.
   372  		// At worse we don't call gchelpwork.
   373  		// Delay the gchelpwork until the epilogue so that it doesn't
   374  		// interfere with the inner working of malloc such as
   375  		// mcache refills that might happen while doing the gchelpwork
   376  		systemstack(gchelpwork)
   377  	}
   378  
   379  	return x
   380  }
   381  
   382  func loadPtrMask(typ *_type) []uint8 {
   383  	var ptrmask *uint8
   384  	nptr := (uintptr(typ.size) + ptrSize - 1) / ptrSize
   385  	if typ.kind&kindGCProg != 0 {
   386  		masksize := nptr
   387  		if masksize%2 != 0 {
   388  			masksize *= 2 // repeated
   389  		}
   390  		masksize = masksize * pointersPerByte / 8 // 4 bits per word
   391  		masksize++                                // unroll flag in the beginning
   392  		if masksize > maxGCMask && typ.gc[1] != 0 {
   393  			// write barriers have not been updated to deal with this case yet.
   394  			throw("maxGCMask too small for now")
   395  		}
   396  		ptrmask = (*uint8)(unsafe.Pointer(uintptr(typ.gc[0])))
   397  		// Check whether the program is already unrolled
   398  		// by checking if the unroll flag byte is set
   399  		maskword := uintptr(atomicloadp(unsafe.Pointer(ptrmask)))
   400  		if *(*uint8)(unsafe.Pointer(&maskword)) == 0 {
   401  			systemstack(func() {
   402  				unrollgcprog_m(typ)
   403  			})
   404  		}
   405  		ptrmask = (*uint8)(add(unsafe.Pointer(ptrmask), 1)) // skip the unroll flag byte
   406  	} else {
   407  		ptrmask = (*uint8)(unsafe.Pointer(typ.gc[0])) // pointer to unrolled mask
   408  	}
   409  	return (*[1 << 30]byte)(unsafe.Pointer(ptrmask))[:(nptr+1)/2]
   410  }
   411  
   412  // implementation of new builtin
   413  func newobject(typ *_type) unsafe.Pointer {
   414  	flags := uint32(0)
   415  	if typ.kind&kindNoPointers != 0 {
   416  		flags |= flagNoScan
   417  	}
   418  	return mallocgc(uintptr(typ.size), typ, flags)
   419  }
   420  
   421  //go:linkname reflect_unsafe_New reflect.unsafe_New
   422  func reflect_unsafe_New(typ *_type) unsafe.Pointer {
   423  	return newobject(typ)
   424  }
   425  
   426  // implementation of make builtin for slices
   427  func newarray(typ *_type, n uintptr) unsafe.Pointer {
   428  	flags := uint32(0)
   429  	if typ.kind&kindNoPointers != 0 {
   430  		flags |= flagNoScan
   431  	}
   432  	if int(n) < 0 || (typ.size > 0 && n > _MaxMem/uintptr(typ.size)) {
   433  		panic("runtime: allocation size out of range")
   434  	}
   435  	return mallocgc(uintptr(typ.size)*n, typ, flags)
   436  }
   437  
   438  //go:linkname reflect_unsafe_NewArray reflect.unsafe_NewArray
   439  func reflect_unsafe_NewArray(typ *_type, n uintptr) unsafe.Pointer {
   440  	return newarray(typ, n)
   441  }
   442  
   443  // rawmem returns a chunk of pointerless memory.  It is
   444  // not zeroed.
   445  func rawmem(size uintptr) unsafe.Pointer {
   446  	return mallocgc(size, nil, flagNoScan|flagNoZero)
   447  }
   448  
   449  func profilealloc(mp *m, x unsafe.Pointer, size uintptr) {
   450  	c := mp.mcache
   451  	rate := MemProfileRate
   452  	if size < uintptr(rate) {
   453  		// pick next profile time
   454  		// If you change this, also change allocmcache.
   455  		if rate > 0x3fffffff { // make 2*rate not overflow
   456  			rate = 0x3fffffff
   457  		}
   458  		next := int32(fastrand1()) % (2 * int32(rate))
   459  		// Subtract the "remainder" of the current allocation.
   460  		// Otherwise objects that are close in size to sampling rate
   461  		// will be under-sampled, because we consistently discard this remainder.
   462  		next -= (int32(size) - c.next_sample)
   463  		if next < 0 {
   464  			next = 0
   465  		}
   466  		c.next_sample = next
   467  	}
   468  
   469  	mProf_Malloc(x, size)
   470  }
   471  
   472  // For now this must be bracketed with a stoptheworld and a starttheworld to ensure
   473  // all go routines see the new barrier.
   474  func gcinstallmarkwb() {
   475  	gcphase = _GCmark
   476  }
   477  
   478  // force = 0 - start concurrent GC
   479  // force = 1 - do STW GC regardless of current heap usage
   480  // force = 2 - go STW GC and eager sweep
   481  func gogc(force int32) {
   482  	// The gc is turned off (via enablegc) until the bootstrap has completed.
   483  	// Also, malloc gets called in the guts of a number of libraries that might be
   484  	// holding locks. To avoid deadlocks during stoptheworld, don't bother
   485  	// trying to run gc while holding a lock. The next mallocgc without a lock
   486  	// will do the gc instead.
   487  
   488  	mp := acquirem()
   489  	if gp := getg(); gp == mp.g0 || mp.locks > 1 || !memstats.enablegc || panicking != 0 || gcpercent < 0 {
   490  		releasem(mp)
   491  		return
   492  	}
   493  	releasem(mp)
   494  	mp = nil
   495  
   496  	if force == 0 {
   497  		lock(&bggc.lock)
   498  		if !bggc.started {
   499  			bggc.working = 1
   500  			bggc.started = true
   501  			go backgroundgc()
   502  		} else if bggc.working == 0 {
   503  			bggc.working = 1
   504  			ready(bggc.g)
   505  		}
   506  		unlock(&bggc.lock)
   507  	} else {
   508  		gcwork(force)
   509  	}
   510  }
   511  
   512  func gcwork(force int32) {
   513  
   514  	semacquire(&worldsema, false)
   515  
   516  	// Pick up the remaining unswept/not being swept spans concurrently
   517  	for gosweepone() != ^uintptr(0) {
   518  		sweep.nbgsweep++
   519  	}
   520  
   521  	// Ok, we're doing it!  Stop everybody else
   522  
   523  	mp := acquirem()
   524  	mp.gcing = 1
   525  	releasem(mp)
   526  	gctimer.count++
   527  	if force == 0 {
   528  		gctimer.cycle.sweepterm = nanotime()
   529  	}
   530  	// Pick up the remaining unswept/not being swept spans before we STW
   531  	for gosweepone() != ^uintptr(0) {
   532  		sweep.nbgsweep++
   533  	}
   534  	systemstack(stoptheworld)
   535  	systemstack(finishsweep_m) // finish sweep before we start concurrent scan.
   536  	if force == 0 {            // Do as much work concurrently as possible
   537  		gcphase = _GCscan
   538  		systemstack(starttheworld)
   539  		gctimer.cycle.scan = nanotime()
   540  		// Do a concurrent heap scan before we stop the world.
   541  		systemstack(gcscan_m)
   542  		gctimer.cycle.installmarkwb = nanotime()
   543  		systemstack(stoptheworld)
   544  		systemstack(gcinstallmarkwb)
   545  		systemstack(starttheworld)
   546  		gctimer.cycle.mark = nanotime()
   547  		systemstack(gcmark_m)
   548  		gctimer.cycle.markterm = nanotime()
   549  		systemstack(stoptheworld)
   550  		systemstack(gcinstalloffwb_m)
   551  	}
   552  
   553  	startTime := nanotime()
   554  	if mp != acquirem() {
   555  		throw("gogc: rescheduled")
   556  	}
   557  
   558  	clearpools()
   559  
   560  	// Run gc on the g0 stack.  We do this so that the g stack
   561  	// we're currently running on will no longer change.  Cuts
   562  	// the root set down a bit (g0 stacks are not scanned, and
   563  	// we don't need to scan gc's internal state).  We also
   564  	// need to switch to g0 so we can shrink the stack.
   565  	n := 1
   566  	if debug.gctrace > 1 {
   567  		n = 2
   568  	}
   569  	eagersweep := force >= 2
   570  	for i := 0; i < n; i++ {
   571  		if i > 0 {
   572  			// refresh start time if doing a second GC
   573  			startTime = nanotime()
   574  		}
   575  		// switch to g0, call gc, then switch back
   576  		systemstack(func() {
   577  			gc_m(startTime, eagersweep)
   578  		})
   579  	}
   580  
   581  	systemstack(func() {
   582  		gccheckmark_m(startTime, eagersweep)
   583  	})
   584  
   585  	// all done
   586  	mp.gcing = 0
   587  
   588  	if force == 0 {
   589  		gctimer.cycle.sweep = nanotime()
   590  	}
   591  
   592  	semrelease(&worldsema)
   593  
   594  	if force == 0 {
   595  		if gctimer.verbose > 1 {
   596  			GCprinttimes()
   597  		} else if gctimer.verbose > 0 {
   598  			calctimes() // ignore result
   599  		}
   600  	}
   601  
   602  	systemstack(starttheworld)
   603  
   604  	releasem(mp)
   605  	mp = nil
   606  
   607  	// now that gc is done, kick off finalizer thread if needed
   608  	if !concurrentSweep {
   609  		// give the queued finalizers, if any, a chance to run
   610  		Gosched()
   611  	}
   612  }
   613  
   614  func GCcheckmarkenable() {
   615  	systemstack(gccheckmarkenable_m)
   616  }
   617  
   618  func GCcheckmarkdisable() {
   619  	systemstack(gccheckmarkdisable_m)
   620  }
   621  
   622  // gctimes records the time in nanoseconds of each phase of the concurrent GC.
   623  type gctimes struct {
   624  	sweepterm     int64 // stw
   625  	scan          int64
   626  	installmarkwb int64 // stw
   627  	mark          int64
   628  	markterm      int64 // stw
   629  	sweep         int64
   630  }
   631  
   632  // gcchronograph holds timer information related to GC phases
   633  // max records the maximum time spent in each GC phase since GCstarttimes.
   634  // total records the total time spent in each GC phase since GCstarttimes.
   635  // cycle records the absolute time (as returned by nanoseconds()) that each GC phase last started at.
   636  type gcchronograph struct {
   637  	count    int64
   638  	verbose  int64
   639  	maxpause int64
   640  	max      gctimes
   641  	total    gctimes
   642  	cycle    gctimes
   643  }
   644  
   645  var gctimer gcchronograph
   646  
   647  // GCstarttimes initializes the gc times. All previous times are lost.
   648  func GCstarttimes(verbose int64) {
   649  	gctimer = gcchronograph{verbose: verbose}
   650  }
   651  
   652  // GCendtimes stops the gc timers.
   653  func GCendtimes() {
   654  	gctimer.verbose = 0
   655  }
   656  
   657  // calctimes converts gctimer.cycle into the elapsed times, updates gctimer.total
   658  // and updates gctimer.max with the max pause time.
   659  func calctimes() gctimes {
   660  	var times gctimes
   661  
   662  	var max = func(a, b int64) int64 {
   663  		if a > b {
   664  			return a
   665  		}
   666  		return b
   667  	}
   668  
   669  	times.sweepterm = gctimer.cycle.scan - gctimer.cycle.sweepterm
   670  	gctimer.total.sweepterm += times.sweepterm
   671  	gctimer.max.sweepterm = max(gctimer.max.sweepterm, times.sweepterm)
   672  	gctimer.maxpause = max(gctimer.maxpause, gctimer.max.sweepterm)
   673  
   674  	times.scan = gctimer.cycle.installmarkwb - gctimer.cycle.scan
   675  	gctimer.total.scan += times.scan
   676  	gctimer.max.scan = max(gctimer.max.scan, times.scan)
   677  
   678  	times.installmarkwb = gctimer.cycle.mark - gctimer.cycle.installmarkwb
   679  	gctimer.total.installmarkwb += times.installmarkwb
   680  	gctimer.max.installmarkwb = max(gctimer.max.installmarkwb, times.installmarkwb)
   681  	gctimer.maxpause = max(gctimer.maxpause, gctimer.max.installmarkwb)
   682  
   683  	times.mark = gctimer.cycle.markterm - gctimer.cycle.mark
   684  	gctimer.total.mark += times.mark
   685  	gctimer.max.mark = max(gctimer.max.mark, times.mark)
   686  
   687  	times.markterm = gctimer.cycle.sweep - gctimer.cycle.markterm
   688  	gctimer.total.markterm += times.markterm
   689  	gctimer.max.markterm = max(gctimer.max.markterm, times.markterm)
   690  	gctimer.maxpause = max(gctimer.maxpause, gctimer.max.markterm)
   691  
   692  	return times
   693  }
   694  
   695  // GCprinttimes prints latency information in nanoseconds about various
   696  // phases in the GC. The information for each phase includes the maximum pause
   697  // and total time since the most recent call to GCstarttimes as well as
   698  // the information from the most recent Concurent GC cycle. Calls from the
   699  // application to runtime.GC() are ignored.
   700  func GCprinttimes() {
   701  	if gctimer.verbose == 0 {
   702  		println("GC timers not enabled")
   703  		return
   704  	}
   705  
   706  	// Explicitly put times on the heap so printPhase can use it.
   707  	times := new(gctimes)
   708  	*times = calctimes()
   709  	cycletime := gctimer.cycle.sweep - gctimer.cycle.sweepterm
   710  	pause := times.sweepterm + times.installmarkwb + times.markterm
   711  	gomaxprocs := GOMAXPROCS(-1)
   712  
   713  	printlock()
   714  	print("GC: #", gctimer.count, " ", cycletime, "ns @", gctimer.cycle.sweepterm, " pause=", pause, " maxpause=", gctimer.maxpause, " goroutines=", allglen, " gomaxprocs=", gomaxprocs, "\n")
   715  	printPhase := func(label string, get func(*gctimes) int64, procs int) {
   716  		print("GC:     ", label, " ", get(times), "ns\tmax=", get(&gctimer.max), "\ttotal=", get(&gctimer.total), "\tprocs=", procs, "\n")
   717  	}
   718  	printPhase("sweep term:", func(t *gctimes) int64 { return t.sweepterm }, gomaxprocs)
   719  	printPhase("scan:      ", func(t *gctimes) int64 { return t.scan }, 1)
   720  	printPhase("install wb:", func(t *gctimes) int64 { return t.installmarkwb }, gomaxprocs)
   721  	printPhase("mark:      ", func(t *gctimes) int64 { return t.mark }, 1)
   722  	printPhase("mark term: ", func(t *gctimes) int64 { return t.markterm }, gomaxprocs)
   723  	printunlock()
   724  }
   725  
   726  // GC runs a garbage collection.
   727  func GC() {
   728  	gogc(2)
   729  }
   730  
   731  // linker-provided
   732  var noptrdata struct{}
   733  var enoptrdata struct{}
   734  var noptrbss struct{}
   735  var enoptrbss struct{}
   736  
   737  // SetFinalizer sets the finalizer associated with x to f.
   738  // When the garbage collector finds an unreachable block
   739  // with an associated finalizer, it clears the association and runs
   740  // f(x) in a separate goroutine.  This makes x reachable again, but
   741  // now without an associated finalizer.  Assuming that SetFinalizer
   742  // is not called again, the next time the garbage collector sees
   743  // that x is unreachable, it will free x.
   744  //
   745  // SetFinalizer(x, nil) clears any finalizer associated with x.
   746  //
   747  // The argument x must be a pointer to an object allocated by
   748  // calling new or by taking the address of a composite literal.
   749  // The argument f must be a function that takes a single argument
   750  // to which x's type can be assigned, and can have arbitrary ignored return
   751  // values. If either of these is not true, SetFinalizer aborts the
   752  // program.
   753  //
   754  // Finalizers are run in dependency order: if A points at B, both have
   755  // finalizers, and they are otherwise unreachable, only the finalizer
   756  // for A runs; once A is freed, the finalizer for B can run.
   757  // If a cyclic structure includes a block with a finalizer, that
   758  // cycle is not guaranteed to be garbage collected and the finalizer
   759  // is not guaranteed to run, because there is no ordering that
   760  // respects the dependencies.
   761  //
   762  // The finalizer for x is scheduled to run at some arbitrary time after
   763  // x becomes unreachable.
   764  // There is no guarantee that finalizers will run before a program exits,
   765  // so typically they are useful only for releasing non-memory resources
   766  // associated with an object during a long-running program.
   767  // For example, an os.File object could use a finalizer to close the
   768  // associated operating system file descriptor when a program discards
   769  // an os.File without calling Close, but it would be a mistake
   770  // to depend on a finalizer to flush an in-memory I/O buffer such as a
   771  // bufio.Writer, because the buffer would not be flushed at program exit.
   772  //
   773  // It is not guaranteed that a finalizer will run if the size of *x is
   774  // zero bytes.
   775  //
   776  // It is not guaranteed that a finalizer will run for objects allocated
   777  // in initializers for package-level variables. Such objects may be
   778  // linker-allocated, not heap-allocated.
   779  //
   780  // A single goroutine runs all finalizers for a program, sequentially.
   781  // If a finalizer must run for a long time, it should do so by starting
   782  // a new goroutine.
   783  func SetFinalizer(obj interface{}, finalizer interface{}) {
   784  	e := (*eface)(unsafe.Pointer(&obj))
   785  	etyp := e._type
   786  	if etyp == nil {
   787  		throw("runtime.SetFinalizer: first argument is nil")
   788  	}
   789  	if etyp.kind&kindMask != kindPtr {
   790  		throw("runtime.SetFinalizer: first argument is " + *etyp._string + ", not pointer")
   791  	}
   792  	ot := (*ptrtype)(unsafe.Pointer(etyp))
   793  	if ot.elem == nil {
   794  		throw("nil elem type!")
   795  	}
   796  
   797  	// find the containing object
   798  	_, base, _ := findObject(e.data)
   799  
   800  	if base == nil {
   801  		// 0-length objects are okay.
   802  		if e.data == unsafe.Pointer(&zerobase) {
   803  			return
   804  		}
   805  
   806  		// Global initializers might be linker-allocated.
   807  		//	var Foo = &Object{}
   808  		//	func main() {
   809  		//		runtime.SetFinalizer(Foo, nil)
   810  		//	}
   811  		// The relevant segments are: noptrdata, data, bss, noptrbss.
   812  		// We cannot assume they are in any order or even contiguous,
   813  		// due to external linking.
   814  		if uintptr(unsafe.Pointer(&noptrdata)) <= uintptr(e.data) && uintptr(e.data) < uintptr(unsafe.Pointer(&enoptrdata)) ||
   815  			uintptr(unsafe.Pointer(&data)) <= uintptr(e.data) && uintptr(e.data) < uintptr(unsafe.Pointer(&edata)) ||
   816  			uintptr(unsafe.Pointer(&bss)) <= uintptr(e.data) && uintptr(e.data) < uintptr(unsafe.Pointer(&ebss)) ||
   817  			uintptr(unsafe.Pointer(&noptrbss)) <= uintptr(e.data) && uintptr(e.data) < uintptr(unsafe.Pointer(&enoptrbss)) {
   818  			return
   819  		}
   820  		throw("runtime.SetFinalizer: pointer not in allocated block")
   821  	}
   822  
   823  	if e.data != base {
   824  		// As an implementation detail we allow to set finalizers for an inner byte
   825  		// of an object if it could come from tiny alloc (see mallocgc for details).
   826  		if ot.elem == nil || ot.elem.kind&kindNoPointers == 0 || ot.elem.size >= maxTinySize {
   827  			throw("runtime.SetFinalizer: pointer not at beginning of allocated block")
   828  		}
   829  	}
   830  
   831  	f := (*eface)(unsafe.Pointer(&finalizer))
   832  	ftyp := f._type
   833  	if ftyp == nil {
   834  		// switch to system stack and remove finalizer
   835  		systemstack(func() {
   836  			removefinalizer(e.data)
   837  		})
   838  		return
   839  	}
   840  
   841  	if ftyp.kind&kindMask != kindFunc {
   842  		throw("runtime.SetFinalizer: second argument is " + *ftyp._string + ", not a function")
   843  	}
   844  	ft := (*functype)(unsafe.Pointer(ftyp))
   845  	ins := *(*[]*_type)(unsafe.Pointer(&ft.in))
   846  	if ft.dotdotdot || len(ins) != 1 {
   847  		throw("runtime.SetFinalizer: cannot pass " + *etyp._string + " to finalizer " + *ftyp._string)
   848  	}
   849  	fint := ins[0]
   850  	switch {
   851  	case fint == etyp:
   852  		// ok - same type
   853  		goto okarg
   854  	case fint.kind&kindMask == kindPtr:
   855  		if (fint.x == nil || fint.x.name == nil || etyp.x == nil || etyp.x.name == nil) && (*ptrtype)(unsafe.Pointer(fint)).elem == ot.elem {
   856  			// ok - not same type, but both pointers,
   857  			// one or the other is unnamed, and same element type, so assignable.
   858  			goto okarg
   859  		}
   860  	case fint.kind&kindMask == kindInterface:
   861  		ityp := (*interfacetype)(unsafe.Pointer(fint))
   862  		if len(ityp.mhdr) == 0 {
   863  			// ok - satisfies empty interface
   864  			goto okarg
   865  		}
   866  		if assertE2I2(ityp, obj, nil) {
   867  			goto okarg
   868  		}
   869  	}
   870  	throw("runtime.SetFinalizer: cannot pass " + *etyp._string + " to finalizer " + *ftyp._string)
   871  okarg:
   872  	// compute size needed for return parameters
   873  	nret := uintptr(0)
   874  	for _, t := range *(*[]*_type)(unsafe.Pointer(&ft.out)) {
   875  		nret = round(nret, uintptr(t.align)) + uintptr(t.size)
   876  	}
   877  	nret = round(nret, ptrSize)
   878  
   879  	// make sure we have a finalizer goroutine
   880  	createfing()
   881  
   882  	systemstack(func() {
   883  		if !addfinalizer(e.data, (*funcval)(f.data), nret, fint, ot) {
   884  			throw("runtime.SetFinalizer: finalizer already set")
   885  		}
   886  	})
   887  }
   888  
   889  // round n up to a multiple of a.  a must be a power of 2.
   890  func round(n, a uintptr) uintptr {
   891  	return (n + a - 1) &^ (a - 1)
   892  }
   893  
   894  // Look up pointer v in heap.  Return the span containing the object,
   895  // the start of the object, and the size of the object.  If the object
   896  // does not exist, return nil, nil, 0.
   897  func findObject(v unsafe.Pointer) (s *mspan, x unsafe.Pointer, n uintptr) {
   898  	c := gomcache()
   899  	c.local_nlookup++
   900  	if ptrSize == 4 && c.local_nlookup >= 1<<30 {
   901  		// purge cache stats to prevent overflow
   902  		lock(&mheap_.lock)
   903  		purgecachedstats(c)
   904  		unlock(&mheap_.lock)
   905  	}
   906  
   907  	// find span
   908  	arena_start := uintptr(unsafe.Pointer(mheap_.arena_start))
   909  	arena_used := uintptr(unsafe.Pointer(mheap_.arena_used))
   910  	if uintptr(v) < arena_start || uintptr(v) >= arena_used {
   911  		return
   912  	}
   913  	p := uintptr(v) >> pageShift
   914  	q := p - arena_start>>pageShift
   915  	s = *(**mspan)(add(unsafe.Pointer(mheap_.spans), q*ptrSize))
   916  	if s == nil {
   917  		return
   918  	}
   919  	x = unsafe.Pointer(uintptr(s.start) << pageShift)
   920  
   921  	if uintptr(v) < uintptr(x) || uintptr(v) >= uintptr(unsafe.Pointer(s.limit)) || s.state != mSpanInUse {
   922  		s = nil
   923  		x = nil
   924  		return
   925  	}
   926  
   927  	n = uintptr(s.elemsize)
   928  	if s.sizeclass != 0 {
   929  		x = add(x, (uintptr(v)-uintptr(x))/n*n)
   930  	}
   931  	return
   932  }
   933  
   934  var fingCreate uint32
   935  
   936  func createfing() {
   937  	// start the finalizer goroutine exactly once
   938  	if fingCreate == 0 && cas(&fingCreate, 0, 1) {
   939  		go runfinq()
   940  	}
   941  }
   942  
   943  // This is the goroutine that runs all of the finalizers
   944  func runfinq() {
   945  	var (
   946  		frame    unsafe.Pointer
   947  		framecap uintptr
   948  	)
   949  
   950  	for {
   951  		lock(&finlock)
   952  		fb := finq
   953  		finq = nil
   954  		if fb == nil {
   955  			gp := getg()
   956  			fing = gp
   957  			fingwait = true
   958  			gp.issystem = true
   959  			goparkunlock(&finlock, "finalizer wait")
   960  			gp.issystem = false
   961  			continue
   962  		}
   963  		unlock(&finlock)
   964  		if raceenabled {
   965  			racefingo()
   966  		}
   967  		for fb != nil {
   968  			for i := int32(0); i < fb.cnt; i++ {
   969  				f := (*finalizer)(add(unsafe.Pointer(&fb.fin), uintptr(i)*unsafe.Sizeof(finalizer{})))
   970  
   971  				framesz := unsafe.Sizeof((interface{})(nil)) + uintptr(f.nret)
   972  				if framecap < framesz {
   973  					// The frame does not contain pointers interesting for GC,
   974  					// all not yet finalized objects are stored in finq.
   975  					// If we do not mark it as FlagNoScan,
   976  					// the last finalized object is not collected.
   977  					frame = mallocgc(framesz, nil, flagNoScan)
   978  					framecap = framesz
   979  				}
   980  
   981  				if f.fint == nil {
   982  					throw("missing type in runfinq")
   983  				}
   984  				switch f.fint.kind & kindMask {
   985  				case kindPtr:
   986  					// direct use of pointer
   987  					*(*unsafe.Pointer)(frame) = f.arg
   988  				case kindInterface:
   989  					ityp := (*interfacetype)(unsafe.Pointer(f.fint))
   990  					// set up with empty interface
   991  					(*eface)(frame)._type = &f.ot.typ
   992  					(*eface)(frame).data = f.arg
   993  					if len(ityp.mhdr) != 0 {
   994  						// convert to interface with methods
   995  						// this conversion is guaranteed to succeed - we checked in SetFinalizer
   996  						assertE2I(ityp, *(*interface{})(frame), (*fInterface)(frame))
   997  					}
   998  				default:
   999  					throw("bad kind in runfinq")
  1000  				}
  1001  				reflectcall(nil, unsafe.Pointer(f.fn), frame, uint32(framesz), uint32(framesz))
  1002  
  1003  				// drop finalizer queue references to finalized object
  1004  				f.fn = nil
  1005  				f.arg = nil
  1006  				f.ot = nil
  1007  			}
  1008  			fb.cnt = 0
  1009  			next := fb.next
  1010  			lock(&finlock)
  1011  			fb.next = finc
  1012  			finc = fb
  1013  			unlock(&finlock)
  1014  			fb = next
  1015  		}
  1016  	}
  1017  }
  1018  
  1019  var persistent struct {
  1020  	lock mutex
  1021  	pos  unsafe.Pointer
  1022  	end  unsafe.Pointer
  1023  }
  1024  
  1025  // Wrapper around sysAlloc that can allocate small chunks.
  1026  // There is no associated free operation.
  1027  // Intended for things like function/type/debug-related persistent data.
  1028  // If align is 0, uses default align (currently 8).
  1029  func persistentalloc(size, align uintptr, stat *uint64) unsafe.Pointer {
  1030  	const (
  1031  		chunk    = 256 << 10
  1032  		maxBlock = 64 << 10 // VM reservation granularity is 64K on windows
  1033  	)
  1034  
  1035  	if align != 0 {
  1036  		if align&(align-1) != 0 {
  1037  			throw("persistentalloc: align is not a power of 2")
  1038  		}
  1039  		if align > _PageSize {
  1040  			throw("persistentalloc: align is too large")
  1041  		}
  1042  	} else {
  1043  		align = 8
  1044  	}
  1045  
  1046  	if size >= maxBlock {
  1047  		return sysAlloc(size, stat)
  1048  	}
  1049  
  1050  	lock(&persistent.lock)
  1051  	persistent.pos = roundup(persistent.pos, align)
  1052  	if uintptr(persistent.pos)+size > uintptr(persistent.end) {
  1053  		persistent.pos = sysAlloc(chunk, &memstats.other_sys)
  1054  		if persistent.pos == nil {
  1055  			unlock(&persistent.lock)
  1056  			throw("runtime: cannot allocate memory")
  1057  		}
  1058  		persistent.end = add(persistent.pos, chunk)
  1059  	}
  1060  	p := persistent.pos
  1061  	persistent.pos = add(persistent.pos, size)
  1062  	unlock(&persistent.lock)
  1063  
  1064  	if stat != &memstats.other_sys {
  1065  		xadd64(stat, int64(size))
  1066  		xadd64(&memstats.other_sys, -int64(size))
  1067  	}
  1068  	return p
  1069  }