github.com/tinygo-org/tinygo@v0.31.3-0.20240404173401-90b0bf646c27/src/runtime/gc_blocks.go (about)

     1  //go:build gc.conservative || gc.precise
     2  
     3  package runtime
     4  
     5  // This memory manager is a textbook mark/sweep implementation, heavily inspired
     6  // by the MicroPython garbage collector.
     7  //
     8  // The memory manager internally uses blocks of 4 pointers big (see
     9  // bytesPerBlock). Every allocation first rounds up to this size to align every
    10  // block. It will first try to find a chain of blocks that is big enough to
    11  // satisfy the allocation. If it finds one, it marks the first one as the "head"
    12  // and the following ones (if any) as the "tail" (see below). If it cannot find
    13  // any free space, it will perform a garbage collection cycle and try again. If
    14  // it still cannot find any free space, it gives up.
    15  //
    16  // Every block has some metadata, which is stored at the end of the heap.
    17  // The four states are "free", "head", "tail", and "mark". During normal
    18  // operation, there are no marked blocks. Every allocated object starts with a
    19  // "head" and is followed by "tail" blocks. The reason for this distinction is
    20  // that this way, the start and end of every object can be found easily.
    21  //
    22  // Metadata is stored in a special area at the end of the heap, in the area
    23  // metadataStart..heapEnd. The actual blocks are stored in
    24  // heapStart..metadataStart.
    25  //
    26  // More information:
    27  // https://aykevl.nl/2020/09/gc-tinygo
    28  // https://github.com/micropython/micropython/wiki/Memory-Manager
    29  // https://github.com/micropython/micropython/blob/master/py/gc.c
    30  // "The Garbage Collection Handbook" by Richard Jones, Antony Hosking, Eliot
    31  // Moss.
    32  
    33  import (
    34  	"internal/task"
    35  	"runtime/interrupt"
    36  	"unsafe"
    37  )
    38  
    39  const gcDebug = false
    40  
    41  // Some globals + constants for the entire GC.
    42  
    43  const (
    44  	wordsPerBlock      = 4 // number of pointers in an allocated block
    45  	bytesPerBlock      = wordsPerBlock * unsafe.Sizeof(heapStart)
    46  	stateBits          = 2 // how many bits a block state takes (see blockState type)
    47  	blocksPerStateByte = 8 / stateBits
    48  	markStackSize      = 4 * unsafe.Sizeof((*int)(nil)) // number of to-be-marked blocks to queue before forcing a rescan
    49  )
    50  
    51  var (
    52  	metadataStart unsafe.Pointer // pointer to the start of the heap metadata
    53  	nextAlloc     gcBlock        // the next block that should be tried by the allocator
    54  	endBlock      gcBlock        // the block just past the end of the available space
    55  	gcTotalAlloc  uint64         // total number of bytes allocated
    56  	gcMallocs     uint64         // total number of allocations
    57  	gcFrees       uint64         // total number of objects freed
    58  )
    59  
    60  // zeroSizedAlloc is just a sentinel that gets returned when allocating 0 bytes.
    61  var zeroSizedAlloc uint8
    62  
    63  // Provide some abstraction over heap blocks.
    64  
    65  // blockState stores the four states in which a block can be. It is two bits in
    66  // size.
    67  type blockState uint8
    68  
    69  const (
    70  	blockStateFree blockState = 0 // 00
    71  	blockStateHead blockState = 1 // 01
    72  	blockStateTail blockState = 2 // 10
    73  	blockStateMark blockState = 3 // 11
    74  	blockStateMask blockState = 3 // 11
    75  )
    76  
    77  // String returns a human-readable version of the block state, for debugging.
    78  func (s blockState) String() string {
    79  	switch s {
    80  	case blockStateFree:
    81  		return "free"
    82  	case blockStateHead:
    83  		return "head"
    84  	case blockStateTail:
    85  		return "tail"
    86  	case blockStateMark:
    87  		return "mark"
    88  	default:
    89  		// must never happen
    90  		return "!err"
    91  	}
    92  }
    93  
    94  // The block number in the pool.
    95  type gcBlock uintptr
    96  
    97  // blockFromAddr returns a block given an address somewhere in the heap (which
    98  // might not be heap-aligned).
    99  func blockFromAddr(addr uintptr) gcBlock {
   100  	if gcAsserts && (addr < heapStart || addr >= uintptr(metadataStart)) {
   101  		runtimePanic("gc: trying to get block from invalid address")
   102  	}
   103  	return gcBlock((addr - heapStart) / bytesPerBlock)
   104  }
   105  
   106  // Return a pointer to the start of the allocated object.
   107  func (b gcBlock) pointer() unsafe.Pointer {
   108  	return unsafe.Pointer(b.address())
   109  }
   110  
   111  // Return the address of the start of the allocated object.
   112  func (b gcBlock) address() uintptr {
   113  	addr := heapStart + uintptr(b)*bytesPerBlock
   114  	if gcAsserts && addr > uintptr(metadataStart) {
   115  		runtimePanic("gc: block pointing inside metadata")
   116  	}
   117  	return addr
   118  }
   119  
   120  // findHead returns the head (first block) of an object, assuming the block
   121  // points to an allocated object. It returns the same block if this block
   122  // already points to the head.
   123  func (b gcBlock) findHead() gcBlock {
   124  	for b.state() == blockStateTail {
   125  		b--
   126  	}
   127  	if gcAsserts {
   128  		if b.state() != blockStateHead && b.state() != blockStateMark {
   129  			runtimePanic("gc: found tail without head")
   130  		}
   131  	}
   132  	return b
   133  }
   134  
   135  // findNext returns the first block just past the end of the tail. This may or
   136  // may not be the head of an object.
   137  func (b gcBlock) findNext() gcBlock {
   138  	if b.state() == blockStateHead || b.state() == blockStateMark {
   139  		b++
   140  	}
   141  	for b.address() < uintptr(metadataStart) && b.state() == blockStateTail {
   142  		b++
   143  	}
   144  	return b
   145  }
   146  
   147  // State returns the current block state.
   148  func (b gcBlock) state() blockState {
   149  	stateBytePtr := (*uint8)(unsafe.Add(metadataStart, b/blocksPerStateByte))
   150  	return blockState(*stateBytePtr>>((b%blocksPerStateByte)*stateBits)) & blockStateMask
   151  }
   152  
   153  // setState sets the current block to the given state, which must contain more
   154  // bits than the current state. Allowed transitions: from free to any state and
   155  // from head to mark.
   156  func (b gcBlock) setState(newState blockState) {
   157  	stateBytePtr := (*uint8)(unsafe.Add(metadataStart, b/blocksPerStateByte))
   158  	*stateBytePtr |= uint8(newState << ((b % blocksPerStateByte) * stateBits))
   159  	if gcAsserts && b.state() != newState {
   160  		runtimePanic("gc: setState() was not successful")
   161  	}
   162  }
   163  
   164  // markFree sets the block state to free, no matter what state it was in before.
   165  func (b gcBlock) markFree() {
   166  	stateBytePtr := (*uint8)(unsafe.Add(metadataStart, b/blocksPerStateByte))
   167  	*stateBytePtr &^= uint8(blockStateMask << ((b % blocksPerStateByte) * stateBits))
   168  	if gcAsserts && b.state() != blockStateFree {
   169  		runtimePanic("gc: markFree() was not successful")
   170  	}
   171  	if gcAsserts {
   172  		*(*[wordsPerBlock]uintptr)(unsafe.Pointer(b.address())) = [wordsPerBlock]uintptr{}
   173  	}
   174  }
   175  
   176  // unmark changes the state of the block from mark to head. It must be marked
   177  // before calling this function.
   178  func (b gcBlock) unmark() {
   179  	if gcAsserts && b.state() != blockStateMark {
   180  		runtimePanic("gc: unmark() on a block that is not marked")
   181  	}
   182  	clearMask := blockStateMask ^ blockStateHead // the bits to clear from the state
   183  	stateBytePtr := (*uint8)(unsafe.Add(metadataStart, b/blocksPerStateByte))
   184  	*stateBytePtr &^= uint8(clearMask << ((b % blocksPerStateByte) * stateBits))
   185  	if gcAsserts && b.state() != blockStateHead {
   186  		runtimePanic("gc: unmark() was not successful")
   187  	}
   188  }
   189  
   190  func isOnHeap(ptr uintptr) bool {
   191  	return ptr >= heapStart && ptr < uintptr(metadataStart)
   192  }
   193  
   194  // Initialize the memory allocator.
   195  // No memory may be allocated before this is called. That means the runtime and
   196  // any packages the runtime depends upon may not allocate memory during package
   197  // initialization.
   198  func initHeap() {
   199  	calculateHeapAddresses()
   200  
   201  	// Set all block states to 'free'.
   202  	metadataSize := heapEnd - uintptr(metadataStart)
   203  	memzero(unsafe.Pointer(metadataStart), metadataSize)
   204  }
   205  
   206  // setHeapEnd is called to expand the heap. The heap can only grow, not shrink.
   207  // Also, the heap should grow substantially each time otherwise growing the heap
   208  // will be expensive.
   209  func setHeapEnd(newHeapEnd uintptr) {
   210  	if gcAsserts && newHeapEnd <= heapEnd {
   211  		runtimePanic("gc: setHeapEnd didn't grow the heap")
   212  	}
   213  
   214  	// Save some old variables we need later.
   215  	oldMetadataStart := metadataStart
   216  	oldMetadataSize := heapEnd - uintptr(metadataStart)
   217  
   218  	// Increase the heap. After setting the new heapEnd, calculateHeapAddresses
   219  	// will update metadataStart and the memcpy will copy the metadata to the
   220  	// new location.
   221  	// The new metadata will be bigger than the old metadata, but a simple
   222  	// memcpy is fine as it only copies the old metadata and the new memory will
   223  	// have been zero initialized.
   224  	heapEnd = newHeapEnd
   225  	calculateHeapAddresses()
   226  	memcpy(metadataStart, oldMetadataStart, oldMetadataSize)
   227  
   228  	// Note: the memcpy above assumes the heap grows enough so that the new
   229  	// metadata does not overlap the old metadata. If that isn't true, memmove
   230  	// should be used to avoid corruption.
   231  	// This assert checks whether that's true.
   232  	if gcAsserts && uintptr(metadataStart) < uintptr(oldMetadataStart)+oldMetadataSize {
   233  		runtimePanic("gc: heap did not grow enough at once")
   234  	}
   235  }
   236  
   237  // calculateHeapAddresses initializes variables such as metadataStart and
   238  // numBlock based on heapStart and heapEnd.
   239  //
   240  // This function can be called again when the heap size increases. The caller is
   241  // responsible for copying the metadata to the new location.
   242  func calculateHeapAddresses() {
   243  	totalSize := heapEnd - heapStart
   244  
   245  	// Allocate some memory to keep 2 bits of information about every block.
   246  	metadataSize := (totalSize + blocksPerStateByte*bytesPerBlock) / (1 + blocksPerStateByte*bytesPerBlock)
   247  	metadataStart = unsafe.Pointer(heapEnd - metadataSize)
   248  
   249  	// Use the rest of the available memory as heap.
   250  	numBlocks := (uintptr(metadataStart) - heapStart) / bytesPerBlock
   251  	endBlock = gcBlock(numBlocks)
   252  	if gcDebug {
   253  		println("heapStart:        ", heapStart)
   254  		println("heapEnd:          ", heapEnd)
   255  		println("total size:       ", totalSize)
   256  		println("metadata size:    ", metadataSize)
   257  		println("metadataStart:    ", metadataStart)
   258  		println("# of blocks:      ", numBlocks)
   259  		println("# of block states:", metadataSize*blocksPerStateByte)
   260  	}
   261  	if gcAsserts && metadataSize*blocksPerStateByte < numBlocks {
   262  		// sanity check
   263  		runtimePanic("gc: metadata array is too small")
   264  	}
   265  }
   266  
   267  // alloc tries to find some free space on the heap, possibly doing a garbage
   268  // collection cycle if needed. If no space is free, it panics.
   269  //
   270  //go:noinline
   271  func alloc(size uintptr, layout unsafe.Pointer) unsafe.Pointer {
   272  	if size == 0 {
   273  		return unsafe.Pointer(&zeroSizedAlloc)
   274  	}
   275  
   276  	if preciseHeap {
   277  		size += align(unsafe.Sizeof(layout))
   278  	}
   279  
   280  	if interrupt.In() {
   281  		runtimePanicAt(returnAddress(0), "heap alloc in interrupt")
   282  	}
   283  
   284  	gcTotalAlloc += uint64(size)
   285  	gcMallocs++
   286  
   287  	neededBlocks := (size + (bytesPerBlock - 1)) / bytesPerBlock
   288  
   289  	// Continue looping until a run of free blocks has been found that fits the
   290  	// requested size.
   291  	index := nextAlloc
   292  	numFreeBlocks := uintptr(0)
   293  	heapScanCount := uint8(0)
   294  	for {
   295  		if index == nextAlloc {
   296  			if heapScanCount == 0 {
   297  				heapScanCount = 1
   298  			} else if heapScanCount == 1 {
   299  				// The entire heap has been searched for free memory, but none
   300  				// could be found. Run a garbage collection cycle to reclaim
   301  				// free memory and try again.
   302  				heapScanCount = 2
   303  				freeBytes := runGC()
   304  				heapSize := uintptr(metadataStart) - heapStart
   305  				if freeBytes < heapSize/3 {
   306  					// Ensure there is at least 33% headroom.
   307  					// This percentage was arbitrarily chosen, and may need to
   308  					// be tuned in the future.
   309  					growHeap()
   310  				}
   311  			} else {
   312  				// Even after garbage collection, no free memory could be found.
   313  				// Try to increase heap size.
   314  				if growHeap() {
   315  					// Success, the heap was increased in size. Try again with a
   316  					// larger heap.
   317  				} else {
   318  					// Unfortunately the heap could not be increased. This
   319  					// happens on baremetal systems for example (where all
   320  					// available RAM has already been dedicated to the heap).
   321  					runtimePanicAt(returnAddress(0), "out of memory")
   322  				}
   323  			}
   324  		}
   325  
   326  		// Wrap around the end of the heap.
   327  		if index == endBlock {
   328  			index = 0
   329  			// Reset numFreeBlocks as allocations cannot wrap.
   330  			numFreeBlocks = 0
   331  			// In rare cases, the initial heap might be so small that there are
   332  			// no blocks at all. In this case, it's better to jump back to the
   333  			// start of the loop and try again, until the GC realizes there is
   334  			// no memory and grows the heap.
   335  			// This can sometimes happen on WebAssembly, where the initial heap
   336  			// is created by whatever is left on the last memory page.
   337  			continue
   338  		}
   339  
   340  		// Is the block we're looking at free?
   341  		if index.state() != blockStateFree {
   342  			// This block is in use. Try again from this point.
   343  			numFreeBlocks = 0
   344  			index++
   345  			continue
   346  		}
   347  		numFreeBlocks++
   348  		index++
   349  
   350  		// Are we finished?
   351  		if numFreeBlocks == neededBlocks {
   352  			// Found a big enough range of free blocks!
   353  			nextAlloc = index
   354  			thisAlloc := index - gcBlock(neededBlocks)
   355  			if gcDebug {
   356  				println("found memory:", thisAlloc.pointer(), int(size))
   357  			}
   358  
   359  			// Set the following blocks as being allocated.
   360  			thisAlloc.setState(blockStateHead)
   361  			for i := thisAlloc + 1; i != nextAlloc; i++ {
   362  				i.setState(blockStateTail)
   363  			}
   364  
   365  			// Return a pointer to this allocation.
   366  			pointer := thisAlloc.pointer()
   367  			if preciseHeap {
   368  				// Store the object layout at the start of the object.
   369  				// TODO: this wastes a little bit of space on systems with
   370  				// larger-than-pointer alignment requirements.
   371  				*(*unsafe.Pointer)(pointer) = layout
   372  				add := align(unsafe.Sizeof(layout))
   373  				pointer = unsafe.Add(pointer, add)
   374  				size -= add
   375  			}
   376  			memzero(pointer, size)
   377  			return pointer
   378  		}
   379  	}
   380  }
   381  
   382  func realloc(ptr unsafe.Pointer, size uintptr) unsafe.Pointer {
   383  	if ptr == nil {
   384  		return alloc(size, nil)
   385  	}
   386  
   387  	ptrAddress := uintptr(ptr)
   388  	endOfTailAddress := blockFromAddr(ptrAddress).findNext().address()
   389  
   390  	// this might be a few bytes longer than the original size of
   391  	// ptr, because we align to full blocks of size bytesPerBlock
   392  	oldSize := endOfTailAddress - ptrAddress
   393  	if size <= oldSize {
   394  		return ptr
   395  	}
   396  
   397  	newAlloc := alloc(size, nil)
   398  	memcpy(newAlloc, ptr, oldSize)
   399  	free(ptr)
   400  
   401  	return newAlloc
   402  }
   403  
   404  func free(ptr unsafe.Pointer) {
   405  	// TODO: free blocks on request, when the compiler knows they're unused.
   406  }
   407  
   408  // GC performs a garbage collection cycle.
   409  func GC() {
   410  	runGC()
   411  }
   412  
   413  // runGC performs a garbage colleciton cycle. It is the internal implementation
   414  // of the runtime.GC() function. The difference is that it returns the number of
   415  // free bytes in the heap after the GC is finished.
   416  func runGC() (freeBytes uintptr) {
   417  	if gcDebug {
   418  		println("running collection cycle...")
   419  	}
   420  
   421  	// Mark phase: mark all reachable objects, recursively.
   422  	markStack()
   423  	findGlobals(markRoots)
   424  
   425  	if baremetal && hasScheduler {
   426  		// Channel operations in interrupts may move task pointers around while we are marking.
   427  		// Therefore we need to scan the runqueue seperately.
   428  		var markedTaskQueue task.Queue
   429  	runqueueScan:
   430  		for !runqueue.Empty() {
   431  			// Pop the next task off of the runqueue.
   432  			t := runqueue.Pop()
   433  
   434  			// Mark the task if it has not already been marked.
   435  			markRoot(uintptr(unsafe.Pointer(&runqueue)), uintptr(unsafe.Pointer(t)))
   436  
   437  			// Push the task onto our temporary queue.
   438  			markedTaskQueue.Push(t)
   439  		}
   440  
   441  		finishMark()
   442  
   443  		// Restore the runqueue.
   444  		i := interrupt.Disable()
   445  		if !runqueue.Empty() {
   446  			// Something new came in while finishing the mark.
   447  			interrupt.Restore(i)
   448  			goto runqueueScan
   449  		}
   450  		runqueue = markedTaskQueue
   451  		interrupt.Restore(i)
   452  	} else {
   453  		finishMark()
   454  	}
   455  
   456  	// Sweep phase: free all non-marked objects and unmark marked objects for
   457  	// the next collection cycle.
   458  	freeBytes = sweep()
   459  
   460  	// Show how much has been sweeped, for debugging.
   461  	if gcDebug {
   462  		dumpHeap()
   463  	}
   464  
   465  	return
   466  }
   467  
   468  // markRoots reads all pointers from start to end (exclusive) and if they look
   469  // like a heap pointer and are unmarked, marks them and scans that object as
   470  // well (recursively). The start and end parameters must be valid pointers and
   471  // must be aligned.
   472  func markRoots(start, end uintptr) {
   473  	if gcDebug {
   474  		println("mark from", start, "to", end, int(end-start))
   475  	}
   476  	if gcAsserts {
   477  		if start >= end {
   478  			runtimePanic("gc: unexpected range to mark")
   479  		}
   480  		if start%unsafe.Alignof(start) != 0 {
   481  			runtimePanic("gc: unaligned start pointer")
   482  		}
   483  		if end%unsafe.Alignof(end) != 0 {
   484  			runtimePanic("gc: unaligned end pointer")
   485  		}
   486  	}
   487  
   488  	// Reduce the end bound to avoid reading too far on platforms where pointer alignment is smaller than pointer size.
   489  	// If the size of the range is 0, then end will be slightly below start after this.
   490  	end -= unsafe.Sizeof(end) - unsafe.Alignof(end)
   491  
   492  	for addr := start; addr < end; addr += unsafe.Alignof(addr) {
   493  		root := *(*uintptr)(unsafe.Pointer(addr))
   494  		markRoot(addr, root)
   495  	}
   496  }
   497  
   498  // stackOverflow is a flag which is set when the GC scans too deep while marking.
   499  // After it is set, all marked allocations must be re-scanned.
   500  var stackOverflow bool
   501  
   502  // startMark starts the marking process on a root and all of its children.
   503  func startMark(root gcBlock) {
   504  	var stack [markStackSize]gcBlock
   505  	stack[0] = root
   506  	root.setState(blockStateMark)
   507  	stackLen := 1
   508  	for stackLen > 0 {
   509  		// Pop a block off of the stack.
   510  		stackLen--
   511  		block := stack[stackLen]
   512  		if gcDebug {
   513  			println("stack popped, remaining stack:", stackLen)
   514  		}
   515  
   516  		// Scan all pointers inside the block.
   517  		scanner := newGCObjectScanner(block)
   518  		if scanner.pointerFree() {
   519  			// This object doesn't contain any pointers.
   520  			// This is a fast path for objects like make([]int, 4096).
   521  			continue
   522  		}
   523  		start, end := block.address(), block.findNext().address()
   524  		if preciseHeap {
   525  			// The first word of the object is just the pointer layout value.
   526  			// Skip it.
   527  			start += align(unsafe.Sizeof(uintptr(0)))
   528  		}
   529  		for addr := start; addr != end; addr += unsafe.Alignof(addr) {
   530  			// Load the word.
   531  			word := *(*uintptr)(unsafe.Pointer(addr))
   532  
   533  			if !scanner.nextIsPointer(word, root.address(), addr) {
   534  				// Not a heap pointer.
   535  				continue
   536  			}
   537  
   538  			// Find the corresponding memory block.
   539  			referencedBlock := blockFromAddr(word)
   540  
   541  			if referencedBlock.state() == blockStateFree {
   542  				// The to-be-marked object doesn't actually exist.
   543  				// This is probably a false positive.
   544  				if gcDebug {
   545  					println("found reference to free memory:", word, "at:", addr)
   546  				}
   547  				continue
   548  			}
   549  
   550  			// Move to the block's head.
   551  			referencedBlock = referencedBlock.findHead()
   552  
   553  			if referencedBlock.state() == blockStateMark {
   554  				// The block has already been marked by something else.
   555  				continue
   556  			}
   557  
   558  			// Mark block.
   559  			if gcDebug {
   560  				println("marking block:", referencedBlock)
   561  			}
   562  			referencedBlock.setState(blockStateMark)
   563  
   564  			if stackLen == len(stack) {
   565  				// The stack is full.
   566  				// It is necessary to rescan all marked blocks once we are done.
   567  				stackOverflow = true
   568  				if gcDebug {
   569  					println("gc stack overflowed")
   570  				}
   571  				continue
   572  			}
   573  
   574  			// Push the pointer onto the stack to be scanned later.
   575  			stack[stackLen] = referencedBlock
   576  			stackLen++
   577  		}
   578  	}
   579  }
   580  
   581  // finishMark finishes the marking process by processing all stack overflows.
   582  func finishMark() {
   583  	for stackOverflow {
   584  		// Re-mark all blocks.
   585  		stackOverflow = false
   586  		for block := gcBlock(0); block < endBlock; block++ {
   587  			if block.state() != blockStateMark {
   588  				// Block is not marked, so we do not need to rescan it.
   589  				continue
   590  			}
   591  
   592  			// Re-mark the block.
   593  			startMark(block)
   594  		}
   595  	}
   596  }
   597  
   598  // mark a GC root at the address addr.
   599  func markRoot(addr, root uintptr) {
   600  	if isOnHeap(root) {
   601  		block := blockFromAddr(root)
   602  		if block.state() == blockStateFree {
   603  			// The to-be-marked object doesn't actually exist.
   604  			// This could either be a dangling pointer (oops!) but most likely
   605  			// just a false positive.
   606  			return
   607  		}
   608  		head := block.findHead()
   609  		if head.state() != blockStateMark {
   610  			if gcDebug {
   611  				println("found unmarked pointer", root, "at address", addr)
   612  			}
   613  			startMark(head)
   614  		}
   615  	}
   616  }
   617  
   618  // Sweep goes through all memory and frees unmarked memory.
   619  // It returns how many bytes are free in the heap after the sweep.
   620  func sweep() (freeBytes uintptr) {
   621  	freeCurrentObject := false
   622  	for block := gcBlock(0); block < endBlock; block++ {
   623  		switch block.state() {
   624  		case blockStateHead:
   625  			// Unmarked head. Free it, including all tail blocks following it.
   626  			block.markFree()
   627  			freeCurrentObject = true
   628  			gcFrees++
   629  			freeBytes += bytesPerBlock
   630  		case blockStateTail:
   631  			if freeCurrentObject {
   632  				// This is a tail object following an unmarked head.
   633  				// Free it now.
   634  				block.markFree()
   635  				freeBytes += bytesPerBlock
   636  			}
   637  		case blockStateMark:
   638  			// This is a marked object. The next tail blocks must not be freed,
   639  			// but the mark bit must be removed so the next GC cycle will
   640  			// collect this object if it is unreferenced then.
   641  			block.unmark()
   642  			freeCurrentObject = false
   643  		case blockStateFree:
   644  			freeBytes += bytesPerBlock
   645  		}
   646  	}
   647  	return
   648  }
   649  
   650  // dumpHeap can be used for debugging purposes. It dumps the state of each heap
   651  // block to standard output.
   652  func dumpHeap() {
   653  	println("heap:")
   654  	for block := gcBlock(0); block < endBlock; block++ {
   655  		switch block.state() {
   656  		case blockStateHead:
   657  			print("*")
   658  		case blockStateTail:
   659  			print("-")
   660  		case blockStateMark:
   661  			print("#")
   662  		default: // free
   663  			print("ยท")
   664  		}
   665  		if block%64 == 63 || block+1 == endBlock {
   666  			println()
   667  		}
   668  	}
   669  }
   670  
   671  // ReadMemStats populates m with memory statistics.
   672  //
   673  // The returned memory statistics are up to date as of the
   674  // call to ReadMemStats. This would not do GC implicitly for you.
   675  func ReadMemStats(m *MemStats) {
   676  	m.HeapIdle = 0
   677  	m.HeapInuse = 0
   678  	for block := gcBlock(0); block < endBlock; block++ {
   679  		bstate := block.state()
   680  		if bstate == blockStateFree {
   681  			m.HeapIdle += uint64(bytesPerBlock)
   682  		} else {
   683  			m.HeapInuse += uint64(bytesPerBlock)
   684  		}
   685  	}
   686  	m.HeapReleased = 0 // always 0, we don't currently release memory back to the OS.
   687  	m.HeapSys = m.HeapInuse + m.HeapIdle
   688  	m.GCSys = uint64(heapEnd - uintptr(metadataStart))
   689  	m.TotalAlloc = gcTotalAlloc
   690  	m.Mallocs = gcMallocs
   691  	m.Frees = gcFrees
   692  	m.Sys = uint64(heapEnd - heapStart)
   693  }
   694  
   695  func SetFinalizer(obj interface{}, finalizer interface{}) {
   696  	// Unimplemented.
   697  }