github.com/reiver/go@v0.0.0-20150109200633-1d0c7792f172/src/runtime/malloc1.go (about)

     1  // Copyright 2009 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // See malloc.h for overview.
     6  //
     7  // TODO(rsc): double-check stats.
     8  
     9  package runtime
    10  
    11  import "unsafe"
    12  
    13  const _MaxArena32 = 2 << 30
    14  
    15  // For use by Go. If it were a C enum it would be made available automatically,
    16  // but the value of MaxMem is too large for enum.
    17  // XXX - uintptr runtime·maxmem = MaxMem;
    18  
    19  func mlookup(v uintptr, base *uintptr, size *uintptr, sp **mspan) int32 {
    20  	_g_ := getg()
    21  
    22  	_g_.m.mcache.local_nlookup++
    23  	if ptrSize == 4 && _g_.m.mcache.local_nlookup >= 1<<30 {
    24  		// purge cache stats to prevent overflow
    25  		lock(&mheap_.lock)
    26  		purgecachedstats(_g_.m.mcache)
    27  		unlock(&mheap_.lock)
    28  	}
    29  
    30  	s := mHeap_LookupMaybe(&mheap_, unsafe.Pointer(v))
    31  	if sp != nil {
    32  		*sp = s
    33  	}
    34  	if s == nil {
    35  		if base != nil {
    36  			*base = 0
    37  		}
    38  		if size != nil {
    39  			*size = 0
    40  		}
    41  		return 0
    42  	}
    43  
    44  	p := uintptr(s.start) << _PageShift
    45  	if s.sizeclass == 0 {
    46  		// Large object.
    47  		if base != nil {
    48  			*base = p
    49  		}
    50  		if size != nil {
    51  			*size = s.npages << _PageShift
    52  		}
    53  		return 1
    54  	}
    55  
    56  	n := s.elemsize
    57  	if base != nil {
    58  		i := (uintptr(v) - uintptr(p)) / n
    59  		*base = p + i*n
    60  	}
    61  	if size != nil {
    62  		*size = n
    63  	}
    64  
    65  	return 1
    66  }
    67  
    68  //go:nosplit
    69  func purgecachedstats(c *mcache) {
    70  	// Protected by either heap or GC lock.
    71  	h := &mheap_
    72  	memstats.heap_alloc += uint64(c.local_cachealloc)
    73  	c.local_cachealloc = 0
    74  	memstats.tinyallocs += uint64(c.local_tinyallocs)
    75  	c.local_tinyallocs = 0
    76  	memstats.nlookup += uint64(c.local_nlookup)
    77  	c.local_nlookup = 0
    78  	h.largefree += uint64(c.local_largefree)
    79  	c.local_largefree = 0
    80  	h.nlargefree += uint64(c.local_nlargefree)
    81  	c.local_nlargefree = 0
    82  	for i := 0; i < len(c.local_nsmallfree); i++ {
    83  		h.nsmallfree[i] += uint64(c.local_nsmallfree[i])
    84  		c.local_nsmallfree[i] = 0
    85  	}
    86  }
    87  
    88  func mallocinit() {
    89  	initSizes()
    90  
    91  	if class_to_size[_TinySizeClass] != _TinySize {
    92  		throw("bad TinySizeClass")
    93  	}
    94  
    95  	var p, bitmapSize, spansSize, pSize, limit uintptr
    96  	var reserved bool
    97  
    98  	// limit = runtime.memlimit();
    99  	// See https://code.google.com/p/go/issues/detail?id=5049
   100  	// TODO(rsc): Fix after 1.1.
   101  	limit = 0
   102  
   103  	// Set up the allocation arena, a contiguous area of memory where
   104  	// allocated data will be found.  The arena begins with a bitmap large
   105  	// enough to hold 4 bits per allocated word.
   106  	if ptrSize == 8 && (limit == 0 || limit > 1<<30) {
   107  		// On a 64-bit machine, allocate from a single contiguous reservation.
   108  		// 128 GB (MaxMem) should be big enough for now.
   109  		//
   110  		// The code will work with the reservation at any address, but ask
   111  		// SysReserve to use 0x0000XXc000000000 if possible (XX=00...7f).
   112  		// Allocating a 128 GB region takes away 37 bits, and the amd64
   113  		// doesn't let us choose the top 17 bits, so that leaves the 11 bits
   114  		// in the middle of 0x00c0 for us to choose.  Choosing 0x00c0 means
   115  		// that the valid memory addresses will begin 0x00c0, 0x00c1, ..., 0x00df.
   116  		// In little-endian, that's c0 00, c1 00, ..., df 00. None of those are valid
   117  		// UTF-8 sequences, and they are otherwise as far away from
   118  		// ff (likely a common byte) as possible.  If that fails, we try other 0xXXc0
   119  		// addresses.  An earlier attempt to use 0x11f8 caused out of memory errors
   120  		// on OS X during thread allocations.  0x00c0 causes conflicts with
   121  		// AddressSanitizer which reserves all memory up to 0x0100.
   122  		// These choices are both for debuggability and to reduce the
   123  		// odds of the conservative garbage collector not collecting memory
   124  		// because some non-pointer block of memory had a bit pattern
   125  		// that matched a memory address.
   126  		//
   127  		// Actually we reserve 136 GB (because the bitmap ends up being 8 GB)
   128  		// but it hardly matters: e0 00 is not valid UTF-8 either.
   129  		//
   130  		// If this fails we fall back to the 32 bit memory mechanism
   131  		arenaSize := round(_MaxMem, _PageSize)
   132  		bitmapSize = arenaSize / (ptrSize * 8 / 4)
   133  		spansSize = arenaSize / _PageSize * ptrSize
   134  		spansSize = round(spansSize, _PageSize)
   135  		for i := 0; i <= 0x7f; i++ {
   136  			p = uintptr(i)<<40 | uintptrMask&(0x00c0<<32)
   137  			pSize = bitmapSize + spansSize + arenaSize + _PageSize
   138  			p = uintptr(sysReserve(unsafe.Pointer(p), pSize, &reserved))
   139  			if p != 0 {
   140  				break
   141  			}
   142  		}
   143  	}
   144  
   145  	if p == 0 {
   146  		// On a 32-bit machine, we can't typically get away
   147  		// with a giant virtual address space reservation.
   148  		// Instead we map the memory information bitmap
   149  		// immediately after the data segment, large enough
   150  		// to handle another 2GB of mappings (256 MB),
   151  		// along with a reservation for an initial arena.
   152  		// When that gets used up, we'll start asking the kernel
   153  		// for any memory anywhere and hope it's in the 2GB
   154  		// following the bitmap (presumably the executable begins
   155  		// near the bottom of memory, so we'll have to use up
   156  		// most of memory before the kernel resorts to giving out
   157  		// memory before the beginning of the text segment).
   158  		//
   159  		// Alternatively we could reserve 512 MB bitmap, enough
   160  		// for 4GB of mappings, and then accept any memory the
   161  		// kernel threw at us, but normally that's a waste of 512 MB
   162  		// of address space, which is probably too much in a 32-bit world.
   163  
   164  		// If we fail to allocate, try again with a smaller arena.
   165  		// This is necessary on Android L where we share a process
   166  		// with ART, which reserves virtual memory aggressively.
   167  		arenaSizes := []uintptr{
   168  			512 << 20,
   169  			256 << 20,
   170  		}
   171  
   172  		for _, arenaSize := range arenaSizes {
   173  			bitmapSize = _MaxArena32 / (ptrSize * 8 / 4)
   174  			spansSize = _MaxArena32 / _PageSize * ptrSize
   175  			if limit > 0 && arenaSize+bitmapSize+spansSize > limit {
   176  				bitmapSize = (limit / 9) &^ ((1 << _PageShift) - 1)
   177  				arenaSize = bitmapSize * 8
   178  				spansSize = arenaSize / _PageSize * ptrSize
   179  			}
   180  			spansSize = round(spansSize, _PageSize)
   181  
   182  			// SysReserve treats the address we ask for, end, as a hint,
   183  			// not as an absolute requirement.  If we ask for the end
   184  			// of the data segment but the operating system requires
   185  			// a little more space before we can start allocating, it will
   186  			// give out a slightly higher pointer.  Except QEMU, which
   187  			// is buggy, as usual: it won't adjust the pointer upward.
   188  			// So adjust it upward a little bit ourselves: 1/4 MB to get
   189  			// away from the running binary image and then round up
   190  			// to a MB boundary.
   191  			p = round(uintptr(unsafe.Pointer(&end))+(1<<18), 1<<20)
   192  			pSize = bitmapSize + spansSize + arenaSize + _PageSize
   193  			p = uintptr(sysReserve(unsafe.Pointer(p), pSize, &reserved))
   194  			if p != 0 {
   195  				break
   196  			}
   197  		}
   198  		if p == 0 {
   199  			throw("runtime: cannot reserve arena virtual address space")
   200  		}
   201  	}
   202  
   203  	// PageSize can be larger than OS definition of page size,
   204  	// so SysReserve can give us a PageSize-unaligned pointer.
   205  	// To overcome this we ask for PageSize more and round up the pointer.
   206  	p1 := round(p, _PageSize)
   207  
   208  	mheap_.spans = (**mspan)(unsafe.Pointer(p1))
   209  	mheap_.bitmap = p1 + spansSize
   210  	mheap_.arena_start = p1 + (spansSize + bitmapSize)
   211  	mheap_.arena_used = mheap_.arena_start
   212  	mheap_.arena_end = p + pSize
   213  	mheap_.arena_reserved = reserved
   214  
   215  	if mheap_.arena_start&(_PageSize-1) != 0 {
   216  		println("bad pagesize", hex(p), hex(p1), hex(spansSize), hex(bitmapSize), hex(_PageSize), "start", hex(mheap_.arena_start))
   217  		throw("misrounded allocation in mallocinit")
   218  	}
   219  
   220  	// Initialize the rest of the allocator.
   221  	mHeap_Init(&mheap_, spansSize)
   222  	_g_ := getg()
   223  	_g_.m.mcache = allocmcache()
   224  }
   225  
   226  func wbshadowinit() {
   227  	// Initialize write barrier shadow heap if we were asked for it
   228  	// and we have enough address space (not on 32-bit).
   229  	if debug.wbshadow == 0 {
   230  		return
   231  	}
   232  	if ptrSize != 8 {
   233  		print("runtime: GODEBUG=wbshadow=1 disabled on 32-bit system\n")
   234  		return
   235  	}
   236  
   237  	var reserved bool
   238  	p1 := sysReserveHigh(mheap_.arena_end-mheap_.arena_start, &reserved)
   239  	if p1 == nil {
   240  		throw("cannot map shadow heap")
   241  	}
   242  	mheap_.shadow_heap = uintptr(p1) - mheap_.arena_start
   243  	sysMap(p1, mheap_.arena_used-mheap_.arena_start, reserved, &memstats.other_sys)
   244  	memmove(p1, unsafe.Pointer(mheap_.arena_start), mheap_.arena_used-mheap_.arena_start)
   245  
   246  	mheap_.shadow_reserved = reserved
   247  	start := ^uintptr(0)
   248  	end := uintptr(0)
   249  	if start > uintptr(unsafe.Pointer(&noptrdata)) {
   250  		start = uintptr(unsafe.Pointer(&noptrdata))
   251  	}
   252  	if start > uintptr(unsafe.Pointer(&data)) {
   253  		start = uintptr(unsafe.Pointer(&data))
   254  	}
   255  	if start > uintptr(unsafe.Pointer(&noptrbss)) {
   256  		start = uintptr(unsafe.Pointer(&noptrbss))
   257  	}
   258  	if start > uintptr(unsafe.Pointer(&bss)) {
   259  		start = uintptr(unsafe.Pointer(&bss))
   260  	}
   261  	if end < uintptr(unsafe.Pointer(&enoptrdata)) {
   262  		end = uintptr(unsafe.Pointer(&enoptrdata))
   263  	}
   264  	if end < uintptr(unsafe.Pointer(&edata)) {
   265  		end = uintptr(unsafe.Pointer(&edata))
   266  	}
   267  	if end < uintptr(unsafe.Pointer(&enoptrbss)) {
   268  		end = uintptr(unsafe.Pointer(&enoptrbss))
   269  	}
   270  	if end < uintptr(unsafe.Pointer(&ebss)) {
   271  		end = uintptr(unsafe.Pointer(&ebss))
   272  	}
   273  	start &^= _PageSize - 1
   274  	end = round(end, _PageSize)
   275  	mheap_.data_start = start
   276  	mheap_.data_end = end
   277  	reserved = false
   278  	p1 = sysReserveHigh(end-start, &reserved)
   279  	if p1 == nil {
   280  		throw("cannot map shadow data")
   281  	}
   282  	mheap_.shadow_data = uintptr(p1) - start
   283  	sysMap(p1, end-start, reserved, &memstats.other_sys)
   284  	memmove(p1, unsafe.Pointer(start), end-start)
   285  
   286  	mheap_.shadow_enabled = true
   287  }
   288  
   289  // sysReserveHigh reserves space somewhere high in the address space.
   290  // sysReserve doesn't actually reserve the full amount requested on
   291  // 64-bit systems, because of problems with ulimit. Instead it checks
   292  // that it can get the first 64 kB and assumes it can grab the rest as
   293  // needed. This doesn't work well with the "let the kernel pick an address"
   294  // mode, so don't do that. Pick a high address instead.
   295  func sysReserveHigh(n uintptr, reserved *bool) unsafe.Pointer {
   296  	if ptrSize == 4 {
   297  		return sysReserve(nil, n, reserved)
   298  	}
   299  
   300  	for i := 0; i <= 0x7f; i++ {
   301  		p := uintptr(i)<<40 | uintptrMask&(0x00c0<<32)
   302  		*reserved = false
   303  		p = uintptr(sysReserve(unsafe.Pointer(p), n, reserved))
   304  		if p != 0 {
   305  			return unsafe.Pointer(p)
   306  		}
   307  	}
   308  
   309  	return sysReserve(nil, n, reserved)
   310  }
   311  
   312  func mHeap_SysAlloc(h *mheap, n uintptr) unsafe.Pointer {
   313  	if n > uintptr(h.arena_end)-uintptr(h.arena_used) {
   314  		// We are in 32-bit mode, maybe we didn't use all possible address space yet.
   315  		// Reserve some more space.
   316  		p_size := round(n+_PageSize, 256<<20)
   317  		new_end := h.arena_end + p_size
   318  		if new_end <= h.arena_start+_MaxArena32 {
   319  			// TODO: It would be bad if part of the arena
   320  			// is reserved and part is not.
   321  			var reserved bool
   322  			p := uintptr(sysReserve((unsafe.Pointer)(h.arena_end), p_size, &reserved))
   323  			if p == h.arena_end {
   324  				h.arena_end = new_end
   325  				h.arena_reserved = reserved
   326  			} else if p+p_size <= h.arena_start+_MaxArena32 {
   327  				// Keep everything page-aligned.
   328  				// Our pages are bigger than hardware pages.
   329  				h.arena_end = p + p_size
   330  				h.arena_used = p + (-uintptr(p) & (_PageSize - 1))
   331  				h.arena_reserved = reserved
   332  			} else {
   333  				var stat uint64
   334  				sysFree((unsafe.Pointer)(p), p_size, &stat)
   335  			}
   336  		}
   337  	}
   338  
   339  	if n <= uintptr(h.arena_end)-uintptr(h.arena_used) {
   340  		// Keep taking from our reservation.
   341  		p := h.arena_used
   342  		sysMap((unsafe.Pointer)(p), n, h.arena_reserved, &memstats.heap_sys)
   343  		h.arena_used += n
   344  		mHeap_MapBits(h)
   345  		mHeap_MapSpans(h)
   346  		if raceenabled {
   347  			racemapshadow((unsafe.Pointer)(p), n)
   348  		}
   349  		if mheap_.shadow_enabled {
   350  			sysMap(unsafe.Pointer(p+mheap_.shadow_heap), n, h.shadow_reserved, &memstats.other_sys)
   351  		}
   352  
   353  		if uintptr(p)&(_PageSize-1) != 0 {
   354  			throw("misrounded allocation in MHeap_SysAlloc")
   355  		}
   356  		return (unsafe.Pointer)(p)
   357  	}
   358  
   359  	// If using 64-bit, our reservation is all we have.
   360  	if uintptr(h.arena_end)-uintptr(h.arena_start) >= _MaxArena32 {
   361  		return nil
   362  	}
   363  
   364  	// On 32-bit, once the reservation is gone we can
   365  	// try to get memory at a location chosen by the OS
   366  	// and hope that it is in the range we allocated bitmap for.
   367  	p_size := round(n, _PageSize) + _PageSize
   368  	p := uintptr(sysAlloc(p_size, &memstats.heap_sys))
   369  	if p == 0 {
   370  		return nil
   371  	}
   372  
   373  	if p < h.arena_start || uintptr(p)+p_size-uintptr(h.arena_start) >= _MaxArena32 {
   374  		print("runtime: memory allocated by OS (", p, ") not in usable range [", hex(h.arena_start), ",", hex(h.arena_start+_MaxArena32), ")\n")
   375  		sysFree((unsafe.Pointer)(p), p_size, &memstats.heap_sys)
   376  		return nil
   377  	}
   378  
   379  	p_end := p + p_size
   380  	p += -p & (_PageSize - 1)
   381  	if uintptr(p)+n > uintptr(h.arena_used) {
   382  		h.arena_used = p + n
   383  		if p_end > h.arena_end {
   384  			h.arena_end = p_end
   385  		}
   386  		mHeap_MapBits(h)
   387  		mHeap_MapSpans(h)
   388  		if raceenabled {
   389  			racemapshadow((unsafe.Pointer)(p), n)
   390  		}
   391  	}
   392  
   393  	if uintptr(p)&(_PageSize-1) != 0 {
   394  		throw("misrounded allocation in MHeap_SysAlloc")
   395  	}
   396  	return (unsafe.Pointer)(p)
   397  }
   398  
   399  var end struct{}
   400  
   401  func largeAlloc(size uintptr, flag uint32) *mspan {
   402  	// print("largeAlloc size=", size, "\n")
   403  
   404  	if size+_PageSize < size {
   405  		throw("out of memory")
   406  	}
   407  	npages := size >> _PageShift
   408  	if size&_PageMask != 0 {
   409  		npages++
   410  	}
   411  	s := mHeap_Alloc(&mheap_, npages, 0, true, flag&_FlagNoZero == 0)
   412  	if s == nil {
   413  		throw("out of memory")
   414  	}
   415  	s.limit = uintptr(s.start)<<_PageShift + size
   416  	v := unsafe.Pointer(uintptr(s.start) << _PageShift)
   417  	// setup for mark sweep
   418  	markspan(v, 0, 0, true)
   419  	return s
   420  }