github.com/moontrade/unsafe@v0.9.1/memory/tlsf/heap.go (about) 1 package tlsf 2 3 import ( 4 "math" 5 "math/bits" 6 "unsafe" 7 ) 8 9 // Heap === Heap (Two-Level Segregate Fit) memory allocator === 10 // 11 // Heap is a general purpose dynamic memory allocator specifically designed to meet 12 // real-time requirements: 13 // 14 // Bounded Response Time - The worst-case execution time (WCET) of memory allocation 15 // and deallocation Has got to be known in advance and be 16 // independent of application data. Allocator Has a constant 17 // cost O(1). 18 // 19 // Fast - Additionally to a bounded cost, the allocator Has to be 20 // efficient and fast enough. Allocator executes a maximum 21 // of 168 processor instructions in a x86 architecture. 22 // Depending on the compiler version and optimisation flags, 23 // it can be slightly lower or higher. 24 // 25 // Efficient Memory Use - Traditionally, real-time systems run for long periods of 26 // time and some (embedded applications), have strong constraints 27 // of memory size. Fragmentation can have a significant impact on 28 // such systems. It can increase dramatically, and degrade the 29 // system performance. A way to measure this efficiency is the 30 // memory fragmentation incurred by the allocator. Allocator has 31 // been tested in hundreds of different loads (real-time tasks, 32 // general purpose applications, etc.) obtaining an average 33 // fragmentation lower than 15 %. The maximum fragmentation 34 // measured is lower than 25%. 35 // 36 // Memory can be added on demand and is a multiple of 64kb pages. Grow is used to allocate new 37 // memory to be added to the allocator. Each Grow must provide a contiguous chunk of memory. 38 // However, the allocator may be comprised of many contiguous chunks which are not contiguous 39 // of each other. There is not a mechanism for shrinking the memory. Supplied Grow function 40 // can effectively limit how big the allocator can get. If a zero pointer is returned it will 41 // cause an out-of-memory situation which is propagated as a nil pointer being returned from 42 // Alloc. It's up to the application to decide how to handle such scenarios. 43 // 44 // see: http://www.gii.upv.es/tlsf/ 45 // see: https://github.com/AssemblyScript/assemblyscript 46 // 47 // - `ffs(x)` is equivalent to `ctz(x)` with x != 0 48 // - `fls(x)` is equivalent to `sizeof(x) * 8 - clz(x) - 1` 49 // 50 // ╒══════════════ Block size interpretation (32-bit) ═════════════╕ 51 // 52 // 3 2 1 53 // 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 bits 54 // 55 // ├─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┼─┴─┴─┴─╫─┴─┴─┴─┤ 56 // │ | FL │ SB = SL + AL │ ◄─ usize 57 // └───────────────────────────────────────────────┴───────╨───────┘ 58 // FL: first level, SL: second level, AL: alignment, SB: small block 59 type Heap struct { 60 root *root 61 HeapStart uintptr 62 HeapEnd uintptr 63 arena uintptr 64 Grow Grow 65 Slot uint8 66 Stats 67 } 68 69 // Stats provides the metrics of an Allocator 70 type Stats struct { 71 HeapSize int64 72 AllocSize int64 73 PeakAllocSize int64 74 FreeSize int64 75 Allocs int32 76 InitialPages int32 77 ConsecutiveLow int32 78 ConsecutiveHigh int32 79 Pages int32 80 Grows int32 81 fragmentation float32 82 } 83 84 func (s *Stats) Fragmentation() float32 { 85 if s.HeapSize == 0 || s.PeakAllocSize == 0 { 86 return 0 87 } 88 pct := float64(s.HeapSize-s.PeakAllocSize) / float64(s.HeapSize) 89 s.fragmentation = float32(math.Floor(pct*100) / 100) 90 return s.fragmentation 91 } 92 93 // Grow provides the ability to Grow the heap and allocate a contiguous 94 // chunk of system memory to add to the allocator. 95 type Grow func(pagesBefore, pagesNeeded int32, minSize uintptr) (pagesAdded int32, start, end uintptr) 96 97 const ( 98 PageSize = uintptr(64 * 1024) 99 100 _TLSFAlignU32 = 2 101 // All allocation sizes and addresses are aligned to 4 or 8 bytes. 102 // 32bit = 2 103 // 64bit = 3 104 // <expr> = bits.UintSize / 8 / 4 + 1 105 _TLSFAlignSizeLog2 uintptr = ((32 << (^uint(0) >> 63)) / 8 / 4) + 1 106 _TLSFSizeofPointer = unsafe.Sizeof(uintptr(0)) 107 108 ALBits uint32 = 4 // 16 bytes to fit up to v128 109 ALSize uintptr = 1 << uintptr(ALBits) 110 ALMask = ALSize - 1 111 112 // Overhead of a memory manager block. 113 BlockOverhead = unsafe.Sizeof(BLOCK{}) 114 // Block constants. A block must have a minimum size of three pointers so it can hold `prev`, 115 // `prev` and `back` if free. 116 BlockMinSize = ((3*_TLSFSizeofPointer + BlockOverhead + ALMask) & ^ALMask) - BlockOverhead 117 // Maximum size of a memory manager block's payload. 118 BlockMaxSize = (1 << 30) - BlockOverhead 119 //BlockMaxSize = (1 << ((_TLSFAlignSizeLog2 + 1)*10)) - BlockOverhead 120 121 _TLSFDebug = false 122 123 _TLSFSLBits uint32 = 4 124 _TLSFSLSize uint32 = 1 << _TLSFSLBits 125 _TLSFSBBits = _TLSFSLBits + ALBits 126 _TLSFSBSize uint32 = 1 << _TLSFSBBits 127 _TLSFFLBits = 31 - _TLSFSBBits 128 129 // [00]: < 256B (SB) [12]: < 1M 130 // [01]: < 512B [13]: < 2M 131 // [02]: < 1K [14]: < 4M 132 // [03]: < 2K [15]: < 8M 133 // [04]: < 4K [16]: < 16M 134 // [05]: < 8K [17]: < 32M 135 // [06]: < 16K [18]: < 64M 136 // [07]: < 32K [19]: < 128M 137 // [08]: < 64K [20]: < 256M 138 // [09]: < 128K [21]: < 512M 139 // [10]: < 256K [22]: <= 1G - OVERHEAD 140 // [11]: < 512K 141 // WASM VMs limit to 2GB total (currently), making one 1G block max 142 // (or three 512M etc.) due to block overhead 143 144 // Tags stored in otherwise unused alignment bits 145 _TLSFFREE uintptr = 1 << 0 146 _TLSFLEFTFREE uintptr = 1 << 1 147 TagsMask = _TLSFFREE | _TLSFLEFTFREE 148 ) 149 150 // Alloc allocates a block of memory that fits the size provided 151 // 152 //goland:noinspection GoVetUnsafePointer 153 func (a *Heap) Alloc(size uintptr) uintptr { 154 if a == nil { 155 panic("nil") 156 } 157 p := uintptr(unsafe.Pointer(a.allocateBlock(size))) 158 if p == 0 { 159 return 0 160 } 161 p = p + BlockOverhead 162 return p 163 } 164 165 // AllocZeroed allocates a block of memory that fits the size provided 166 // 167 //goland:noinspection GoVetUnsafePointer 168 func (a *Heap) AllocZeroed(size uintptr) uintptr { 169 p := uintptr(unsafe.Pointer(a.allocateBlock(size))) 170 if p == 0 { 171 return 0 172 } 173 p = p + BlockOverhead 174 Zero(unsafe.Pointer(p), size) 175 return p 176 } 177 178 // Realloc determines the best way to resize an allocation. 179 func (a *Heap) Realloc(ptr uintptr, size uintptr) uintptr { 180 p := uintptr(unsafe.Pointer(a.moveBlock(checkUsedBlock(ptr), size))) 181 if p == 0 { 182 return 0 183 } 184 return p + BlockOverhead 185 } 186 187 // Free release the allocation back into the free list. 188 // 189 //goland:noinspection GoVetUnsafePointer 190 func (a *Heap) Free(ptr uintptr) { 191 //println("Free", uint(ptr)) 192 //a.freeBlock((*tlsfBlock)(unsafe.Pointer(ptr - BlockOverhead))) 193 a.freeBlock(checkUsedBlock(ptr)) 194 } 195 196 //goland:noinspection GoVetUnsafePointer 197 func SizeOf(ptr uintptr) uintptr { 198 return ((*tlsfBlock)(unsafe.Pointer(ptr - BlockOverhead))).MMInfo & ^TagsMask 199 } 200 201 // Bootstrap bootstraps the Allocator with the initial block of contiguous memory 202 // that at least fits the minimum required to fit the bitmap. 203 // 204 //goland:noinspection GoVetUnsafePointer 205 func Bootstrap(start, end uintptr, pages int32, grow Grow) *Heap { 206 start = (start + unsafe.Alignof(unsafe.Pointer(nil)) - 1) &^ (unsafe.Alignof(unsafe.Pointer(nil)) - 1) 207 208 //if a.T { 209 //println("Bootstrap", "pages", pages, uint(start), uint(end), uint(end-start)) 210 //} 211 // init allocator 212 a := (*Heap)(unsafe.Pointer(start)) 213 *a = Heap{ 214 HeapStart: start, 215 HeapEnd: end, 216 Stats: Stats{ 217 InitialPages: pages, 218 Pages: pages, 219 }, 220 Grow: grow, 221 } 222 223 // init root 224 rootOffset := unsafe.Sizeof(Heap{}) + ((start + ALMask) & ^ALMask) 225 a.root = (*root)(unsafe.Pointer(rootOffset)) 226 a.root.init() 227 228 // add initial memory 229 a.addMemory(rootOffset+RootSize, end) 230 return a 231 } 232 233 // Memory manager 234 235 // ╒════════════ Memory manager block layout (32-bit) ═════════════╕ 236 // 237 // 3 2 1 238 // 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 bits 239 // 240 // ├─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┤ 241 // │ MM info │ -4 242 // ╞>ptr═══════════════════════════════════════════════════════════╡ 243 // │ ... │ 244 type BLOCK struct { 245 MMInfo uintptr 246 } 247 248 // ╒════════════════════ Block layout (32-bit) ════════════════════╕ 249 // 250 // 3 2 1 251 // 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 bits 252 // 253 // ├─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┼─┼─┤ ┐ 254 // │ size │L│F│ ◄─┐ info overhead 255 // ╞>ptr═══════════════════════════════════════════════════════╧═╧═╡ │ ┘ 256 // │ if free: ◄ prev │ ◄─┤ usize 257 // ├───────────────────────────────────────────────────────────────┤ │ 258 // │ if free: next ► │ ◄─┤ 259 // ├───────────────────────────────────────────────────────────────┤ │ 260 // │ ... │ │ >= 0 261 // ├───────────────────────────────────────────────────────────────┤ │ 262 // │ if free: back ▲ │ ◄─┘ 263 // └───────────────────────────────────────────────────────────────┘ >= MIN SIZE 264 // F: FREE, L: LEFTFREE 265 type tlsfBlock struct { 266 BLOCK 267 // Previous free block, if any. Only valid if free, otherwise part of payload. 268 //prev *Block 269 prev uintptr 270 // Next free block, if any. Only valid if free, otherwise part of payload. 271 //next *Block 272 next uintptr 273 274 // If the block is free, there is a 'back'reference at its end pointing at its start. 275 } 276 277 // Gets the left block of a block. Only valid if the left block is free. 278 func (block *tlsfBlock) getFreeLeft() *tlsfBlock { 279 return *(**tlsfBlock)(unsafe.Pointer(uintptr(unsafe.Pointer(block)) - _TLSFSizeofPointer)) 280 } 281 282 // Gets the right block of a block by advancing to the right by its size. 283 func (block *tlsfBlock) getRight() *tlsfBlock { 284 return (*tlsfBlock)(unsafe.Pointer(uintptr(unsafe.Pointer(block)) + BlockOverhead + (block.MMInfo & ^TagsMask))) 285 } 286 287 // ╒═════════════════════ Root layout (32-bit) ════════════════════╕ 288 // 289 // 3 2 1 290 // 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 bits 291 // 292 // ├─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┤ ┐ 293 // │ 0 | flMap S│ ◄────┐ 294 // ╞═══════════════════════════════════════════════════════════════╡ │ 295 // │ slMap[0] S │ ◄─┐ │ 296 // ├───────────────────────────────────────────────────────────────┤ │ │ 297 // │ slMap[1] │ ◄─┤ │ 298 // ├───────────────────────────────────────────────────────────────┤ uint32 │ 299 // │ slMap[22] │ ◄─┘ │ 300 // ╞═══════════════════════════════════════════════════════════════╡ usize 301 // │ head[0] │ ◄────┤ 302 // ├───────────────────────────────────────────────────────────────┤ │ 303 // │ ... │ ◄────┤ 304 // ├───────────────────────────────────────────────────────────────┤ │ 305 // │ head[367] │ ◄────┤ 306 // ╞═══════════════════════════════════════════════════════════════╡ │ 307 // │ tail │ ◄────┘ 308 // └───────────────────────────────────────────────────────────────┘ SIZE ┘ 309 // S: Small blocks map 310 type root struct { 311 flMap uintptr 312 } 313 314 func (r *root) init() { 315 r.flMap = 0 316 r.setTail(nil) 317 for fl := uintptr(0); fl < uintptr(_TLSFFLBits); fl++ { 318 r.setSL(fl, 0) 319 for sl := uint32(0); sl < _TLSFSLSize; sl++ { 320 r.setHead(fl, sl, nil) 321 } 322 } 323 } 324 325 const ( 326 SLStart = _TLSFSizeofPointer 327 SLEnd = SLStart + (uintptr(_TLSFFLBits) << _TLSFAlignU32) 328 HLStart = (SLEnd + ALMask) &^ ALMask 329 HLEnd = HLStart + uintptr(_TLSFFLBits)*uintptr(_TLSFSLSize)*_TLSFSizeofPointer 330 RootSize = HLEnd + _TLSFSizeofPointer 331 ) 332 333 // Gets the second level map of the specified first level. 334 func (r *root) getSL(fl uintptr) uint32 { 335 return *(*uint32)(unsafe.Pointer(uintptr(unsafe.Pointer(r)) + (fl << _TLSFAlignU32) + SLStart)) 336 } 337 338 // Sets the second level map of the specified first level. 339 func (r *root) setSL(fl uintptr, slMap uint32) { 340 *(*uint32)(unsafe.Pointer(uintptr(unsafe.Pointer(r)) + (fl << _TLSFAlignU32) + SLStart)) = slMap 341 } 342 343 // Gets the head of the free list for the specified combination of first and second level. 344 func (r *root) getHead(fl uintptr, sl uint32) *tlsfBlock { 345 return *(**tlsfBlock)(unsafe.Pointer(uintptr(unsafe.Pointer(r)) + HLStart + 346 (((fl << _TLSFSLBits) + uintptr(sl)) << _TLSFAlignSizeLog2))) 347 } 348 349 // Sets the head of the free list for the specified combination of first and second level. 350 func (r *root) setHead(fl uintptr, sl uint32, head *tlsfBlock) { 351 *(*uintptr)(unsafe.Pointer(uintptr(unsafe.Pointer(r)) + HLStart + 352 (((fl << _TLSFSLBits) + uintptr(sl)) << _TLSFAlignSizeLog2))) = uintptr(unsafe.Pointer(head)) 353 } 354 355 // Gets the tail block. 356 func (r *root) getTail() *tlsfBlock { 357 return *(**tlsfBlock)(unsafe.Pointer(uintptr(unsafe.Pointer(r)) + HLEnd)) 358 } 359 360 // Sets the tail block. 361 func (r *root) setTail(tail *tlsfBlock) { 362 *(*uintptr)(unsafe.Pointer(uintptr(unsafe.Pointer(r)) + HLEnd)) = uintptr(unsafe.Pointer(tail)) 363 } 364 365 // Inserts a previously used block back into the free list. 366 func (a *Heap) insertBlock(block *tlsfBlock) { 367 var ( 368 r = a.root 369 blockInfo = block.MMInfo 370 right = block.getRight() 371 rightInfo = right.MMInfo 372 ) 373 //(blockInfo & FREE) 374 375 // merge with right block if also free 376 if rightInfo&_TLSFFREE != 0 { 377 a.removeBlock(right) 378 blockInfo = blockInfo + BlockOverhead + (rightInfo & ^TagsMask) // keep block tags 379 block.MMInfo = blockInfo 380 right = block.getRight() 381 rightInfo = right.MMInfo 382 // 'back' is Add below 383 } 384 385 // merge with left block if also free 386 if blockInfo&_TLSFLEFTFREE != 0 { 387 left := block.getFreeLeft() 388 leftInfo := left.MMInfo 389 if _TLSFDebug { 390 assert(leftInfo&_TLSFFREE != 0, "must be free according to right tags") 391 } 392 a.removeBlock(left) 393 block = left 394 blockInfo = leftInfo + BlockOverhead + (blockInfo & ^TagsMask) // keep left tags 395 block.MMInfo = blockInfo 396 // 'back' is Add below 397 } 398 399 right.MMInfo = rightInfo | _TLSFLEFTFREE 400 // reference to right is no longer used now, hence rightInfo is not synced 401 402 // we now know the size of the block 403 size := blockInfo & ^TagsMask 404 405 // Add 'back' to itself at the end of block 406 *(*uintptr)(unsafe.Pointer(uintptr(unsafe.Pointer(right)) - _TLSFSizeofPointer)) = uintptr(unsafe.Pointer(block)) 407 408 // mapping_insert 409 var ( 410 fl uintptr 411 sl uint32 412 ) 413 if size < uintptr(_TLSFSBSize) { 414 fl = 0 415 sl = uint32(size >> ALBits) 416 } else { 417 const inv = _TLSFSizeofPointer*8 - 1 418 boundedSize := min(size, BlockMaxSize) 419 fl = inv - clz(boundedSize) 420 sl = uint32((boundedSize >> (fl - uintptr(_TLSFSLBits))) ^ (1 << _TLSFSLBits)) 421 fl -= uintptr(_TLSFSBBits) - 1 422 } 423 424 // perform insertion 425 head := r.getHead(fl, sl) 426 block.prev = 0 427 block.next = uintptr(unsafe.Pointer(head)) 428 if head != nil { 429 head.prev = uintptr(unsafe.Pointer(block)) 430 } 431 r.setHead(fl, sl, block) 432 433 // update first and second level maps 434 r.flMap |= 1 << fl 435 r.setSL(fl, r.getSL(fl)|(1<<sl)) 436 } 437 438 //goland:noinspection GoVetUnsafePointer 439 func (a *Heap) removeBlock(block *tlsfBlock) { 440 r := a.root 441 blockInfo := block.MMInfo 442 if _TLSFDebug { 443 assert(blockInfo&_TLSFFREE != 0, "must be free") 444 } 445 size := blockInfo & ^TagsMask 446 if _TLSFDebug { 447 assert(size >= BlockMinSize, "must be valid") 448 } 449 450 // mapping_insert 451 var ( 452 fl uintptr 453 sl uint32 454 ) 455 if size < uintptr(_TLSFSBSize) { 456 fl = 0 457 sl = uint32(size >> ALBits) 458 } else { 459 const inv = _TLSFSizeofPointer*8 - 1 460 boundedSize := min(size, BlockMaxSize) 461 fl = inv - clz(boundedSize) 462 sl = uint32((boundedSize >> (fl - uintptr(_TLSFSLBits))) ^ (1 << uintptr(_TLSFSLBits))) 463 fl -= uintptr(_TLSFSBBits) - 1 464 } 465 if _TLSFDebug { 466 assert(fl < uintptr(_TLSFFLBits) && sl < _TLSFSLSize, "fl/sl out of range") 467 } 468 469 // link previous and prev free block 470 var ( 471 prev = block.prev 472 next = block.next 473 ) 474 if prev != 0 { 475 (*tlsfBlock)(unsafe.Pointer(prev)).next = next 476 } 477 if next != 0 { 478 (*tlsfBlock)(unsafe.Pointer(next)).prev = prev 479 } 480 481 // update head if we are removing it 482 if block == r.getHead(fl, sl) { 483 r.setHead(fl, sl, (*tlsfBlock)(unsafe.Pointer(next))) 484 485 // clear second level map if head is empty now 486 if next == 0 { 487 slMap := r.getSL(fl) 488 slMap &= ^(1 << sl) 489 r.setSL(fl, slMap) 490 491 // clear first level map if second level is empty now 492 if slMap == 0 { 493 r.flMap &= ^(1 << fl) 494 } 495 } 496 } 497 // note: does not alter left/back because it is likely that splitting 498 // is performed afterwards, invalidating those changes. so, the caller 499 // must perform those updates. 500 } 501 502 // Searches for a free block of at least the specified size. 503 func (a *Heap) searchBlock(size uintptr) *tlsfBlock { 504 // mapping_search 505 var ( 506 fl uintptr 507 sl uint32 508 r = a.root 509 ) 510 if size < uintptr(_TLSFSBSize) { 511 fl = 0 512 sl = uint32(size >> ALBits) 513 } else { 514 const ( 515 halfMaxSize = BlockMaxSize >> 1 // don't round last fl 516 inv = _TLSFSizeofPointer*8 - 1 517 invRound = inv - uintptr(_TLSFSLBits) 518 ) 519 520 var requestSize uintptr 521 if size < halfMaxSize { 522 requestSize = size + (1 << (invRound - clz(size))) - 1 523 } else { 524 requestSize = size 525 } 526 527 fl = inv - clz(requestSize) 528 sl = uint32((requestSize >> (fl - uintptr(_TLSFSLBits))) ^ (1 << _TLSFSLBits)) 529 fl -= uintptr(_TLSFSBBits) - 1 530 } 531 if _TLSFDebug { 532 assert(fl < uintptr(_TLSFFLBits) && sl < _TLSFSLSize, "fl/sl out of range") 533 } 534 535 // search second level 536 var ( 537 slMap = r.getSL(fl) & (^uint32(0) << sl) 538 head *tlsfBlock 539 ) 540 if slMap == 0 { 541 // search prev larger first level 542 flMap := r.flMap & (^uintptr(0) << (fl + 1)) 543 if flMap == 0 { 544 head = nil 545 } else { 546 fl = ctz(flMap) 547 slMap = r.getSL(fl) 548 if _TLSFDebug { 549 assert(slMap != 0, "can't be zero if fl points here") 550 } 551 head = r.getHead(fl, ctz32(slMap)) 552 } 553 } else { 554 head = r.getHead(fl, ctz32(slMap)) 555 } 556 557 return head 558 } 559 560 func (a *Heap) prepareBlock(block *tlsfBlock, size uintptr) { 561 blockInfo := block.MMInfo 562 if _TLSFDebug { 563 assert(((size+BlockOverhead)&ALMask) == 0, 564 "size must be aligned so the New block is") 565 } 566 // split if the block can hold another MINSIZE block incl. overhead 567 remaining := (blockInfo & ^TagsMask) - size 568 if remaining >= BlockOverhead+BlockMinSize { 569 block.MMInfo = size | (blockInfo & _TLSFLEFTFREE) // also discards FREE 570 571 spare := (*tlsfBlock)(unsafe.Pointer(uintptr(unsafe.Pointer(block)) + BlockOverhead + size)) 572 spare.MMInfo = (remaining - BlockOverhead) | _TLSFFREE // not LEFTFREE 573 a.insertBlock(spare) // also sets 'back' 574 575 // otherwise tag block as no longer FREE and right as no longer LEFTFREE 576 } else { 577 block.MMInfo = blockInfo & ^_TLSFFREE 578 block.getRight().MMInfo &= ^_TLSFLEFTFREE 579 } 580 } 581 582 // growMemory grows the pool by a number of 64kb pages to fit the required size 583 func (a *Heap) growMemory(size uintptr) bool { 584 if a.Grow == nil { 585 return false 586 } 587 // Here, both rounding performed in searchBlock ... 588 const halfMaxSize = BlockMaxSize >> 1 589 if size < halfMaxSize { // don't round last fl 590 const invRound = (_TLSFSizeofPointer*8 - 1) - uintptr(_TLSFSLBits) 591 size += (1 << (invRound - clz(size))) - 1 592 } 593 // and additional BLOCK_OVERHEAD must be taken into account. If we are going 594 // to merge with the tail block, that's one time, otherwise it's two times. 595 var ( 596 pagesBefore = a.Pages 597 offset uintptr = 0 598 ) 599 if BlockOverhead != uintptr(unsafe.Pointer(a.root.getTail())) { 600 offset = 1 601 } 602 size += BlockOverhead << ((uintptr(pagesBefore) << 16) - offset) 603 pagesNeeded := ((int32(size) + 0xffff) & ^0xffff) >> 16 604 605 addedPages, start, end := a.Grow(pagesBefore, pagesNeeded, size) 606 if start == 0 || end == 0 { 607 return false 608 } 609 if addedPages == 0 { 610 addedPages = int32((end - start) / PageSize) 611 if (end-start)%PageSize > 0 { 612 addedPages++ 613 } 614 } 615 a.Pages += addedPages 616 a.HeapEnd = end 617 a.addMemory(start, end) 618 return true 619 } 620 621 // addMemory adds the newly allocated memory to the Allocator bitmaps 622 // 623 //goland:noinspection GoVetUnsafePointer 624 func (a *Heap) addMemory(start, end uintptr) bool { 625 if _TLSFDebug { 626 assert(start <= end, "start must be <= end") 627 } 628 start = ((start + BlockOverhead + ALMask) & ^ALMask) - BlockOverhead 629 end &= ^ALMask 630 631 var tail = a.root.getTail() 632 var tailInfo uintptr = 0 633 if tail != nil { // more memory 634 if _TLSFDebug { 635 assert(start >= uintptr(unsafe.Pointer(tail))+BlockOverhead, "out of bounds") 636 } 637 638 // merge with current tail if adjacent 639 const offsetToTail = ALSize 640 if start-offsetToTail == uintptr(unsafe.Pointer(tail)) { 641 start -= offsetToTail 642 tailInfo = tail.MMInfo 643 } else { 644 // We don't do this, but a user might `memory.Grow` manually 645 // leading to non-adjacent pages managed by Allocator. 646 } 647 } else if _TLSFDebug { // first memory 648 assert(start >= uintptr(unsafe.Pointer(a.root))+RootSize, "starts after root") 649 } 650 651 // check if size is large enough for a free block and the tail block 652 var size = end - start 653 if size < BlockOverhead+BlockMinSize+BlockOverhead { 654 return false 655 } 656 657 // left size is total minus its own and the zero-length tail's header 658 var ( 659 leftSize = size - 2*BlockOverhead 660 left = (*tlsfBlock)(unsafe.Pointer(start)) 661 ) 662 left.MMInfo = leftSize | _TLSFFREE | (tailInfo & _TLSFLEFTFREE) 663 left.prev = 0 664 left.next = 0 665 666 // tail is a zero-length used block 667 tail = (*tlsfBlock)(unsafe.Pointer(start + BlockOverhead + leftSize)) 668 tail.MMInfo = 0 | _TLSFLEFTFREE 669 a.root.setTail(tail) 670 671 a.FreeSize += int64(leftSize) 672 a.HeapSize += int64(end - start) 673 674 // also merges with free left before tail / sets 'back' 675 a.insertBlock(left) 676 677 return true 678 } 679 680 // Computes the size (excl. header) of a block. 681 func computeSize(size uintptr) uintptr { 682 // Size must be large enough and aligned minus preceeding overhead 683 if size <= BlockMinSize { 684 return BlockMinSize 685 } else { 686 return ((size + BlockOverhead + ALMask) & ^ALMask) - BlockOverhead 687 } 688 } 689 690 // Prepares and checks an allocation size. 691 func prepareSize(size uintptr) uintptr { 692 if size > BlockMaxSize { 693 panic("allocation too large") 694 } 695 return computeSize(size) 696 } 697 698 // Allocates a block of the specified size. 699 func (a *Heap) allocateBlock(size uintptr) *tlsfBlock { 700 var payloadSize = prepareSize(size) 701 var block = a.searchBlock(payloadSize) 702 if block == nil { 703 if !a.growMemory(payloadSize) { 704 return nil 705 } 706 block = a.searchBlock(payloadSize) 707 if _TLSFDebug { 708 assert(block != nil, "block must be found now") 709 } 710 if block == nil { 711 return nil 712 } 713 } 714 if _TLSFDebug { 715 assert((block.MMInfo & ^TagsMask) >= payloadSize, "must fit") 716 } 717 718 a.removeBlock(block) 719 a.prepareBlock(block, payloadSize) 720 721 // update stats 722 payloadSize = block.MMInfo & ^TagsMask 723 a.AllocSize += int64(payloadSize) 724 if a.AllocSize > a.PeakAllocSize { 725 a.PeakAllocSize = a.AllocSize 726 } 727 a.FreeSize -= int64(payloadSize) 728 a.Allocs++ 729 730 // return block 731 return block 732 } 733 734 func (a *Heap) reallocateBlock(block *tlsfBlock, size uintptr) *tlsfBlock { 735 var ( 736 payloadSize = prepareSize(size) 737 blockInfo = block.MMInfo 738 blockSize = blockInfo & ^TagsMask 739 ) 740 741 // possibly split and update runtime size if it still fits 742 if payloadSize <= blockSize { 743 a.prepareBlock(block, payloadSize) 744 //if (isDefined(ASC_RTRACE)) { 745 // if (payloadSize != blockSize) onresize(block, BLOCK_OVERHEAD + blockSize); 746 //} 747 return block 748 } 749 750 // merge with right free block if merger is large enough 751 var ( 752 right = block.getRight() 753 rightInfo = right.MMInfo 754 ) 755 if rightInfo&_TLSFFREE != 0 { 756 mergeSize := blockSize + BlockOverhead + (rightInfo & ^TagsMask) 757 if mergeSize >= payloadSize { 758 a.removeBlock(right) 759 block.MMInfo = (blockInfo & TagsMask) | mergeSize 760 a.prepareBlock(block, payloadSize) 761 //if (isDefined(ASC_RTRACE)) onresize(block, BLOCK_OVERHEAD + blockSize); 762 return block 763 } 764 } 765 766 // otherwise, move the block 767 return a.moveBlock(block, size) 768 } 769 770 func (a *Heap) moveBlock(block *tlsfBlock, newSize uintptr) *tlsfBlock { 771 newBlock := a.allocateBlock(newSize) 772 if newBlock == nil { 773 return nil 774 } 775 776 Copy(unsafe.Pointer(uintptr(unsafe.Pointer(newBlock))+BlockOverhead), 777 unsafe.Pointer(uintptr(unsafe.Pointer(block))+BlockOverhead), 778 block.MMInfo & ^TagsMask) 779 780 a.freeBlock(block) 781 //maybeFreeBlock(a, block) 782 783 return newBlock 784 } 785 786 func (a *Heap) freeBlock(block *tlsfBlock) { 787 size := block.MMInfo & ^TagsMask 788 a.FreeSize += int64(size) 789 a.AllocSize -= int64(size) 790 a.Allocs-- 791 792 block.MMInfo = block.MMInfo | _TLSFFREE 793 a.insertBlock(block) 794 } 795 796 func min(l, r uintptr) uintptr { 797 if l < r { 798 return l 799 } 800 return r 801 } 802 803 func clz(value uintptr) uintptr { 804 return uintptr(bits.LeadingZeros(uint(value))) 805 } 806 807 func ctz(value uintptr) uintptr { 808 return uintptr(bits.TrailingZeros(uint(value))) 809 } 810 811 func ctz32(value uint32) uint32 { 812 return uint32(bits.TrailingZeros32(value)) 813 } 814 815 //goland:noinspection GoVetUnsafePointer 816 func checkUsedBlock(ptr uintptr) *tlsfBlock { 817 block := (*tlsfBlock)(unsafe.Pointer(ptr - BlockOverhead)) 818 if !(ptr != 0 && ((ptr & ALMask) == 0) && ((block.MMInfo & _TLSFFREE) == 0)) { 819 panic("used block is not valid to be freed or reallocated") 820 } 821 return block 822 } 823 824 func PrintDebugInfo() { 825 println("ALIGNOF_U32 ", int64(_TLSFAlignU32)) 826 println("ALIGN_SIZE_LOG2 ", int64(_TLSFAlignSizeLog2)) 827 println("U32_MAX ", ^uint32(0)) 828 println("PTR_MAX ", ^uintptr(0)) 829 println("AL_BITS ", int64(ALBits)) 830 println("AL_SIZE ", int64(ALSize)) 831 println("AL_MASK ", int64(ALMask)) 832 println("BLOCK_OVERHEAD ", int64(BlockOverhead)) 833 println("BLOCK_MAXSIZE ", int64(BlockMaxSize)) 834 println("SL_BITS ", int64(_TLSFSLBits)) 835 println("SL_SIZE ", int64(_TLSFSLSize)) 836 println("SB_BITS ", int64(_TLSFSBBits)) 837 println("SB_SIZE ", int64(_TLSFSBSize)) 838 println("FL_BITS ", int64(_TLSFFLBits)) 839 println("FREE ", int64(_TLSFFREE)) 840 println("LEFTFREE ", int64(_TLSFLEFTFREE)) 841 println("TAGS_MASK ", int64(TagsMask)) 842 println("BLOCK_MINSIZE ", int64(BlockMinSize)) 843 println("SL_START ", int64(SLStart)) 844 println("SL_END ", int64(SLEnd)) 845 println("HL_START ", int64(HLStart)) 846 println("HL_END ", int64(HLEnd)) 847 println("ROOT_SIZE ", int64(RootSize)) 848 }