github.com/bytedance/gopkg@v0.0.0-20240514070511-01b2cbcf35e1/collection/lscq/asm_arm64.go (about)

     1  //go:build arm64 && !gccgo && !appengine
     2  // +build arm64,!gccgo,!appengine
     3  
     4  package lscq
     5  
     6  import (
     7  	"golang.org/x/sys/cpu"
     8  	"runtime"
     9  	"unsafe"
    10  )
    11  
    12  var arm64HasAtomics = detectArm64HasAtomics()
    13  
    14  type uint128 [2]uint64
    15  
    16  func compareAndSwapUint128(addr *uint128, old1, old2, new1, new2 uint64) (swapped bool)
    17  
    18  func loadUint128(addr *uint128) (val uint128)
    19  
    20  func loadSCQNodePointer(addr unsafe.Pointer) (val scqNodePointer)
    21  
    22  func loadSCQNodeUint64(addr unsafe.Pointer) (val scqNodeUint64)
    23  
    24  func atomicTestAndSetFirstBit(addr *uint64) (val uint64)
    25  
    26  func atomicTestAndSetSecondBit(addr *uint64) (val uint64)
    27  
    28  func resetNode(addr unsafe.Pointer)
    29  
    30  //go:nosplit
    31  func compareAndSwapSCQNodePointer(addr *scqNodePointer, old, new scqNodePointer) (swapped bool) {
    32  	// Ref: src/runtime/atomic_pointer.go:sync_atomic_CompareAndSwapPointer
    33  	if runtimeEnableWriteBarrier() {
    34  		runtimeatomicwb(&addr.data, new.data)
    35  	}
    36  	return compareAndSwapUint128((*uint128)(runtimenoescape(unsafe.Pointer(addr))), old.flags, uint64(uintptr(old.data)), new.flags, uint64(uintptr(new.data)))
    37  }
    38  
    39  func compareAndSwapSCQNodeUint64(addr *scqNodeUint64, old, new scqNodeUint64) (swapped bool) {
    40  	return compareAndSwapUint128((*uint128)(unsafe.Pointer(addr)), old.flags, old.data, new.flags, new.data)
    41  }
    42  
    43  func runtimeEnableWriteBarrier() bool
    44  
    45  //go:linkname runtimeatomicwb runtime.atomicwb
    46  //go:noescape
    47  func runtimeatomicwb(ptr *unsafe.Pointer, new unsafe.Pointer)
    48  
    49  //go:linkname runtimenoescape runtime.noescape
    50  func runtimenoescape(p unsafe.Pointer) unsafe.Pointer
    51  
    52  //go:nosplit
    53  func atomicWriteBarrier(ptr *unsafe.Pointer) {
    54  	// For SCQ dequeue only. (fastpath)
    55  	if runtimeEnableWriteBarrier() {
    56  		runtimeatomicwb(ptr, nil)
    57  	}
    58  }
    59  
    60  //go:linkname sysctlEnabled internal/cpu.sysctlEnabled
    61  func sysctlEnabled(name []byte) bool
    62  
    63  func detectArm64HasAtomics() bool {
    64  	switch runtime.GOOS {
    65  	case "darwin":
    66  		return sysctlEnabled([]byte("hw.optional.armv8_1_atomics\x00"))
    67  	default:
    68  		return cpu.ARM64.HasATOMICS
    69  	}
    70  }