github.com/songzhibin97/gkit@v1.2.13/structure/lscq/asm_arm64.go (about)

     1  //go:build arm64 && !gccgo && !appengine
     2  // +build arm64,!gccgo,!appengine
     3  
     4  package lscq
     5  
     6  import (
     7  	"runtime"
     8  	"unsafe"
     9  
    10  	"golang.org/x/sys/cpu"
    11  )
    12  
    13  var arm64HasAtomics = detectArm64HasAtomics()
    14  
    15  type uint128 [2]uint64
    16  
    17  func compareAndSwapUint128(addr *uint128, old1, old2, new1, new2 uint64) (swapped bool)
    18  
    19  func loadUint128(addr *uint128) (val uint128)
    20  
    21  func loadSCQNodePointer(addr unsafe.Pointer) (val scqNodePointer)
    22  
    23  func loadSCQNodeUint64(addr unsafe.Pointer) (val scqNodeUint64)
    24  
    25  func atomicTestAndSetFirstBit(addr *uint64) (val uint64)
    26  
    27  func atomicTestAndSetSecondBit(addr *uint64) (val uint64)
    28  
    29  func resetNode(addr unsafe.Pointer)
    30  
    31  //go:nosplit
    32  func compareAndSwapSCQNodePointer(addr *scqNodePointer, old, new scqNodePointer) (swapped bool) {
    33  	// Ref: src/runtime/atomic_pointer.go:sync_atomic_CompareAndSwapPointer
    34  	if runtimeEnableWriteBarrier() {
    35  		runtimeatomicwb(&addr.data, new.data)
    36  	}
    37  	return compareAndSwapUint128((*uint128)(runtimenoescape(unsafe.Pointer(addr))), old.flags, uint64(uintptr(old.data)), new.flags, uint64(uintptr(new.data)))
    38  }
    39  
    40  func compareAndSwapSCQNodeUint64(addr *scqNodeUint64, old, new scqNodeUint64) (swapped bool) {
    41  	return compareAndSwapUint128((*uint128)(unsafe.Pointer(addr)), old.flags, old.data, new.flags, new.data)
    42  }
    43  
    44  func runtimeEnableWriteBarrier() bool
    45  
    46  //go:linkname runtimeatomicwb runtime.atomicwb
    47  //go:noescape
    48  func runtimeatomicwb(ptr *unsafe.Pointer, new unsafe.Pointer)
    49  
    50  //go:linkname runtimenoescape runtime.noescape
    51  func runtimenoescape(p unsafe.Pointer) unsafe.Pointer
    52  
    53  //go:nosplit
    54  func atomicWriteBarrier(ptr *unsafe.Pointer) {
    55  	// For SCQ dequeue only. (fastpath)
    56  	if runtimeEnableWriteBarrier() {
    57  		runtimeatomicwb(ptr, nil)
    58  	}
    59  }
    60  
    61  //go:linkname sysctlEnabled internal/cpu.sysctlEnabled
    62  func sysctlEnabled(name []byte) bool
    63  
    64  func detectArm64HasAtomics() bool {
    65  	switch runtime.GOOS {
    66  	case "darwin":
    67  		return sysctlEnabled([]byte("hw.optional.armv8_1_atomics\x00"))
    68  	default:
    69  		return cpu.ARM64.HasATOMICS
    70  	}
    71  }