github.com/bytedance/gopkg@v0.0.0-20240514070511-01b2cbcf35e1/collection/lscq/asm.go (about)

     1  // Copyright 2021 ByteDance Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  //go:build amd64 && !gccgo && !appengine
    16  // +build amd64,!gccgo,!appengine
    17  
    18  package lscq
    19  
    20  import (
    21  	"unsafe"
    22  )
    23  
    24  type uint128 [2]uint64
    25  
    26  func compareAndSwapUint128(addr *uint128, old1, old2, new1, new2 uint64) (swapped bool)
    27  
    28  func loadUint128(addr *uint128) (val uint128)
    29  
    30  func loadSCQNodePointer(addr unsafe.Pointer) (val scqNodePointer)
    31  
    32  func loadSCQNodeUint64(addr unsafe.Pointer) (val scqNodeUint64)
    33  
    34  func atomicTestAndSetFirstBit(addr *uint64) (val uint64)
    35  
    36  func atomicTestAndSetSecondBit(addr *uint64) (val uint64)
    37  
    38  func resetNode(addr unsafe.Pointer)
    39  
    40  //go:nosplit
    41  func compareAndSwapSCQNodePointer(addr *scqNodePointer, old, new scqNodePointer) (swapped bool) {
    42  	// Ref: src/runtime/atomic_pointer.go:sync_atomic_CompareAndSwapPointer
    43  	if runtimeEnableWriteBarrier() {
    44  		runtimeatomicwb(&addr.data, new.data)
    45  	}
    46  	return compareAndSwapUint128((*uint128)(runtimenoescape(unsafe.Pointer(addr))), old.flags, uint64(uintptr(old.data)), new.flags, uint64(uintptr(new.data)))
    47  }
    48  
    49  func compareAndSwapSCQNodeUint64(addr *scqNodeUint64, old, new scqNodeUint64) (swapped bool) {
    50  	return compareAndSwapUint128((*uint128)(unsafe.Pointer(addr)), old.flags, old.data, new.flags, new.data)
    51  }
    52  
    53  func runtimeEnableWriteBarrier() bool
    54  
    55  //go:linkname runtimeatomicwb runtime.atomicwb
    56  //go:noescape
    57  func runtimeatomicwb(ptr *unsafe.Pointer, new unsafe.Pointer)
    58  
    59  //go:linkname runtimenoescape runtime.noescape
    60  func runtimenoescape(p unsafe.Pointer) unsafe.Pointer
    61  
    62  //go:nosplit
    63  func atomicWriteBarrier(ptr *unsafe.Pointer) {
    64  	// For SCQ dequeue only. (fastpath)
    65  	if runtimeEnableWriteBarrier() {
    66  		runtimeatomicwb(ptr, nil)
    67  	}
    68  }