github.com/mtsmfm/go/src@v0.0.0-20221020090648-44bdcb9f8fde/runtime/atomic_pointer.go (about)

     1  // Copyright 2009 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package runtime
     6  
     7  import (
     8  	"runtime/internal/atomic"
     9  	"unsafe"
    10  )
    11  
    12  // These functions cannot have go:noescape annotations,
    13  // because while ptr does not escape, new does.
    14  // If new is marked as not escaping, the compiler will make incorrect
    15  // escape analysis decisions about the pointer value being stored.
    16  
    17  // atomicwb performs a write barrier before an atomic pointer write.
    18  // The caller should guard the call with "if writeBarrier.enabled".
    19  //
    20  //go:nosplit
    21  func atomicwb(ptr *unsafe.Pointer, new unsafe.Pointer) {
    22  	slot := (*uintptr)(unsafe.Pointer(ptr))
    23  	if !getg().m.p.ptr().wbBuf.putFast(*slot, uintptr(new)) {
    24  		wbBufFlush(slot, uintptr(new))
    25  	}
    26  }
    27  
    28  // atomicstorep performs *ptr = new atomically and invokes a write barrier.
    29  //
    30  //go:nosplit
    31  func atomicstorep(ptr unsafe.Pointer, new unsafe.Pointer) {
    32  	if writeBarrier.enabled {
    33  		atomicwb((*unsafe.Pointer)(ptr), new)
    34  	}
    35  	atomic.StorepNoWB(noescape(ptr), new)
    36  }
    37  
    38  // atomic_storePointer is the implementation of runtime/internal/UnsafePointer.Store
    39  // (like StoreNoWB but with the write barrier).
    40  //
    41  //go:nosplit
    42  //go:linkname atomic_storePointer runtime/internal/atomic.storePointer
    43  func atomic_storePointer(ptr *unsafe.Pointer, new unsafe.Pointer) {
    44  	atomicstorep(unsafe.Pointer(ptr), new)
    45  }
    46  
    47  // atomic_casPointer is the implementation of runtime/internal/UnsafePointer.CompareAndSwap
    48  // (like CompareAndSwapNoWB but with the write barrier).
    49  //
    50  //go:nosplit
    51  //go:linkname atomic_casPointer runtime/internal/atomic.casPointer
    52  func atomic_casPointer(ptr *unsafe.Pointer, old, new unsafe.Pointer) bool {
    53  	if writeBarrier.enabled {
    54  		atomicwb(ptr, new)
    55  	}
    56  	return atomic.Casp1(ptr, old, new)
    57  }
    58  
    59  // Like above, but implement in terms of sync/atomic's uintptr operations.
    60  // We cannot just call the runtime routines, because the race detector expects
    61  // to be able to intercept the sync/atomic forms but not the runtime forms.
    62  
    63  //go:linkname sync_atomic_StoreUintptr sync/atomic.StoreUintptr
    64  func sync_atomic_StoreUintptr(ptr *uintptr, new uintptr)
    65  
    66  //go:linkname sync_atomic_StorePointer sync/atomic.StorePointer
    67  //go:nosplit
    68  func sync_atomic_StorePointer(ptr *unsafe.Pointer, new unsafe.Pointer) {
    69  	if writeBarrier.enabled {
    70  		atomicwb(ptr, new)
    71  	}
    72  	sync_atomic_StoreUintptr((*uintptr)(unsafe.Pointer(ptr)), uintptr(new))
    73  }
    74  
    75  //go:linkname sync_atomic_SwapUintptr sync/atomic.SwapUintptr
    76  func sync_atomic_SwapUintptr(ptr *uintptr, new uintptr) uintptr
    77  
    78  //go:linkname sync_atomic_SwapPointer sync/atomic.SwapPointer
    79  //go:nosplit
    80  func sync_atomic_SwapPointer(ptr *unsafe.Pointer, new unsafe.Pointer) unsafe.Pointer {
    81  	if writeBarrier.enabled {
    82  		atomicwb(ptr, new)
    83  	}
    84  	old := unsafe.Pointer(sync_atomic_SwapUintptr((*uintptr)(noescape(unsafe.Pointer(ptr))), uintptr(new)))
    85  	return old
    86  }
    87  
    88  //go:linkname sync_atomic_CompareAndSwapUintptr sync/atomic.CompareAndSwapUintptr
    89  func sync_atomic_CompareAndSwapUintptr(ptr *uintptr, old, new uintptr) bool
    90  
    91  //go:linkname sync_atomic_CompareAndSwapPointer sync/atomic.CompareAndSwapPointer
    92  //go:nosplit
    93  func sync_atomic_CompareAndSwapPointer(ptr *unsafe.Pointer, old, new unsafe.Pointer) bool {
    94  	if writeBarrier.enabled {
    95  		atomicwb(ptr, new)
    96  	}
    97  	return sync_atomic_CompareAndSwapUintptr((*uintptr)(noescape(unsafe.Pointer(ptr))), uintptr(old), uintptr(new))
    98  }