github.com/primecitizens/pcz/std@v0.2.1/core/atomic/atomic_arm64.s (about)

     1  // SPDX-License-Identifier: Apache-2.0
     2  // Copyright 2023 The Prime Citizens
     3  //
     4  // Copyright 2014 The Go Authors. All rights reserved.
     5  // Use of this source code is governed by a BSD-style
     6  // license that can be found in the LICENSE file.
     7  
     8  //go:build arm64
     9  
    10  #include "go_asm.h"
    11  #include "textflag.h"
    12  
    13  #define HAS_ATOMICS_R4 \
    14  	MOVW ·useARM64Atomics(SB), R4
    15  
    16  TEXT ·PublicationBarrier(SB),NOSPLIT|NOFRAME,$0-0
    17  	DMB $0xe // DMB ST
    18  	RET
    19  
    20  //
    21  // Store
    22  //
    23  
    24  TEXT ·Store8(SB),NOSPLIT,$0-9
    25  	MOVD ptr+0(FP), R0
    26  	MOVB val+8(FP), R1
    27  	STLRB R1, (R0)
    28  	RET
    29  
    30  TEXT ·Store32(SB),NOSPLIT,$0-12
    31  	MOVD ptr+0(FP), R0
    32  	MOVW val+8(FP), R1
    33  	STLRW R1, (R0)
    34  	RET
    35  
    36  TEXT ·Store64(SB),NOSPLIT,$0-16
    37  	MOVD ptr+0(FP), R0
    38  	MOVD val+8(FP), R1
    39  	STLR R1, (R0)
    40  	RET
    41  
    42  TEXT ·StoreUintptr(SB),NOSPLIT,$0-16
    43  	B ·Store64(SB)
    44  
    45  TEXT ·StorePointer(SB),NOSPLIT,$0-16
    46  	B ·Store64(SB)
    47  
    48  TEXT ·StoreInt32(SB),NOSPLIT,$0-12
    49  	B ·Store32(SB)
    50  
    51  TEXT ·StoreInt64(SB),NOSPLIT,$0-16
    52  	B ·Store64(SB)
    53  
    54  //
    55  // StoreRel
    56  //
    57  
    58  
    59  TEXT ·StoreRelUintptr(SB),NOSPLIT,$0-16
    60  	B ·Store64(SB)
    61  
    62  TEXT ·StoreRel32(SB),NOSPLIT,$0-12
    63  	B ·Store32(SB)
    64  
    65  TEXT ·StoreRel64(SB),NOSPLIT,$0-16
    66  	B ·Store64(SB)
    67  
    68  //
    69  // Load
    70  //
    71  
    72  // uint8 ·Load8(uint8 volatile* addr)
    73  TEXT ·Load8(SB),NOSPLIT,$0-9
    74  	MOVD ptr+0(FP), R0
    75  	LDARB (R0), R0
    76  	MOVB R0, ret+8(FP)
    77  	RET
    78  
    79  // uint32 ·Load32(uint32 volatile* addr)
    80  TEXT ·Load32(SB),NOSPLIT,$0-12
    81  	MOVD ptr+0(FP), R0
    82  	LDARW (R0), R0
    83  	MOVW R0, ret+8(FP)
    84  	RET
    85  
    86  
    87  // uint64 ·Load64(uint64 volatile* addr)
    88  TEXT ·Load64(SB),NOSPLIT,$0-16
    89  	MOVD ptr+0(FP), R0
    90  	LDAR (R0), R0
    91  	MOVD R0, ret+8(FP)
    92  	RET
    93  
    94  TEXT ·LoadUintptr(SB),NOSPLIT,$0-16
    95  	B ·Load64(SB)
    96  
    97  // void *·LoadPointer(void *volatile *addr)
    98  TEXT ·LoadPointer(SB),NOSPLIT,$0-16
    99  	MOVD ptr+0(FP), R0
   100  	LDAR (R0), R0
   101  	MOVD R0, ret+8(FP)
   102  	RET
   103  
   104  TEXT ·LoadUint(SB),NOSPLIT,$0-16
   105  	B ·Load64(SB)
   106  
   107  TEXT ·LoadInt32(SB),NOSPLIT,$0-12
   108  	B ·Load32(SB)
   109  
   110  TEXT ·LoadInt64(SB),NOSPLIT,$0-16
   111  	B ·Load64(SB)
   112  
   113  //
   114  // LoadAcq
   115  //
   116  
   117  // uint32 ·LoadAcq32(uint32 volatile* addr)
   118  TEXT ·LoadAcq32(SB),NOSPLIT,$0-12
   119  	B ·Load32(SB)
   120  
   121  // uint64 ·LoadAcqUintptr(uint64 volatile* addr)
   122  TEXT ·LoadAcq64(SB),NOSPLIT,$0-16
   123  	B ·Load64(SB)
   124  
   125  // uintptr ·LoadAcq64(uintptr volatile* addr)
   126  TEXT ·LoadAcqUintptr(SB),NOSPLIT,$0-16
   127  	B ·Load64(SB)
   128  
   129  //
   130  // bitwise
   131  //
   132  
   133  TEXT ·And8(SB),NOSPLIT,$0-9
   134  	MOVD ptr+0(FP), R0
   135  	MOVB val+8(FP), R1
   136  	HAS_ATOMICS_R4
   137  	CBZ 	R4, load_store_loop
   138  	MVN 	R1, R2
   139  	LDCLRALB R2, (R0), R3
   140  	RET
   141  load_store_loop:
   142  	LDAXRB (R0), R2
   143  	AND R1, R2
   144  	STLXRB R2, (R0), R3
   145  	CBNZ R3, load_store_loop
   146  	RET
   147  
   148  // func And32(addr *uint32, v uint32)
   149  TEXT ·And32(SB),NOSPLIT,$0-12
   150  	MOVD ptr+0(FP), R0
   151  	MOVW val+8(FP), R1
   152  	HAS_ATOMICS_R4
   153  	CBZ 	R4, load_store_loop
   154  	MVN 	R1, R2
   155  	LDCLRALW R2, (R0), R3
   156  	RET
   157  load_store_loop:
   158  	LDAXRW (R0), R2
   159  	AND R1, R2
   160  	STLXRW R2, (R0), R3
   161  	CBNZ R3, load_store_loop
   162  	RET
   163  
   164  TEXT ·Or8(SB),NOSPLIT,$0-9
   165  	MOVD ptr+0(FP), R0
   166  	MOVB val+8(FP), R1
   167  	HAS_ATOMICS_R4
   168  	CBZ 	R4, load_store_loop
   169  	LDORALB R1, (R0), R2
   170  	RET
   171  load_store_loop:
   172  	LDAXRB (R0), R2
   173  	ORR R1, R2
   174  	STLXRB R2, (R0), R3
   175  	CBNZ R3, load_store_loop
   176  	RET
   177  
   178  // func Or32(addr *uint32, v uint32)
   179  TEXT ·Or32(SB),NOSPLIT,$0-12
   180  	MOVD ptr+0(FP), R0
   181  	MOVW val+8(FP), R1
   182  	HAS_ATOMICS_R4
   183  	CBZ 	R4, load_store_loop
   184  	LDORALW R1, (R0), R2
   185  	RET
   186  load_store_loop:
   187  	LDAXRW (R0), R2
   188  	ORR R1, R2
   189  	STLXRW R2, (R0), R3
   190  	CBNZ R3, load_store_loop
   191  	RET
   192  
   193  //
   194  // Swap
   195  //
   196  
   197  // uint32 Swap32(ptr *uint32, new uint32)
   198  TEXT ·Swap32(SB),NOSPLIT,$0-20
   199  	MOVD ptr+0(FP), R0
   200  	MOVW new+8(FP), R1
   201  	HAS_ATOMICS_R4
   202  	CBZ 	R4, load_store_loop
   203  	SWPALW R1, (R0), R2
   204  	MOVW R2, ret+16(FP)
   205  	RET
   206  load_store_loop:
   207  	LDAXRW (R0), R2
   208  	STLXRW R1, (R0), R3
   209  	CBNZ R3, load_store_loop
   210  	MOVW R2, ret+16(FP)
   211  	RET
   212  
   213  // uint64 Swap64(ptr *uint64, new uint64)
   214  TEXT ·Swap64(SB),NOSPLIT,$0-24
   215  	MOVD ptr+0(FP), R0
   216  	MOVD new+8(FP), R1
   217  	HAS_ATOMICS_R4
   218  	CBZ 	R4, load_store_loop
   219  	SWPALD R1, (R0), R2
   220  	MOVD R2, ret+16(FP)
   221  	RET
   222  load_store_loop:
   223  	LDAXR (R0), R2
   224  	STLXR R1, (R0), R3
   225  	CBNZ R3, load_store_loop
   226  	MOVD R2, ret+16(FP)
   227  	RET
   228  
   229  TEXT ·SwapUintptr(SB),NOSPLIT,$0-24
   230  	B ·Swap64(SB)
   231  
   232  TEXT ·SwapInt32(SB),NOSPLIT,$0-20
   233  	B ·Swap32(SB)
   234  
   235  TEXT ·SwapInt64(SB),NOSPLIT,$0-24
   236  	B ·Swap64(SB)
   237  
   238  //
   239  // Add
   240  //
   241  
   242  // uint32 Add32(uint32 volatile *ptr, int32 delta)
   243  TEXT ·Add32(SB),NOSPLIT,$0-20
   244  	MOVD ptr+0(FP), R0
   245  	MOVW delta+8(FP), R1
   246  	HAS_ATOMICS_R4
   247  	CBZ 	R4, load_store_loop
   248  	LDADDALW R1, (R0), R2
   249  	ADD 	R1, R2
   250  	MOVW R2, ret+16(FP)
   251  	RET
   252  load_store_loop:
   253  	LDAXRW (R0), R2
   254  	ADDW R2, R1, R2
   255  	STLXRW R2, (R0), R3
   256  	CBNZ R3, load_store_loop
   257  	MOVW R2, ret+16(FP)
   258  	RET
   259  
   260  // uint64 Add64(uint64 volatile *ptr, int64 delta)
   261  TEXT ·Add64(SB),NOSPLIT,$0-24
   262  	MOVD ptr+0(FP), R0
   263  	MOVD delta+8(FP), R1
   264  	HAS_ATOMICS_R4
   265  	CBZ 	R4, load_store_loop
   266  	LDADDALD R1, (R0), R2
   267  	ADD 	R1, R2
   268  	MOVD R2, ret+16(FP)
   269  	RET
   270  load_store_loop:
   271  	LDAXR (R0), R2
   272  	ADD R2, R1, R2
   273  	STLXR R2, (R0), R3
   274  	CBNZ R3, load_store_loop
   275  	MOVD R2, ret+16(FP)
   276  	RET
   277  
   278  TEXT ·AddUintptr(SB),NOSPLIT,$0-24
   279  	B ·Add64(SB)
   280  
   281  TEXT ·AddInt32(SB),NOSPLIT,$0-20
   282  	B ·Add32(SB)
   283  
   284  TEXT ·AddInt64(SB),NOSPLIT,$0-24
   285  	B ·Add64(SB)
   286  
   287  //
   288  // Compare and swap
   289  //
   290  
   291  // bool Cas32(uint32 *ptr, uint32 old, uint32 new)
   292  TEXT ·Cas32(SB),NOSPLIT,$0-17
   293  	MOVD ptr+0(FP), R0
   294  	MOVW old+8(FP), R1
   295  	MOVW new+12(FP), R2
   296  	HAS_ATOMICS_R4
   297  	CBZ 	R4, load_store_loop
   298  	MOVD R1, R3
   299  	CASALW R3, (R0), R2
   300  	CMP 	R1, R3
   301  	CSET EQ, R0
   302  	MOVB R0, ret+16(FP)
   303  	RET
   304  load_store_loop:
   305  	LDAXRW (R0), R3
   306  	CMPW R1, R3
   307  	BNE ok
   308  	STLXRW R2, (R0), R3
   309  	CBNZ R3, load_store_loop
   310  ok:
   311  	CSET EQ, R0
   312  	MOVB R0, ret+16(FP)
   313  	RET
   314  
   315  // bool ·Cas64(uint64 *ptr, uint64 old, uint64 new)
   316  TEXT ·Cas64(SB),NOSPLIT,$0-25
   317  	MOVD ptr+0(FP), R0
   318  	MOVD old+8(FP), R1
   319  	MOVD new+16(FP), R2
   320  	HAS_ATOMICS_R4
   321  	CBZ 	R4, load_store_loop
   322  	MOVD R1, R3
   323  	CASALD R3, (R0), R2
   324  	CMP 	R1, R3
   325  	CSET EQ, R0
   326  	MOVB R0, ret+24(FP)
   327  	RET
   328  load_store_loop:
   329  	LDAXR (R0), R3
   330  	CMP R1, R3
   331  	BNE ok
   332  	STLXR R2, (R0), R3
   333  	CBNZ R3, load_store_loop
   334  ok:
   335  	CSET EQ, R0
   336  	MOVB R0, ret+24(FP)
   337  	RET
   338  
   339  TEXT ·CasUintptr(SB),NOSPLIT,$0-25
   340  	B ·Cas64(SB)
   341  
   342  TEXT ·CasUnsafePointer(SB),NOSPLIT,$0-25
   343  	B ·Cas64(SB)
   344  
   345  TEXT ·CasInt32(SB),NOSPLIT,$0-17
   346  	B ·Cas32(SB)
   347  
   348  TEXT ·CasInt64(SB),NOSPLIT,$0-25
   349  	B ·Cas64(SB)
   350  
   351  //
   352  // CasRel
   353  //
   354  
   355  TEXT ·CasRel32(SB),NOSPLIT,$0-17
   356  	B ·Cas32(SB)