github.com/ice-blockchain/go/src@v0.0.0-20240403114104-1564d284e521/runtime/internal/atomic/atomic_arm64.s (about)

     1  // Copyright 2014 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  #include "go_asm.h"
     6  #include "textflag.h"
     7  
     8  TEXT ·Casint32(SB), NOSPLIT, $0-17
     9  	B	·Cas(SB)
    10  
    11  TEXT ·Casint64(SB), NOSPLIT, $0-25
    12  	B	·Cas64(SB)
    13  
    14  TEXT ·Casuintptr(SB), NOSPLIT, $0-25
    15  	B	·Cas64(SB)
    16  
    17  TEXT ·CasRel(SB), NOSPLIT, $0-17
    18  	B	·Cas(SB)
    19  
    20  TEXT ·Loadint32(SB), NOSPLIT, $0-12
    21  	B	·Load(SB)
    22  
    23  TEXT ·Loadint64(SB), NOSPLIT, $0-16
    24  	B	·Load64(SB)
    25  
    26  TEXT ·Loaduintptr(SB), NOSPLIT, $0-16
    27  	B	·Load64(SB)
    28  
    29  TEXT ·Loaduint(SB), NOSPLIT, $0-16
    30  	B	·Load64(SB)
    31  
    32  TEXT ·Storeint32(SB), NOSPLIT, $0-12
    33  	B	·Store(SB)
    34  
    35  TEXT ·Storeint64(SB), NOSPLIT, $0-16
    36  	B	·Store64(SB)
    37  
    38  TEXT ·Storeuintptr(SB), NOSPLIT, $0-16
    39  	B	·Store64(SB)
    40  
    41  TEXT ·Xaddint32(SB), NOSPLIT, $0-20
    42  	B	·Xadd(SB)
    43  
    44  TEXT ·Xaddint64(SB), NOSPLIT, $0-24
    45  	B	·Xadd64(SB)
    46  
    47  TEXT ·Xadduintptr(SB), NOSPLIT, $0-24
    48  	B	·Xadd64(SB)
    49  
    50  TEXT ·Casp1(SB), NOSPLIT, $0-25
    51  	B ·Cas64(SB)
    52  
    53  // uint32 ·Load(uint32 volatile* addr)
    54  TEXT ·Load(SB),NOSPLIT,$0-12
    55  	MOVD	ptr+0(FP), R0
    56  	LDARW	(R0), R0
    57  	MOVW	R0, ret+8(FP)
    58  	RET
    59  
    60  // uint8 ·Load8(uint8 volatile* addr)
    61  TEXT ·Load8(SB),NOSPLIT,$0-9
    62  	MOVD	ptr+0(FP), R0
    63  	LDARB	(R0), R0
    64  	MOVB	R0, ret+8(FP)
    65  	RET
    66  
    67  // uint64 ·Load64(uint64 volatile* addr)
    68  TEXT ·Load64(SB),NOSPLIT,$0-16
    69  	MOVD	ptr+0(FP), R0
    70  	LDAR	(R0), R0
    71  	MOVD	R0, ret+8(FP)
    72  	RET
    73  
    74  // void *·Loadp(void *volatile *addr)
    75  TEXT ·Loadp(SB),NOSPLIT,$0-16
    76  	MOVD	ptr+0(FP), R0
    77  	LDAR	(R0), R0
    78  	MOVD	R0, ret+8(FP)
    79  	RET
    80  
    81  // uint32 ·LoadAcq(uint32 volatile* addr)
    82  TEXT ·LoadAcq(SB),NOSPLIT,$0-12
    83  	B	·Load(SB)
    84  
    85  // uint64 ·LoadAcquintptr(uint64 volatile* addr)
    86  TEXT ·LoadAcq64(SB),NOSPLIT,$0-16
    87  	B	·Load64(SB)
    88  
    89  // uintptr ·LoadAcq64(uintptr volatile* addr)
    90  TEXT ·LoadAcquintptr(SB),NOSPLIT,$0-16
    91  	B	·Load64(SB)
    92  
    93  TEXT ·StorepNoWB(SB), NOSPLIT, $0-16
    94  	B	·Store64(SB)
    95  
    96  TEXT ·StoreRel(SB), NOSPLIT, $0-12
    97  	B	·Store(SB)
    98  
    99  TEXT ·StoreRel64(SB), NOSPLIT, $0-16
   100  	B	·Store64(SB)
   101  
   102  TEXT ·StoreReluintptr(SB), NOSPLIT, $0-16
   103  	B	·Store64(SB)
   104  
   105  TEXT ·Store(SB), NOSPLIT, $0-12
   106  	MOVD	ptr+0(FP), R0
   107  	MOVW	val+8(FP), R1
   108  	STLRW	R1, (R0)
   109  	RET
   110  
   111  TEXT ·Store8(SB), NOSPLIT, $0-9
   112  	MOVD	ptr+0(FP), R0
   113  	MOVB	val+8(FP), R1
   114  	STLRB	R1, (R0)
   115  	RET
   116  
   117  TEXT ·Store64(SB), NOSPLIT, $0-16
   118  	MOVD	ptr+0(FP), R0
   119  	MOVD	val+8(FP), R1
   120  	STLR	R1, (R0)
   121  	RET
   122  
   123  // uint32 Xchg(ptr *uint32, new uint32)
   124  // Atomically:
   125  //	old := *ptr;
   126  //	*ptr = new;
   127  //	return old;
   128  TEXT ·Xchg(SB), NOSPLIT, $0-20
   129  	MOVD	ptr+0(FP), R0
   130  	MOVW	new+8(FP), R1
   131  #ifndef GOARM64_LSE
   132  	MOVBU	internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
   133  	CBZ 	R4, load_store_loop
   134  #endif
   135  	SWPALW	R1, (R0), R2
   136  	MOVW	R2, ret+16(FP)
   137  	RET
   138  #ifndef GOARM64_LSE
   139  load_store_loop:
   140  	LDAXRW	(R0), R2
   141  	STLXRW	R1, (R0), R3
   142  	CBNZ	R3, load_store_loop
   143  	MOVW	R2, ret+16(FP)
   144  	RET
   145  #endif
   146  
   147  // uint64 Xchg64(ptr *uint64, new uint64)
   148  // Atomically:
   149  //	old := *ptr;
   150  //	*ptr = new;
   151  //	return old;
   152  TEXT ·Xchg64(SB), NOSPLIT, $0-24
   153  	MOVD	ptr+0(FP), R0
   154  	MOVD	new+8(FP), R1
   155  #ifndef GOARM64_LSE
   156  	MOVBU	internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
   157  	CBZ 	R4, load_store_loop
   158  #endif
   159  	SWPALD	R1, (R0), R2
   160  	MOVD	R2, ret+16(FP)
   161  	RET
   162  #ifndef GOARM64_LSE
   163  load_store_loop:
   164  	LDAXR	(R0), R2
   165  	STLXR	R1, (R0), R3
   166  	CBNZ	R3, load_store_loop
   167  	MOVD	R2, ret+16(FP)
   168  	RET
   169  #endif
   170  
   171  // bool Cas(uint32 *ptr, uint32 old, uint32 new)
   172  // Atomically:
   173  //	if(*val == old){
   174  //		*val = new;
   175  //		return 1;
   176  //	} else
   177  //		return 0;
   178  TEXT ·Cas(SB), NOSPLIT, $0-17
   179  	MOVD	ptr+0(FP), R0
   180  	MOVW	old+8(FP), R1
   181  	MOVW	new+12(FP), R2
   182  #ifndef GOARM64_LSE
   183  	MOVBU	internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
   184  	CBZ 	R4, load_store_loop
   185  #endif
   186  	MOVD	R1, R3
   187  	CASALW	R3, (R0), R2
   188  	CMP 	R1, R3
   189  	CSET	EQ, R0
   190  	MOVB	R0, ret+16(FP)
   191  	RET
   192  #ifndef GOARM64_LSE
   193  load_store_loop:
   194  	LDAXRW	(R0), R3
   195  	CMPW	R1, R3
   196  	BNE	ok
   197  	STLXRW	R2, (R0), R3
   198  	CBNZ	R3, load_store_loop
   199  ok:
   200  	CSET	EQ, R0
   201  	MOVB	R0, ret+16(FP)
   202  	RET
   203  #endif
   204  
   205  // bool ·Cas64(uint64 *ptr, uint64 old, uint64 new)
   206  // Atomically:
   207  //      if(*val == old){
   208  //              *val = new;
   209  //              return 1;
   210  //      } else {
   211  //              return 0;
   212  //      }
   213  TEXT ·Cas64(SB), NOSPLIT, $0-25
   214  	MOVD	ptr+0(FP), R0
   215  	MOVD	old+8(FP), R1
   216  	MOVD	new+16(FP), R2
   217  #ifndef GOARM64_LSE
   218  	MOVBU	internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
   219  	CBZ 	R4, load_store_loop
   220  #endif
   221  	MOVD	R1, R3
   222  	CASALD	R3, (R0), R2
   223  	CMP 	R1, R3
   224  	CSET	EQ, R0
   225  	MOVB	R0, ret+24(FP)
   226  	RET
   227  #ifndef GOARM64_LSE
   228  load_store_loop:
   229  	LDAXR	(R0), R3
   230  	CMP	R1, R3
   231  	BNE	ok
   232  	STLXR	R2, (R0), R3
   233  	CBNZ	R3, load_store_loop
   234  ok:
   235  	CSET	EQ, R0
   236  	MOVB	R0, ret+24(FP)
   237  	RET
   238  #endif
   239  
   240  // uint32 xadd(uint32 volatile *ptr, int32 delta)
   241  // Atomically:
   242  //      *val += delta;
   243  //      return *val;
   244  TEXT ·Xadd(SB), NOSPLIT, $0-20
   245  	MOVD	ptr+0(FP), R0
   246  	MOVW	delta+8(FP), R1
   247  #ifndef GOARM64_LSE
   248  	MOVBU	internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
   249  	CBZ 	R4, load_store_loop
   250  #endif
   251  	LDADDALW	R1, (R0), R2
   252  	ADD 	R1, R2
   253  	MOVW	R2, ret+16(FP)
   254  	RET
   255  #ifndef GOARM64_LSE
   256  load_store_loop:
   257  	LDAXRW	(R0), R2
   258  	ADDW	R2, R1, R2
   259  	STLXRW	R2, (R0), R3
   260  	CBNZ	R3, load_store_loop
   261  	MOVW	R2, ret+16(FP)
   262  	RET
   263  #endif
   264  
   265  // uint64 Xadd64(uint64 volatile *ptr, int64 delta)
   266  // Atomically:
   267  //      *val += delta;
   268  //      return *val;
   269  TEXT ·Xadd64(SB), NOSPLIT, $0-24
   270  	MOVD	ptr+0(FP), R0
   271  	MOVD	delta+8(FP), R1
   272  #ifndef GOARM64_LSE
   273  	MOVBU	internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
   274  	CBZ 	R4, load_store_loop
   275  #endif
   276  	LDADDALD	R1, (R0), R2
   277  	ADD 	R1, R2
   278  	MOVD	R2, ret+16(FP)
   279  	RET
   280  #ifndef GOARM64_LSE
   281  load_store_loop:
   282  	LDAXR	(R0), R2
   283  	ADD	R2, R1, R2
   284  	STLXR	R2, (R0), R3
   285  	CBNZ	R3, load_store_loop
   286  	MOVD	R2, ret+16(FP)
   287  	RET
   288  #endif
   289  
   290  TEXT ·Xchgint32(SB), NOSPLIT, $0-20
   291  	B	·Xchg(SB)
   292  
   293  TEXT ·Xchgint64(SB), NOSPLIT, $0-24
   294  	B	·Xchg64(SB)
   295  
   296  TEXT ·Xchguintptr(SB), NOSPLIT, $0-24
   297  	B	·Xchg64(SB)
   298  
   299  TEXT ·And8(SB), NOSPLIT, $0-9
   300  	MOVD	ptr+0(FP), R0
   301  	MOVB	val+8(FP), R1
   302  #ifndef GOARM64_LSE
   303  	MOVBU	internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
   304  	CBZ 	R4, load_store_loop
   305  #endif
   306  	MVN 	R1, R2
   307  	LDCLRALB	R2, (R0), R3
   308  	RET
   309  #ifndef GOARM64_LSE
   310  load_store_loop:
   311  	LDAXRB	(R0), R2
   312  	AND	R1, R2
   313  	STLXRB	R2, (R0), R3
   314  	CBNZ	R3, load_store_loop
   315  	RET
   316  #endif
   317  
   318  TEXT ·Or8(SB), NOSPLIT, $0-9
   319  	MOVD	ptr+0(FP), R0
   320  	MOVB	val+8(FP), R1
   321  #ifndef GOARM64_LSE
   322  	MOVBU	internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
   323  	CBZ 	R4, load_store_loop
   324  #endif
   325  	LDORALB	R1, (R0), R2
   326  	RET
   327  #ifndef GOARM64_LSE
   328  load_store_loop:
   329  	LDAXRB	(R0), R2
   330  	ORR	R1, R2
   331  	STLXRB	R2, (R0), R3
   332  	CBNZ	R3, load_store_loop
   333  	RET
   334  #endif
   335  
   336  // func And(addr *uint32, v uint32)
   337  TEXT ·And(SB), NOSPLIT, $0-12
   338  	MOVD	ptr+0(FP), R0
   339  	MOVW	val+8(FP), R1
   340  #ifndef GOARM64_LSE
   341  	MOVBU	internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
   342  	CBZ 	R4, load_store_loop
   343  #endif
   344  	MVN 	R1, R2
   345  	LDCLRALW	R2, (R0), R3
   346  	RET
   347  #ifndef GOARM64_LSE
   348  load_store_loop:
   349  	LDAXRW	(R0), R2
   350  	AND	R1, R2
   351  	STLXRW	R2, (R0), R3
   352  	CBNZ	R3, load_store_loop
   353  	RET
   354  #endif
   355  
   356  // func Or(addr *uint32, v uint32)
   357  TEXT ·Or(SB), NOSPLIT, $0-12
   358  	MOVD	ptr+0(FP), R0
   359  	MOVW	val+8(FP), R1
   360  #ifndef GOARM64_LSE
   361  	MOVBU	internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
   362  	CBZ 	R4, load_store_loop
   363  #endif
   364  	LDORALW	R1, (R0), R2
   365  	RET
   366  #ifndef GOARM64_LSE
   367  load_store_loop:
   368  	LDAXRW	(R0), R2
   369  	ORR	R1, R2
   370  	STLXRW	R2, (R0), R3
   371  	CBNZ	R3, load_store_loop
   372  	RET
   373  #endif
   374  
   375  // func Or32(addr *uint32, v uint32) old uint32
   376  TEXT ·Or32(SB), NOSPLIT, $0-20
   377  	MOVD	ptr+0(FP), R0
   378  	MOVW	val+8(FP), R1
   379  #ifndef GOARM64_LSE
   380  	MOVBU	internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
   381  	CBZ 	R4, load_store_loop
   382  #endif
   383  	LDORALW	R1, (R0), R2
   384  	MOVD	R2, ret+16(FP)
   385  	RET
   386  #ifndef GOARM64_LSE
   387  load_store_loop:
   388  	LDAXRW	(R0), R2
   389  	ORR	R1, R2, R3
   390  	STLXRW	R3, (R0), R4
   391  	CBNZ	R4, load_store_loop
   392  	MOVD R2, ret+16(FP)
   393  	RET
   394  #endif
   395  
   396  // func And32(addr *uint32, v uint32) old uint32
   397  TEXT ·And32(SB), NOSPLIT, $0-20
   398  	MOVD	ptr+0(FP), R0
   399  	MOVW	val+8(FP), R1
   400  #ifndef GOARM64_LSE
   401  	MOVBU	internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
   402  	CBZ 	R4, load_store_loop
   403  #endif
   404  	MVN 	R1, R2
   405  	LDCLRALW	R2, (R0), R3
   406  	MOVD	R3, ret+16(FP)
   407  	RET
   408  #ifndef GOARM64_LSE
   409  load_store_loop:
   410  	LDAXRW	(R0), R2
   411  	AND	R1, R2, R3
   412  	STLXRW	R3, (R0), R4
   413  	CBNZ	R4, load_store_loop
   414  	MOVD R2, ret+16(FP)
   415  	RET
   416  #endif
   417  
   418  // func Or64(addr *uint64, v uint64) old uint64
   419  TEXT ·Or64(SB), NOSPLIT, $0-24
   420  	MOVD	ptr+0(FP), R0
   421  	MOVD	val+8(FP), R1
   422  #ifndef GOARM64_LSE
   423  	MOVBU	internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
   424  	CBZ 	R4, load_store_loop
   425  #endif
   426  	LDORALD	R1, (R0), R2
   427  	MOVD	R2, ret+16(FP)
   428  	RET
   429  #ifndef GOARM64_LSE
   430  load_store_loop:
   431  	LDAXR	(R0), R2
   432  	ORR	R1, R2, R3
   433  	STLXR	R3, (R0), R4
   434  	CBNZ	R4, load_store_loop
   435  	MOVD 	R2, ret+16(FP)
   436  	RET
   437  #endif
   438  
   439  // func And64(addr *uint64, v uint64) old uint64
   440  TEXT ·And64(SB), NOSPLIT, $0-24
   441  	MOVD	ptr+0(FP), R0
   442  	MOVD	val+8(FP), R1
   443  #ifndef GOARM64_LSE
   444  	MOVBU	internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
   445  	CBZ 	R4, load_store_loop
   446  #endif
   447  	MVN 	R1, R2
   448  	LDCLRALD	R2, (R0), R3
   449  	MOVD	R3, ret+16(FP)
   450  	RET
   451  #ifndef GOARM64_LSE
   452  load_store_loop:
   453  	LDAXR	(R0), R2
   454  	AND	R1, R2, R3
   455  	STLXR	R3, (R0), R4
   456  	CBNZ	R4, load_store_loop
   457  	MOVD 	R2, ret+16(FP)
   458  	RET
   459  #endif
   460  
   461  // func Anduintptr(addr *uintptr, v uintptr) old uintptr
   462  TEXT ·Anduintptr(SB), NOSPLIT, $0-24
   463  	B	·And64(SB)
   464  
   465  // func Oruintptr(addr *uintptr, v uintptr) old uintptr
   466  TEXT ·Oruintptr(SB), NOSPLIT, $0-24
   467  	B	·Or64(SB)