Text file src/runtime/internal/atomic/atomic_riscv64.s

     1  // Copyright 2014 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // RISC-V's atomic operations have two bits, aq ("acquire") and rl ("release"),
     6  // which may be toggled on and off. Their precise semantics are defined in
     7  // section 6.3 of the specification, but the basic idea is as follows:
     8  //
     9  //   - If neither aq nor rl is set, the CPU may reorder the atomic arbitrarily.
    10  //     It guarantees only that it will execute atomically.
    11  //
    12  //   - If aq is set, the CPU may move the instruction backward, but not forward.
    13  //
    14  //   - If rl is set, the CPU may move the instruction forward, but not backward.
    15  //
    16  //   - If both are set, the CPU may not reorder the instruction at all.
    17  //
    18  // These four modes correspond to other well-known memory models on other CPUs.
    19  // On ARM, aq corresponds to a dmb ishst, aq+rl corresponds to a dmb ish. On
    20  // Intel, aq corresponds to an lfence, rl to an sfence, and aq+rl to an mfence
    21  // (or a lock prefix).
    22  //
    23  // Go's memory model requires that
    24  //   - if a read happens after a write, the read must observe the write, and
    25  //     that
    26  //   - if a read happens concurrently with a write, the read may observe the
    27  //     write.
    28  // aq is sufficient to guarantee this, so that's what we use here. (This jibes
    29  // with ARM, which uses dmb ishst.)
    30  
    31  #include "textflag.h"
    32  
    33  // func Cas(ptr *uint64, old, new uint64) bool
    34  // Atomically:
    35  //      if(*val == old){
    36  //              *val = new;
    37  //              return 1;
    38  //      } else {
    39  //              return 0;
    40  //      }
    41  TEXT ·Cas(SB), NOSPLIT, $0-17
    42  	MOV	ptr+0(FP), A0
    43  	MOVW	old+8(FP), A1
    44  	MOVW	new+12(FP), A2
    45  cas_again:
    46  	LRW	(A0), A3
    47  	BNE	A3, A1, cas_fail
    48  	SCW	A2, (A0), A4
    49  	BNE	A4, ZERO, cas_again
    50  	MOV	$1, A0
    51  	MOVB	A0, ret+16(FP)
    52  	RET
    53  cas_fail:
    54  	MOV	$0, A0
    55  	MOV	A0, ret+16(FP)
    56  	RET
    57  
    58  // func Cas64(ptr *uint64, old, new uint64) bool
    59  TEXT ·Cas64(SB), NOSPLIT, $0-25
    60  	MOV	ptr+0(FP), A0
    61  	MOV	old+8(FP), A1
    62  	MOV	new+16(FP), A2
    63  cas_again:
    64  	LRD	(A0), A3
    65  	BNE	A3, A1, cas_fail
    66  	SCD	A2, (A0), A4
    67  	BNE	A4, ZERO, cas_again
    68  	MOV	$1, A0
    69  	MOVB	A0, ret+24(FP)
    70  	RET
    71  cas_fail:
    72  	MOVB	ZERO, ret+24(FP)
    73  	RET
    74  
    75  // func Load(ptr *uint32) uint32
    76  TEXT ·Load(SB),NOSPLIT|NOFRAME,$0-12
    77  	MOV	ptr+0(FP), A0
    78  	LRW	(A0), A0
    79  	MOVW	A0, ret+8(FP)
    80  	RET
    81  
    82  // func Load8(ptr *uint8) uint8
    83  TEXT ·Load8(SB),NOSPLIT|NOFRAME,$0-9
    84  	MOV	ptr+0(FP), A0
    85  	FENCE
    86  	MOVBU	(A0), A1
    87  	FENCE
    88  	MOVB	A1, ret+8(FP)
    89  	RET
    90  
    91  // func Load64(ptr *uint64) uint64
    92  TEXT ·Load64(SB),NOSPLIT|NOFRAME,$0-16
    93  	MOV	ptr+0(FP), A0
    94  	LRD	(A0), A0
    95  	MOV	A0, ret+8(FP)
    96  	RET
    97  
    98  // func Store(ptr *uint32, val uint32)
    99  TEXT ·Store(SB), NOSPLIT, $0-12
   100  	MOV	ptr+0(FP), A0
   101  	MOVW	val+8(FP), A1
   102  	AMOSWAPW A1, (A0), ZERO
   103  	RET
   104  
   105  // func Store8(ptr *uint8, val uint8)
   106  TEXT ·Store8(SB), NOSPLIT, $0-9
   107  	MOV	ptr+0(FP), A0
   108  	MOVBU	val+8(FP), A1
   109  	FENCE
   110  	MOVB	A1, (A0)
   111  	FENCE
   112  	RET
   113  
   114  // func Store64(ptr *uint64, val uint64)
   115  TEXT ·Store64(SB), NOSPLIT, $0-16
   116  	MOV	ptr+0(FP), A0
   117  	MOV	val+8(FP), A1
   118  	AMOSWAPD A1, (A0), ZERO
   119  	RET
   120  
   121  TEXT ·Casp1(SB), NOSPLIT, $0-25
   122  	JMP	·Cas64(SB)
   123  
   124  TEXT ·Casint32(SB),NOSPLIT,$0-17
   125  	JMP	·Cas(SB)
   126  
   127  TEXT ·Casint64(SB),NOSPLIT,$0-25
   128  	JMP	·Cas64(SB)
   129  
   130  TEXT ·Casuintptr(SB),NOSPLIT,$0-25
   131  	JMP	·Cas64(SB)
   132  
   133  TEXT ·CasRel(SB), NOSPLIT, $0-17
   134  	JMP	·Cas(SB)
   135  
   136  TEXT ·Loaduintptr(SB),NOSPLIT,$0-16
   137  	JMP	·Load64(SB)
   138  
   139  TEXT ·Storeint32(SB),NOSPLIT,$0-12
   140  	JMP	·Store(SB)
   141  
   142  TEXT ·Storeint64(SB),NOSPLIT,$0-16
   143  	JMP	·Store64(SB)
   144  
   145  TEXT ·Storeuintptr(SB),NOSPLIT,$0-16
   146  	JMP	·Store64(SB)
   147  
   148  TEXT ·Loaduint(SB),NOSPLIT,$0-16
   149  	JMP ·Loaduintptr(SB)
   150  
   151  TEXT ·Loadint32(SB),NOSPLIT,$0-12
   152  	JMP ·Load(SB)
   153  
   154  TEXT ·Loadint64(SB),NOSPLIT,$0-16
   155  	JMP ·Load64(SB)
   156  
   157  TEXT ·Xaddint32(SB),NOSPLIT,$0-20
   158  	JMP ·Xadd(SB)
   159  
   160  TEXT ·Xaddint64(SB),NOSPLIT,$0-24
   161  	MOV	ptr+0(FP), A0
   162  	MOV	delta+8(FP), A1
   163  	AMOADDD A1, (A0), A0
   164  	ADD	A0, A1, A0
   165  	MOVW	A0, ret+16(FP)
   166  	RET
   167  
   168  TEXT ·LoadAcq(SB),NOSPLIT|NOFRAME,$0-12
   169  	JMP	·Load(SB)
   170  
   171  TEXT ·LoadAcq64(SB),NOSPLIT|NOFRAME,$0-16
   172  	JMP	·Load64(SB)
   173  
   174  TEXT ·LoadAcquintptr(SB),NOSPLIT|NOFRAME,$0-16
   175  	JMP	·Load64(SB)
   176  
   177  // func Loadp(ptr unsafe.Pointer) unsafe.Pointer
   178  TEXT ·Loadp(SB),NOSPLIT,$0-16
   179  	JMP	·Load64(SB)
   180  
   181  // func StorepNoWB(ptr unsafe.Pointer, val unsafe.Pointer)
   182  TEXT ·StorepNoWB(SB), NOSPLIT, $0-16
   183  	JMP	·Store64(SB)
   184  
   185  TEXT ·StoreRel(SB), NOSPLIT, $0-12
   186  	JMP	·Store(SB)
   187  
   188  TEXT ·StoreRel64(SB), NOSPLIT, $0-16
   189  	JMP	·Store64(SB)
   190  
   191  TEXT ·StoreReluintptr(SB), NOSPLIT, $0-16
   192  	JMP	·Store64(SB)
   193  
   194  // func Xchg(ptr *uint32, new uint32) uint32
   195  TEXT ·Xchg(SB), NOSPLIT, $0-20
   196  	MOV	ptr+0(FP), A0
   197  	MOVW	new+8(FP), A1
   198  	AMOSWAPW A1, (A0), A1
   199  	MOVW	A1, ret+16(FP)
   200  	RET
   201  
   202  // func Xchg64(ptr *uint64, new uint64) uint64
   203  TEXT ·Xchg64(SB), NOSPLIT, $0-24
   204  	MOV	ptr+0(FP), A0
   205  	MOV	new+8(FP), A1
   206  	AMOSWAPD A1, (A0), A1
   207  	MOV	A1, ret+16(FP)
   208  	RET
   209  
   210  // Atomically:
   211  //      *val += delta;
   212  //      return *val;
   213  
   214  // func Xadd(ptr *uint32, delta int32) uint32
   215  TEXT ·Xadd(SB), NOSPLIT, $0-20
   216  	MOV	ptr+0(FP), A0
   217  	MOVW	delta+8(FP), A1
   218  	AMOADDW A1, (A0), A2
   219  	ADD	A2,A1,A0
   220  	MOVW	A0, ret+16(FP)
   221  	RET
   222  
   223  // func Xadd64(ptr *uint64, delta int64) uint64
   224  TEXT ·Xadd64(SB), NOSPLIT, $0-24
   225  	MOV	ptr+0(FP), A0
   226  	MOV	delta+8(FP), A1
   227  	AMOADDD A1, (A0), A2
   228  	ADD	A2, A1, A0
   229  	MOV	A0, ret+16(FP)
   230  	RET
   231  
   232  // func Xadduintptr(ptr *uintptr, delta uintptr) uintptr
   233  TEXT ·Xadduintptr(SB), NOSPLIT, $0-24
   234  	JMP	·Xadd64(SB)
   235  
   236  // func Xchgint32(ptr *int32, new int32) int32
   237  TEXT ·Xchgint32(SB), NOSPLIT, $0-20
   238  	JMP	·Xchg(SB)
   239  
   240  // func Xchgint64(ptr *int64, new int64) int64
   241  TEXT ·Xchgint64(SB), NOSPLIT, $0-24
   242  	JMP	·Xchg64(SB)
   243  
   244  // func Xchguintptr(ptr *uintptr, new uintptr) uintptr
   245  TEXT ·Xchguintptr(SB), NOSPLIT, $0-24
   246  	JMP	·Xchg64(SB)
   247  
   248  // func And8(ptr *uint8, val uint8)
   249  TEXT ·And8(SB), NOSPLIT, $0-9
   250  	MOV	ptr+0(FP), A0
   251  	MOVBU	val+8(FP), A1
   252  	AND	$3, A0, A2
   253  	AND	$-4, A0
   254  	SLL	$3, A2
   255  	XOR	$255, A1
   256  	SLL	A2, A1
   257  	XOR	$-1, A1
   258  	AMOANDW A1, (A0), ZERO
   259  	RET
   260  
   261  // func Or8(ptr *uint8, val uint8)
   262  TEXT ·Or8(SB), NOSPLIT, $0-9
   263  	MOV	ptr+0(FP), A0
   264  	MOVBU	val+8(FP), A1
   265  	AND	$3, A0, A2
   266  	AND	$-4, A0
   267  	SLL	$3, A2
   268  	SLL	A2, A1
   269  	AMOORW	A1, (A0), ZERO
   270  	RET
   271  
   272  // func And(ptr *uint32, val uint32)
   273  TEXT ·And(SB), NOSPLIT, $0-12
   274  	MOV	ptr+0(FP), A0
   275  	MOVW	val+8(FP), A1
   276  	AMOANDW	A1, (A0), ZERO
   277  	RET
   278  
   279  // func Or(ptr *uint32, val uint32)
   280  TEXT ·Or(SB), NOSPLIT, $0-12
   281  	MOV	ptr+0(FP), A0
   282  	MOVW	val+8(FP), A1
   283  	AMOORW	A1, (A0), ZERO
   284  	RET
   285  
   286  // func Or32(ptr *uint32, val uint32) uint32
   287  TEXT ·Or32(SB), NOSPLIT, $0-20
   288  	MOV	ptr+0(FP), A0
   289  	MOVW	val+8(FP), A1
   290  	AMOORW	A1, (A0), A2
   291  	MOVW	A2, ret+16(FP)
   292  	RET
   293  
   294  // func And32(ptr *uint32, val uint32) uint32
   295  TEXT ·And32(SB), NOSPLIT, $0-20
   296  	MOV	ptr+0(FP), A0
   297  	MOVW	val+8(FP), A1
   298  	AMOANDW	A1, (A0), A2
   299  	MOVW	A2, ret+16(FP)
   300  	RET
   301  
   302  // func Or64(ptr *uint64, val uint64) uint64
   303  TEXT ·Or64(SB), NOSPLIT, $0-24
   304  	MOV	ptr+0(FP), A0
   305  	MOV	val+8(FP), A1
   306  	AMOORD	A1, (A0), A2
   307  	MOV	A2, ret+16(FP)
   308  	RET
   309  
   310  // func And64(ptr *uint64, val uint64) uint64
   311  TEXT ·And64(SB), NOSPLIT, $0-24
   312  	MOV	ptr+0(FP), A0
   313  	MOV	val+8(FP), A1
   314  	AMOANDD	A1, (A0), A2
   315  	MOV	A2, ret+16(FP)
   316  	RET
   317  
   318  // func Anduintptr(ptr *uintptr, val uintptr) uintptr
   319  TEXT ·Anduintptr(SB), NOSPLIT, $0-24
   320  	JMP	·And64(SB)
   321  
   322  // func Oruintptr(ptr *uintptr, val uintptr) uintptr
   323  TEXT ·Oruintptr(SB), NOSPLIT, $0-24
   324  	JMP	·Or64(SB)
   325  

View as plain text