1 // Copyright 2015 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4
5 #include "go_asm.h"
6 #include "textflag.h"
7 #include "funcdata.h"
8
9 // bool armcas(int32 *val, int32 old, int32 new)
10 // Atomically:
11 // if(*val == old){
12 // *val = new;
13 // return 1;
14 // }else
15 // return 0;
16 //
17 // To implement ·cas in sys_$GOOS_arm.s
18 // using the native instructions, use:
19 //
20 // TEXT ·cas(SB),NOSPLIT,$0
21 // B ·armcas(SB)
22 //
23 TEXT ·armcas(SB),NOSPLIT,$0-13
24 MOVW ptr+0(FP), R1
25 MOVW old+4(FP), R2
26 MOVW new+8(FP), R3
27 casl:
28 LDREX (R1), R0
29 CMP R0, R2
30 BNE casfail
31
32 #ifndef GOARM_7
33 MOVB internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11
34 CMP $0, R11
35 BEQ 2(PC)
36 #endif
37 DMB MB_ISHST
38
39 STREX R3, (R1), R0
40 CMP $0, R0
41 BNE casl
42 MOVW $1, R0
43
44 #ifndef GOARM_7
45 CMP $0, R11
46 BEQ 2(PC)
47 #endif
48 DMB MB_ISH
49
50 MOVB R0, ret+12(FP)
51 RET
52 casfail:
53 MOVW $0, R0
54 MOVB R0, ret+12(FP)
55 RET
56
57 // stubs
58
59 TEXT ·Loadp(SB),NOSPLIT|NOFRAME,$0-8
60 B ·Load(SB)
61
62 TEXT ·LoadAcq(SB),NOSPLIT|NOFRAME,$0-8
63 B ·Load(SB)
64
65 TEXT ·LoadAcquintptr(SB),NOSPLIT|NOFRAME,$0-8
66 B ·Load(SB)
67
68 TEXT ·Casint32(SB),NOSPLIT,$0-13
69 B ·Cas(SB)
70
71 TEXT ·Casint64(SB),NOSPLIT,$-4-21
72 B ·Cas64(SB)
73
74 TEXT ·Casuintptr(SB),NOSPLIT,$0-13
75 B ·Cas(SB)
76
77 TEXT ·Casp1(SB),NOSPLIT,$0-13
78 B ·Cas(SB)
79
80 TEXT ·CasRel(SB),NOSPLIT,$0-13
81 B ·Cas(SB)
82
83 TEXT ·Loadint32(SB),NOSPLIT,$0-8
84 B ·Load(SB)
85
86 TEXT ·Loadint64(SB),NOSPLIT,$-4-12
87 B ·Load64(SB)
88
89 TEXT ·Loaduintptr(SB),NOSPLIT,$0-8
90 B ·Load(SB)
91
92 TEXT ·Loaduint(SB),NOSPLIT,$0-8
93 B ·Load(SB)
94
95 TEXT ·Storeint32(SB),NOSPLIT,$0-8
96 B ·Store(SB)
97
98 TEXT ·Storeint64(SB),NOSPLIT,$0-12
99 B ·Store64(SB)
100
101 TEXT ·Storeuintptr(SB),NOSPLIT,$0-8
102 B ·Store(SB)
103
104 TEXT ·StorepNoWB(SB),NOSPLIT,$0-8
105 B ·Store(SB)
106
107 TEXT ·StoreRel(SB),NOSPLIT,$0-8
108 B ·Store(SB)
109
110 TEXT ·StoreReluintptr(SB),NOSPLIT,$0-8
111 B ·Store(SB)
112
113 TEXT ·Xaddint32(SB),NOSPLIT,$0-12
114 B ·Xadd(SB)
115
116 TEXT ·Xaddint64(SB),NOSPLIT,$-4-20
117 B ·Xadd64(SB)
118
119 TEXT ·Xadduintptr(SB),NOSPLIT,$0-12
120 B ·Xadd(SB)
121
122 TEXT ·Xchgint32(SB),NOSPLIT,$0-12
123 B ·Xchg(SB)
124
125 TEXT ·Xchgint64(SB),NOSPLIT,$-4-20
126 B ·Xchg64(SB)
127
128 // 64-bit atomics
129 // The native ARM implementations use LDREXD/STREXD, which are
130 // available on ARMv6k or later. We use them only on ARMv7.
131 // On older ARM, we use Go implementations which simulate 64-bit
132 // atomics with locks.
133 TEXT armCas64<>(SB),NOSPLIT,$0-21
134 // addr is already in R1
135 MOVW old_lo+4(FP), R2
136 MOVW old_hi+8(FP), R3
137 MOVW new_lo+12(FP), R4
138 MOVW new_hi+16(FP), R5
139 cas64loop:
140 LDREXD (R1), R6 // loads R6 and R7
141 CMP R2, R6
142 BNE cas64fail
143 CMP R3, R7
144 BNE cas64fail
145
146 DMB MB_ISHST
147
148 STREXD R4, (R1), R0 // stores R4 and R5
149 CMP $0, R0
150 BNE cas64loop
151 MOVW $1, R0
152
153 DMB MB_ISH
154
155 MOVBU R0, swapped+20(FP)
156 RET
157 cas64fail:
158 MOVW $0, R0
159 MOVBU R0, swapped+20(FP)
160 RET
161
162 TEXT armXadd64<>(SB),NOSPLIT,$0-20
163 // addr is already in R1
164 MOVW delta_lo+4(FP), R2
165 MOVW delta_hi+8(FP), R3
166
167 add64loop:
168 LDREXD (R1), R4 // loads R4 and R5
169 ADD.S R2, R4
170 ADC R3, R5
171
172 DMB MB_ISHST
173
174 STREXD R4, (R1), R0 // stores R4 and R5
175 CMP $0, R0
176 BNE add64loop
177
178 DMB MB_ISH
179
180 MOVW R4, new_lo+12(FP)
181 MOVW R5, new_hi+16(FP)
182 RET
183
184 TEXT armXchg64<>(SB),NOSPLIT,$0-20
185 // addr is already in R1
186 MOVW new_lo+4(FP), R2
187 MOVW new_hi+8(FP), R3
188
189 swap64loop:
190 LDREXD (R1), R4 // loads R4 and R5
191
192 DMB MB_ISHST
193
194 STREXD R2, (R1), R0 // stores R2 and R3
195 CMP $0, R0
196 BNE swap64loop
197
198 DMB MB_ISH
199
200 MOVW R4, old_lo+12(FP)
201 MOVW R5, old_hi+16(FP)
202 RET
203
204 TEXT armLoad64<>(SB),NOSPLIT,$0-12
205 // addr is already in R1
206
207 LDREXD (R1), R2 // loads R2 and R3
208 DMB MB_ISH
209
210 MOVW R2, val_lo+4(FP)
211 MOVW R3, val_hi+8(FP)
212 RET
213
214 TEXT armStore64<>(SB),NOSPLIT,$0-12
215 // addr is already in R1
216 MOVW val_lo+4(FP), R2
217 MOVW val_hi+8(FP), R3
218
219 store64loop:
220 LDREXD (R1), R4 // loads R4 and R5
221
222 DMB MB_ISHST
223
224 STREXD R2, (R1), R0 // stores R2 and R3
225 CMP $0, R0
226 BNE store64loop
227
228 DMB MB_ISH
229 RET
230
231 // The following functions all panic if their address argument isn't
232 // 8-byte aligned. Since we're calling back into Go code to do this,
233 // we have to cooperate with stack unwinding. In the normal case, the
234 // functions tail-call into the appropriate implementation, which
235 // means they must not open a frame. Hence, when they go down the
236 // panic path, at that point they push the LR to create a real frame
237 // (they don't need to pop it because panic won't return; however, we
238 // do need to set the SP delta back).
239
240 // Check if R1 is 8-byte aligned, panic if not.
241 // Clobbers R2.
242 #define CHECK_ALIGN \
243 AND.S $7, R1, R2 \
244 BEQ 4(PC) \
245 MOVW.W R14, -4(R13) /* prepare a real frame */ \
246 BL ·panicUnaligned(SB) \
247 ADD $4, R13 /* compensate SP delta */
248
249 TEXT ·Cas64(SB),NOSPLIT,$-4-21
250 NO_LOCAL_POINTERS
251 MOVW addr+0(FP), R1
252 CHECK_ALIGN
253
254 #ifndef GOARM_7
255 MOVB internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11
256 CMP $1, R11
257 BEQ 2(PC)
258 JMP ·goCas64(SB)
259 #endif
260 JMP armCas64<>(SB)
261
262 TEXT ·Xadd64(SB),NOSPLIT,$-4-20
263 NO_LOCAL_POINTERS
264 MOVW addr+0(FP), R1
265 CHECK_ALIGN
266
267 #ifndef GOARM_7
268 MOVB internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11
269 CMP $1, R11
270 BEQ 2(PC)
271 JMP ·goXadd64(SB)
272 #endif
273 JMP armXadd64<>(SB)
274
275 TEXT ·Xchg64(SB),NOSPLIT,$-4-20
276 NO_LOCAL_POINTERS
277 MOVW addr+0(FP), R1
278 CHECK_ALIGN
279
280 #ifndef GOARM_7
281 MOVB internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11
282 CMP $1, R11
283 BEQ 2(PC)
284 JMP ·goXchg64(SB)
285 #endif
286 JMP armXchg64<>(SB)
287
288 TEXT ·Load64(SB),NOSPLIT,$-4-12
289 NO_LOCAL_POINTERS
290 MOVW addr+0(FP), R1
291 CHECK_ALIGN
292
293 #ifndef GOARM_7
294 MOVB internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11
295 CMP $1, R11
296 BEQ 2(PC)
297 JMP ·goLoad64(SB)
298 #endif
299 JMP armLoad64<>(SB)
300
301 TEXT ·Store64(SB),NOSPLIT,$-4-12
302 NO_LOCAL_POINTERS
303 MOVW addr+0(FP), R1
304 CHECK_ALIGN
305
306 #ifndef GOARM_7
307 MOVB internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11
308 CMP $1, R11
309 BEQ 2(PC)
310 JMP ·goStore64(SB)
311 #endif
312 JMP armStore64<>(SB)
313
View as plain text