1 // Copyright 2014 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4
5 #include "go_asm.h"
6 #include "textflag.h"
7
8 TEXT ·Casint32(SB), NOSPLIT, $0-17
9 B ·Cas(SB)
10
11 TEXT ·Casint64(SB), NOSPLIT, $0-25
12 B ·Cas64(SB)
13
14 TEXT ·Casuintptr(SB), NOSPLIT, $0-25
15 B ·Cas64(SB)
16
17 TEXT ·CasRel(SB), NOSPLIT, $0-17
18 B ·Cas(SB)
19
20 TEXT ·Loadint32(SB), NOSPLIT, $0-12
21 B ·Load(SB)
22
23 TEXT ·Loadint64(SB), NOSPLIT, $0-16
24 B ·Load64(SB)
25
26 TEXT ·Loaduintptr(SB), NOSPLIT, $0-16
27 B ·Load64(SB)
28
29 TEXT ·Loaduint(SB), NOSPLIT, $0-16
30 B ·Load64(SB)
31
32 TEXT ·Storeint32(SB), NOSPLIT, $0-12
33 B ·Store(SB)
34
35 TEXT ·Storeint64(SB), NOSPLIT, $0-16
36 B ·Store64(SB)
37
38 TEXT ·Storeuintptr(SB), NOSPLIT, $0-16
39 B ·Store64(SB)
40
41 TEXT ·Xaddint32(SB), NOSPLIT, $0-20
42 B ·Xadd(SB)
43
44 TEXT ·Xaddint64(SB), NOSPLIT, $0-24
45 B ·Xadd64(SB)
46
47 TEXT ·Xadduintptr(SB), NOSPLIT, $0-24
48 B ·Xadd64(SB)
49
50 TEXT ·Casp1(SB), NOSPLIT, $0-25
51 B ·Cas64(SB)
52
53 // uint32 ·Load(uint32 volatile* addr)
54 TEXT ·Load(SB),NOSPLIT,$0-12
55 MOVD ptr+0(FP), R0
56 LDARW (R0), R0
57 MOVW R0, ret+8(FP)
58 RET
59
60 // uint8 ·Load8(uint8 volatile* addr)
61 TEXT ·Load8(SB),NOSPLIT,$0-9
62 MOVD ptr+0(FP), R0
63 LDARB (R0), R0
64 MOVB R0, ret+8(FP)
65 RET
66
67 // uint64 ·Load64(uint64 volatile* addr)
68 TEXT ·Load64(SB),NOSPLIT,$0-16
69 MOVD ptr+0(FP), R0
70 LDAR (R0), R0
71 MOVD R0, ret+8(FP)
72 RET
73
74 // void *·Loadp(void *volatile *addr)
75 TEXT ·Loadp(SB),NOSPLIT,$0-16
76 MOVD ptr+0(FP), R0
77 LDAR (R0), R0
78 MOVD R0, ret+8(FP)
79 RET
80
81 // uint32 ·LoadAcq(uint32 volatile* addr)
82 TEXT ·LoadAcq(SB),NOSPLIT,$0-12
83 B ·Load(SB)
84
85 // uint64 ·LoadAcquintptr(uint64 volatile* addr)
86 TEXT ·LoadAcq64(SB),NOSPLIT,$0-16
87 B ·Load64(SB)
88
89 // uintptr ·LoadAcq64(uintptr volatile* addr)
90 TEXT ·LoadAcquintptr(SB),NOSPLIT,$0-16
91 B ·Load64(SB)
92
93 TEXT ·StorepNoWB(SB), NOSPLIT, $0-16
94 B ·Store64(SB)
95
96 TEXT ·StoreRel(SB), NOSPLIT, $0-12
97 B ·Store(SB)
98
99 TEXT ·StoreRel64(SB), NOSPLIT, $0-16
100 B ·Store64(SB)
101
102 TEXT ·StoreReluintptr(SB), NOSPLIT, $0-16
103 B ·Store64(SB)
104
105 TEXT ·Store(SB), NOSPLIT, $0-12
106 MOVD ptr+0(FP), R0
107 MOVW val+8(FP), R1
108 STLRW R1, (R0)
109 RET
110
111 TEXT ·Store8(SB), NOSPLIT, $0-9
112 MOVD ptr+0(FP), R0
113 MOVB val+8(FP), R1
114 STLRB R1, (R0)
115 RET
116
117 TEXT ·Store64(SB), NOSPLIT, $0-16
118 MOVD ptr+0(FP), R0
119 MOVD val+8(FP), R1
120 STLR R1, (R0)
121 RET
122
123 // uint32 Xchg(ptr *uint32, new uint32)
124 // Atomically:
125 // old := *ptr;
126 // *ptr = new;
127 // return old;
128 TEXT ·Xchg(SB), NOSPLIT, $0-20
129 MOVD ptr+0(FP), R0
130 MOVW new+8(FP), R1
131 MOVBU internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
132 CBZ R4, load_store_loop
133 SWPALW R1, (R0), R2
134 MOVW R2, ret+16(FP)
135 RET
136 load_store_loop:
137 LDAXRW (R0), R2
138 STLXRW R1, (R0), R3
139 CBNZ R3, load_store_loop
140 MOVW R2, ret+16(FP)
141 RET
142
143 // uint64 Xchg64(ptr *uint64, new uint64)
144 // Atomically:
145 // old := *ptr;
146 // *ptr = new;
147 // return old;
148 TEXT ·Xchg64(SB), NOSPLIT, $0-24
149 MOVD ptr+0(FP), R0
150 MOVD new+8(FP), R1
151 MOVBU internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
152 CBZ R4, load_store_loop
153 SWPALD R1, (R0), R2
154 MOVD R2, ret+16(FP)
155 RET
156 load_store_loop:
157 LDAXR (R0), R2
158 STLXR R1, (R0), R3
159 CBNZ R3, load_store_loop
160 MOVD R2, ret+16(FP)
161 RET
162
163 // bool Cas(uint32 *ptr, uint32 old, uint32 new)
164 // Atomically:
165 // if(*val == old){
166 // *val = new;
167 // return 1;
168 // } else
169 // return 0;
170 TEXT ·Cas(SB), NOSPLIT, $0-17
171 MOVD ptr+0(FP), R0
172 MOVW old+8(FP), R1
173 MOVW new+12(FP), R2
174 MOVBU internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
175 CBZ R4, load_store_loop
176 MOVD R1, R3
177 CASALW R3, (R0), R2
178 CMP R1, R3
179 CSET EQ, R0
180 MOVB R0, ret+16(FP)
181 RET
182 load_store_loop:
183 LDAXRW (R0), R3
184 CMPW R1, R3
185 BNE ok
186 STLXRW R2, (R0), R3
187 CBNZ R3, load_store_loop
188 ok:
189 CSET EQ, R0
190 MOVB R0, ret+16(FP)
191 RET
192
193 // bool ·Cas64(uint64 *ptr, uint64 old, uint64 new)
194 // Atomically:
195 // if(*val == old){
196 // *val = new;
197 // return 1;
198 // } else {
199 // return 0;
200 // }
201 TEXT ·Cas64(SB), NOSPLIT, $0-25
202 MOVD ptr+0(FP), R0
203 MOVD old+8(FP), R1
204 MOVD new+16(FP), R2
205 MOVBU internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
206 CBZ R4, load_store_loop
207 MOVD R1, R3
208 CASALD R3, (R0), R2
209 CMP R1, R3
210 CSET EQ, R0
211 MOVB R0, ret+24(FP)
212 RET
213 load_store_loop:
214 LDAXR (R0), R3
215 CMP R1, R3
216 BNE ok
217 STLXR R2, (R0), R3
218 CBNZ R3, load_store_loop
219 ok:
220 CSET EQ, R0
221 MOVB R0, ret+24(FP)
222 RET
223
224 // uint32 xadd(uint32 volatile *ptr, int32 delta)
225 // Atomically:
226 // *val += delta;
227 // return *val;
228 TEXT ·Xadd(SB), NOSPLIT, $0-20
229 MOVD ptr+0(FP), R0
230 MOVW delta+8(FP), R1
231 MOVBU internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
232 CBZ R4, load_store_loop
233 LDADDALW R1, (R0), R2
234 ADD R1, R2
235 MOVW R2, ret+16(FP)
236 RET
237 load_store_loop:
238 LDAXRW (R0), R2
239 ADDW R2, R1, R2
240 STLXRW R2, (R0), R3
241 CBNZ R3, load_store_loop
242 MOVW R2, ret+16(FP)
243 RET
244
245 // uint64 Xadd64(uint64 volatile *ptr, int64 delta)
246 // Atomically:
247 // *val += delta;
248 // return *val;
249 TEXT ·Xadd64(SB), NOSPLIT, $0-24
250 MOVD ptr+0(FP), R0
251 MOVD delta+8(FP), R1
252 MOVBU internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
253 CBZ R4, load_store_loop
254 LDADDALD R1, (R0), R2
255 ADD R1, R2
256 MOVD R2, ret+16(FP)
257 RET
258 load_store_loop:
259 LDAXR (R0), R2
260 ADD R2, R1, R2
261 STLXR R2, (R0), R3
262 CBNZ R3, load_store_loop
263 MOVD R2, ret+16(FP)
264 RET
265
266 TEXT ·Xchgint32(SB), NOSPLIT, $0-20
267 B ·Xchg(SB)
268
269 TEXT ·Xchgint64(SB), NOSPLIT, $0-24
270 B ·Xchg64(SB)
271
272 TEXT ·Xchguintptr(SB), NOSPLIT, $0-24
273 B ·Xchg64(SB)
274
275 TEXT ·And8(SB), NOSPLIT, $0-9
276 MOVD ptr+0(FP), R0
277 MOVB val+8(FP), R1
278 MOVBU internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
279 CBZ R4, load_store_loop
280 MVN R1, R2
281 LDCLRALB R2, (R0), R3
282 RET
283 load_store_loop:
284 LDAXRB (R0), R2
285 AND R1, R2
286 STLXRB R2, (R0), R3
287 CBNZ R3, load_store_loop
288 RET
289
290 TEXT ·Or8(SB), NOSPLIT, $0-9
291 MOVD ptr+0(FP), R0
292 MOVB val+8(FP), R1
293 MOVBU internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
294 CBZ R4, load_store_loop
295 LDORALB R1, (R0), R2
296 RET
297 load_store_loop:
298 LDAXRB (R0), R2
299 ORR R1, R2
300 STLXRB R2, (R0), R3
301 CBNZ R3, load_store_loop
302 RET
303
304 // func And(addr *uint32, v uint32)
305 TEXT ·And(SB), NOSPLIT, $0-12
306 MOVD ptr+0(FP), R0
307 MOVW val+8(FP), R1
308 MOVBU internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
309 CBZ R4, load_store_loop
310 MVN R1, R2
311 LDCLRALW R2, (R0), R3
312 RET
313 load_store_loop:
314 LDAXRW (R0), R2
315 AND R1, R2
316 STLXRW R2, (R0), R3
317 CBNZ R3, load_store_loop
318 RET
319
320 // func Or(addr *uint32, v uint32)
321 TEXT ·Or(SB), NOSPLIT, $0-12
322 MOVD ptr+0(FP), R0
323 MOVW val+8(FP), R1
324 MOVBU internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
325 CBZ R4, load_store_loop
326 LDORALW R1, (R0), R2
327 RET
328 load_store_loop:
329 LDAXRW (R0), R2
330 ORR R1, R2
331 STLXRW R2, (R0), R3
332 CBNZ R3, load_store_loop
333 RET
334
335 // func Or32(addr *uint32, v uint32) old uint32
336 TEXT ·Or32(SB), NOSPLIT, $0-20
337 MOVD ptr+0(FP), R0
338 MOVW val+8(FP), R1
339 MOVBU internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
340 CBZ R4, load_store_loop
341 LDORALW R1, (R0), R2
342 MOVD R2, ret+16(FP)
343 RET
344 load_store_loop:
345 LDAXRW (R0), R2
346 ORR R1, R2, R3
347 STLXRW R3, (R0), R4
348 CBNZ R4, load_store_loop
349 MOVD R2, ret+16(FP)
350 RET
351
352 // func And32(addr *uint32, v uint32) old uint32
353 TEXT ·And32(SB), NOSPLIT, $0-20
354 MOVD ptr+0(FP), R0
355 MOVW val+8(FP), R1
356 MOVBU internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
357 CBZ R4, load_store_loop
358 MVN R1, R2
359 LDCLRALW R2, (R0), R3
360 MOVD R3, ret+16(FP)
361 RET
362 load_store_loop:
363 LDAXRW (R0), R2
364 AND R1, R2, R3
365 STLXRW R3, (R0), R4
366 CBNZ R4, load_store_loop
367 MOVD R2, ret+16(FP)
368 RET
369
370 // func Or64(addr *uint64, v uint64) old uint64
371 TEXT ·Or64(SB), NOSPLIT, $0-24
372 MOVD ptr+0(FP), R0
373 MOVD val+8(FP), R1
374 MOVBU internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
375 CBZ R4, load_store_loop
376 LDORALD R1, (R0), R2
377 MOVD R2, ret+16(FP)
378 RET
379 load_store_loop:
380 LDAXR (R0), R2
381 ORR R1, R2, R3
382 STLXR R3, (R0), R4
383 CBNZ R4, load_store_loop
384 MOVD R2, ret+16(FP)
385 RET
386
387 // func And64(addr *uint64, v uint64) old uint64
388 TEXT ·And64(SB), NOSPLIT, $0-24
389 MOVD ptr+0(FP), R0
390 MOVD val+8(FP), R1
391 MOVBU internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
392 CBZ R4, load_store_loop
393 MVN R1, R2
394 LDCLRALD R2, (R0), R3
395 MOVD R3, ret+16(FP)
396 RET
397 load_store_loop:
398 LDAXR (R0), R2
399 AND R1, R2, R3
400 STLXR R3, (R0), R4
401 CBNZ R4, load_store_loop
402 MOVD R2, ret+16(FP)
403 RET
404
405 // func Anduintptr(addr *uintptr, v uintptr) old uintptr
406 TEXT ·Anduintptr(SB), NOSPLIT, $0-24
407 B ·And64(SB)
408
409 // func Oruintptr(addr *uintptr, v uintptr) old uintptr
410 TEXT ·Oruintptr(SB), NOSPLIT, $0-24
411 B ·Or64(SB)
412
View as plain text