...
Run Format

Text file src/sync/atomic/asm_arm.s

Documentation: sync/atomic

     1	// Copyright 2011 The Go Authors. All rights reserved.
     2	// Use of this source code is governed by a BSD-style
     3	// license that can be found in the LICENSE file.
     4	
     5	// +build !race
     6	
     7	#include "textflag.h"
     8	
     9	// ARM atomic operations, for use by asm_$(GOOS)_arm.s.
    10	
    11	#define DMB_ISHST_7 \
    12		MOVB	runtime·goarm(SB), R11; \
    13		CMP	$7, R11; \
    14		BLT	2(PC); \
    15		WORD	$0xf57ff05a	// dmb ishst
    16	
    17	#define DMB_ISH_7 \
    18		MOVB	runtime·goarm(SB), R11; \
    19		CMP	$7, R11; \
    20		BLT	2(PC); \
    21		WORD	$0xf57ff05b	// dmb ish
    22	
    23	TEXT ·armCompareAndSwapUint32(SB),NOSPLIT,$0-13
    24		MOVW	addr+0(FP), R1
    25		MOVW	old+4(FP), R2
    26		MOVW	new+8(FP), R3
    27	casloop:
    28		// LDREX and STREX were introduced in ARMv6.
    29		LDREX	(R1), R0
    30		CMP	R0, R2
    31		BNE	casfail
    32		DMB_ISHST_7
    33		STREX	R3, (R1), R0
    34		CMP	$0, R0
    35		BNE	casloop
    36		MOVW	$1, R0
    37		DMB_ISH_7
    38		MOVBU	R0, swapped+12(FP)
    39		RET
    40	casfail:
    41		MOVW	$0, R0
    42		MOVBU	R0, swapped+12(FP)
    43		RET
    44	
    45	TEXT ·armCompareAndSwapUint64(SB),NOSPLIT,$0-21
    46		BL	fastCheck64<>(SB)
    47		MOVW	addr+0(FP), R1
    48		// make unaligned atomic access panic
    49		AND.S	$7, R1, R2
    50		BEQ 	2(PC)
    51		MOVW	R2, (R2)
    52		MOVW	old_lo+4(FP), R2
    53		MOVW	old_hi+8(FP), R3
    54		MOVW	new_lo+12(FP), R4
    55		MOVW	new_hi+16(FP), R5
    56	cas64loop:
    57		// LDREXD and STREXD were introduced in ARMv6k.
    58		LDREXD	(R1), R6  // loads R6 and R7
    59		CMP	R2, R6
    60		BNE	cas64fail
    61		CMP	R3, R7
    62		BNE	cas64fail
    63		DMB_ISHST_7
    64		STREXD	R4, (R1), R0	// stores R4 and R5
    65		CMP	$0, R0
    66		BNE	cas64loop
    67		MOVW	$1, R0
    68		DMB_ISH_7
    69		MOVBU	R0, swapped+20(FP)
    70		RET
    71	cas64fail:
    72		MOVW	$0, R0
    73		MOVBU	R0, swapped+20(FP)
    74		RET
    75	
    76	TEXT ·armAddUint32(SB),NOSPLIT,$0-12
    77		MOVW	addr+0(FP), R1
    78		MOVW	delta+4(FP), R2
    79	addloop:
    80		// LDREX and STREX were introduced in ARMv6.
    81		LDREX	(R1), R3
    82		ADD	R2, R3
    83		DMB_ISHST_7
    84		STREX	R3, (R1), R0
    85		CMP	$0, R0
    86		BNE	addloop
    87		DMB_ISH_7
    88		MOVW	R3, new+8(FP)
    89		RET
    90	
    91	TEXT ·armAddUint64(SB),NOSPLIT,$0-20
    92		BL	fastCheck64<>(SB)
    93		MOVW	addr+0(FP), R1
    94		// make unaligned atomic access panic
    95		AND.S	$7, R1, R2
    96		BEQ 	2(PC)
    97		MOVW	R2, (R2)
    98		MOVW	delta_lo+4(FP), R2
    99		MOVW	delta_hi+8(FP), R3
   100	add64loop:
   101		// LDREXD and STREXD were introduced in ARMv6k.
   102		LDREXD	(R1), R4	// loads R4 and R5
   103		ADD.S	R2, R4
   104		ADC	R3, R5
   105		DMB_ISHST_7
   106		STREXD	R4, (R1), R0	// stores R4 and R5
   107		CMP	$0, R0
   108		BNE	add64loop
   109		DMB_ISH_7
   110		MOVW	R4, new_lo+12(FP)
   111		MOVW	R5, new_hi+16(FP)
   112		RET
   113	
   114	TEXT ·armSwapUint32(SB),NOSPLIT,$0-12
   115		MOVW	addr+0(FP), R1
   116		MOVW	new+4(FP), R2
   117	swaploop:
   118		// LDREX and STREX were introduced in ARMv6.
   119		LDREX	(R1), R3
   120		DMB_ISHST_7
   121		STREX	R2, (R1), R0
   122		CMP	$0, R0
   123		BNE	swaploop
   124		DMB_ISH_7
   125		MOVW	R3, old+8(FP)
   126		RET
   127	
   128	TEXT ·armSwapUint64(SB),NOSPLIT,$0-20
   129		BL	fastCheck64<>(SB)
   130		MOVW	addr+0(FP), R1
   131		// make unaligned atomic access panic
   132		AND.S	$7, R1, R2
   133		BEQ 	2(PC)
   134		MOVW	R2, (R2)
   135		MOVW	new_lo+4(FP), R2
   136		MOVW	new_hi+8(FP), R3
   137	swap64loop:
   138		// LDREXD and STREXD were introduced in ARMv6k.
   139		LDREXD	(R1), R4	// loads R4 and R5
   140		DMB_ISHST_7
   141		STREXD	R2, (R1), R0	// stores R2 and R3
   142		CMP	$0, R0
   143		BNE	swap64loop
   144		DMB_ISH_7
   145		MOVW	R4, old_lo+12(FP)
   146		MOVW	R5, old_hi+16(FP)
   147		RET
   148	
   149	TEXT ·armLoadUint64(SB),NOSPLIT,$0-12
   150		BL	fastCheck64<>(SB)
   151		MOVW	addr+0(FP), R1
   152		// make unaligned atomic access panic
   153		AND.S	$7, R1, R2
   154		BEQ 	2(PC)
   155		MOVW	R2, (R2)
   156	load64loop:
   157		LDREXD	(R1), R2	// loads R2 and R3
   158		DMB_ISHST_7
   159		STREXD	R2, (R1), R0	// stores R2 and R3
   160		CMP	$0, R0
   161		BNE	load64loop
   162		DMB_ISH_7
   163		MOVW	R2, val_lo+4(FP)
   164		MOVW	R3, val_hi+8(FP)
   165		RET
   166	
   167	TEXT ·armStoreUint64(SB),NOSPLIT,$0-12
   168		BL	fastCheck64<>(SB)
   169		MOVW	addr+0(FP), R1
   170		// make unaligned atomic access panic
   171		AND.S	$7, R1, R2
   172		BEQ 	2(PC)
   173		MOVW	R2, (R2)
   174		MOVW	val_lo+4(FP), R2
   175		MOVW	val_hi+8(FP), R3
   176	store64loop:
   177		LDREXD	(R1), R4	// loads R4 and R5
   178		DMB_ISHST_7
   179		STREXD	R2, (R1), R0	// stores R2 and R3
   180		CMP	$0, R0
   181		BNE	store64loop
   182		DMB_ISH_7
   183		RET
   184	
   185	// Check for broken 64-bit LDREXD as found in QEMU.
   186	// LDREXD followed by immediate STREXD should succeed.
   187	// If it fails, try a few times just to be sure (maybe our thread got
   188	// rescheduled between the two instructions) and then panic.
   189	// A bug in some copies of QEMU makes STREXD never succeed,
   190	// which will make uses of the 64-bit atomic operations loop forever.
   191	// If things are working, set okLDREXD to avoid future checks.
   192	// https://bugs.launchpad.net/qemu/+bug/670883.
   193	TEXT	check64<>(SB),NOSPLIT,$16-0
   194		MOVW	$10, R1
   195		// 8-aligned stack address scratch space.
   196		MOVW	$8(R13), R5
   197		AND	$~7, R5
   198	loop:
   199		LDREXD	(R5), R2
   200		STREXD	R2, (R5), R0
   201		CMP	$0, R0
   202		BEQ	ok
   203		SUB	$1, R1
   204		CMP	$0, R1
   205		BNE	loop
   206		// Must be buggy QEMU.
   207		BL	·panic64(SB)
   208	ok:
   209		RET
   210	
   211	// Fast, cached version of check. No frame, just MOVW CMP RET after first time.
   212	TEXT	fastCheck64<>(SB),NOSPLIT,$-4
   213		MOVW	ok64<>(SB), R0
   214		CMP	$0, R0	// have we been here before?
   215		RET.NE
   216		B	slowCheck64<>(SB)
   217	
   218	TEXT slowCheck64<>(SB),NOSPLIT,$0-0
   219		BL	check64<>(SB)
   220		// Still here, must be okay.
   221		MOVW	$1, R0
   222		MOVW	R0, ok64<>(SB)
   223		RET
   224	
   225	GLOBL ok64<>(SB), NOPTR, $4

View as plain text