...
Run Format

Text file src/runtime/asm_arm64.s

Documentation: runtime

     1	// Copyright 2015 The Go Authors. All rights reserved.
     2	// Use of this source code is governed by a BSD-style
     3	// license that can be found in the LICENSE file.
     4	
     5	#include "go_asm.h"
     6	#include "go_tls.h"
     7	#include "tls_arm64.h"
     8	#include "funcdata.h"
     9	#include "textflag.h"
    10	
    11	TEXT runtime·rt0_go(SB),NOSPLIT,$0
    12		// SP = stack; R0 = argc; R1 = argv
    13	
    14		SUB	$32, RSP
    15		MOVW	R0, 8(RSP) // argc
    16		MOVD	R1, 16(RSP) // argv
    17	
    18		// create istack out of the given (operating system) stack.
    19		// _cgo_init may update stackguard.
    20		MOVD	$runtime·g0(SB), g
    21		MOVD RSP, R7
    22		MOVD	$(-64*1024)(R7), R0
    23		MOVD	R0, g_stackguard0(g)
    24		MOVD	R0, g_stackguard1(g)
    25		MOVD	R0, (g_stack+stack_lo)(g)
    26		MOVD	R7, (g_stack+stack_hi)(g)
    27	
    28		// if there is a _cgo_init, call it using the gcc ABI.
    29		MOVD	_cgo_init(SB), R12
    30		CMP	$0, R12
    31		BEQ	nocgo
    32	
    33		MRS_TPIDR_R0			// load TLS base pointer
    34		MOVD	R0, R3			// arg 3: TLS base pointer
    35	#ifdef TLSG_IS_VARIABLE
    36		MOVD	$runtime·tls_g(SB), R2 	// arg 2: &tls_g
    37	#else
    38		MOVD	$0, R2		        // arg 2: not used when using platform's TLS
    39	#endif
    40		MOVD	$setg_gcc<>(SB), R1	// arg 1: setg
    41		MOVD	g, R0			// arg 0: G
    42		BL	(R12)
    43		MOVD	_cgo_init(SB), R12
    44		CMP	$0, R12
    45		BEQ	nocgo
    46	
    47	nocgo:
    48		// update stackguard after _cgo_init
    49		MOVD	(g_stack+stack_lo)(g), R0
    50		ADD	$const__StackGuard, R0
    51		MOVD	R0, g_stackguard0(g)
    52		MOVD	R0, g_stackguard1(g)
    53	
    54		// set the per-goroutine and per-mach "registers"
    55		MOVD	$runtime·m0(SB), R0
    56	
    57		// save m->g0 = g0
    58		MOVD	g, m_g0(R0)
    59		// save m0 to g0->m
    60		MOVD	R0, g_m(g)
    61	
    62		BL	runtime·check(SB)
    63	
    64		MOVW	8(RSP), R0	// copy argc
    65		MOVW	R0, -8(RSP)
    66		MOVD	16(RSP), R0		// copy argv
    67		MOVD	R0, 0(RSP)
    68		BL	runtime·args(SB)
    69		BL	runtime·osinit(SB)
    70		BL	runtime·schedinit(SB)
    71	
    72		// create a new goroutine to start program
    73		MOVD	$runtime·mainPC(SB), R0		// entry
    74		MOVD	RSP, R7
    75		MOVD.W	$0, -8(R7)
    76		MOVD.W	R0, -8(R7)
    77		MOVD.W	$0, -8(R7)
    78		MOVD.W	$0, -8(R7)
    79		MOVD	R7, RSP
    80		BL	runtime·newproc(SB)
    81		ADD	$32, RSP
    82	
    83		// start this M
    84		BL	runtime·mstart(SB)
    85	
    86		MOVD	$0, R0
    87		MOVD	R0, (R0)	// boom
    88		UNDEF
    89	
    90	DATA	runtime·mainPC+0(SB)/8,$runtime·main(SB)
    91	GLOBL	runtime·mainPC(SB),RODATA,$8
    92	
    93	TEXT runtime·breakpoint(SB),NOSPLIT|NOFRAME,$0-0
    94		BRK
    95		RET
    96	
    97	TEXT runtime·asminit(SB),NOSPLIT|NOFRAME,$0-0
    98		RET
    99	
   100	/*
   101	 *  go-routine
   102	 */
   103	
   104	// void gosave(Gobuf*)
   105	// save state in Gobuf; setjmp
   106	TEXT runtime·gosave(SB), NOSPLIT|NOFRAME, $0-8
   107		MOVD	buf+0(FP), R3
   108		MOVD	RSP, R0
   109		MOVD	R0, gobuf_sp(R3)
   110		MOVD	LR, gobuf_pc(R3)
   111		MOVD	g, gobuf_g(R3)
   112		MOVD	ZR, gobuf_lr(R3)
   113		MOVD	ZR, gobuf_ret(R3)
   114		// Assert ctxt is zero. See func save.
   115		MOVD	gobuf_ctxt(R3), R0
   116		CMP	$0, R0
   117		BEQ	2(PC)
   118		CALL	runtime·badctxt(SB)
   119		RET
   120	
   121	// void gogo(Gobuf*)
   122	// restore state from Gobuf; longjmp
   123	TEXT runtime·gogo(SB), NOSPLIT, $24-8
   124		MOVD	buf+0(FP), R5
   125		MOVD	gobuf_g(R5), g
   126		BL	runtime·save_g(SB)
   127	
   128		MOVD	0(g), R4	// make sure g is not nil
   129		MOVD	gobuf_sp(R5), R0
   130		MOVD	R0, RSP
   131		MOVD	gobuf_lr(R5), LR
   132		MOVD	gobuf_ret(R5), R0
   133		MOVD	gobuf_ctxt(R5), R26
   134		MOVD	$0, gobuf_sp(R5)
   135		MOVD	$0, gobuf_ret(R5)
   136		MOVD	$0, gobuf_lr(R5)
   137		MOVD	$0, gobuf_ctxt(R5)
   138		CMP	ZR, ZR // set condition codes for == test, needed by stack split
   139		MOVD	gobuf_pc(R5), R6
   140		B	(R6)
   141	
   142	// void mcall(fn func(*g))
   143	// Switch to m->g0's stack, call fn(g).
   144	// Fn must never return. It should gogo(&g->sched)
   145	// to keep running g.
   146	TEXT runtime·mcall(SB), NOSPLIT|NOFRAME, $0-8
   147		// Save caller state in g->sched
   148		MOVD	RSP, R0
   149		MOVD	R0, (g_sched+gobuf_sp)(g)
   150		MOVD	LR, (g_sched+gobuf_pc)(g)
   151		MOVD	$0, (g_sched+gobuf_lr)(g)
   152		MOVD	g, (g_sched+gobuf_g)(g)
   153	
   154		// Switch to m->g0 & its stack, call fn.
   155		MOVD	g, R3
   156		MOVD	g_m(g), R8
   157		MOVD	m_g0(R8), g
   158		BL	runtime·save_g(SB)
   159		CMP	g, R3
   160		BNE	2(PC)
   161		B	runtime·badmcall(SB)
   162		MOVD	fn+0(FP), R26			// context
   163		MOVD	0(R26), R4			// code pointer
   164		MOVD	(g_sched+gobuf_sp)(g), R0
   165		MOVD	R0, RSP	// sp = m->g0->sched.sp
   166		MOVD	R3, -8(RSP)
   167		MOVD	$0, -16(RSP)
   168		SUB	$16, RSP
   169		BL	(R4)
   170		B	runtime·badmcall2(SB)
   171	
   172	// systemstack_switch is a dummy routine that systemstack leaves at the bottom
   173	// of the G stack. We need to distinguish the routine that
   174	// lives at the bottom of the G stack from the one that lives
   175	// at the top of the system stack because the one at the top of
   176	// the system stack terminates the stack walk (see topofstack()).
   177	TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0
   178		UNDEF
   179		BL	(LR)	// make sure this function is not leaf
   180		RET
   181	
   182	// func systemstack(fn func())
   183	TEXT runtime·systemstack(SB), NOSPLIT, $0-8
   184		MOVD	fn+0(FP), R3	// R3 = fn
   185		MOVD	R3, R26		// context
   186		MOVD	g_m(g), R4	// R4 = m
   187	
   188		MOVD	m_gsignal(R4), R5	// R5 = gsignal
   189		CMP	g, R5
   190		BEQ	noswitch
   191	
   192		MOVD	m_g0(R4), R5	// R5 = g0
   193		CMP	g, R5
   194		BEQ	noswitch
   195	
   196		MOVD	m_curg(R4), R6
   197		CMP	g, R6
   198		BEQ	switch
   199	
   200		// Bad: g is not gsignal, not g0, not curg. What is it?
   201		// Hide call from linker nosplit analysis.
   202		MOVD	$runtime·badsystemstack(SB), R3
   203		BL	(R3)
   204		B	runtime·abort(SB)
   205	
   206	switch:
   207		// save our state in g->sched. Pretend to
   208		// be systemstack_switch if the G stack is scanned.
   209		MOVD	$runtime·systemstack_switch(SB), R6
   210		ADD	$8, R6	// get past prologue
   211		MOVD	R6, (g_sched+gobuf_pc)(g)
   212		MOVD	RSP, R0
   213		MOVD	R0, (g_sched+gobuf_sp)(g)
   214		MOVD	$0, (g_sched+gobuf_lr)(g)
   215		MOVD	g, (g_sched+gobuf_g)(g)
   216	
   217		// switch to g0
   218		MOVD	R5, g
   219		BL	runtime·save_g(SB)
   220		MOVD	(g_sched+gobuf_sp)(g), R3
   221		// make it look like mstart called systemstack on g0, to stop traceback
   222		SUB	$16, R3
   223		AND	$~15, R3
   224		MOVD	$runtime·mstart(SB), R4
   225		MOVD	R4, 0(R3)
   226		MOVD	R3, RSP
   227	
   228		// call target function
   229		MOVD	0(R26), R3	// code pointer
   230		BL	(R3)
   231	
   232		// switch back to g
   233		MOVD	g_m(g), R3
   234		MOVD	m_curg(R3), g
   235		BL	runtime·save_g(SB)
   236		MOVD	(g_sched+gobuf_sp)(g), R0
   237		MOVD	R0, RSP
   238		MOVD	$0, (g_sched+gobuf_sp)(g)
   239		RET
   240	
   241	noswitch:
   242		// already on m stack, just call directly
   243		// Using a tail call here cleans up tracebacks since we won't stop
   244		// at an intermediate systemstack.
   245		MOVD	0(R26), R3	// code pointer
   246		MOVD.P	16(RSP), R30	// restore LR
   247		B	(R3)
   248	
   249	/*
   250	 * support for morestack
   251	 */
   252	
   253	// Called during function prolog when more stack is needed.
   254	// Caller has already loaded:
   255	// R3 prolog's LR (R30)
   256	//
   257	// The traceback routines see morestack on a g0 as being
   258	// the top of a stack (for example, morestack calling newstack
   259	// calling the scheduler calling newm calling gc), so we must
   260	// record an argument size. For that purpose, it has no arguments.
   261	TEXT runtime·morestack(SB),NOSPLIT|NOFRAME,$0-0
   262		// Cannot grow scheduler stack (m->g0).
   263		MOVD	g_m(g), R8
   264		MOVD	m_g0(R8), R4
   265		CMP	g, R4
   266		BNE	3(PC)
   267		BL	runtime·badmorestackg0(SB)
   268		B	runtime·abort(SB)
   269	
   270		// Cannot grow signal stack (m->gsignal).
   271		MOVD	m_gsignal(R8), R4
   272		CMP	g, R4
   273		BNE	3(PC)
   274		BL	runtime·badmorestackgsignal(SB)
   275		B	runtime·abort(SB)
   276	
   277		// Called from f.
   278		// Set g->sched to context in f
   279		MOVD	RSP, R0
   280		MOVD	R0, (g_sched+gobuf_sp)(g)
   281		MOVD	LR, (g_sched+gobuf_pc)(g)
   282		MOVD	R3, (g_sched+gobuf_lr)(g)
   283		MOVD	R26, (g_sched+gobuf_ctxt)(g)
   284	
   285		// Called from f.
   286		// Set m->morebuf to f's callers.
   287		MOVD	R3, (m_morebuf+gobuf_pc)(R8)	// f's caller's PC
   288		MOVD	RSP, R0
   289		MOVD	R0, (m_morebuf+gobuf_sp)(R8)	// f's caller's RSP
   290		MOVD	g, (m_morebuf+gobuf_g)(R8)
   291	
   292		// Call newstack on m->g0's stack.
   293		MOVD	m_g0(R8), g
   294		BL	runtime·save_g(SB)
   295		MOVD	(g_sched+gobuf_sp)(g), R0
   296		MOVD	R0, RSP
   297		MOVD.W	$0, -16(RSP)	// create a call frame on g0 (saved LR; keep 16-aligned)
   298		BL	runtime·newstack(SB)
   299	
   300		// Not reached, but make sure the return PC from the call to newstack
   301		// is still in this function, and not the beginning of the next.
   302		UNDEF
   303	
   304	TEXT runtime·morestack_noctxt(SB),NOSPLIT|NOFRAME,$0-0
   305		MOVW	$0, R26
   306		B runtime·morestack(SB)
   307	
   308	// reflectcall: call a function with the given argument list
   309	// func call(argtype *_type, f *FuncVal, arg *byte, argsize, retoffset uint32).
   310	// we don't have variable-sized frames, so we use a small number
   311	// of constant-sized-frame functions to encode a few bits of size in the pc.
   312	// Caution: ugly multiline assembly macros in your future!
   313	
   314	#define DISPATCH(NAME,MAXSIZE)		\
   315		MOVD	$MAXSIZE, R27;		\
   316		CMP	R27, R16;		\
   317		BGT	3(PC);			\
   318		MOVD	$NAME(SB), R27;	\
   319		B	(R27)
   320	// Note: can't just "B NAME(SB)" - bad inlining results.
   321	
   322	TEXT reflect·call(SB), NOSPLIT, $0-0
   323		B	·reflectcall(SB)
   324	
   325	TEXT ·reflectcall(SB), NOSPLIT|NOFRAME, $0-32
   326		MOVWU argsize+24(FP), R16
   327		DISPATCH(runtime·call32, 32)
   328		DISPATCH(runtime·call64, 64)
   329		DISPATCH(runtime·call128, 128)
   330		DISPATCH(runtime·call256, 256)
   331		DISPATCH(runtime·call512, 512)
   332		DISPATCH(runtime·call1024, 1024)
   333		DISPATCH(runtime·call2048, 2048)
   334		DISPATCH(runtime·call4096, 4096)
   335		DISPATCH(runtime·call8192, 8192)
   336		DISPATCH(runtime·call16384, 16384)
   337		DISPATCH(runtime·call32768, 32768)
   338		DISPATCH(runtime·call65536, 65536)
   339		DISPATCH(runtime·call131072, 131072)
   340		DISPATCH(runtime·call262144, 262144)
   341		DISPATCH(runtime·call524288, 524288)
   342		DISPATCH(runtime·call1048576, 1048576)
   343		DISPATCH(runtime·call2097152, 2097152)
   344		DISPATCH(runtime·call4194304, 4194304)
   345		DISPATCH(runtime·call8388608, 8388608)
   346		DISPATCH(runtime·call16777216, 16777216)
   347		DISPATCH(runtime·call33554432, 33554432)
   348		DISPATCH(runtime·call67108864, 67108864)
   349		DISPATCH(runtime·call134217728, 134217728)
   350		DISPATCH(runtime·call268435456, 268435456)
   351		DISPATCH(runtime·call536870912, 536870912)
   352		DISPATCH(runtime·call1073741824, 1073741824)
   353		MOVD	$runtime·badreflectcall(SB), R0
   354		B	(R0)
   355	
   356	#define CALLFN(NAME,MAXSIZE)			\
   357	TEXT NAME(SB), WRAPPER, $MAXSIZE-24;		\
   358		NO_LOCAL_POINTERS;			\
   359		/* copy arguments to stack */		\
   360		MOVD	arg+16(FP), R3;			\
   361		MOVWU	argsize+24(FP), R4;		\
   362		ADD	$8, RSP, R5;			\
   363		BIC	$0xf, R4, R6;			\
   364		CBZ	R6, 6(PC);			\
   365		/* if R6=(argsize&~15) != 0 */		\
   366		ADD	R6, R5, R6;			\
   367		/* copy 16 bytes a time */		\
   368		LDP.P	16(R3), (R7, R8);		\
   369		STP.P	(R7, R8), 16(R5);		\
   370		CMP	R5, R6;				\
   371		BNE	-3(PC);				\
   372		AND	$0xf, R4, R6;			\
   373		CBZ	R6, 6(PC);			\
   374		/* if R6=(argsize&15) != 0 */		\
   375		ADD	R6, R5, R6;			\
   376		/* copy 1 byte a time for the rest */	\
   377		MOVBU.P	1(R3), R7;			\
   378		MOVBU.P	R7, 1(R5);			\
   379		CMP	R5, R6;				\
   380		BNE	-3(PC);				\
   381		/* call function */			\
   382		MOVD	f+8(FP), R26;			\
   383		MOVD	(R26), R0;			\
   384		PCDATA  $PCDATA_StackMapIndex, $0;	\
   385		BL	(R0);				\
   386		/* copy return values back */		\
   387		MOVD	argtype+0(FP), R7;		\
   388		MOVD	arg+16(FP), R3;			\
   389		MOVWU	n+24(FP), R4;			\
   390		MOVWU	retoffset+28(FP), R6;		\
   391		ADD	$8, RSP, R5;			\
   392		ADD	R6, R5; 			\
   393		ADD	R6, R3;				\
   394		SUB	R6, R4;				\
   395		BL	callRet<>(SB);			\
   396		RET
   397	
   398	// callRet copies return values back at the end of call*. This is a
   399	// separate function so it can allocate stack space for the arguments
   400	// to reflectcallmove. It does not follow the Go ABI; it expects its
   401	// arguments in registers.
   402	TEXT callRet<>(SB), NOSPLIT, $40-0
   403		MOVD	R7, 8(RSP)
   404		MOVD	R3, 16(RSP)
   405		MOVD	R5, 24(RSP)
   406		MOVD	R4, 32(RSP)
   407		BL	runtime·reflectcallmove(SB)
   408		RET
   409	
   410	// These have 8 added to make the overall frame size a multiple of 16,
   411	// as required by the ABI. (There is another +8 for the saved LR.)
   412	CALLFN(·call32, 40 )
   413	CALLFN(·call64, 72 )
   414	CALLFN(·call128, 136 )
   415	CALLFN(·call256, 264 )
   416	CALLFN(·call512, 520 )
   417	CALLFN(·call1024, 1032 )
   418	CALLFN(·call2048, 2056 )
   419	CALLFN(·call4096, 4104 )
   420	CALLFN(·call8192, 8200 )
   421	CALLFN(·call16384, 16392 )
   422	CALLFN(·call32768, 32776 )
   423	CALLFN(·call65536, 65544 )
   424	CALLFN(·call131072, 131080 )
   425	CALLFN(·call262144, 262152 )
   426	CALLFN(·call524288, 524296 )
   427	CALLFN(·call1048576, 1048584 )
   428	CALLFN(·call2097152, 2097160 )
   429	CALLFN(·call4194304, 4194312 )
   430	CALLFN(·call8388608, 8388616 )
   431	CALLFN(·call16777216, 16777224 )
   432	CALLFN(·call33554432, 33554440 )
   433	CALLFN(·call67108864, 67108872 )
   434	CALLFN(·call134217728, 134217736 )
   435	CALLFN(·call268435456, 268435464 )
   436	CALLFN(·call536870912, 536870920 )
   437	CALLFN(·call1073741824, 1073741832 )
   438	
   439	// func aeshash32(p unsafe.Pointer, h uintptr) uintptr
   440	TEXT runtime·aeshash32(SB),NOSPLIT|NOFRAME,$0-24
   441		MOVD	p+0(FP), R0
   442		MOVD	h+8(FP), R1
   443		MOVD	$ret+16(FP), R2
   444		MOVD	$runtime·aeskeysched+0(SB), R3
   445	
   446		VEOR	V0.B16, V0.B16, V0.B16
   447		VLD1	(R3), [V2.B16]
   448		VLD1	(R0), V0.S[1]
   449		VMOV	R1, V0.S[0]
   450	
   451		AESE	V2.B16, V0.B16
   452		AESMC	V0.B16, V0.B16
   453		AESE	V2.B16, V0.B16
   454		AESMC	V0.B16, V0.B16
   455		AESE	V2.B16, V0.B16
   456	
   457		VST1	[V0.D1], (R2)
   458		RET
   459	
   460	// func aeshash64(p unsafe.Pointer, h uintptr) uintptr
   461	TEXT runtime·aeshash64(SB),NOSPLIT|NOFRAME,$0-24
   462		MOVD	p+0(FP), R0
   463		MOVD	h+8(FP), R1
   464		MOVD	$ret+16(FP), R2
   465		MOVD	$runtime·aeskeysched+0(SB), R3
   466	
   467		VEOR	V0.B16, V0.B16, V0.B16
   468		VLD1	(R3), [V2.B16]
   469		VLD1	(R0), V0.D[1]
   470		VMOV	R1, V0.D[0]
   471	
   472		AESE	V2.B16, V0.B16
   473		AESMC	V0.B16, V0.B16
   474		AESE	V2.B16, V0.B16
   475		AESMC	V0.B16, V0.B16
   476		AESE	V2.B16, V0.B16
   477	
   478		VST1	[V0.D1], (R2)
   479		RET
   480	
   481	// func aeshash(p unsafe.Pointer, h, size uintptr) uintptr
   482	TEXT runtime·aeshash(SB),NOSPLIT|NOFRAME,$0-32
   483		MOVD	p+0(FP), R0
   484		MOVD	s+16(FP), R1
   485		MOVWU	h+8(FP), R3
   486		MOVD	$ret+24(FP), R2
   487		B	aeshashbody<>(SB)
   488	
   489	// func aeshashstr(p unsafe.Pointer, h uintptr) uintptr
   490	TEXT runtime·aeshashstr(SB),NOSPLIT|NOFRAME,$0-24
   491		MOVD	p+0(FP), R10 // string pointer
   492		LDP	(R10), (R0, R1) //string data/ length
   493		MOVWU	h+8(FP), R3
   494		MOVD	$ret+16(FP), R2 // return adddress
   495		B	aeshashbody<>(SB)
   496	
   497	// R0: data
   498	// R1: length (maximum 32 bits)
   499	// R2: address to put return value
   500	// R3: seed data
   501	TEXT aeshashbody<>(SB),NOSPLIT|NOFRAME,$0
   502		VEOR	V30.B16, V30.B16, V30.B16
   503		VMOV	R3, V30.S[0]
   504		VMOV	R1, V30.S[1] // load length into seed
   505	
   506		MOVD	$runtime·aeskeysched+0(SB), R4
   507		VLD1.P	16(R4), [V0.B16]
   508		AESE	V30.B16, V0.B16
   509		AESMC	V0.B16, V0.B16
   510		CMP	$16, R1
   511		BLO	aes0to15
   512		BEQ	aes16
   513		CMP	$32, R1
   514		BLS	aes17to32
   515		CMP	$64, R1
   516		BLS	aes33to64
   517		CMP	$128, R1
   518		BLS	aes65to128
   519		B	aes129plus
   520	
   521	aes0to15:
   522		CMP	$0, R1
   523		BEQ	aes0
   524		VEOR	V2.B16, V2.B16, V2.B16
   525		TBZ	$3, R1, less_than_8
   526		VLD1.P	8(R0), V2.D[0]
   527	
   528	less_than_8:
   529		TBZ	$2, R1, less_than_4
   530		VLD1.P	4(R0), V2.S[2]
   531	
   532	less_than_4:
   533		TBZ	$1, R1, less_than_2
   534		VLD1.P	2(R0), V2.H[6]
   535	
   536	less_than_2:
   537		TBZ	$0, R1, done
   538		VLD1	(R0), V2.B[14]
   539	done:
   540		AESE	V0.B16, V2.B16
   541		AESMC	V2.B16, V2.B16
   542		AESE	V0.B16, V2.B16
   543		AESMC	V2.B16, V2.B16
   544		AESE	V0.B16, V2.B16
   545	
   546		VST1	[V2.D1], (R2)
   547		RET
   548	aes0:
   549		VST1	[V0.D1], (R2)
   550		RET
   551	aes16:
   552		VLD1	(R0), [V2.B16]
   553		B	done
   554	
   555	aes17to32:
   556		// make second seed
   557		VLD1	(R4), [V1.B16]
   558		AESE	V30.B16, V1.B16
   559		AESMC	V1.B16, V1.B16
   560		SUB	$16, R1, R10
   561		VLD1.P	(R0)(R10), [V2.B16]
   562		VLD1	(R0), [V3.B16]
   563	
   564		AESE	V0.B16, V2.B16
   565		AESMC	V2.B16, V2.B16
   566		AESE	V1.B16, V3.B16
   567		AESMC	V3.B16, V3.B16
   568	
   569		AESE	V0.B16, V2.B16
   570		AESMC	V2.B16, V2.B16
   571		AESE	V1.B16, V3.B16
   572		AESMC	V3.B16, V3.B16
   573	
   574		AESE	V0.B16, V2.B16
   575		AESE	V1.B16, V3.B16
   576	
   577		VEOR	V3.B16, V2.B16, V2.B16
   578		VST1	[V2.D1], (R2)
   579		RET
   580	
   581	aes33to64:
   582		VLD1	(R4), [V1.B16, V2.B16, V3.B16]
   583		AESE	V30.B16, V1.B16
   584		AESMC	V1.B16, V1.B16
   585		AESE	V30.B16, V2.B16
   586		AESMC	V2.B16, V2.B16
   587		AESE	V30.B16, V3.B16
   588		AESMC	V3.B16, V3.B16
   589		SUB	$32, R1, R10
   590	
   591		VLD1.P	(R0)(R10), [V4.B16, V5.B16]
   592		VLD1	(R0), [V6.B16, V7.B16]
   593	
   594		AESE	V0.B16, V4.B16
   595		AESMC	V4.B16, V4.B16
   596		AESE	V1.B16, V5.B16
   597		AESMC	V5.B16, V5.B16
   598		AESE	V2.B16, V6.B16
   599		AESMC	V6.B16, V6.B16
   600		AESE	V3.B16, V7.B16
   601		AESMC	V7.B16, V7.B16
   602	
   603		AESE	V0.B16, V4.B16
   604		AESMC	V4.B16, V4.B16
   605		AESE	V1.B16, V5.B16
   606		AESMC	V5.B16, V5.B16
   607		AESE	V2.B16, V6.B16
   608		AESMC	V6.B16, V6.B16
   609		AESE	V3.B16, V7.B16
   610		AESMC	V7.B16, V7.B16
   611	
   612		AESE	V0.B16, V4.B16
   613		AESE	V1.B16, V5.B16
   614		AESE	V2.B16, V6.B16
   615		AESE	V3.B16, V7.B16
   616	
   617		VEOR	V6.B16, V4.B16, V4.B16
   618		VEOR	V7.B16, V5.B16, V5.B16
   619		VEOR	V5.B16, V4.B16, V4.B16
   620	
   621		VST1	[V4.D1], (R2)
   622		RET
   623	
   624	aes65to128:
   625		VLD1.P	64(R4), [V1.B16, V2.B16, V3.B16, V4.B16]
   626		VLD1	(R4), [V5.B16, V6.B16, V7.B16]
   627		AESE	V30.B16, V1.B16
   628		AESMC	V1.B16, V1.B16
   629		AESE	V30.B16, V2.B16
   630		AESMC	V2.B16, V2.B16
   631		AESE	V30.B16, V3.B16
   632		AESMC	V3.B16, V3.B16
   633		AESE	V30.B16, V4.B16
   634		AESMC	V4.B16, V4.B16
   635		AESE	V30.B16, V5.B16
   636		AESMC	V5.B16, V5.B16
   637		AESE	V30.B16, V6.B16
   638		AESMC	V6.B16, V6.B16
   639		AESE	V30.B16, V7.B16
   640		AESMC	V7.B16, V7.B16
   641	
   642		SUB	$64, R1, R10
   643		VLD1.P	(R0)(R10), [V8.B16, V9.B16, V10.B16, V11.B16]
   644		VLD1	(R0), [V12.B16, V13.B16, V14.B16, V15.B16]
   645		AESE	V0.B16,	 V8.B16
   646		AESMC	V8.B16,  V8.B16
   647		AESE	V1.B16,	 V9.B16
   648		AESMC	V9.B16,  V9.B16
   649		AESE	V2.B16, V10.B16
   650		AESMC	V10.B16,  V10.B16
   651		AESE	V3.B16, V11.B16
   652		AESMC	V11.B16,  V11.B16
   653		AESE	V4.B16, V12.B16
   654		AESMC	V12.B16,  V12.B16
   655		AESE	V5.B16, V13.B16
   656		AESMC	V13.B16,  V13.B16
   657		AESE	V6.B16, V14.B16
   658		AESMC	V14.B16,  V14.B16
   659		AESE	V7.B16, V15.B16
   660		AESMC	V15.B16,  V15.B16
   661	
   662		AESE	V0.B16,	 V8.B16
   663		AESMC	V8.B16,  V8.B16
   664		AESE	V1.B16,	 V9.B16
   665		AESMC	V9.B16,  V9.B16
   666		AESE	V2.B16, V10.B16
   667		AESMC	V10.B16,  V10.B16
   668		AESE	V3.B16, V11.B16
   669		AESMC	V11.B16,  V11.B16
   670		AESE	V4.B16, V12.B16
   671		AESMC	V12.B16,  V12.B16
   672		AESE	V5.B16, V13.B16
   673		AESMC	V13.B16,  V13.B16
   674		AESE	V6.B16, V14.B16
   675		AESMC	V14.B16,  V14.B16
   676		AESE	V7.B16, V15.B16
   677		AESMC	V15.B16,  V15.B16
   678	
   679		AESE	V0.B16,	 V8.B16
   680		AESE	V1.B16,	 V9.B16
   681		AESE	V2.B16, V10.B16
   682		AESE	V3.B16, V11.B16
   683		AESE	V4.B16, V12.B16
   684		AESE	V5.B16, V13.B16
   685		AESE	V6.B16, V14.B16
   686		AESE	V7.B16, V15.B16
   687	
   688		VEOR	V12.B16, V8.B16, V8.B16
   689		VEOR	V13.B16, V9.B16, V9.B16
   690		VEOR	V14.B16, V10.B16, V10.B16
   691		VEOR	V15.B16, V11.B16, V11.B16
   692		VEOR	V10.B16, V8.B16, V8.B16
   693		VEOR	V11.B16, V9.B16, V9.B16
   694		VEOR	V9.B16, V8.B16, V8.B16
   695	
   696		VST1	[V8.D1], (R2)
   697		RET
   698	
   699	aes129plus:
   700		PRFM (R0), PLDL1KEEP
   701		VLD1.P	64(R4), [V1.B16, V2.B16, V3.B16, V4.B16]
   702		VLD1	(R4), [V5.B16, V6.B16, V7.B16]
   703		AESE	V30.B16, V1.B16
   704		AESMC	V1.B16, V1.B16
   705		AESE	V30.B16, V2.B16
   706		AESMC	V2.B16, V2.B16
   707		AESE	V30.B16, V3.B16
   708		AESMC	V3.B16, V3.B16
   709		AESE	V30.B16, V4.B16
   710		AESMC	V4.B16, V4.B16
   711		AESE	V30.B16, V5.B16
   712		AESMC	V5.B16, V5.B16
   713		AESE	V30.B16, V6.B16
   714		AESMC	V6.B16, V6.B16
   715		AESE	V30.B16, V7.B16
   716		AESMC	V7.B16, V7.B16
   717		ADD	R0, R1, R10
   718		SUB	$128, R10, R10
   719		VLD1.P	64(R10), [V8.B16, V9.B16, V10.B16, V11.B16]
   720		VLD1	(R10), [V12.B16, V13.B16, V14.B16, V15.B16]
   721		SUB	$1, R1, R1
   722		LSR	$7, R1, R1
   723	
   724	aesloop:
   725		AESE	V8.B16,	 V0.B16
   726		AESMC	V0.B16,  V0.B16
   727		AESE	V9.B16,	 V1.B16
   728		AESMC	V1.B16,  V1.B16
   729		AESE	V10.B16, V2.B16
   730		AESMC	V2.B16,  V2.B16
   731		AESE	V11.B16, V3.B16
   732		AESMC	V3.B16,  V3.B16
   733		AESE	V12.B16, V4.B16
   734		AESMC	V4.B16,  V4.B16
   735		AESE	V13.B16, V5.B16
   736		AESMC	V5.B16,  V5.B16
   737		AESE	V14.B16, V6.B16
   738		AESMC	V6.B16,  V6.B16
   739		AESE	V15.B16, V7.B16
   740		AESMC	V7.B16,  V7.B16
   741	
   742		VLD1.P	64(R0), [V8.B16, V9.B16, V10.B16, V11.B16]
   743		AESE	V8.B16,	 V0.B16
   744		AESMC	V0.B16,  V0.B16
   745		AESE	V9.B16,	 V1.B16
   746		AESMC	V1.B16,  V1.B16
   747		AESE	V10.B16, V2.B16
   748		AESMC	V2.B16,  V2.B16
   749		AESE	V11.B16, V3.B16
   750		AESMC	V3.B16,  V3.B16
   751	
   752		VLD1.P	64(R0), [V12.B16, V13.B16, V14.B16, V15.B16]
   753		AESE	V12.B16, V4.B16
   754		AESMC	V4.B16,  V4.B16
   755		AESE	V13.B16, V5.B16
   756		AESMC	V5.B16,  V5.B16
   757		AESE	V14.B16, V6.B16
   758		AESMC	V6.B16,  V6.B16
   759		AESE	V15.B16, V7.B16
   760		AESMC	V7.B16,  V7.B16
   761		SUB	$1, R1, R1
   762		CBNZ	R1, aesloop
   763	
   764		AESE	V8.B16,	 V0.B16
   765		AESMC	V0.B16,  V0.B16
   766		AESE	V9.B16,	 V1.B16
   767		AESMC	V1.B16,  V1.B16
   768		AESE	V10.B16, V2.B16
   769		AESMC	V2.B16,  V2.B16
   770		AESE	V11.B16, V3.B16
   771		AESMC	V3.B16,  V3.B16
   772		AESE	V12.B16, V4.B16
   773		AESMC	V4.B16,  V4.B16
   774		AESE	V13.B16, V5.B16
   775		AESMC	V5.B16,  V5.B16
   776		AESE	V14.B16, V6.B16
   777		AESMC	V6.B16,  V6.B16
   778		AESE	V15.B16, V7.B16
   779		AESMC	V7.B16,  V7.B16
   780	
   781		AESE	V8.B16,	 V0.B16
   782		AESMC	V0.B16,  V0.B16
   783		AESE	V9.B16,	 V1.B16
   784		AESMC	V1.B16,  V1.B16
   785		AESE	V10.B16, V2.B16
   786		AESMC	V2.B16,  V2.B16
   787		AESE	V11.B16, V3.B16
   788		AESMC	V3.B16,  V3.B16
   789		AESE	V12.B16, V4.B16
   790		AESMC	V4.B16,  V4.B16
   791		AESE	V13.B16, V5.B16
   792		AESMC	V5.B16,  V5.B16
   793		AESE	V14.B16, V6.B16
   794		AESMC	V6.B16,  V6.B16
   795		AESE	V15.B16, V7.B16
   796		AESMC	V7.B16,  V7.B16
   797	
   798		AESE	V8.B16,	 V0.B16
   799		AESE	V9.B16,	 V1.B16
   800		AESE	V10.B16, V2.B16
   801		AESE	V11.B16, V3.B16
   802		AESE	V12.B16, V4.B16
   803		AESE	V13.B16, V5.B16
   804		AESE	V14.B16, V6.B16
   805		AESE	V15.B16, V7.B16
   806	
   807		VEOR	V0.B16, V1.B16, V0.B16
   808		VEOR	V2.B16, V3.B16, V2.B16
   809		VEOR	V4.B16, V5.B16, V4.B16
   810		VEOR	V6.B16, V7.B16, V6.B16
   811		VEOR	V0.B16, V2.B16, V0.B16
   812		VEOR	V4.B16, V6.B16, V4.B16
   813		VEOR	V4.B16, V0.B16, V0.B16
   814	
   815		VST1	[V0.D1], (R2)
   816		RET
   817	
   818	TEXT runtime·procyield(SB),NOSPLIT,$0-0
   819		MOVWU	cycles+0(FP), R0
   820	again:
   821		YIELD
   822		SUBW	$1, R0
   823		CBNZ	R0, again
   824		RET
   825	
   826	// void jmpdefer(fv, sp);
   827	// called from deferreturn.
   828	// 1. grab stored LR for caller
   829	// 2. sub 4 bytes to get back to BL deferreturn
   830	// 3. BR to fn
   831	TEXT runtime·jmpdefer(SB), NOSPLIT|NOFRAME, $0-16
   832		MOVD	0(RSP), R0
   833		SUB	$4, R0
   834		MOVD	R0, LR
   835	
   836		MOVD	fv+0(FP), R26
   837		MOVD	argp+8(FP), R0
   838		MOVD	R0, RSP
   839		SUB	$8, RSP
   840		MOVD	0(R26), R3
   841		B	(R3)
   842	
   843	// Save state of caller into g->sched. Smashes R0.
   844	TEXT gosave<>(SB),NOSPLIT|NOFRAME,$0
   845		MOVD	LR, (g_sched+gobuf_pc)(g)
   846		MOVD RSP, R0
   847		MOVD	R0, (g_sched+gobuf_sp)(g)
   848		MOVD	$0, (g_sched+gobuf_lr)(g)
   849		MOVD	$0, (g_sched+gobuf_ret)(g)
   850		// Assert ctxt is zero. See func save.
   851		MOVD	(g_sched+gobuf_ctxt)(g), R0
   852		CMP	$0, R0
   853		BEQ	2(PC)
   854		CALL	runtime·badctxt(SB)
   855		RET
   856	
   857	// func asmcgocall(fn, arg unsafe.Pointer) int32
   858	// Call fn(arg) on the scheduler stack,
   859	// aligned appropriately for the gcc ABI.
   860	// See cgocall.go for more details.
   861	TEXT ·asmcgocall(SB),NOSPLIT,$0-20
   862		MOVD	fn+0(FP), R1
   863		MOVD	arg+8(FP), R0
   864	
   865		MOVD	RSP, R2		// save original stack pointer
   866		CMP	$0, g
   867		BEQ	nosave
   868		MOVD	g, R4
   869	
   870		// Figure out if we need to switch to m->g0 stack.
   871		// We get called to create new OS threads too, and those
   872		// come in on the m->g0 stack already.
   873		MOVD	g_m(g), R8
   874		MOVD	m_gsignal(R8), R3
   875		CMP	R3, g
   876		BEQ	nosave
   877		MOVD	m_g0(R8), R3
   878		CMP	R3, g
   879		BEQ	nosave
   880	
   881		// Switch to system stack.
   882		MOVD	R0, R9	// gosave<> and save_g might clobber R0
   883		BL	gosave<>(SB)
   884		MOVD	R3, g
   885		BL	runtime·save_g(SB)
   886		MOVD	(g_sched+gobuf_sp)(g), R0
   887		MOVD	R0, RSP
   888		MOVD	R9, R0
   889	
   890		// Now on a scheduling stack (a pthread-created stack).
   891		// Save room for two of our pointers /*, plus 32 bytes of callee
   892		// save area that lives on the caller stack. */
   893		MOVD	RSP, R13
   894		SUB	$16, R13
   895		MOVD	R13, RSP
   896		MOVD	R4, 0(RSP)	// save old g on stack
   897		MOVD	(g_stack+stack_hi)(R4), R4
   898		SUB	R2, R4
   899		MOVD	R4, 8(RSP)	// save depth in old g stack (can't just save SP, as stack might be copied during a callback)
   900		BL	(R1)
   901		MOVD	R0, R9
   902	
   903		// Restore g, stack pointer. R0 is errno, so don't touch it
   904		MOVD	0(RSP), g
   905		BL	runtime·save_g(SB)
   906		MOVD	(g_stack+stack_hi)(g), R5
   907		MOVD	8(RSP), R6
   908		SUB	R6, R5
   909		MOVD	R9, R0
   910		MOVD	R5, RSP
   911	
   912		MOVW	R0, ret+16(FP)
   913		RET
   914	
   915	nosave:
   916		// Running on a system stack, perhaps even without a g.
   917		// Having no g can happen during thread creation or thread teardown
   918		// (see needm/dropm on Solaris, for example).
   919		// This code is like the above sequence but without saving/restoring g
   920		// and without worrying about the stack moving out from under us
   921		// (because we're on a system stack, not a goroutine stack).
   922		// The above code could be used directly if already on a system stack,
   923		// but then the only path through this code would be a rare case on Solaris.
   924		// Using this code for all "already on system stack" calls exercises it more,
   925		// which should help keep it correct.
   926		MOVD	RSP, R13
   927		SUB	$16, R13
   928		MOVD	R13, RSP
   929		MOVD	$0, R4
   930		MOVD	R4, 0(RSP)	// Where above code stores g, in case someone looks during debugging.
   931		MOVD	R2, 8(RSP)	// Save original stack pointer.
   932		BL	(R1)
   933		// Restore stack pointer.
   934		MOVD	8(RSP), R2
   935		MOVD	R2, RSP	
   936		MOVD	R0, ret+16(FP)
   937		RET
   938	
   939	// cgocallback(void (*fn)(void*), void *frame, uintptr framesize, uintptr ctxt)
   940	// Turn the fn into a Go func (by taking its address) and call
   941	// cgocallback_gofunc.
   942	TEXT runtime·cgocallback(SB),NOSPLIT,$40-32
   943		MOVD	$fn+0(FP), R0
   944		MOVD	R0, 8(RSP)
   945		MOVD	frame+8(FP), R0
   946		MOVD	R0, 16(RSP)
   947		MOVD	framesize+16(FP), R0
   948		MOVD	R0, 24(RSP)
   949		MOVD	ctxt+24(FP), R0
   950		MOVD	R0, 32(RSP)
   951		MOVD	$runtime·cgocallback_gofunc(SB), R0
   952		BL	(R0)
   953		RET
   954	
   955	// cgocallback_gofunc(FuncVal*, void *frame, uintptr framesize, uintptr ctxt)
   956	// See cgocall.go for more details.
   957	TEXT ·cgocallback_gofunc(SB),NOSPLIT,$24-32
   958		NO_LOCAL_POINTERS
   959	
   960		// Load g from thread-local storage.
   961		MOVB	runtime·iscgo(SB), R3
   962		CMP	$0, R3
   963		BEQ	nocgo
   964		BL	runtime·load_g(SB)
   965	nocgo:
   966	
   967		// If g is nil, Go did not create the current thread.
   968		// Call needm to obtain one for temporary use.
   969		// In this case, we're running on the thread stack, so there's
   970		// lots of space, but the linker doesn't know. Hide the call from
   971		// the linker analysis by using an indirect call.
   972		CMP	$0, g
   973		BEQ	needm
   974	
   975		MOVD	g_m(g), R8
   976		MOVD	R8, savedm-8(SP)
   977		B	havem
   978	
   979	needm:
   980		MOVD	g, savedm-8(SP) // g is zero, so is m.
   981		MOVD	$runtime·needm(SB), R0
   982		BL	(R0)
   983	
   984		// Set m->sched.sp = SP, so that if a panic happens
   985		// during the function we are about to execute, it will
   986		// have a valid SP to run on the g0 stack.
   987		// The next few lines (after the havem label)
   988		// will save this SP onto the stack and then write
   989		// the same SP back to m->sched.sp. That seems redundant,
   990		// but if an unrecovered panic happens, unwindm will
   991		// restore the g->sched.sp from the stack location
   992		// and then systemstack will try to use it. If we don't set it here,
   993		// that restored SP will be uninitialized (typically 0) and
   994		// will not be usable.
   995		MOVD	g_m(g), R8
   996		MOVD	m_g0(R8), R3
   997		MOVD	RSP, R0
   998		MOVD	R0, (g_sched+gobuf_sp)(R3)
   999	
  1000	havem:
  1001		// Now there's a valid m, and we're running on its m->g0.
  1002		// Save current m->g0->sched.sp on stack and then set it to SP.
  1003		// Save current sp in m->g0->sched.sp in preparation for
  1004		// switch back to m->curg stack.
  1005		// NOTE: unwindm knows that the saved g->sched.sp is at 16(RSP) aka savedsp-16(SP).
  1006		// Beware that the frame size is actually 32.
  1007		MOVD	m_g0(R8), R3
  1008		MOVD	(g_sched+gobuf_sp)(R3), R4
  1009		MOVD	R4, savedsp-16(SP)
  1010		MOVD	RSP, R0
  1011		MOVD	R0, (g_sched+gobuf_sp)(R3)
  1012	
  1013		// Switch to m->curg stack and call runtime.cgocallbackg.
  1014		// Because we are taking over the execution of m->curg
  1015		// but *not* resuming what had been running, we need to
  1016		// save that information (m->curg->sched) so we can restore it.
  1017		// We can restore m->curg->sched.sp easily, because calling
  1018		// runtime.cgocallbackg leaves SP unchanged upon return.
  1019		// To save m->curg->sched.pc, we push it onto the stack.
  1020		// This has the added benefit that it looks to the traceback
  1021		// routine like cgocallbackg is going to return to that
  1022		// PC (because the frame we allocate below has the same
  1023		// size as cgocallback_gofunc's frame declared above)
  1024		// so that the traceback will seamlessly trace back into
  1025		// the earlier calls.
  1026		//
  1027		// In the new goroutine, -8(SP) is unused (where SP refers to
  1028		// m->curg's SP while we're setting it up, before we've adjusted it).
  1029		MOVD	m_curg(R8), g
  1030		BL	runtime·save_g(SB)
  1031		MOVD	(g_sched+gobuf_sp)(g), R4 // prepare stack as R4
  1032		MOVD	(g_sched+gobuf_pc)(g), R5
  1033		MOVD	R5, -(24+8)(R4)
  1034		MOVD	ctxt+24(FP), R0
  1035		MOVD	R0, -(16+8)(R4)
  1036		MOVD	$-(24+8)(R4), R0 // maintain 16-byte SP alignment
  1037		MOVD	R0, RSP
  1038		BL	runtime·cgocallbackg(SB)
  1039	
  1040		// Restore g->sched (== m->curg->sched) from saved values.
  1041		MOVD	0(RSP), R5
  1042		MOVD	R5, (g_sched+gobuf_pc)(g)
  1043		MOVD	RSP, R4
  1044		ADD	$(24+8), R4, R4
  1045		MOVD	R4, (g_sched+gobuf_sp)(g)
  1046	
  1047		// Switch back to m->g0's stack and restore m->g0->sched.sp.
  1048		// (Unlike m->curg, the g0 goroutine never uses sched.pc,
  1049		// so we do not have to restore it.)
  1050		MOVD	g_m(g), R8
  1051		MOVD	m_g0(R8), g
  1052		BL	runtime·save_g(SB)
  1053		MOVD	(g_sched+gobuf_sp)(g), R0
  1054		MOVD	R0, RSP
  1055		MOVD	savedsp-16(SP), R4
  1056		MOVD	R4, (g_sched+gobuf_sp)(g)
  1057	
  1058		// If the m on entry was nil, we called needm above to borrow an m
  1059		// for the duration of the call. Since the call is over, return it with dropm.
  1060		MOVD	savedm-8(SP), R6
  1061		CMP	$0, R6
  1062		BNE	droppedm
  1063		MOVD	$runtime·dropm(SB), R0
  1064		BL	(R0)
  1065	droppedm:
  1066	
  1067		// Done!
  1068		RET
  1069	
  1070	// Called from cgo wrappers, this function returns g->m->curg.stack.hi.
  1071	// Must obey the gcc calling convention.
  1072	TEXT _cgo_topofstack(SB),NOSPLIT,$24
  1073		// g (R28) and REGTMP (R27)  might be clobbered by load_g. They
  1074		// are callee-save in the gcc calling convention, so save them.
  1075		MOVD	R27, savedR27-8(SP)
  1076		MOVD	g, saveG-16(SP)
  1077	
  1078		BL	runtime·load_g(SB)
  1079		MOVD	g_m(g), R0
  1080		MOVD	m_curg(R0), R0
  1081		MOVD	(g_stack+stack_hi)(R0), R0
  1082	
  1083		MOVD	saveG-16(SP), g
  1084		MOVD	savedR28-8(SP), R27
  1085		RET
  1086	
  1087	// void setg(G*); set g. for use by needm.
  1088	TEXT runtime·setg(SB), NOSPLIT, $0-8
  1089		MOVD	gg+0(FP), g
  1090		// This only happens if iscgo, so jump straight to save_g
  1091		BL	runtime·save_g(SB)
  1092		RET
  1093	
  1094	// void setg_gcc(G*); set g called from gcc
  1095	TEXT setg_gcc<>(SB),NOSPLIT,$8
  1096		MOVD	R0, g
  1097		MOVD	R27, savedR27-8(SP)
  1098		BL	runtime·save_g(SB)
  1099		MOVD	savedR27-8(SP), R27
  1100		RET
  1101	
  1102	TEXT runtime·abort(SB),NOSPLIT|NOFRAME,$0-0
  1103		MOVD	ZR, R0
  1104		MOVD	(R0), R0
  1105		UNDEF
  1106	
  1107	TEXT runtime·return0(SB), NOSPLIT, $0
  1108		MOVW	$0, R0
  1109		RET
  1110	
  1111	// The top-most function running on a goroutine
  1112	// returns to goexit+PCQuantum.
  1113	TEXT runtime·goexit(SB),NOSPLIT|NOFRAME,$0-0
  1114		MOVD	R0, R0	// NOP
  1115		BL	runtime·goexit1(SB)	// does not return
  1116	
  1117	TEXT runtime·sigreturn(SB),NOSPLIT,$0-0
  1118		RET
  1119	
  1120	// This is called from .init_array and follows the platform, not Go, ABI.
  1121	TEXT runtime·addmoduledata(SB),NOSPLIT,$0-0
  1122		SUB	$0x10, RSP
  1123		MOVD	R27, 8(RSP) // The access to global variables below implicitly uses R27, which is callee-save
  1124		MOVD	runtime·lastmoduledatap(SB), R1
  1125		MOVD	R0, moduledata_next(R1)
  1126		MOVD	R0, runtime·lastmoduledatap(SB)
  1127		MOVD	8(RSP), R27
  1128		ADD	$0x10, RSP
  1129		RET
  1130	
  1131	TEXT ·checkASM(SB),NOSPLIT,$0-1
  1132		MOVW	$1, R3
  1133		MOVB	R3, ret+0(FP)
  1134		RET
  1135	
  1136	// gcWriteBarrier performs a heap pointer write and informs the GC.
  1137	//
  1138	// gcWriteBarrier does NOT follow the Go ABI. It takes two arguments:
  1139	// - R2 is the destination of the write
  1140	// - R3 is the value being written at R2
  1141	// It clobbers condition codes.
  1142	// It does not clobber any general-purpose registers,
  1143	// but may clobber others (e.g., floating point registers)
  1144	// The act of CALLing gcWriteBarrier will clobber R30 (LR).
  1145	TEXT runtime·gcWriteBarrier(SB),NOSPLIT,$216
  1146		// Save the registers clobbered by the fast path.
  1147		MOVD	R0, 200(RSP)
  1148		MOVD	R1, 208(RSP)
  1149		MOVD	g_m(g), R0
  1150		MOVD	m_p(R0), R0
  1151		MOVD	(p_wbBuf+wbBuf_next)(R0), R1
  1152		// Increment wbBuf.next position.
  1153		ADD	$16, R1
  1154		MOVD	R1, (p_wbBuf+wbBuf_next)(R0)
  1155		MOVD	(p_wbBuf+wbBuf_end)(R0), R0
  1156		CMP	R1, R0
  1157		// Record the write.
  1158		MOVD	R3, -16(R1)	// Record value
  1159		MOVD	(R2), R0	// TODO: This turns bad writes into bad reads.
  1160		MOVD	R0, -8(R1)	// Record *slot
  1161		// Is the buffer full? (flags set in CMP above)
  1162		BEQ	flush
  1163	ret:
  1164		MOVD	200(RSP), R0
  1165		MOVD	208(RSP), R1
  1166		// Do the write.
  1167		MOVD	R3, (R2)
  1168		RET
  1169	
  1170	flush:
  1171		// Save all general purpose registers since these could be
  1172		// clobbered by wbBufFlush and were not saved by the caller.
  1173		MOVD	R2, 8(RSP)	// Also first argument to wbBufFlush
  1174		MOVD	R3, 16(RSP)	// Also second argument to wbBufFlush
  1175		// R0 already saved
  1176		// R1 already saved
  1177		MOVD	R4, 24(RSP)
  1178		MOVD	R5, 32(RSP)
  1179		MOVD	R6, 40(RSP)
  1180		MOVD	R7, 48(RSP)
  1181		MOVD	R8, 56(RSP)
  1182		MOVD	R9, 64(RSP)
  1183		MOVD	R10, 72(RSP)
  1184		MOVD	R11, 80(RSP)
  1185		MOVD	R12, 88(RSP)
  1186		MOVD	R13, 96(RSP)
  1187		MOVD	R14, 104(RSP)
  1188		MOVD	R15, 112(RSP)
  1189		MOVD	R16, 120(RSP)
  1190		MOVD	R17, 128(RSP)
  1191		// R18 is unused.
  1192		MOVD	R19, 136(RSP)
  1193		MOVD	R20, 144(RSP)
  1194		MOVD	R21, 152(RSP)
  1195		MOVD	R22, 160(RSP)
  1196		MOVD	R23, 168(RSP)
  1197		MOVD	R24, 176(RSP)
  1198		MOVD	R25, 184(RSP)
  1199		MOVD	R26, 192(RSP)
  1200		// R27 is temp register.
  1201		// R28 is g.
  1202		// R29 is frame pointer (unused).
  1203		// R30 is LR, which was saved by the prologue.
  1204		// R31 is SP.
  1205	
  1206		// This takes arguments R2 and R3.
  1207		CALL	runtime·wbBufFlush(SB)
  1208	
  1209		MOVD	8(RSP), R2
  1210		MOVD	16(RSP), R3
  1211		MOVD	24(RSP), R4
  1212		MOVD	32(RSP), R5
  1213		MOVD	40(RSP), R6
  1214		MOVD	48(RSP), R7
  1215		MOVD	56(RSP), R8
  1216		MOVD	64(RSP), R9
  1217		MOVD	72(RSP), R10
  1218		MOVD	80(RSP), R11
  1219		MOVD	88(RSP), R12
  1220		MOVD	96(RSP), R13
  1221		MOVD	104(RSP), R14
  1222		MOVD	112(RSP), R15
  1223		MOVD	120(RSP), R16
  1224		MOVD	128(RSP), R17
  1225		MOVD	136(RSP), R19
  1226		MOVD	144(RSP), R20
  1227		MOVD	152(RSP), R21
  1228		MOVD	160(RSP), R22
  1229		MOVD	168(RSP), R23
  1230		MOVD	176(RSP), R24
  1231		MOVD	184(RSP), R25
  1232		MOVD	192(RSP), R26
  1233		JMP	ret

View as plain text