...
Run Format

Text file src/runtime/asm_arm64.s

Documentation: runtime

     1	// Copyright 2015 The Go Authors. All rights reserved.
     2	// Use of this source code is governed by a BSD-style
     3	// license that can be found in the LICENSE file.
     4	
     5	#include "go_asm.h"
     6	#include "go_tls.h"
     7	#include "tls_arm64.h"
     8	#include "funcdata.h"
     9	#include "textflag.h"
    10	
    11	TEXT runtime·rt0_go(SB),NOSPLIT,$0
    12		// SP = stack; R0 = argc; R1 = argv
    13	
    14		SUB	$32, RSP
    15		MOVW	R0, 8(RSP) // argc
    16		MOVD	R1, 16(RSP) // argv
    17	
    18		// create istack out of the given (operating system) stack.
    19		// _cgo_init may update stackguard.
    20		MOVD	$runtime·g0(SB), g
    21		MOVD RSP, R7
    22		MOVD	$(-64*1024)(R7), R0
    23		MOVD	R0, g_stackguard0(g)
    24		MOVD	R0, g_stackguard1(g)
    25		MOVD	R0, (g_stack+stack_lo)(g)
    26		MOVD	R7, (g_stack+stack_hi)(g)
    27	
    28		// if there is a _cgo_init, call it using the gcc ABI.
    29		MOVD	_cgo_init(SB), R12
    30		CMP	$0, R12
    31		BEQ	nocgo
    32	
    33		MRS_TPIDR_R0			// load TLS base pointer
    34		MOVD	R0, R3			// arg 3: TLS base pointer
    35	#ifdef TLSG_IS_VARIABLE
    36		MOVD	$runtime·tls_g(SB), R2 	// arg 2: &tls_g
    37	#else
    38		MOVD	$0, R2		        // arg 2: not used when using platform's TLS
    39	#endif
    40		MOVD	$setg_gcc<>(SB), R1	// arg 1: setg
    41		MOVD	g, R0			// arg 0: G
    42		BL	(R12)
    43		MOVD	_cgo_init(SB), R12
    44		CMP	$0, R12
    45		BEQ	nocgo
    46	
    47	nocgo:
    48		// update stackguard after _cgo_init
    49		MOVD	(g_stack+stack_lo)(g), R0
    50		ADD	$const__StackGuard, R0
    51		MOVD	R0, g_stackguard0(g)
    52		MOVD	R0, g_stackguard1(g)
    53	
    54		// set the per-goroutine and per-mach "registers"
    55		MOVD	$runtime·m0(SB), R0
    56	
    57		// save m->g0 = g0
    58		MOVD	g, m_g0(R0)
    59		// save m0 to g0->m
    60		MOVD	R0, g_m(g)
    61	
    62		BL	runtime·check(SB)
    63	
    64		MOVW	8(RSP), R0	// copy argc
    65		MOVW	R0, -8(RSP)
    66		MOVD	16(RSP), R0		// copy argv
    67		MOVD	R0, 0(RSP)
    68		BL	runtime·args(SB)
    69		BL	runtime·osinit(SB)
    70		BL	runtime·schedinit(SB)
    71	
    72		// create a new goroutine to start program
    73		MOVD	$runtime·mainPC(SB), R0		// entry
    74		MOVD	RSP, R7
    75		MOVD.W	$0, -8(R7)
    76		MOVD.W	R0, -8(R7)
    77		MOVD.W	$0, -8(R7)
    78		MOVD.W	$0, -8(R7)
    79		MOVD	R7, RSP
    80		BL	runtime·newproc(SB)
    81		ADD	$32, RSP
    82	
    83		// start this M
    84		BL	runtime·mstart(SB)
    85	
    86		MOVD	$0, R0
    87		MOVD	R0, (R0)	// boom
    88		UNDEF
    89	
    90	DATA	runtime·mainPC+0(SB)/8,$runtime·main(SB)
    91	GLOBL	runtime·mainPC(SB),RODATA,$8
    92	
    93	TEXT runtime·breakpoint(SB),NOSPLIT,$-8-0
    94		BRK
    95		RET
    96	
    97	TEXT runtime·asminit(SB),NOSPLIT,$-8-0
    98		RET
    99	
   100	/*
   101	 *  go-routine
   102	 */
   103	
   104	// void gosave(Gobuf*)
   105	// save state in Gobuf; setjmp
   106	TEXT runtime·gosave(SB), NOSPLIT, $-8-8
   107		MOVD	buf+0(FP), R3
   108		MOVD	RSP, R0
   109		MOVD	R0, gobuf_sp(R3)
   110		MOVD	LR, gobuf_pc(R3)
   111		MOVD	g, gobuf_g(R3)
   112		MOVD	ZR, gobuf_lr(R3)
   113		MOVD	ZR, gobuf_ret(R3)
   114		// Assert ctxt is zero. See func save.
   115		MOVD	gobuf_ctxt(R3), R0
   116		CMP	$0, R0
   117		BEQ	2(PC)
   118		CALL	runtime·badctxt(SB)
   119		RET
   120	
   121	// void gogo(Gobuf*)
   122	// restore state from Gobuf; longjmp
   123	TEXT runtime·gogo(SB), NOSPLIT, $24-8
   124		MOVD	buf+0(FP), R5
   125	
   126		// If ctxt is not nil, invoke deletion barrier before overwriting.
   127		MOVD	gobuf_ctxt(R5), R0
   128		CMP	$0, R0
   129		BEQ	nilctxt
   130		MOVD	$gobuf_ctxt(R5), R0
   131		MOVD	R0, 8(RSP)
   132		MOVD	ZR, 16(RSP)
   133		BL	runtime·writebarrierptr_prewrite(SB)
   134		MOVD	buf+0(FP), R5
   135	
   136	nilctxt:
   137		MOVD	gobuf_g(R5), g
   138		BL	runtime·save_g(SB)
   139	
   140		MOVD	0(g), R4	// make sure g is not nil
   141		MOVD	gobuf_sp(R5), R0
   142		MOVD	R0, RSP
   143		MOVD	gobuf_lr(R5), LR
   144		MOVD	gobuf_ret(R5), R0
   145		MOVD	gobuf_ctxt(R5), R26
   146		MOVD	$0, gobuf_sp(R5)
   147		MOVD	$0, gobuf_ret(R5)
   148		MOVD	$0, gobuf_lr(R5)
   149		MOVD	$0, gobuf_ctxt(R5)
   150		CMP	ZR, ZR // set condition codes for == test, needed by stack split
   151		MOVD	gobuf_pc(R5), R6
   152		B	(R6)
   153	
   154	// void mcall(fn func(*g))
   155	// Switch to m->g0's stack, call fn(g).
   156	// Fn must never return. It should gogo(&g->sched)
   157	// to keep running g.
   158	TEXT runtime·mcall(SB), NOSPLIT, $-8-8
   159		// Save caller state in g->sched
   160		MOVD	RSP, R0
   161		MOVD	R0, (g_sched+gobuf_sp)(g)
   162		MOVD	LR, (g_sched+gobuf_pc)(g)
   163		MOVD	$0, (g_sched+gobuf_lr)(g)
   164		MOVD	g, (g_sched+gobuf_g)(g)
   165	
   166		// Switch to m->g0 & its stack, call fn.
   167		MOVD	g, R3
   168		MOVD	g_m(g), R8
   169		MOVD	m_g0(R8), g
   170		BL	runtime·save_g(SB)
   171		CMP	g, R3
   172		BNE	2(PC)
   173		B	runtime·badmcall(SB)
   174		MOVD	fn+0(FP), R26			// context
   175		MOVD	0(R26), R4			// code pointer
   176		MOVD	(g_sched+gobuf_sp)(g), R0
   177		MOVD	R0, RSP	// sp = m->g0->sched.sp
   178		MOVD	R3, -8(RSP)
   179		MOVD	$0, -16(RSP)
   180		SUB	$16, RSP
   181		BL	(R4)
   182		B	runtime·badmcall2(SB)
   183	
   184	// systemstack_switch is a dummy routine that systemstack leaves at the bottom
   185	// of the G stack. We need to distinguish the routine that
   186	// lives at the bottom of the G stack from the one that lives
   187	// at the top of the system stack because the one at the top of
   188	// the system stack terminates the stack walk (see topofstack()).
   189	TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0
   190		UNDEF
   191		BL	(LR)	// make sure this function is not leaf
   192		RET
   193	
   194	// func systemstack(fn func())
   195	TEXT runtime·systemstack(SB), NOSPLIT, $0-8
   196		MOVD	fn+0(FP), R3	// R3 = fn
   197		MOVD	R3, R26		// context
   198		MOVD	g_m(g), R4	// R4 = m
   199	
   200		MOVD	m_gsignal(R4), R5	// R5 = gsignal
   201		CMP	g, R5
   202		BEQ	noswitch
   203	
   204		MOVD	m_g0(R4), R5	// R5 = g0
   205		CMP	g, R5
   206		BEQ	noswitch
   207	
   208		MOVD	m_curg(R4), R6
   209		CMP	g, R6
   210		BEQ	switch
   211	
   212		// Bad: g is not gsignal, not g0, not curg. What is it?
   213		// Hide call from linker nosplit analysis.
   214		MOVD	$runtime·badsystemstack(SB), R3
   215		BL	(R3)
   216	
   217	switch:
   218		// save our state in g->sched. Pretend to
   219		// be systemstack_switch if the G stack is scanned.
   220		MOVD	$runtime·systemstack_switch(SB), R6
   221		ADD	$8, R6	// get past prologue
   222		MOVD	R6, (g_sched+gobuf_pc)(g)
   223		MOVD	RSP, R0
   224		MOVD	R0, (g_sched+gobuf_sp)(g)
   225		MOVD	$0, (g_sched+gobuf_lr)(g)
   226		MOVD	g, (g_sched+gobuf_g)(g)
   227	
   228		// switch to g0
   229		MOVD	R5, g
   230		BL	runtime·save_g(SB)
   231		MOVD	(g_sched+gobuf_sp)(g), R3
   232		// make it look like mstart called systemstack on g0, to stop traceback
   233		SUB	$16, R3
   234		AND	$~15, R3
   235		MOVD	$runtime·mstart(SB), R4
   236		MOVD	R4, 0(R3)
   237		MOVD	R3, RSP
   238	
   239		// call target function
   240		MOVD	0(R26), R3	// code pointer
   241		BL	(R3)
   242	
   243		// switch back to g
   244		MOVD	g_m(g), R3
   245		MOVD	m_curg(R3), g
   246		BL	runtime·save_g(SB)
   247		MOVD	(g_sched+gobuf_sp)(g), R0
   248		MOVD	R0, RSP
   249		MOVD	$0, (g_sched+gobuf_sp)(g)
   250		RET
   251	
   252	noswitch:
   253		// already on m stack, just call directly
   254		MOVD	0(R26), R3	// code pointer
   255		BL	(R3)
   256		RET
   257	
   258	/*
   259	 * support for morestack
   260	 */
   261	
   262	// Called during function prolog when more stack is needed.
   263	// Caller has already loaded:
   264	// R3 prolog's LR (R30)
   265	//
   266	// The traceback routines see morestack on a g0 as being
   267	// the top of a stack (for example, morestack calling newstack
   268	// calling the scheduler calling newm calling gc), so we must
   269	// record an argument size. For that purpose, it has no arguments.
   270	TEXT runtime·morestack(SB),NOSPLIT,$-8-0
   271		// Cannot grow scheduler stack (m->g0).
   272		MOVD	g_m(g), R8
   273		MOVD	m_g0(R8), R4
   274		CMP	g, R4
   275		BNE	3(PC)
   276		BL	runtime·badmorestackg0(SB)
   277		B	runtime·abort(SB)
   278	
   279		// Cannot grow signal stack (m->gsignal).
   280		MOVD	m_gsignal(R8), R4
   281		CMP	g, R4
   282		BNE	3(PC)
   283		BL	runtime·badmorestackgsignal(SB)
   284		B	runtime·abort(SB)
   285	
   286		// Called from f.
   287		// Set g->sched to context in f
   288		MOVD	RSP, R0
   289		MOVD	R0, (g_sched+gobuf_sp)(g)
   290		MOVD	LR, (g_sched+gobuf_pc)(g)
   291		MOVD	R3, (g_sched+gobuf_lr)(g)
   292		// newstack will fill gobuf.ctxt.
   293	
   294		// Called from f.
   295		// Set m->morebuf to f's callers.
   296		MOVD	R3, (m_morebuf+gobuf_pc)(R8)	// f's caller's PC
   297		MOVD	RSP, R0
   298		MOVD	R0, (m_morebuf+gobuf_sp)(R8)	// f's caller's RSP
   299		MOVD	g, (m_morebuf+gobuf_g)(R8)
   300	
   301		// Call newstack on m->g0's stack.
   302		MOVD	m_g0(R8), g
   303		BL	runtime·save_g(SB)
   304		MOVD	(g_sched+gobuf_sp)(g), R0
   305		MOVD	R0, RSP
   306		MOVD.W	$0, -16(RSP)	// create a call frame on g0
   307		MOVD	R26, 8(RSP)	// ctxt argument
   308		BL	runtime·newstack(SB)
   309	
   310		// Not reached, but make sure the return PC from the call to newstack
   311		// is still in this function, and not the beginning of the next.
   312		UNDEF
   313	
   314	TEXT runtime·morestack_noctxt(SB),NOSPLIT,$-4-0
   315		MOVW	$0, R26
   316		B runtime·morestack(SB)
   317	
   318	// reflectcall: call a function with the given argument list
   319	// func call(argtype *_type, f *FuncVal, arg *byte, argsize, retoffset uint32).
   320	// we don't have variable-sized frames, so we use a small number
   321	// of constant-sized-frame functions to encode a few bits of size in the pc.
   322	// Caution: ugly multiline assembly macros in your future!
   323	
   324	#define DISPATCH(NAME,MAXSIZE)		\
   325		MOVD	$MAXSIZE, R27;		\
   326		CMP	R27, R16;		\
   327		BGT	3(PC);			\
   328		MOVD	$NAME(SB), R27;	\
   329		B	(R27)
   330	// Note: can't just "B NAME(SB)" - bad inlining results.
   331	
   332	TEXT reflect·call(SB), NOSPLIT, $0-0
   333		B	·reflectcall(SB)
   334	
   335	TEXT ·reflectcall(SB), NOSPLIT, $-8-32
   336		MOVWU argsize+24(FP), R16
   337		DISPATCH(runtime·call32, 32)
   338		DISPATCH(runtime·call64, 64)
   339		DISPATCH(runtime·call128, 128)
   340		DISPATCH(runtime·call256, 256)
   341		DISPATCH(runtime·call512, 512)
   342		DISPATCH(runtime·call1024, 1024)
   343		DISPATCH(runtime·call2048, 2048)
   344		DISPATCH(runtime·call4096, 4096)
   345		DISPATCH(runtime·call8192, 8192)
   346		DISPATCH(runtime·call16384, 16384)
   347		DISPATCH(runtime·call32768, 32768)
   348		DISPATCH(runtime·call65536, 65536)
   349		DISPATCH(runtime·call131072, 131072)
   350		DISPATCH(runtime·call262144, 262144)
   351		DISPATCH(runtime·call524288, 524288)
   352		DISPATCH(runtime·call1048576, 1048576)
   353		DISPATCH(runtime·call2097152, 2097152)
   354		DISPATCH(runtime·call4194304, 4194304)
   355		DISPATCH(runtime·call8388608, 8388608)
   356		DISPATCH(runtime·call16777216, 16777216)
   357		DISPATCH(runtime·call33554432, 33554432)
   358		DISPATCH(runtime·call67108864, 67108864)
   359		DISPATCH(runtime·call134217728, 134217728)
   360		DISPATCH(runtime·call268435456, 268435456)
   361		DISPATCH(runtime·call536870912, 536870912)
   362		DISPATCH(runtime·call1073741824, 1073741824)
   363		MOVD	$runtime·badreflectcall(SB), R0
   364		B	(R0)
   365	
   366	#define CALLFN(NAME,MAXSIZE)			\
   367	TEXT NAME(SB), WRAPPER, $MAXSIZE-24;		\
   368		NO_LOCAL_POINTERS;			\
   369		/* copy arguments to stack */		\
   370		MOVD	arg+16(FP), R3;			\
   371		MOVWU	argsize+24(FP), R4;			\
   372		MOVD	RSP, R5;				\
   373		ADD	$(8-1), R5;			\
   374		SUB	$1, R3;				\
   375		ADD	R5, R4;				\
   376		CMP	R5, R4;				\
   377		BEQ	4(PC);				\
   378		MOVBU.W	1(R3), R6;			\
   379		MOVBU.W	R6, 1(R5);			\
   380		B	-4(PC);				\
   381		/* call function */			\
   382		MOVD	f+8(FP), R26;			\
   383		MOVD	(R26), R0;			\
   384		PCDATA  $PCDATA_StackMapIndex, $0;	\
   385		BL	(R0);				\
   386		/* copy return values back */		\
   387		MOVD	argtype+0(FP), R7;		\
   388		MOVD	arg+16(FP), R3;			\
   389		MOVWU	n+24(FP), R4;			\
   390		MOVWU	retoffset+28(FP), R6;		\
   391		ADD	$8, RSP, R5;			\
   392		ADD	R6, R5; 			\
   393		ADD	R6, R3;				\
   394		SUB	R6, R4;				\
   395		BL	callRet<>(SB);			\
   396		RET
   397	
   398	// callRet copies return values back at the end of call*. This is a
   399	// separate function so it can allocate stack space for the arguments
   400	// to reflectcallmove. It does not follow the Go ABI; it expects its
   401	// arguments in registers.
   402	TEXT callRet<>(SB), NOSPLIT, $40-0
   403		MOVD	R7, 8(RSP)
   404		MOVD	R3, 16(RSP)
   405		MOVD	R5, 24(RSP)
   406		MOVD	R4, 32(RSP)
   407		BL	runtime·reflectcallmove(SB)
   408		RET
   409	
   410	// These have 8 added to make the overall frame size a multiple of 16,
   411	// as required by the ABI. (There is another +8 for the saved LR.)
   412	CALLFN(·call32, 40 )
   413	CALLFN(·call64, 72 )
   414	CALLFN(·call128, 136 )
   415	CALLFN(·call256, 264 )
   416	CALLFN(·call512, 520 )
   417	CALLFN(·call1024, 1032 )
   418	CALLFN(·call2048, 2056 )
   419	CALLFN(·call4096, 4104 )
   420	CALLFN(·call8192, 8200 )
   421	CALLFN(·call16384, 16392 )
   422	CALLFN(·call32768, 32776 )
   423	CALLFN(·call65536, 65544 )
   424	CALLFN(·call131072, 131080 )
   425	CALLFN(·call262144, 262152 )
   426	CALLFN(·call524288, 524296 )
   427	CALLFN(·call1048576, 1048584 )
   428	CALLFN(·call2097152, 2097160 )
   429	CALLFN(·call4194304, 4194312 )
   430	CALLFN(·call8388608, 8388616 )
   431	CALLFN(·call16777216, 16777224 )
   432	CALLFN(·call33554432, 33554440 )
   433	CALLFN(·call67108864, 67108872 )
   434	CALLFN(·call134217728, 134217736 )
   435	CALLFN(·call268435456, 268435464 )
   436	CALLFN(·call536870912, 536870920 )
   437	CALLFN(·call1073741824, 1073741832 )
   438	
   439	// AES hashing not implemented for ARM64, issue #10109.
   440	TEXT runtime·aeshash(SB),NOSPLIT,$-8-0
   441		MOVW	$0, R0
   442		MOVW	(R0), R1
   443	TEXT runtime·aeshash32(SB),NOSPLIT,$-8-0
   444		MOVW	$0, R0
   445		MOVW	(R0), R1
   446	TEXT runtime·aeshash64(SB),NOSPLIT,$-8-0
   447		MOVW	$0, R0
   448		MOVW	(R0), R1
   449	TEXT runtime·aeshashstr(SB),NOSPLIT,$-8-0
   450		MOVW	$0, R0
   451		MOVW	(R0), R1
   452		
   453	TEXT runtime·procyield(SB),NOSPLIT,$0-0
   454		MOVWU	cycles+0(FP), R0
   455	again:
   456		YIELD
   457		SUBW	$1, R0
   458		CBNZ	R0, again
   459		RET
   460	
   461	// void jmpdefer(fv, sp);
   462	// called from deferreturn.
   463	// 1. grab stored LR for caller
   464	// 2. sub 4 bytes to get back to BL deferreturn
   465	// 3. BR to fn
   466	TEXT runtime·jmpdefer(SB), NOSPLIT, $-8-16
   467		MOVD	0(RSP), R0
   468		SUB	$4, R0
   469		MOVD	R0, LR
   470	
   471		MOVD	fv+0(FP), R26
   472		MOVD	argp+8(FP), R0
   473		MOVD	R0, RSP
   474		SUB	$8, RSP
   475		MOVD	0(R26), R3
   476		B	(R3)
   477	
   478	// Save state of caller into g->sched. Smashes R0.
   479	TEXT gosave<>(SB),NOSPLIT,$-8
   480		MOVD	LR, (g_sched+gobuf_pc)(g)
   481		MOVD RSP, R0
   482		MOVD	R0, (g_sched+gobuf_sp)(g)
   483		MOVD	$0, (g_sched+gobuf_lr)(g)
   484		MOVD	$0, (g_sched+gobuf_ret)(g)
   485		// Assert ctxt is zero. See func save.
   486		MOVD	(g_sched+gobuf_ctxt)(g), R0
   487		CMP	$0, R0
   488		BEQ	2(PC)
   489		CALL	runtime·badctxt(SB)
   490		RET
   491	
   492	// func asmcgocall(fn, arg unsafe.Pointer) int32
   493	// Call fn(arg) on the scheduler stack,
   494	// aligned appropriately for the gcc ABI.
   495	// See cgocall.go for more details.
   496	TEXT ·asmcgocall(SB),NOSPLIT,$0-20
   497		MOVD	fn+0(FP), R1
   498		MOVD	arg+8(FP), R0
   499	
   500		MOVD	RSP, R2		// save original stack pointer
   501		MOVD	g, R4
   502	
   503		// Figure out if we need to switch to m->g0 stack.
   504		// We get called to create new OS threads too, and those
   505		// come in on the m->g0 stack already.
   506		MOVD	g_m(g), R8
   507		MOVD	m_g0(R8), R3
   508		CMP	R3, g
   509		BEQ	g0
   510		MOVD	R0, R9	// gosave<> and save_g might clobber R0
   511		BL	gosave<>(SB)
   512		MOVD	R3, g
   513		BL	runtime·save_g(SB)
   514		MOVD	(g_sched+gobuf_sp)(g), R0
   515		MOVD	R0, RSP
   516		MOVD	R9, R0
   517	
   518		// Now on a scheduling stack (a pthread-created stack).
   519	g0:
   520		// Save room for two of our pointers /*, plus 32 bytes of callee
   521		// save area that lives on the caller stack. */
   522		MOVD	RSP, R13
   523		SUB	$16, R13
   524		MOVD	R13, RSP
   525		MOVD	R4, 0(RSP)	// save old g on stack
   526		MOVD	(g_stack+stack_hi)(R4), R4
   527		SUB	R2, R4
   528		MOVD	R4, 8(RSP)	// save depth in old g stack (can't just save SP, as stack might be copied during a callback)
   529		BL	(R1)
   530		MOVD	R0, R9
   531	
   532		// Restore g, stack pointer. R0 is errno, so don't touch it
   533		MOVD	0(RSP), g
   534		BL	runtime·save_g(SB)
   535		MOVD	(g_stack+stack_hi)(g), R5
   536		MOVD	8(RSP), R6
   537		SUB	R6, R5
   538		MOVD	R9, R0
   539		MOVD	R5, RSP
   540	
   541		MOVW	R0, ret+16(FP)
   542		RET
   543	
   544	// cgocallback(void (*fn)(void*), void *frame, uintptr framesize, uintptr ctxt)
   545	// Turn the fn into a Go func (by taking its address) and call
   546	// cgocallback_gofunc.
   547	TEXT runtime·cgocallback(SB),NOSPLIT,$40-32
   548		MOVD	$fn+0(FP), R0
   549		MOVD	R0, 8(RSP)
   550		MOVD	frame+8(FP), R0
   551		MOVD	R0, 16(RSP)
   552		MOVD	framesize+16(FP), R0
   553		MOVD	R0, 24(RSP)
   554		MOVD	ctxt+24(FP), R0
   555		MOVD	R0, 32(RSP)
   556		MOVD	$runtime·cgocallback_gofunc(SB), R0
   557		BL	(R0)
   558		RET
   559	
   560	// cgocallback_gofunc(FuncVal*, void *frame, uintptr framesize, uintptr ctxt)
   561	// See cgocall.go for more details.
   562	TEXT ·cgocallback_gofunc(SB),NOSPLIT,$24-32
   563		NO_LOCAL_POINTERS
   564	
   565		// Load g from thread-local storage.
   566		MOVB	runtime·iscgo(SB), R3
   567		CMP	$0, R3
   568		BEQ	nocgo
   569		BL	runtime·load_g(SB)
   570	nocgo:
   571	
   572		// If g is nil, Go did not create the current thread.
   573		// Call needm to obtain one for temporary use.
   574		// In this case, we're running on the thread stack, so there's
   575		// lots of space, but the linker doesn't know. Hide the call from
   576		// the linker analysis by using an indirect call.
   577		CMP	$0, g
   578		BEQ	needm
   579	
   580		MOVD	g_m(g), R8
   581		MOVD	R8, savedm-8(SP)
   582		B	havem
   583	
   584	needm:
   585		MOVD	g, savedm-8(SP) // g is zero, so is m.
   586		MOVD	$runtime·needm(SB), R0
   587		BL	(R0)
   588	
   589		// Set m->sched.sp = SP, so that if a panic happens
   590		// during the function we are about to execute, it will
   591		// have a valid SP to run on the g0 stack.
   592		// The next few lines (after the havem label)
   593		// will save this SP onto the stack and then write
   594		// the same SP back to m->sched.sp. That seems redundant,
   595		// but if an unrecovered panic happens, unwindm will
   596		// restore the g->sched.sp from the stack location
   597		// and then systemstack will try to use it. If we don't set it here,
   598		// that restored SP will be uninitialized (typically 0) and
   599		// will not be usable.
   600		MOVD	g_m(g), R8
   601		MOVD	m_g0(R8), R3
   602		MOVD	RSP, R0
   603		MOVD	R0, (g_sched+gobuf_sp)(R3)
   604	
   605	havem:
   606		// Now there's a valid m, and we're running on its m->g0.
   607		// Save current m->g0->sched.sp on stack and then set it to SP.
   608		// Save current sp in m->g0->sched.sp in preparation for
   609		// switch back to m->curg stack.
   610		// NOTE: unwindm knows that the saved g->sched.sp is at 16(RSP) aka savedsp-16(SP).
   611		// Beware that the frame size is actually 32.
   612		MOVD	m_g0(R8), R3
   613		MOVD	(g_sched+gobuf_sp)(R3), R4
   614		MOVD	R4, savedsp-16(SP)
   615		MOVD	RSP, R0
   616		MOVD	R0, (g_sched+gobuf_sp)(R3)
   617	
   618		// Switch to m->curg stack and call runtime.cgocallbackg.
   619		// Because we are taking over the execution of m->curg
   620		// but *not* resuming what had been running, we need to
   621		// save that information (m->curg->sched) so we can restore it.
   622		// We can restore m->curg->sched.sp easily, because calling
   623		// runtime.cgocallbackg leaves SP unchanged upon return.
   624		// To save m->curg->sched.pc, we push it onto the stack.
   625		// This has the added benefit that it looks to the traceback
   626		// routine like cgocallbackg is going to return to that
   627		// PC (because the frame we allocate below has the same
   628		// size as cgocallback_gofunc's frame declared above)
   629		// so that the traceback will seamlessly trace back into
   630		// the earlier calls.
   631		//
   632		// In the new goroutine, -8(SP) is unused (where SP refers to
   633		// m->curg's SP while we're setting it up, before we've adjusted it).
   634		MOVD	m_curg(R8), g
   635		BL	runtime·save_g(SB)
   636		MOVD	(g_sched+gobuf_sp)(g), R4 // prepare stack as R4
   637		MOVD	(g_sched+gobuf_pc)(g), R5
   638		MOVD	R5, -(24+8)(R4)
   639		MOVD	ctxt+24(FP), R0
   640		MOVD	R0, -(16+8)(R4)
   641		MOVD	$-(24+8)(R4), R0 // maintain 16-byte SP alignment
   642		MOVD	R0, RSP
   643		BL	runtime·cgocallbackg(SB)
   644	
   645		// Restore g->sched (== m->curg->sched) from saved values.
   646		MOVD	0(RSP), R5
   647		MOVD	R5, (g_sched+gobuf_pc)(g)
   648		MOVD	RSP, R4
   649		ADD	$(24+8), R4, R4
   650		MOVD	R4, (g_sched+gobuf_sp)(g)
   651	
   652		// Switch back to m->g0's stack and restore m->g0->sched.sp.
   653		// (Unlike m->curg, the g0 goroutine never uses sched.pc,
   654		// so we do not have to restore it.)
   655		MOVD	g_m(g), R8
   656		MOVD	m_g0(R8), g
   657		BL	runtime·save_g(SB)
   658		MOVD	(g_sched+gobuf_sp)(g), R0
   659		MOVD	R0, RSP
   660		MOVD	savedsp-16(SP), R4
   661		MOVD	R4, (g_sched+gobuf_sp)(g)
   662	
   663		// If the m on entry was nil, we called needm above to borrow an m
   664		// for the duration of the call. Since the call is over, return it with dropm.
   665		MOVD	savedm-8(SP), R6
   666		CMP	$0, R6
   667		BNE	droppedm
   668		MOVD	$runtime·dropm(SB), R0
   669		BL	(R0)
   670	droppedm:
   671	
   672		// Done!
   673		RET
   674	
   675	// Called from cgo wrappers, this function returns g->m->curg.stack.hi.
   676	// Must obey the gcc calling convention.
   677	TEXT _cgo_topofstack(SB),NOSPLIT,$24
   678		// g (R28) and REGTMP (R27)  might be clobbered by load_g. They
   679		// are callee-save in the gcc calling convention, so save them.
   680		MOVD	R27, savedR27-8(SP)
   681		MOVD	g, saveG-16(SP)
   682	
   683		BL	runtime·load_g(SB)
   684		MOVD	g_m(g), R0
   685		MOVD	m_curg(R0), R0
   686		MOVD	(g_stack+stack_hi)(R0), R0
   687	
   688		MOVD	saveG-16(SP), g
   689		MOVD	savedR28-8(SP), R27
   690		RET
   691	
   692	// void setg(G*); set g. for use by needm.
   693	TEXT runtime·setg(SB), NOSPLIT, $0-8
   694		MOVD	gg+0(FP), g
   695		// This only happens if iscgo, so jump straight to save_g
   696		BL	runtime·save_g(SB)
   697		RET
   698	
   699	// void setg_gcc(G*); set g called from gcc
   700	TEXT setg_gcc<>(SB),NOSPLIT,$8
   701		MOVD	R0, g
   702		MOVD	R27, savedR27-8(SP)
   703		BL	runtime·save_g(SB)
   704		MOVD	savedR27-8(SP), R27
   705		RET
   706	
   707	TEXT runtime·getcallerpc(SB),NOSPLIT,$8-16
   708		MOVD	16(RSP), R0		// LR saved by caller
   709		MOVD	R0, ret+8(FP)
   710		RET
   711	
   712	TEXT runtime·abort(SB),NOSPLIT,$-8-0
   713		B	(ZR)
   714		UNDEF
   715	
   716	// memhash_varlen(p unsafe.Pointer, h seed) uintptr
   717	// redirects to memhash(p, h, size) using the size
   718	// stored in the closure.
   719	TEXT runtime·memhash_varlen(SB),NOSPLIT,$40-24
   720		GO_ARGS
   721		NO_LOCAL_POINTERS
   722		MOVD	p+0(FP), R3
   723		MOVD	h+8(FP), R4
   724		MOVD	8(R26), R5
   725		MOVD	R3, 8(RSP)
   726		MOVD	R4, 16(RSP)
   727		MOVD	R5, 24(RSP)
   728		BL	runtime·memhash(SB)
   729		MOVD	32(RSP), R3
   730		MOVD	R3, ret+16(FP)
   731		RET
   732	
   733	// memequal(p, q unsafe.Pointer, size uintptr) bool
   734	TEXT runtime·memequal(SB),NOSPLIT,$-8-25
   735		MOVD	a+0(FP), R1
   736		MOVD	b+8(FP), R2
   737		MOVD	size+16(FP), R3
   738		ADD	R1, R3, R6
   739		MOVD	$1, R0
   740		MOVB	R0, ret+24(FP)
   741		CMP	R1, R2
   742		BEQ	done
   743	loop:
   744		CMP	R1, R6
   745		BEQ	done
   746		MOVBU.P	1(R1), R4
   747		MOVBU.P	1(R2), R5
   748		CMP	R4, R5
   749		BEQ	loop
   750	
   751		MOVB	$0, ret+24(FP)
   752	done:
   753		RET
   754	
   755	// memequal_varlen(a, b unsafe.Pointer) bool
   756	TEXT runtime·memequal_varlen(SB),NOSPLIT,$40-17
   757		MOVD	a+0(FP), R3
   758		MOVD	b+8(FP), R4
   759		CMP	R3, R4
   760		BEQ	eq
   761		MOVD	8(R26), R5    // compiler stores size at offset 8 in the closure
   762		MOVD	R3, 8(RSP)
   763		MOVD	R4, 16(RSP)
   764		MOVD	R5, 24(RSP)
   765		BL	runtime·memequal(SB)
   766		MOVBU	32(RSP), R3
   767		MOVB	R3, ret+16(FP)
   768		RET
   769	eq:
   770		MOVD	$1, R3
   771		MOVB	R3, ret+16(FP)
   772		RET
   773	
   774	TEXT runtime·cmpstring(SB),NOSPLIT,$-4-40
   775		MOVD	s1_base+0(FP), R2
   776		MOVD	s1_len+8(FP), R0
   777		MOVD	s2_base+16(FP), R3
   778		MOVD	s2_len+24(FP), R1
   779		ADD	$40, RSP, R7
   780		B	runtime·cmpbody<>(SB)
   781	
   782	TEXT bytes·Compare(SB),NOSPLIT,$-4-56
   783		MOVD	s1+0(FP), R2
   784		MOVD	s1+8(FP), R0
   785		MOVD	s2+24(FP), R3
   786		MOVD	s2+32(FP), R1
   787		ADD	$56, RSP, R7
   788		B	runtime·cmpbody<>(SB)
   789	
   790	// On entry:
   791	// R0 is the length of s1
   792	// R1 is the length of s2
   793	// R2 points to the start of s1
   794	// R3 points to the start of s2
   795	// R7 points to return value (-1/0/1 will be written here)
   796	//
   797	// On exit:
   798	// R4, R5, and R6 are clobbered
   799	TEXT runtime·cmpbody<>(SB),NOSPLIT,$-4-0
   800		CMP	R2, R3
   801		BEQ	samebytes // same starting pointers; compare lengths
   802		CMP	R0, R1
   803		CSEL    LT, R1, R0, R6 // R6 is min(R0, R1)
   804	
   805		ADD	R2, R6	// R2 is current byte in s1, R6 is last byte in s1 to compare
   806	loop:
   807		CMP	R2, R6
   808		BEQ	samebytes // all compared bytes were the same; compare lengths
   809		MOVBU.P	1(R2), R4
   810		MOVBU.P	1(R3), R5
   811		CMP	R4, R5
   812		BEQ	loop
   813		// bytes differed
   814		MOVD	$1, R4
   815		CSNEG	LT, R4, R4, R4
   816		MOVD	R4, (R7)
   817		RET
   818	samebytes:
   819		MOVD	$1, R4
   820		CMP	R0, R1
   821		CSNEG	LT, R4, R4, R4
   822		CSEL	EQ, ZR, R4, R4
   823		MOVD	R4, (R7)
   824		RET
   825	
   826	// eqstring tests whether two strings are equal.
   827	// The compiler guarantees that strings passed
   828	// to eqstring have equal length.
   829	// See runtime_test.go:eqstring_generic for
   830	// equivalent Go code.
   831	TEXT runtime·eqstring(SB),NOSPLIT,$0-33
   832		MOVD	s1_base+0(FP), R0
   833		MOVD	s1_len+8(FP), R1
   834		MOVD	s2_base+16(FP), R2
   835		ADD	R0, R1		// end
   836	loop:
   837		CMP	R0, R1
   838		BEQ	equal		// reaches the end
   839		MOVBU.P	1(R0), R4
   840		MOVBU.P	1(R2), R5
   841		CMP	R4, R5
   842		BEQ	loop
   843	notequal:
   844		MOVB	ZR, ret+32(FP)
   845		RET
   846	equal:
   847		MOVD	$1, R0
   848		MOVB	R0, ret+32(FP)
   849		RET
   850	
   851	//
   852	// functions for other packages
   853	//
   854	TEXT bytes·IndexByte(SB),NOSPLIT,$0-40
   855		MOVD	b+0(FP), R0
   856		MOVD	b_len+8(FP), R1
   857		MOVBU	c+24(FP), R2	// byte to find
   858		MOVD	R0, R4		// store base for later
   859		ADD	R0, R1		// end
   860	loop:
   861		CMP	R0, R1
   862		BEQ	notfound
   863		MOVBU.P	1(R0), R3
   864		CMP	R2, R3
   865		BNE	loop
   866	
   867		SUB	$1, R0		// R0 will be one beyond the position we want
   868		SUB	R4, R0		// remove base
   869		MOVD	R0, ret+32(FP)
   870		RET
   871	
   872	notfound:
   873		MOVD	$-1, R0
   874		MOVD	R0, ret+32(FP)
   875		RET
   876	
   877	TEXT strings·IndexByte(SB),NOSPLIT,$0-32
   878		MOVD	s+0(FP), R0
   879		MOVD	s_len+8(FP), R1
   880		MOVBU	c+16(FP), R2	// byte to find
   881		MOVD	R0, R4		// store base for later
   882		ADD	R0, R1		// end
   883	loop:
   884		CMP	R0, R1
   885		BEQ	notfound
   886		MOVBU.P	1(R0), R3
   887		CMP	R2, R3
   888		BNE	loop
   889	
   890		SUB	$1, R0		// R0 will be one beyond the position we want
   891		SUB	R4, R0		// remove base
   892		MOVD	R0, ret+24(FP)
   893		RET
   894	
   895	notfound:
   896		MOVD	$-1, R0
   897		MOVD	R0, ret+24(FP)
   898		RET
   899	
   900	// TODO: share code with memequal?
   901	TEXT bytes·Equal(SB),NOSPLIT,$0-49
   902		MOVD	a_len+8(FP), R1
   903		MOVD	b_len+32(FP), R3
   904		CMP	R1, R3		// unequal lengths are not equal
   905		BNE	notequal
   906		MOVD	a+0(FP), R0
   907		MOVD	b+24(FP), R2
   908		ADD	R0, R1		// end
   909	loop:
   910		CMP	R0, R1
   911		BEQ	equal		// reaches the end
   912		MOVBU.P	1(R0), R4
   913		MOVBU.P	1(R2), R5
   914		CMP	R4, R5
   915		BEQ	loop
   916	notequal:
   917		MOVB	ZR, ret+48(FP)
   918		RET
   919	equal:
   920		MOVD	$1, R0
   921		MOVB	R0, ret+48(FP)
   922		RET
   923	
   924	TEXT runtime·return0(SB), NOSPLIT, $0
   925		MOVW	$0, R0
   926		RET
   927	
   928	// The top-most function running on a goroutine
   929	// returns to goexit+PCQuantum.
   930	TEXT runtime·goexit(SB),NOSPLIT,$-8-0
   931		MOVD	R0, R0	// NOP
   932		BL	runtime·goexit1(SB)	// does not return
   933	
   934	// TODO(aram): use PRFM here.
   935	TEXT runtime·prefetcht0(SB),NOSPLIT,$0-8
   936		RET
   937	
   938	TEXT runtime·prefetcht1(SB),NOSPLIT,$0-8
   939		RET
   940	
   941	TEXT runtime·prefetcht2(SB),NOSPLIT,$0-8
   942		RET
   943	
   944	TEXT runtime·prefetchnta(SB),NOSPLIT,$0-8
   945		RET
   946	
   947	TEXT runtime·sigreturn(SB),NOSPLIT,$0-0
   948		RET
   949	
   950	// This is called from .init_array and follows the platform, not Go, ABI.
   951	TEXT runtime·addmoduledata(SB),NOSPLIT,$0-0
   952		SUB	$0x10, RSP
   953		MOVD	R27, 8(RSP) // The access to global variables below implicitly uses R27, which is callee-save
   954		MOVD	runtime·lastmoduledatap(SB), R1
   955		MOVD	R0, moduledata_next(R1)
   956		MOVD	R0, runtime·lastmoduledatap(SB)
   957		MOVD	8(RSP), R27
   958		ADD	$0x10, RSP
   959		RET
   960	
   961	TEXT ·checkASM(SB),NOSPLIT,$0-1
   962		MOVW	$1, R3
   963		MOVB	R3, ret+0(FP)
   964		RET

View as plain text