Black Lives Matter. Support the Equal Justice Initiative.

Text file src/runtime/asm_amd64.s

Documentation: runtime

     1  // Copyright 2009 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  #include "go_asm.h"
     6  #include "go_tls.h"
     7  #include "funcdata.h"
     8  #include "textflag.h"
     9  
    10  // _rt0_amd64 is common startup code for most amd64 systems when using
    11  // internal linking. This is the entry point for the program from the
    12  // kernel for an ordinary -buildmode=exe program. The stack holds the
    13  // number of arguments and the C-style argv.
    14  TEXT _rt0_amd64(SB),NOSPLIT,$-8
    15  	MOVQ	0(SP), DI	// argc
    16  	LEAQ	8(SP), SI	// argv
    17  	JMP	runtime·rt0_go(SB)
    18  
    19  // main is common startup code for most amd64 systems when using
    20  // external linking. The C startup code will call the symbol "main"
    21  // passing argc and argv in the usual C ABI registers DI and SI.
    22  TEXT main(SB),NOSPLIT,$-8
    23  	JMP	runtime·rt0_go(SB)
    24  
    25  // _rt0_amd64_lib is common startup code for most amd64 systems when
    26  // using -buildmode=c-archive or -buildmode=c-shared. The linker will
    27  // arrange to invoke this function as a global constructor (for
    28  // c-archive) or when the shared library is loaded (for c-shared).
    29  // We expect argc and argv to be passed in the usual C ABI registers
    30  // DI and SI.
    31  TEXT _rt0_amd64_lib(SB),NOSPLIT,$0x50
    32  	// Align stack per ELF ABI requirements.
    33  	MOVQ	SP, AX
    34  	ANDQ	$~15, SP
    35  	// Save C ABI callee-saved registers, as caller may need them.
    36  	MOVQ	BX, 0x10(SP)
    37  	MOVQ	BP, 0x18(SP)
    38  	MOVQ	R12, 0x20(SP)
    39  	MOVQ	R13, 0x28(SP)
    40  	MOVQ	R14, 0x30(SP)
    41  	MOVQ	R15, 0x38(SP)
    42  	MOVQ	AX, 0x40(SP)
    43  
    44  	MOVQ	DI, _rt0_amd64_lib_argc<>(SB)
    45  	MOVQ	SI, _rt0_amd64_lib_argv<>(SB)
    46  
    47  	// Synchronous initialization.
    48  	CALL	runtime·libpreinit(SB)
    49  
    50  	// Create a new thread to finish Go runtime initialization.
    51  	MOVQ	_cgo_sys_thread_create(SB), AX
    52  	TESTQ	AX, AX
    53  	JZ	nocgo
    54  	MOVQ	$_rt0_amd64_lib_go(SB), DI
    55  	MOVQ	$0, SI
    56  	CALL	AX
    57  	JMP	restore
    58  
    59  nocgo:
    60  	MOVQ	$0x800000, 0(SP)		// stacksize
    61  	MOVQ	$_rt0_amd64_lib_go(SB), AX
    62  	MOVQ	AX, 8(SP)			// fn
    63  	CALL	runtime·newosproc0(SB)
    64  
    65  restore:
    66  	MOVQ	0x10(SP), BX
    67  	MOVQ	0x18(SP), BP
    68  	MOVQ	0x20(SP), R12
    69  	MOVQ	0x28(SP), R13
    70  	MOVQ	0x30(SP), R14
    71  	MOVQ	0x38(SP), R15
    72  	MOVQ	0x40(SP), SP
    73  	RET
    74  
    75  // _rt0_amd64_lib_go initializes the Go runtime.
    76  // This is started in a separate thread by _rt0_amd64_lib.
    77  TEXT _rt0_amd64_lib_go(SB),NOSPLIT,$0
    78  	MOVQ	_rt0_amd64_lib_argc<>(SB), DI
    79  	MOVQ	_rt0_amd64_lib_argv<>(SB), SI
    80  	JMP	runtime·rt0_go(SB)
    81  
    82  DATA _rt0_amd64_lib_argc<>(SB)/8, $0
    83  GLOBL _rt0_amd64_lib_argc<>(SB),NOPTR, $8
    84  DATA _rt0_amd64_lib_argv<>(SB)/8, $0
    85  GLOBL _rt0_amd64_lib_argv<>(SB),NOPTR, $8
    86  
    87  // Defined as ABIInternal since it does not use the stack-based Go ABI (and
    88  // in addition there are no calls to this entry point from Go code).
    89  TEXT runtime·rt0_go<ABIInternal>(SB),NOSPLIT,$0
    90  	// copy arguments forward on an even stack
    91  	MOVQ	DI, AX		// argc
    92  	MOVQ	SI, BX		// argv
    93  	SUBQ	$(4*8+7), SP		// 2args 2auto
    94  	ANDQ	$~15, SP
    95  	MOVQ	AX, 16(SP)
    96  	MOVQ	BX, 24(SP)
    97  
    98  	// create istack out of the given (operating system) stack.
    99  	// _cgo_init may update stackguard.
   100  	MOVQ	$runtime·g0(SB), DI
   101  	LEAQ	(-64*1024+104)(SP), BX
   102  	MOVQ	BX, g_stackguard0(DI)
   103  	MOVQ	BX, g_stackguard1(DI)
   104  	MOVQ	BX, (g_stack+stack_lo)(DI)
   105  	MOVQ	SP, (g_stack+stack_hi)(DI)
   106  
   107  	// find out information about the processor we're on
   108  	MOVL	$0, AX
   109  	CPUID
   110  	MOVL	AX, SI
   111  	CMPL	AX, $0
   112  	JE	nocpuinfo
   113  
   114  	// Figure out how to serialize RDTSC.
   115  	// On Intel processors LFENCE is enough. AMD requires MFENCE.
   116  	// Don't know about the rest, so let's do MFENCE.
   117  	CMPL	BX, $0x756E6547  // "Genu"
   118  	JNE	notintel
   119  	CMPL	DX, $0x49656E69  // "ineI"
   120  	JNE	notintel
   121  	CMPL	CX, $0x6C65746E  // "ntel"
   122  	JNE	notintel
   123  	MOVB	$1, runtime·isIntel(SB)
   124  	MOVB	$1, runtime·lfenceBeforeRdtsc(SB)
   125  notintel:
   126  
   127  	// Load EAX=1 cpuid flags
   128  	MOVL	$1, AX
   129  	CPUID
   130  	MOVL	AX, runtime·processorVersionInfo(SB)
   131  
   132  nocpuinfo:
   133  	// if there is an _cgo_init, call it.
   134  	MOVQ	_cgo_init(SB), AX
   135  	TESTQ	AX, AX
   136  	JZ	needtls
   137  	// arg 1: g0, already in DI
   138  	MOVQ	$setg_gcc<>(SB), SI // arg 2: setg_gcc
   139  #ifdef GOOS_android
   140  	MOVQ	$runtime·tls_g(SB), DX 	// arg 3: &tls_g
   141  	// arg 4: TLS base, stored in slot 0 (Android's TLS_SLOT_SELF).
   142  	// Compensate for tls_g (+16).
   143  	MOVQ	-16(TLS), CX
   144  #else
   145  	MOVQ	$0, DX	// arg 3, 4: not used when using platform's TLS
   146  	MOVQ	$0, CX
   147  #endif
   148  #ifdef GOOS_windows
   149  	// Adjust for the Win64 calling convention.
   150  	MOVQ	CX, R9 // arg 4
   151  	MOVQ	DX, R8 // arg 3
   152  	MOVQ	SI, DX // arg 2
   153  	MOVQ	DI, CX // arg 1
   154  #endif
   155  	CALL	AX
   156  
   157  	// update stackguard after _cgo_init
   158  	MOVQ	$runtime·g0(SB), CX
   159  	MOVQ	(g_stack+stack_lo)(CX), AX
   160  	ADDQ	$const__StackGuard, AX
   161  	MOVQ	AX, g_stackguard0(CX)
   162  	MOVQ	AX, g_stackguard1(CX)
   163  
   164  #ifndef GOOS_windows
   165  	JMP ok
   166  #endif
   167  needtls:
   168  #ifdef GOOS_plan9
   169  	// skip TLS setup on Plan 9
   170  	JMP ok
   171  #endif
   172  #ifdef GOOS_solaris
   173  	// skip TLS setup on Solaris
   174  	JMP ok
   175  #endif
   176  #ifdef GOOS_illumos
   177  	// skip TLS setup on illumos
   178  	JMP ok
   179  #endif
   180  #ifdef GOOS_darwin
   181  	// skip TLS setup on Darwin
   182  	JMP ok
   183  #endif
   184  #ifdef GOOS_openbsd
   185  	// skip TLS setup on OpenBSD
   186  	JMP ok
   187  #endif
   188  
   189  	LEAQ	runtime·m0+m_tls(SB), DI
   190  	CALL	runtime·settls(SB)
   191  
   192  	// store through it, to make sure it works
   193  	get_tls(BX)
   194  	MOVQ	$0x123, g(BX)
   195  	MOVQ	runtime·m0+m_tls(SB), AX
   196  	CMPQ	AX, $0x123
   197  	JEQ 2(PC)
   198  	CALL	runtime·abort(SB)
   199  ok:
   200  	// set the per-goroutine and per-mach "registers"
   201  	get_tls(BX)
   202  	LEAQ	runtime·g0(SB), CX
   203  	MOVQ	CX, g(BX)
   204  	LEAQ	runtime·m0(SB), AX
   205  
   206  	// save m->g0 = g0
   207  	MOVQ	CX, m_g0(AX)
   208  	// save m0 to g0->m
   209  	MOVQ	AX, g_m(CX)
   210  
   211  	CLD				// convention is D is always left cleared
   212  	CALL	runtime·check(SB)
   213  
   214  	MOVL	16(SP), AX		// copy argc
   215  	MOVL	AX, 0(SP)
   216  	MOVQ	24(SP), AX		// copy argv
   217  	MOVQ	AX, 8(SP)
   218  	CALL	runtime·args(SB)
   219  	CALL	runtime·osinit(SB)
   220  	CALL	runtime·schedinit(SB)
   221  
   222  	// create a new goroutine to start program
   223  	MOVQ	$runtime·mainPC(SB), AX		// entry
   224  	PUSHQ	AX
   225  	PUSHQ	$0			// arg size
   226  	CALL	runtime·newproc(SB)
   227  	POPQ	AX
   228  	POPQ	AX
   229  
   230  	// start this M
   231  	CALL	runtime·mstart(SB)
   232  
   233  	CALL	runtime·abort(SB)	// mstart should never return
   234  	RET
   235  
   236  	// Prevent dead-code elimination of debugCallV1, which is
   237  	// intended to be called by debuggers.
   238  	MOVQ	$runtime·debugCallV1<ABIInternal>(SB), AX
   239  	RET
   240  
   241  // mainPC is a function value for runtime.main, to be passed to newproc.
   242  // The reference to runtime.main is made via ABIInternal, since the
   243  // actual function (not the ABI0 wrapper) is needed by newproc.
   244  DATA	runtime·mainPC+0(SB)/8,$runtime·main<ABIInternal>(SB)
   245  GLOBL	runtime·mainPC(SB),RODATA,$8
   246  
   247  TEXT runtime·breakpoint(SB),NOSPLIT,$0-0
   248  	BYTE	$0xcc
   249  	RET
   250  
   251  TEXT runtime·asminit(SB),NOSPLIT,$0-0
   252  	// No per-thread init.
   253  	RET
   254  
   255  /*
   256   *  go-routine
   257   */
   258  
   259  // func gosave(buf *gobuf)
   260  // save state in Gobuf; setjmp
   261  TEXT runtime·gosave(SB), NOSPLIT, $0-8
   262  	MOVQ	buf+0(FP), AX		// gobuf
   263  	LEAQ	buf+0(FP), BX		// caller's SP
   264  	MOVQ	BX, gobuf_sp(AX)
   265  	MOVQ	0(SP), BX		// caller's PC
   266  	MOVQ	BX, gobuf_pc(AX)
   267  	MOVQ	$0, gobuf_ret(AX)
   268  	MOVQ	BP, gobuf_bp(AX)
   269  	// Assert ctxt is zero. See func save.
   270  	MOVQ	gobuf_ctxt(AX), BX
   271  	TESTQ	BX, BX
   272  	JZ	2(PC)
   273  	CALL	runtime·badctxt(SB)
   274  	get_tls(CX)
   275  	MOVQ	g(CX), BX
   276  	MOVQ	BX, gobuf_g(AX)
   277  	RET
   278  
   279  // func gogo(buf *gobuf)
   280  // restore state from Gobuf; longjmp
   281  TEXT runtime·gogo(SB), NOSPLIT, $16-8
   282  	MOVQ	buf+0(FP), BX		// gobuf
   283  	MOVQ	gobuf_g(BX), DX
   284  	MOVQ	0(DX), CX		// make sure g != nil
   285  	get_tls(CX)
   286  	MOVQ	DX, g(CX)
   287  	MOVQ	gobuf_sp(BX), SP	// restore SP
   288  	MOVQ	gobuf_ret(BX), AX
   289  	MOVQ	gobuf_ctxt(BX), DX
   290  	MOVQ	gobuf_bp(BX), BP
   291  	MOVQ	$0, gobuf_sp(BX)	// clear to help garbage collector
   292  	MOVQ	$0, gobuf_ret(BX)
   293  	MOVQ	$0, gobuf_ctxt(BX)
   294  	MOVQ	$0, gobuf_bp(BX)
   295  	MOVQ	gobuf_pc(BX), BX
   296  	JMP	BX
   297  
   298  // func mcall(fn func(*g))
   299  // Switch to m->g0's stack, call fn(g).
   300  // Fn must never return. It should gogo(&g->sched)
   301  // to keep running g.
   302  TEXT runtime·mcall(SB), NOSPLIT, $0-8
   303  	MOVQ	fn+0(FP), DI
   304  
   305  	get_tls(CX)
   306  	MOVQ	g(CX), AX	// save state in g->sched
   307  	MOVQ	0(SP), BX	// caller's PC
   308  	MOVQ	BX, (g_sched+gobuf_pc)(AX)
   309  	LEAQ	fn+0(FP), BX	// caller's SP
   310  	MOVQ	BX, (g_sched+gobuf_sp)(AX)
   311  	MOVQ	AX, (g_sched+gobuf_g)(AX)
   312  	MOVQ	BP, (g_sched+gobuf_bp)(AX)
   313  
   314  	// switch to m->g0 & its stack, call fn
   315  	MOVQ	g(CX), BX
   316  	MOVQ	g_m(BX), BX
   317  	MOVQ	m_g0(BX), SI
   318  	CMPQ	SI, AX	// if g == m->g0 call badmcall
   319  	JNE	3(PC)
   320  	MOVQ	$runtime·badmcall(SB), AX
   321  	JMP	AX
   322  	MOVQ	SI, g(CX)	// g = m->g0
   323  	MOVQ	(g_sched+gobuf_sp)(SI), SP	// sp = m->g0->sched.sp
   324  	PUSHQ	AX
   325  	MOVQ	DI, DX
   326  	MOVQ	0(DI), DI
   327  	CALL	DI
   328  	POPQ	AX
   329  	MOVQ	$runtime·badmcall2(SB), AX
   330  	JMP	AX
   331  	RET
   332  
   333  // systemstack_switch is a dummy routine that systemstack leaves at the bottom
   334  // of the G stack. We need to distinguish the routine that
   335  // lives at the bottom of the G stack from the one that lives
   336  // at the top of the system stack because the one at the top of
   337  // the system stack terminates the stack walk (see topofstack()).
   338  TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0
   339  	RET
   340  
   341  // func systemstack(fn func())
   342  TEXT runtime·systemstack(SB), NOSPLIT, $0-8
   343  	MOVQ	fn+0(FP), DI	// DI = fn
   344  	get_tls(CX)
   345  	MOVQ	g(CX), AX	// AX = g
   346  	MOVQ	g_m(AX), BX	// BX = m
   347  
   348  	CMPQ	AX, m_gsignal(BX)
   349  	JEQ	noswitch
   350  
   351  	MOVQ	m_g0(BX), DX	// DX = g0
   352  	CMPQ	AX, DX
   353  	JEQ	noswitch
   354  
   355  	CMPQ	AX, m_curg(BX)
   356  	JNE	bad
   357  
   358  	// switch stacks
   359  	// save our state in g->sched. Pretend to
   360  	// be systemstack_switch if the G stack is scanned.
   361  	MOVQ	$runtime·systemstack_switch(SB), SI
   362  	MOVQ	SI, (g_sched+gobuf_pc)(AX)
   363  	MOVQ	SP, (g_sched+gobuf_sp)(AX)
   364  	MOVQ	AX, (g_sched+gobuf_g)(AX)
   365  	MOVQ	BP, (g_sched+gobuf_bp)(AX)
   366  
   367  	// switch to g0
   368  	MOVQ	DX, g(CX)
   369  	MOVQ	(g_sched+gobuf_sp)(DX), BX
   370  	// make it look like mstart called systemstack on g0, to stop traceback
   371  	SUBQ	$8, BX
   372  	MOVQ	$runtime·mstart(SB), DX
   373  	MOVQ	DX, 0(BX)
   374  	MOVQ	BX, SP
   375  
   376  	// call target function
   377  	MOVQ	DI, DX
   378  	MOVQ	0(DI), DI
   379  	CALL	DI
   380  
   381  	// switch back to g
   382  	get_tls(CX)
   383  	MOVQ	g(CX), AX
   384  	MOVQ	g_m(AX), BX
   385  	MOVQ	m_curg(BX), AX
   386  	MOVQ	AX, g(CX)
   387  	MOVQ	(g_sched+gobuf_sp)(AX), SP
   388  	MOVQ	$0, (g_sched+gobuf_sp)(AX)
   389  	RET
   390  
   391  noswitch:
   392  	// already on m stack; tail call the function
   393  	// Using a tail call here cleans up tracebacks since we won't stop
   394  	// at an intermediate systemstack.
   395  	MOVQ	DI, DX
   396  	MOVQ	0(DI), DI
   397  	JMP	DI
   398  
   399  bad:
   400  	// Bad: g is not gsignal, not g0, not curg. What is it?
   401  	MOVQ	$runtime·badsystemstack(SB), AX
   402  	CALL	AX
   403  	INT	$3
   404  
   405  
   406  /*
   407   * support for morestack
   408   */
   409  
   410  // Called during function prolog when more stack is needed.
   411  //
   412  // The traceback routines see morestack on a g0 as being
   413  // the top of a stack (for example, morestack calling newstack
   414  // calling the scheduler calling newm calling gc), so we must
   415  // record an argument size. For that purpose, it has no arguments.
   416  TEXT runtime·morestack(SB),NOSPLIT,$0-0
   417  	// Cannot grow scheduler stack (m->g0).
   418  	get_tls(CX)
   419  	MOVQ	g(CX), BX
   420  	MOVQ	g_m(BX), BX
   421  	MOVQ	m_g0(BX), SI
   422  	CMPQ	g(CX), SI
   423  	JNE	3(PC)
   424  	CALL	runtime·badmorestackg0(SB)
   425  	CALL	runtime·abort(SB)
   426  
   427  	// Cannot grow signal stack (m->gsignal).
   428  	MOVQ	m_gsignal(BX), SI
   429  	CMPQ	g(CX), SI
   430  	JNE	3(PC)
   431  	CALL	runtime·badmorestackgsignal(SB)
   432  	CALL	runtime·abort(SB)
   433  
   434  	// Called from f.
   435  	// Set m->morebuf to f's caller.
   436  	NOP	SP	// tell vet SP changed - stop checking offsets
   437  	MOVQ	8(SP), AX	// f's caller's PC
   438  	MOVQ	AX, (m_morebuf+gobuf_pc)(BX)
   439  	LEAQ	16(SP), AX	// f's caller's SP
   440  	MOVQ	AX, (m_morebuf+gobuf_sp)(BX)
   441  	get_tls(CX)
   442  	MOVQ	g(CX), SI
   443  	MOVQ	SI, (m_morebuf+gobuf_g)(BX)
   444  
   445  	// Set g->sched to context in f.
   446  	MOVQ	0(SP), AX // f's PC
   447  	MOVQ	AX, (g_sched+gobuf_pc)(SI)
   448  	MOVQ	SI, (g_sched+gobuf_g)(SI)
   449  	LEAQ	8(SP), AX // f's SP
   450  	MOVQ	AX, (g_sched+gobuf_sp)(SI)
   451  	MOVQ	BP, (g_sched+gobuf_bp)(SI)
   452  	MOVQ	DX, (g_sched+gobuf_ctxt)(SI)
   453  
   454  	// Call newstack on m->g0's stack.
   455  	MOVQ	m_g0(BX), BX
   456  	MOVQ	BX, g(CX)
   457  	MOVQ	(g_sched+gobuf_sp)(BX), SP
   458  	CALL	runtime·newstack(SB)
   459  	CALL	runtime·abort(SB)	// crash if newstack returns
   460  	RET
   461  
   462  // morestack but not preserving ctxt.
   463  TEXT runtime·morestack_noctxt(SB),NOSPLIT,$0
   464  	MOVL	$0, DX
   465  	JMP	runtime·morestack(SB)
   466  
   467  // reflectcall: call a function with the given argument list
   468  // func call(argtype *_type, f *FuncVal, arg *byte, argsize, retoffset uint32).
   469  // we don't have variable-sized frames, so we use a small number
   470  // of constant-sized-frame functions to encode a few bits of size in the pc.
   471  // Caution: ugly multiline assembly macros in your future!
   472  
   473  #define DISPATCH(NAME,MAXSIZE)		\
   474  	CMPQ	CX, $MAXSIZE;		\
   475  	JA	3(PC);			\
   476  	MOVQ	$NAME(SB), AX;		\
   477  	JMP	AX
   478  // Note: can't just "JMP NAME(SB)" - bad inlining results.
   479  
   480  TEXT ·reflectcall<ABIInternal>(SB), NOSPLIT, $0-32
   481  	MOVLQZX argsize+24(FP), CX
   482  	DISPATCH(runtime·call16, 16)
   483  	DISPATCH(runtime·call32, 32)
   484  	DISPATCH(runtime·call64, 64)
   485  	DISPATCH(runtime·call128, 128)
   486  	DISPATCH(runtime·call256, 256)
   487  	DISPATCH(runtime·call512, 512)
   488  	DISPATCH(runtime·call1024, 1024)
   489  	DISPATCH(runtime·call2048, 2048)
   490  	DISPATCH(runtime·call4096, 4096)
   491  	DISPATCH(runtime·call8192, 8192)
   492  	DISPATCH(runtime·call16384, 16384)
   493  	DISPATCH(runtime·call32768, 32768)
   494  	DISPATCH(runtime·call65536, 65536)
   495  	DISPATCH(runtime·call131072, 131072)
   496  	DISPATCH(runtime·call262144, 262144)
   497  	DISPATCH(runtime·call524288, 524288)
   498  	DISPATCH(runtime·call1048576, 1048576)
   499  	DISPATCH(runtime·call2097152, 2097152)
   500  	DISPATCH(runtime·call4194304, 4194304)
   501  	DISPATCH(runtime·call8388608, 8388608)
   502  	DISPATCH(runtime·call16777216, 16777216)
   503  	DISPATCH(runtime·call33554432, 33554432)
   504  	DISPATCH(runtime·call67108864, 67108864)
   505  	DISPATCH(runtime·call134217728, 134217728)
   506  	DISPATCH(runtime·call268435456, 268435456)
   507  	DISPATCH(runtime·call536870912, 536870912)
   508  	DISPATCH(runtime·call1073741824, 1073741824)
   509  	MOVQ	$runtime·badreflectcall(SB), AX
   510  	JMP	AX
   511  
   512  #define CALLFN(NAME,MAXSIZE)			\
   513  TEXT NAME(SB), WRAPPER, $MAXSIZE-32;		\
   514  	NO_LOCAL_POINTERS;			\
   515  	/* copy arguments to stack */		\
   516  	MOVQ	argptr+16(FP), SI;		\
   517  	MOVLQZX argsize+24(FP), CX;		\
   518  	MOVQ	SP, DI;				\
   519  	REP;MOVSB;				\
   520  	/* call function */			\
   521  	MOVQ	f+8(FP), DX;			\
   522  	PCDATA  $PCDATA_StackMapIndex, $0;	\
   523  	MOVQ	(DX), AX;			\
   524  	CALL	AX;				\
   525  	/* copy return values back */		\
   526  	MOVQ	argtype+0(FP), DX;		\
   527  	MOVQ	argptr+16(FP), DI;		\
   528  	MOVLQZX	argsize+24(FP), CX;		\
   529  	MOVLQZX	retoffset+28(FP), BX;		\
   530  	MOVQ	SP, SI;				\
   531  	ADDQ	BX, DI;				\
   532  	ADDQ	BX, SI;				\
   533  	SUBQ	BX, CX;				\
   534  	CALL	callRet<>(SB);			\
   535  	RET
   536  
   537  // callRet copies return values back at the end of call*. This is a
   538  // separate function so it can allocate stack space for the arguments
   539  // to reflectcallmove. It does not follow the Go ABI; it expects its
   540  // arguments in registers.
   541  TEXT callRet<>(SB), NOSPLIT, $32-0
   542  	NO_LOCAL_POINTERS
   543  	MOVQ	DX, 0(SP)
   544  	MOVQ	DI, 8(SP)
   545  	MOVQ	SI, 16(SP)
   546  	MOVQ	CX, 24(SP)
   547  	CALL	runtime·reflectcallmove(SB)
   548  	RET
   549  
   550  CALLFN(·call16, 16)
   551  CALLFN(·call32, 32)
   552  CALLFN(·call64, 64)
   553  CALLFN(·call128, 128)
   554  CALLFN(·call256, 256)
   555  CALLFN(·call512, 512)
   556  CALLFN(·call1024, 1024)
   557  CALLFN(·call2048, 2048)
   558  CALLFN(·call4096, 4096)
   559  CALLFN(·call8192, 8192)
   560  CALLFN(·call16384, 16384)
   561  CALLFN(·call32768, 32768)
   562  CALLFN(·call65536, 65536)
   563  CALLFN(·call131072, 131072)
   564  CALLFN(·call262144, 262144)
   565  CALLFN(·call524288, 524288)
   566  CALLFN(·call1048576, 1048576)
   567  CALLFN(·call2097152, 2097152)
   568  CALLFN(·call4194304, 4194304)
   569  CALLFN(·call8388608, 8388608)
   570  CALLFN(·call16777216, 16777216)
   571  CALLFN(·call33554432, 33554432)
   572  CALLFN(·call67108864, 67108864)
   573  CALLFN(·call134217728, 134217728)
   574  CALLFN(·call268435456, 268435456)
   575  CALLFN(·call536870912, 536870912)
   576  CALLFN(·call1073741824, 1073741824)
   577  
   578  TEXT runtime·procyield(SB),NOSPLIT,$0-0
   579  	MOVL	cycles+0(FP), AX
   580  again:
   581  	PAUSE
   582  	SUBL	$1, AX
   583  	JNZ	again
   584  	RET
   585  
   586  
   587  TEXT ·publicationBarrier(SB),NOSPLIT,$0-0
   588  	// Stores are already ordered on x86, so this is just a
   589  	// compile barrier.
   590  	RET
   591  
   592  // func jmpdefer(fv *funcval, argp uintptr)
   593  // argp is a caller SP.
   594  // called from deferreturn.
   595  // 1. pop the caller
   596  // 2. sub 5 bytes from the callers return
   597  // 3. jmp to the argument
   598  TEXT runtime·jmpdefer(SB), NOSPLIT, $0-16
   599  	MOVQ	fv+0(FP), DX	// fn
   600  	MOVQ	argp+8(FP), BX	// caller sp
   601  	LEAQ	-8(BX), SP	// caller sp after CALL
   602  	MOVQ	-8(SP), BP	// restore BP as if deferreturn returned (harmless if framepointers not in use)
   603  	SUBQ	$5, (SP)	// return to CALL again
   604  	MOVQ	0(DX), BX
   605  	JMP	BX	// but first run the deferred function
   606  
   607  // Save state of caller into g->sched. Smashes R8, R9.
   608  TEXT gosave<>(SB),NOSPLIT,$0
   609  	get_tls(R8)
   610  	MOVQ	g(R8), R8
   611  	MOVQ	0(SP), R9
   612  	MOVQ	R9, (g_sched+gobuf_pc)(R8)
   613  	LEAQ	8(SP), R9
   614  	MOVQ	R9, (g_sched+gobuf_sp)(R8)
   615  	MOVQ	$0, (g_sched+gobuf_ret)(R8)
   616  	MOVQ	BP, (g_sched+gobuf_bp)(R8)
   617  	// Assert ctxt is zero. See func save.
   618  	MOVQ	(g_sched+gobuf_ctxt)(R8), R9
   619  	TESTQ	R9, R9
   620  	JZ	2(PC)
   621  	CALL	runtime·badctxt(SB)
   622  	RET
   623  
   624  // func asmcgocall(fn, arg unsafe.Pointer) int32
   625  // Call fn(arg) on the scheduler stack,
   626  // aligned appropriately for the gcc ABI.
   627  // See cgocall.go for more details.
   628  TEXT ·asmcgocall(SB),NOSPLIT,$0-20
   629  	MOVQ	fn+0(FP), AX
   630  	MOVQ	arg+8(FP), BX
   631  
   632  	MOVQ	SP, DX
   633  
   634  	// Figure out if we need to switch to m->g0 stack.
   635  	// We get called to create new OS threads too, and those
   636  	// come in on the m->g0 stack already.
   637  	get_tls(CX)
   638  	MOVQ	g(CX), R8
   639  	CMPQ	R8, $0
   640  	JEQ	nosave
   641  	MOVQ	g_m(R8), R8
   642  	MOVQ	m_g0(R8), SI
   643  	MOVQ	g(CX), DI
   644  	CMPQ	SI, DI
   645  	JEQ	nosave
   646  	MOVQ	m_gsignal(R8), SI
   647  	CMPQ	SI, DI
   648  	JEQ	nosave
   649  
   650  	// Switch to system stack.
   651  	MOVQ	m_g0(R8), SI
   652  	CALL	gosave<>(SB)
   653  	MOVQ	SI, g(CX)
   654  	MOVQ	(g_sched+gobuf_sp)(SI), SP
   655  
   656  	// Now on a scheduling stack (a pthread-created stack).
   657  	// Make sure we have enough room for 4 stack-backed fast-call
   658  	// registers as per windows amd64 calling convention.
   659  	SUBQ	$64, SP
   660  	ANDQ	$~15, SP	// alignment for gcc ABI
   661  	MOVQ	DI, 48(SP)	// save g
   662  	MOVQ	(g_stack+stack_hi)(DI), DI
   663  	SUBQ	DX, DI
   664  	MOVQ	DI, 40(SP)	// save depth in stack (can't just save SP, as stack might be copied during a callback)
   665  	MOVQ	BX, DI		// DI = first argument in AMD64 ABI
   666  	MOVQ	BX, CX		// CX = first argument in Win64
   667  	CALL	AX
   668  
   669  	// Restore registers, g, stack pointer.
   670  	get_tls(CX)
   671  	MOVQ	48(SP), DI
   672  	MOVQ	(g_stack+stack_hi)(DI), SI
   673  	SUBQ	40(SP), SI
   674  	MOVQ	DI, g(CX)
   675  	MOVQ	SI, SP
   676  
   677  	MOVL	AX, ret+16(FP)
   678  	RET
   679  
   680  nosave:
   681  	// Running on a system stack, perhaps even without a g.
   682  	// Having no g can happen during thread creation or thread teardown
   683  	// (see needm/dropm on Solaris, for example).
   684  	// This code is like the above sequence but without saving/restoring g
   685  	// and without worrying about the stack moving out from under us
   686  	// (because we're on a system stack, not a goroutine stack).
   687  	// The above code could be used directly if already on a system stack,
   688  	// but then the only path through this code would be a rare case on Solaris.
   689  	// Using this code for all "already on system stack" calls exercises it more,
   690  	// which should help keep it correct.
   691  	SUBQ	$64, SP
   692  	ANDQ	$~15, SP
   693  	MOVQ	$0, 48(SP)		// where above code stores g, in case someone looks during debugging
   694  	MOVQ	DX, 40(SP)	// save original stack pointer
   695  	MOVQ	BX, DI		// DI = first argument in AMD64 ABI
   696  	MOVQ	BX, CX		// CX = first argument in Win64
   697  	CALL	AX
   698  	MOVQ	40(SP), SI	// restore original stack pointer
   699  	MOVQ	SI, SP
   700  	MOVL	AX, ret+16(FP)
   701  	RET
   702  
   703  // func cgocallback(fn, frame unsafe.Pointer, ctxt uintptr)
   704  // See cgocall.go for more details.
   705  TEXT ·cgocallback(SB),NOSPLIT,$24-24
   706  	NO_LOCAL_POINTERS
   707  
   708  	// If g is nil, Go did not create the current thread.
   709  	// Call needm to obtain one m for temporary use.
   710  	// In this case, we're running on the thread stack, so there's
   711  	// lots of space, but the linker doesn't know. Hide the call from
   712  	// the linker analysis by using an indirect call through AX.
   713  	get_tls(CX)
   714  #ifdef GOOS_windows
   715  	MOVL	$0, BX
   716  	CMPQ	CX, $0
   717  	JEQ	2(PC)
   718  #endif
   719  	MOVQ	g(CX), BX
   720  	CMPQ	BX, $0
   721  	JEQ	needm
   722  	MOVQ	g_m(BX), BX
   723  	MOVQ	BX, savedm-8(SP)	// saved copy of oldm
   724  	JMP	havem
   725  needm:
   726  	MOVQ    $runtime·needm(SB), AX
   727  	CALL	AX
   728  	MOVQ	$0, savedm-8(SP) // dropm on return
   729  	get_tls(CX)
   730  	MOVQ	g(CX), BX
   731  	MOVQ	g_m(BX), BX
   732  
   733  	// Set m->sched.sp = SP, so that if a panic happens
   734  	// during the function we are about to execute, it will
   735  	// have a valid SP to run on the g0 stack.
   736  	// The next few lines (after the havem label)
   737  	// will save this SP onto the stack and then write
   738  	// the same SP back to m->sched.sp. That seems redundant,
   739  	// but if an unrecovered panic happens, unwindm will
   740  	// restore the g->sched.sp from the stack location
   741  	// and then systemstack will try to use it. If we don't set it here,
   742  	// that restored SP will be uninitialized (typically 0) and
   743  	// will not be usable.
   744  	MOVQ	m_g0(BX), SI
   745  	MOVQ	SP, (g_sched+gobuf_sp)(SI)
   746  
   747  havem:
   748  	// Now there's a valid m, and we're running on its m->g0.
   749  	// Save current m->g0->sched.sp on stack and then set it to SP.
   750  	// Save current sp in m->g0->sched.sp in preparation for
   751  	// switch back to m->curg stack.
   752  	// NOTE: unwindm knows that the saved g->sched.sp is at 0(SP).
   753  	MOVQ	m_g0(BX), SI
   754  	MOVQ	(g_sched+gobuf_sp)(SI), AX
   755  	MOVQ	AX, 0(SP)
   756  	MOVQ	SP, (g_sched+gobuf_sp)(SI)
   757  
   758  	// Switch to m->curg stack and call runtime.cgocallbackg.
   759  	// Because we are taking over the execution of m->curg
   760  	// but *not* resuming what had been running, we need to
   761  	// save that information (m->curg->sched) so we can restore it.
   762  	// We can restore m->curg->sched.sp easily, because calling
   763  	// runtime.cgocallbackg leaves SP unchanged upon return.
   764  	// To save m->curg->sched.pc, we push it onto the curg stack and
   765  	// open a frame the same size as cgocallback's g0 frame.
   766  	// Once we switch to the curg stack, the pushed PC will appear
   767  	// to be the return PC of cgocallback, so that the traceback
   768  	// will seamlessly trace back into the earlier calls.
   769  	MOVQ	m_curg(BX), SI
   770  	MOVQ	SI, g(CX)
   771  	MOVQ	(g_sched+gobuf_sp)(SI), DI  // prepare stack as DI
   772  	MOVQ	(g_sched+gobuf_pc)(SI), BX
   773  	MOVQ	BX, -8(DI)  // "push" return PC on the g stack
   774  	// Gather our arguments into registers.
   775  	MOVQ	fn+0(FP), BX
   776  	MOVQ	frame+8(FP), CX
   777  	MOVQ	ctxt+16(FP), DX
   778  	// Compute the size of the frame, including return PC and, if
   779  	// GOEXPERIMENT=framepointer, the saved base pointer
   780  	LEAQ	fn+0(FP), AX
   781  	SUBQ	SP, AX   // AX is our actual frame size
   782  	SUBQ	AX, DI   // Allocate the same frame size on the g stack
   783  	MOVQ	DI, SP
   784  
   785  	MOVQ	BX, 0(SP)
   786  	MOVQ	CX, 8(SP)
   787  	MOVQ	DX, 16(SP)
   788  	CALL	runtime·cgocallbackg(SB)
   789  
   790  	// Compute the size of the frame again. FP and SP have
   791  	// completely different values here than they did above,
   792  	// but only their difference matters.
   793  	LEAQ	fn+0(FP), AX
   794  	SUBQ	SP, AX
   795  
   796  	// Restore g->sched (== m->curg->sched) from saved values.
   797  	get_tls(CX)
   798  	MOVQ	g(CX), SI
   799  	MOVQ	SP, DI
   800  	ADDQ	AX, DI
   801  	MOVQ	-8(DI), BX
   802  	MOVQ	BX, (g_sched+gobuf_pc)(SI)
   803  	MOVQ	DI, (g_sched+gobuf_sp)(SI)
   804  
   805  	// Switch back to m->g0's stack and restore m->g0->sched.sp.
   806  	// (Unlike m->curg, the g0 goroutine never uses sched.pc,
   807  	// so we do not have to restore it.)
   808  	MOVQ	g(CX), BX
   809  	MOVQ	g_m(BX), BX
   810  	MOVQ	m_g0(BX), SI
   811  	MOVQ	SI, g(CX)
   812  	MOVQ	(g_sched+gobuf_sp)(SI), SP
   813  	MOVQ	0(SP), AX
   814  	MOVQ	AX, (g_sched+gobuf_sp)(SI)
   815  
   816  	// If the m on entry was nil, we called needm above to borrow an m
   817  	// for the duration of the call. Since the call is over, return it with dropm.
   818  	MOVQ	savedm-8(SP), BX
   819  	CMPQ	BX, $0
   820  	JNE 3(PC)
   821  	MOVQ	$runtime·dropm(SB), AX
   822  	CALL	AX
   823  
   824  	// Done!
   825  	RET
   826  
   827  // func setg(gg *g)
   828  // set g. for use by needm.
   829  TEXT runtime·setg(SB), NOSPLIT, $0-8
   830  	MOVQ	gg+0(FP), BX
   831  #ifdef GOOS_windows
   832  	CMPQ	BX, $0
   833  	JNE	settls
   834  	MOVQ	$0, 0x28(GS)
   835  	RET
   836  settls:
   837  	MOVQ	g_m(BX), AX
   838  	LEAQ	m_tls(AX), AX
   839  	MOVQ	AX, 0x28(GS)
   840  #endif
   841  	get_tls(CX)
   842  	MOVQ	BX, g(CX)
   843  	RET
   844  
   845  // void setg_gcc(G*); set g called from gcc.
   846  TEXT setg_gcc<>(SB),NOSPLIT,$0
   847  	get_tls(AX)
   848  	MOVQ	DI, g(AX)
   849  	RET
   850  
   851  TEXT runtime·abort(SB),NOSPLIT,$0-0
   852  	INT	$3
   853  loop:
   854  	JMP	loop
   855  
   856  // check that SP is in range [g->stack.lo, g->stack.hi)
   857  TEXT runtime·stackcheck(SB), NOSPLIT, $0-0
   858  	get_tls(CX)
   859  	MOVQ	g(CX), AX
   860  	CMPQ	(g_stack+stack_hi)(AX), SP
   861  	JHI	2(PC)
   862  	CALL	runtime·abort(SB)
   863  	CMPQ	SP, (g_stack+stack_lo)(AX)
   864  	JHI	2(PC)
   865  	CALL	runtime·abort(SB)
   866  	RET
   867  
   868  // func cputicks() int64
   869  TEXT runtime·cputicks(SB),NOSPLIT,$0-0
   870  	CMPB	runtime·lfenceBeforeRdtsc(SB), $1
   871  	JNE	mfence
   872  	LFENCE
   873  	JMP	done
   874  mfence:
   875  	MFENCE
   876  done:
   877  	RDTSC
   878  	SHLQ	$32, DX
   879  	ADDQ	DX, AX
   880  	MOVQ	AX, ret+0(FP)
   881  	RET
   882  
   883  // func memhash(p unsafe.Pointer, h, s uintptr) uintptr
   884  // hash function using AES hardware instructions
   885  TEXT runtime·memhash(SB),NOSPLIT,$0-32
   886  	CMPB	runtime·useAeshash(SB), $0
   887  	JEQ	noaes
   888  	MOVQ	p+0(FP), AX	// ptr to data
   889  	MOVQ	s+16(FP), CX	// size
   890  	LEAQ	ret+24(FP), DX
   891  	JMP	aeshashbody<>(SB)
   892  noaes:
   893  	JMP	runtime·memhashFallback(SB)
   894  
   895  // func strhash(p unsafe.Pointer, h uintptr) uintptr
   896  TEXT runtime·strhash(SB),NOSPLIT,$0-24
   897  	CMPB	runtime·useAeshash(SB), $0
   898  	JEQ	noaes
   899  	MOVQ	p+0(FP), AX	// ptr to string struct
   900  	MOVQ	8(AX), CX	// length of string
   901  	MOVQ	(AX), AX	// string data
   902  	LEAQ	ret+16(FP), DX
   903  	JMP	aeshashbody<>(SB)
   904  noaes:
   905  	JMP	runtime·strhashFallback(SB)
   906  
   907  // AX: data
   908  // CX: length
   909  // DX: address to put return value
   910  TEXT aeshashbody<>(SB),NOSPLIT,$0-0
   911  	// Fill an SSE register with our seeds.
   912  	MOVQ	h+8(FP), X0			// 64 bits of per-table hash seed
   913  	PINSRW	$4, CX, X0			// 16 bits of length
   914  	PSHUFHW $0, X0, X0			// repeat length 4 times total
   915  	MOVO	X0, X1				// save unscrambled seed
   916  	PXOR	runtime·aeskeysched(SB), X0	// xor in per-process seed
   917  	AESENC	X0, X0				// scramble seed
   918  
   919  	CMPQ	CX, $16
   920  	JB	aes0to15
   921  	JE	aes16
   922  	CMPQ	CX, $32
   923  	JBE	aes17to32
   924  	CMPQ	CX, $64
   925  	JBE	aes33to64
   926  	CMPQ	CX, $128
   927  	JBE	aes65to128
   928  	JMP	aes129plus
   929  
   930  aes0to15:
   931  	TESTQ	CX, CX
   932  	JE	aes0
   933  
   934  	ADDQ	$16, AX
   935  	TESTW	$0xff0, AX
   936  	JE	endofpage
   937  
   938  	// 16 bytes loaded at this address won't cross
   939  	// a page boundary, so we can load it directly.
   940  	MOVOU	-16(AX), X1
   941  	ADDQ	CX, CX
   942  	MOVQ	$masks<>(SB), AX
   943  	PAND	(AX)(CX*8), X1
   944  final1:
   945  	PXOR	X0, X1	// xor data with seed
   946  	AESENC	X1, X1	// scramble combo 3 times
   947  	AESENC	X1, X1
   948  	AESENC	X1, X1
   949  	MOVQ	X1, (DX)
   950  	RET
   951  
   952  endofpage:
   953  	// address ends in 1111xxxx. Might be up against
   954  	// a page boundary, so load ending at last byte.
   955  	// Then shift bytes down using pshufb.
   956  	MOVOU	-32(AX)(CX*1), X1
   957  	ADDQ	CX, CX
   958  	MOVQ	$shifts<>(SB), AX
   959  	PSHUFB	(AX)(CX*8), X1
   960  	JMP	final1
   961  
   962  aes0:
   963  	// Return scrambled input seed
   964  	AESENC	X0, X0
   965  	MOVQ	X0, (DX)
   966  	RET
   967  
   968  aes16:
   969  	MOVOU	(AX), X1
   970  	JMP	final1
   971  
   972  aes17to32:
   973  	// make second starting seed
   974  	PXOR	runtime·aeskeysched+16(SB), X1
   975  	AESENC	X1, X1
   976  
   977  	// load data to be hashed
   978  	MOVOU	(AX), X2
   979  	MOVOU	-16(AX)(CX*1), X3
   980  
   981  	// xor with seed
   982  	PXOR	X0, X2
   983  	PXOR	X1, X3
   984  
   985  	// scramble 3 times
   986  	AESENC	X2, X2
   987  	AESENC	X3, X3
   988  	AESENC	X2, X2
   989  	AESENC	X3, X3
   990  	AESENC	X2, X2
   991  	AESENC	X3, X3
   992  
   993  	// combine results
   994  	PXOR	X3, X2
   995  	MOVQ	X2, (DX)
   996  	RET
   997  
   998  aes33to64:
   999  	// make 3 more starting seeds
  1000  	MOVO	X1, X2
  1001  	MOVO	X1, X3
  1002  	PXOR	runtime·aeskeysched+16(SB), X1
  1003  	PXOR	runtime·aeskeysched+32(SB), X2
  1004  	PXOR	runtime·aeskeysched+48(SB), X3
  1005  	AESENC	X1, X1
  1006  	AESENC	X2, X2
  1007  	AESENC	X3, X3
  1008  
  1009  	MOVOU	(AX), X4
  1010  	MOVOU	16(AX), X5
  1011  	MOVOU	-32(AX)(CX*1), X6
  1012  	MOVOU	-16(AX)(CX*1), X7
  1013  
  1014  	PXOR	X0, X4
  1015  	PXOR	X1, X5
  1016  	PXOR	X2, X6
  1017  	PXOR	X3, X7
  1018  
  1019  	AESENC	X4, X4
  1020  	AESENC	X5, X5
  1021  	AESENC	X6, X6
  1022  	AESENC	X7, X7
  1023  
  1024  	AESENC	X4, X4
  1025  	AESENC	X5, X5
  1026  	AESENC	X6, X6
  1027  	AESENC	X7, X7
  1028  
  1029  	AESENC	X4, X4
  1030  	AESENC	X5, X5
  1031  	AESENC	X6, X6
  1032  	AESENC	X7, X7
  1033  
  1034  	PXOR	X6, X4
  1035  	PXOR	X7, X5
  1036  	PXOR	X5, X4
  1037  	MOVQ	X4, (DX)
  1038  	RET
  1039  
  1040  aes65to128:
  1041  	// make 7 more starting seeds
  1042  	MOVO	X1, X2
  1043  	MOVO	X1, X3
  1044  	MOVO	X1, X4
  1045  	MOVO	X1, X5
  1046  	MOVO	X1, X6
  1047  	MOVO	X1, X7
  1048  	PXOR	runtime·aeskeysched+16(SB), X1
  1049  	PXOR	runtime·aeskeysched+32(SB), X2
  1050  	PXOR	runtime·aeskeysched+48(SB), X3
  1051  	PXOR	runtime·aeskeysched+64(SB), X4
  1052  	PXOR	runtime·aeskeysched+80(SB), X5
  1053  	PXOR	runtime·aeskeysched+96(SB), X6
  1054  	PXOR	runtime·aeskeysched+112(SB), X7
  1055  	AESENC	X1, X1
  1056  	AESENC	X2, X2
  1057  	AESENC	X3, X3
  1058  	AESENC	X4, X4
  1059  	AESENC	X5, X5
  1060  	AESENC	X6, X6
  1061  	AESENC	X7, X7
  1062  
  1063  	// load data
  1064  	MOVOU	(AX), X8
  1065  	MOVOU	16(AX), X9
  1066  	MOVOU	32(AX), X10
  1067  	MOVOU	48(AX), X11
  1068  	MOVOU	-64(AX)(CX*1), X12
  1069  	MOVOU	-48(AX)(CX*1), X13
  1070  	MOVOU	-32(AX)(CX*1), X14
  1071  	MOVOU	-16(AX)(CX*1), X15
  1072  
  1073  	// xor with seed
  1074  	PXOR	X0, X8
  1075  	PXOR	X1, X9
  1076  	PXOR	X2, X10
  1077  	PXOR	X3, X11
  1078  	PXOR	X4, X12
  1079  	PXOR	X5, X13
  1080  	PXOR	X6, X14
  1081  	PXOR	X7, X15
  1082  
  1083  	// scramble 3 times
  1084  	AESENC	X8, X8
  1085  	AESENC	X9, X9
  1086  	AESENC	X10, X10
  1087  	AESENC	X11, X11
  1088  	AESENC	X12, X12
  1089  	AESENC	X13, X13
  1090  	AESENC	X14, X14
  1091  	AESENC	X15, X15
  1092  
  1093  	AESENC	X8, X8
  1094  	AESENC	X9, X9
  1095  	AESENC	X10, X10
  1096  	AESENC	X11, X11
  1097  	AESENC	X12, X12
  1098  	AESENC	X13, X13
  1099  	AESENC	X14, X14
  1100  	AESENC	X15, X15
  1101  
  1102  	AESENC	X8, X8
  1103  	AESENC	X9, X9
  1104  	AESENC	X10, X10
  1105  	AESENC	X11, X11
  1106  	AESENC	X12, X12
  1107  	AESENC	X13, X13
  1108  	AESENC	X14, X14
  1109  	AESENC	X15, X15
  1110  
  1111  	// combine results
  1112  	PXOR	X12, X8
  1113  	PXOR	X13, X9
  1114  	PXOR	X14, X10
  1115  	PXOR	X15, X11
  1116  	PXOR	X10, X8
  1117  	PXOR	X11, X9
  1118  	PXOR	X9, X8
  1119  	MOVQ	X8, (DX)
  1120  	RET
  1121  
  1122  aes129plus:
  1123  	// make 7 more starting seeds
  1124  	MOVO	X1, X2
  1125  	MOVO	X1, X3
  1126  	MOVO	X1, X4
  1127  	MOVO	X1, X5
  1128  	MOVO	X1, X6
  1129  	MOVO	X1, X7
  1130  	PXOR	runtime·aeskeysched+16(SB), X1
  1131  	PXOR	runtime·aeskeysched+32(SB), X2
  1132  	PXOR	runtime·aeskeysched+48(SB), X3
  1133  	PXOR	runtime·aeskeysched+64(SB), X4
  1134  	PXOR	runtime·aeskeysched+80(SB), X5
  1135  	PXOR	runtime·aeskeysched+96(SB), X6
  1136  	PXOR	runtime·aeskeysched+112(SB), X7
  1137  	AESENC	X1, X1
  1138  	AESENC	X2, X2
  1139  	AESENC	X3, X3
  1140  	AESENC	X4, X4
  1141  	AESENC	X5, X5
  1142  	AESENC	X6, X6
  1143  	AESENC	X7, X7
  1144  
  1145  	// start with last (possibly overlapping) block
  1146  	MOVOU	-128(AX)(CX*1), X8
  1147  	MOVOU	-112(AX)(CX*1), X9
  1148  	MOVOU	-96(AX)(CX*1), X10
  1149  	MOVOU	-80(AX)(CX*1), X11
  1150  	MOVOU	-64(AX)(CX*1), X12
  1151  	MOVOU	-48(AX)(CX*1), X13
  1152  	MOVOU	-32(AX)(CX*1), X14
  1153  	MOVOU	-16(AX)(CX*1), X15
  1154  
  1155  	// xor in seed
  1156  	PXOR	X0, X8
  1157  	PXOR	X1, X9
  1158  	PXOR	X2, X10
  1159  	PXOR	X3, X11
  1160  	PXOR	X4, X12
  1161  	PXOR	X5, X13
  1162  	PXOR	X6, X14
  1163  	PXOR	X7, X15
  1164  
  1165  	// compute number of remaining 128-byte blocks
  1166  	DECQ	CX
  1167  	SHRQ	$7, CX
  1168  
  1169  aesloop:
  1170  	// scramble state
  1171  	AESENC	X8, X8
  1172  	AESENC	X9, X9
  1173  	AESENC	X10, X10
  1174  	AESENC	X11, X11
  1175  	AESENC	X12, X12
  1176  	AESENC	X13, X13
  1177  	AESENC	X14, X14
  1178  	AESENC	X15, X15
  1179  
  1180  	// scramble state, xor in a block
  1181  	MOVOU	(AX), X0
  1182  	MOVOU	16(AX), X1
  1183  	MOVOU	32(AX), X2
  1184  	MOVOU	48(AX), X3
  1185  	AESENC	X0, X8
  1186  	AESENC	X1, X9
  1187  	AESENC	X2, X10
  1188  	AESENC	X3, X11
  1189  	MOVOU	64(AX), X4
  1190  	MOVOU	80(AX), X5
  1191  	MOVOU	96(AX), X6
  1192  	MOVOU	112(AX), X7
  1193  	AESENC	X4, X12
  1194  	AESENC	X5, X13
  1195  	AESENC	X6, X14
  1196  	AESENC	X7, X15
  1197  
  1198  	ADDQ	$128, AX
  1199  	DECQ	CX
  1200  	JNE	aesloop
  1201  
  1202  	// 3 more scrambles to finish
  1203  	AESENC	X8, X8
  1204  	AESENC	X9, X9
  1205  	AESENC	X10, X10
  1206  	AESENC	X11, X11
  1207  	AESENC	X12, X12
  1208  	AESENC	X13, X13
  1209  	AESENC	X14, X14
  1210  	AESENC	X15, X15
  1211  	AESENC	X8, X8
  1212  	AESENC	X9, X9
  1213  	AESENC	X10, X10
  1214  	AESENC	X11, X11
  1215  	AESENC	X12, X12
  1216  	AESENC	X13, X13
  1217  	AESENC	X14, X14
  1218  	AESENC	X15, X15
  1219  	AESENC	X8, X8
  1220  	AESENC	X9, X9
  1221  	AESENC	X10, X10
  1222  	AESENC	X11, X11
  1223  	AESENC	X12, X12
  1224  	AESENC	X13, X13
  1225  	AESENC	X14, X14
  1226  	AESENC	X15, X15
  1227  
  1228  	PXOR	X12, X8
  1229  	PXOR	X13, X9
  1230  	PXOR	X14, X10
  1231  	PXOR	X15, X11
  1232  	PXOR	X10, X8
  1233  	PXOR	X11, X9
  1234  	PXOR	X9, X8
  1235  	MOVQ	X8, (DX)
  1236  	RET
  1237  
  1238  // func memhash32(p unsafe.Pointer, h uintptr) uintptr
  1239  TEXT runtime·memhash32(SB),NOSPLIT,$0-24
  1240  	CMPB	runtime·useAeshash(SB), $0
  1241  	JEQ	noaes
  1242  	MOVQ	p+0(FP), AX	// ptr to data
  1243  	MOVQ	h+8(FP), X0	// seed
  1244  	PINSRD	$2, (AX), X0	// data
  1245  	AESENC	runtime·aeskeysched+0(SB), X0
  1246  	AESENC	runtime·aeskeysched+16(SB), X0
  1247  	AESENC	runtime·aeskeysched+32(SB), X0
  1248  	MOVQ	X0, ret+16(FP)
  1249  	RET
  1250  noaes:
  1251  	JMP	runtime·memhash32Fallback(SB)
  1252  
  1253  // func memhash64(p unsafe.Pointer, h uintptr) uintptr
  1254  TEXT runtime·memhash64(SB),NOSPLIT,$0-24
  1255  	CMPB	runtime·useAeshash(SB), $0
  1256  	JEQ	noaes
  1257  	MOVQ	p+0(FP), AX	// ptr to data
  1258  	MOVQ	h+8(FP), X0	// seed
  1259  	PINSRQ	$1, (AX), X0	// data
  1260  	AESENC	runtime·aeskeysched+0(SB), X0
  1261  	AESENC	runtime·aeskeysched+16(SB), X0
  1262  	AESENC	runtime·aeskeysched+32(SB), X0
  1263  	MOVQ	X0, ret+16(FP)
  1264  	RET
  1265  noaes:
  1266  	JMP	runtime·memhash64Fallback(SB)
  1267  
  1268  // simple mask to get rid of data in the high part of the register.
  1269  DATA masks<>+0x00(SB)/8, $0x0000000000000000
  1270  DATA masks<>+0x08(SB)/8, $0x0000000000000000
  1271  DATA masks<>+0x10(SB)/8, $0x00000000000000ff
  1272  DATA masks<>+0x18(SB)/8, $0x0000000000000000
  1273  DATA masks<>+0x20(SB)/8, $0x000000000000ffff
  1274  DATA masks<>+0x28(SB)/8, $0x0000000000000000
  1275  DATA masks<>+0x30(SB)/8, $0x0000000000ffffff
  1276  DATA masks<>+0x38(SB)/8, $0x0000000000000000
  1277  DATA masks<>+0x40(SB)/8, $0x00000000ffffffff
  1278  DATA masks<>+0x48(SB)/8, $0x0000000000000000
  1279  DATA masks<>+0x50(SB)/8, $0x000000ffffffffff
  1280  DATA masks<>+0x58(SB)/8, $0x0000000000000000
  1281  DATA masks<>+0x60(SB)/8, $0x0000ffffffffffff
  1282  DATA masks<>+0x68(SB)/8, $0x0000000000000000
  1283  DATA masks<>+0x70(SB)/8, $0x00ffffffffffffff
  1284  DATA masks<>+0x78(SB)/8, $0x0000000000000000
  1285  DATA masks<>+0x80(SB)/8, $0xffffffffffffffff
  1286  DATA masks<>+0x88(SB)/8, $0x0000000000000000
  1287  DATA masks<>+0x90(SB)/8, $0xffffffffffffffff
  1288  DATA masks<>+0x98(SB)/8, $0x00000000000000ff
  1289  DATA masks<>+0xa0(SB)/8, $0xffffffffffffffff
  1290  DATA masks<>+0xa8(SB)/8, $0x000000000000ffff
  1291  DATA masks<>+0xb0(SB)/8, $0xffffffffffffffff
  1292  DATA masks<>+0xb8(SB)/8, $0x0000000000ffffff
  1293  DATA masks<>+0xc0(SB)/8, $0xffffffffffffffff
  1294  DATA masks<>+0xc8(SB)/8, $0x00000000ffffffff
  1295  DATA masks<>+0xd0(SB)/8, $0xffffffffffffffff
  1296  DATA masks<>+0xd8(SB)/8, $0x000000ffffffffff
  1297  DATA masks<>+0xe0(SB)/8, $0xffffffffffffffff
  1298  DATA masks<>+0xe8(SB)/8, $0x0000ffffffffffff
  1299  DATA masks<>+0xf0(SB)/8, $0xffffffffffffffff
  1300  DATA masks<>+0xf8(SB)/8, $0x00ffffffffffffff
  1301  GLOBL masks<>(SB),RODATA,$256
  1302  
  1303  // func checkASM() bool
  1304  TEXT ·checkASM(SB),NOSPLIT,$0-1
  1305  	// check that masks<>(SB) and shifts<>(SB) are aligned to 16-byte
  1306  	MOVQ	$masks<>(SB), AX
  1307  	MOVQ	$shifts<>(SB), BX
  1308  	ORQ	BX, AX
  1309  	TESTQ	$15, AX
  1310  	SETEQ	ret+0(FP)
  1311  	RET
  1312  
  1313  // these are arguments to pshufb. They move data down from
  1314  // the high bytes of the register to the low bytes of the register.
  1315  // index is how many bytes to move.
  1316  DATA shifts<>+0x00(SB)/8, $0x0000000000000000
  1317  DATA shifts<>+0x08(SB)/8, $0x0000000000000000
  1318  DATA shifts<>+0x10(SB)/8, $0xffffffffffffff0f
  1319  DATA shifts<>+0x18(SB)/8, $0xffffffffffffffff
  1320  DATA shifts<>+0x20(SB)/8, $0xffffffffffff0f0e
  1321  DATA shifts<>+0x28(SB)/8, $0xffffffffffffffff
  1322  DATA shifts<>+0x30(SB)/8, $0xffffffffff0f0e0d
  1323  DATA shifts<>+0x38(SB)/8, $0xffffffffffffffff
  1324  DATA shifts<>+0x40(SB)/8, $0xffffffff0f0e0d0c
  1325  DATA shifts<>+0x48(SB)/8, $0xffffffffffffffff
  1326  DATA shifts<>+0x50(SB)/8, $0xffffff0f0e0d0c0b
  1327  DATA shifts<>+0x58(SB)/8, $0xffffffffffffffff
  1328  DATA shifts<>+0x60(SB)/8, $0xffff0f0e0d0c0b0a
  1329  DATA shifts<>+0x68(SB)/8, $0xffffffffffffffff
  1330  DATA shifts<>+0x70(SB)/8, $0xff0f0e0d0c0b0a09
  1331  DATA shifts<>+0x78(SB)/8, $0xffffffffffffffff
  1332  DATA shifts<>+0x80(SB)/8, $0x0f0e0d0c0b0a0908
  1333  DATA shifts<>+0x88(SB)/8, $0xffffffffffffffff
  1334  DATA shifts<>+0x90(SB)/8, $0x0e0d0c0b0a090807
  1335  DATA shifts<>+0x98(SB)/8, $0xffffffffffffff0f
  1336  DATA shifts<>+0xa0(SB)/8, $0x0d0c0b0a09080706
  1337  DATA shifts<>+0xa8(SB)/8, $0xffffffffffff0f0e
  1338  DATA shifts<>+0xb0(SB)/8, $0x0c0b0a0908070605
  1339  DATA shifts<>+0xb8(SB)/8, $0xffffffffff0f0e0d
  1340  DATA shifts<>+0xc0(SB)/8, $0x0b0a090807060504
  1341  DATA shifts<>+0xc8(SB)/8, $0xffffffff0f0e0d0c
  1342  DATA shifts<>+0xd0(SB)/8, $0x0a09080706050403
  1343  DATA shifts<>+0xd8(SB)/8, $0xffffff0f0e0d0c0b
  1344  DATA shifts<>+0xe0(SB)/8, $0x0908070605040302
  1345  DATA shifts<>+0xe8(SB)/8, $0xffff0f0e0d0c0b0a
  1346  DATA shifts<>+0xf0(SB)/8, $0x0807060504030201
  1347  DATA shifts<>+0xf8(SB)/8, $0xff0f0e0d0c0b0a09
  1348  GLOBL shifts<>(SB),RODATA,$256
  1349  
  1350  TEXT runtime·return0(SB), NOSPLIT, $0
  1351  	MOVL	$0, AX
  1352  	RET
  1353  
  1354  
  1355  // Called from cgo wrappers, this function returns g->m->curg.stack.hi.
  1356  // Must obey the gcc calling convention.
  1357  TEXT _cgo_topofstack(SB),NOSPLIT,$0
  1358  	get_tls(CX)
  1359  	MOVQ	g(CX), AX
  1360  	MOVQ	g_m(AX), AX
  1361  	MOVQ	m_curg(AX), AX
  1362  	MOVQ	(g_stack+stack_hi)(AX), AX
  1363  	RET
  1364  
  1365  // The top-most function running on a goroutine
  1366  // returns to goexit+PCQuantum. Defined as ABIInternal
  1367  // so as to make it identifiable to traceback (this
  1368  // function it used as a sentinel; traceback wants to
  1369  // see the func PC, not a wrapper PC).
  1370  TEXT runtime·goexit<ABIInternal>(SB),NOSPLIT,$0-0
  1371  	BYTE	$0x90	// NOP
  1372  	CALL	runtime·goexit1(SB)	// does not return
  1373  	// traceback from goexit1 must hit code range of goexit
  1374  	BYTE	$0x90	// NOP
  1375  
  1376  // This is called from .init_array and follows the platform, not Go, ABI.
  1377  TEXT runtime·addmoduledata(SB),NOSPLIT,$0-0
  1378  	PUSHQ	R15 // The access to global variables below implicitly uses R15, which is callee-save
  1379  	MOVQ	runtime·lastmoduledatap(SB), AX
  1380  	MOVQ	DI, moduledata_next(AX)
  1381  	MOVQ	DI, runtime·lastmoduledatap(SB)
  1382  	POPQ	R15
  1383  	RET
  1384  
  1385  // gcWriteBarrier performs a heap pointer write and informs the GC.
  1386  //
  1387  // gcWriteBarrier does NOT follow the Go ABI. It takes two arguments:
  1388  // - DI is the destination of the write
  1389  // - AX is the value being written at DI
  1390  // It clobbers FLAGS. It does not clobber any general-purpose registers,
  1391  // but may clobber others (e.g., SSE registers).
  1392  // Defined as ABIInternal since it does not use the stack-based Go ABI.
  1393  TEXT runtime·gcWriteBarrier<ABIInternal>(SB),NOSPLIT,$120
  1394  	// Save the registers clobbered by the fast path. This is slightly
  1395  	// faster than having the caller spill these.
  1396  	MOVQ	R14, 104(SP)
  1397  	MOVQ	R13, 112(SP)
  1398  	// TODO: Consider passing g.m.p in as an argument so they can be shared
  1399  	// across a sequence of write barriers.
  1400  	get_tls(R13)
  1401  	MOVQ	g(R13), R13
  1402  	MOVQ	g_m(R13), R13
  1403  	MOVQ	m_p(R13), R13
  1404  	MOVQ	(p_wbBuf+wbBuf_next)(R13), R14
  1405  	// Increment wbBuf.next position.
  1406  	LEAQ	16(R14), R14
  1407  	MOVQ	R14, (p_wbBuf+wbBuf_next)(R13)
  1408  	CMPQ	R14, (p_wbBuf+wbBuf_end)(R13)
  1409  	// Record the write.
  1410  	MOVQ	AX, -16(R14)	// Record value
  1411  	// Note: This turns bad pointer writes into bad
  1412  	// pointer reads, which could be confusing. We could avoid
  1413  	// reading from obviously bad pointers, which would
  1414  	// take care of the vast majority of these. We could
  1415  	// patch this up in the signal handler, or use XCHG to
  1416  	// combine the read and the write.
  1417  	MOVQ	(DI), R13
  1418  	MOVQ	R13, -8(R14)	// Record *slot
  1419  	// Is the buffer full? (flags set in CMPQ above)
  1420  	JEQ	flush
  1421  ret:
  1422  	MOVQ	104(SP), R14
  1423  	MOVQ	112(SP), R13
  1424  	// Do the write.
  1425  	MOVQ	AX, (DI)
  1426  	RET
  1427  
  1428  flush:
  1429  	// Save all general purpose registers since these could be
  1430  	// clobbered by wbBufFlush and were not saved by the caller.
  1431  	// It is possible for wbBufFlush to clobber other registers
  1432  	// (e.g., SSE registers), but the compiler takes care of saving
  1433  	// those in the caller if necessary. This strikes a balance
  1434  	// with registers that are likely to be used.
  1435  	//
  1436  	// We don't have type information for these, but all code under
  1437  	// here is NOSPLIT, so nothing will observe these.
  1438  	//
  1439  	// TODO: We could strike a different balance; e.g., saving X0
  1440  	// and not saving GP registers that are less likely to be used.
  1441  	MOVQ	DI, 0(SP)	// Also first argument to wbBufFlush
  1442  	MOVQ	AX, 8(SP)	// Also second argument to wbBufFlush
  1443  	MOVQ	BX, 16(SP)
  1444  	MOVQ	CX, 24(SP)
  1445  	MOVQ	DX, 32(SP)
  1446  	// DI already saved
  1447  	MOVQ	SI, 40(SP)
  1448  	MOVQ	BP, 48(SP)
  1449  	MOVQ	R8, 56(SP)
  1450  	MOVQ	R9, 64(SP)
  1451  	MOVQ	R10, 72(SP)
  1452  	MOVQ	R11, 80(SP)
  1453  	MOVQ	R12, 88(SP)
  1454  	// R13 already saved
  1455  	// R14 already saved
  1456  	MOVQ	R15, 96(SP)
  1457  
  1458  	// This takes arguments DI and AX
  1459  	CALL	runtime·wbBufFlush(SB)
  1460  
  1461  	MOVQ	0(SP), DI
  1462  	MOVQ	8(SP), AX
  1463  	MOVQ	16(SP), BX
  1464  	MOVQ	24(SP), CX
  1465  	MOVQ	32(SP), DX
  1466  	MOVQ	40(SP), SI
  1467  	MOVQ	48(SP), BP
  1468  	MOVQ	56(SP), R8
  1469  	MOVQ	64(SP), R9
  1470  	MOVQ	72(SP), R10
  1471  	MOVQ	80(SP), R11
  1472  	MOVQ	88(SP), R12
  1473  	MOVQ	96(SP), R15
  1474  	JMP	ret
  1475  
  1476  // gcWriteBarrierCX is gcWriteBarrier, but with args in DI and CX.
  1477  // Defined as ABIInternal since it does not use the stable Go ABI.
  1478  TEXT runtime·gcWriteBarrierCX<ABIInternal>(SB),NOSPLIT,$0
  1479  	XCHGQ CX, AX
  1480  	CALL runtime·gcWriteBarrier<ABIInternal>(SB)
  1481  	XCHGQ CX, AX
  1482  	RET
  1483  
  1484  // gcWriteBarrierDX is gcWriteBarrier, but with args in DI and DX.
  1485  // Defined as ABIInternal since it does not use the stable Go ABI.
  1486  TEXT runtime·gcWriteBarrierDX<ABIInternal>(SB),NOSPLIT,$0
  1487  	XCHGQ DX, AX
  1488  	CALL runtime·gcWriteBarrier<ABIInternal>(SB)
  1489  	XCHGQ DX, AX
  1490  	RET
  1491  
  1492  // gcWriteBarrierBX is gcWriteBarrier, but with args in DI and BX.
  1493  // Defined as ABIInternal since it does not use the stable Go ABI.
  1494  TEXT runtime·gcWriteBarrierBX<ABIInternal>(SB),NOSPLIT,$0
  1495  	XCHGQ BX, AX
  1496  	CALL runtime·gcWriteBarrier<ABIInternal>(SB)
  1497  	XCHGQ BX, AX
  1498  	RET
  1499  
  1500  // gcWriteBarrierBP is gcWriteBarrier, but with args in DI and BP.
  1501  // Defined as ABIInternal since it does not use the stable Go ABI.
  1502  TEXT runtime·gcWriteBarrierBP<ABIInternal>(SB),NOSPLIT,$0
  1503  	XCHGQ BP, AX
  1504  	CALL runtime·gcWriteBarrier<ABIInternal>(SB)
  1505  	XCHGQ BP, AX
  1506  	RET
  1507  
  1508  // gcWriteBarrierSI is gcWriteBarrier, but with args in DI and SI.
  1509  // Defined as ABIInternal since it does not use the stable Go ABI.
  1510  TEXT runtime·gcWriteBarrierSI<ABIInternal>(SB),NOSPLIT,$0
  1511  	XCHGQ SI, AX
  1512  	CALL runtime·gcWriteBarrier<ABIInternal>(SB)
  1513  	XCHGQ SI, AX
  1514  	RET
  1515  
  1516  // gcWriteBarrierR8 is gcWriteBarrier, but with args in DI and R8.
  1517  // Defined as ABIInternal since it does not use the stable Go ABI.
  1518  TEXT runtime·gcWriteBarrierR8<ABIInternal>(SB),NOSPLIT,$0
  1519  	XCHGQ R8, AX
  1520  	CALL runtime·gcWriteBarrier<ABIInternal>(SB)
  1521  	XCHGQ R8, AX
  1522  	RET
  1523  
  1524  // gcWriteBarrierR9 is gcWriteBarrier, but with args in DI and R9.
  1525  // Defined as ABIInternal since it does not use the stable Go ABI.
  1526  TEXT runtime·gcWriteBarrierR9<ABIInternal>(SB),NOSPLIT,$0
  1527  	XCHGQ R9, AX
  1528  	CALL runtime·gcWriteBarrier<ABIInternal>(SB)
  1529  	XCHGQ R9, AX
  1530  	RET
  1531  
  1532  DATA	debugCallFrameTooLarge<>+0x00(SB)/20, $"call frame too large"
  1533  GLOBL	debugCallFrameTooLarge<>(SB), RODATA, $20	// Size duplicated below
  1534  
  1535  // debugCallV1 is the entry point for debugger-injected function
  1536  // calls on running goroutines. It informs the runtime that a
  1537  // debug call has been injected and creates a call frame for the
  1538  // debugger to fill in.
  1539  //
  1540  // To inject a function call, a debugger should:
  1541  // 1. Check that the goroutine is in state _Grunning and that
  1542  //    there are at least 256 bytes free on the stack.
  1543  // 2. Push the current PC on the stack (updating SP).
  1544  // 3. Write the desired argument frame size at SP-16 (using the SP
  1545  //    after step 2).
  1546  // 4. Save all machine registers (including flags and XMM reigsters)
  1547  //    so they can be restored later by the debugger.
  1548  // 5. Set the PC to debugCallV1 and resume execution.
  1549  //
  1550  // If the goroutine is in state _Grunnable, then it's not generally
  1551  // safe to inject a call because it may return out via other runtime
  1552  // operations. Instead, the debugger should unwind the stack to find
  1553  // the return to non-runtime code, add a temporary breakpoint there,
  1554  // and inject the call once that breakpoint is hit.
  1555  //
  1556  // If the goroutine is in any other state, it's not safe to inject a call.
  1557  //
  1558  // This function communicates back to the debugger by setting RAX and
  1559  // invoking INT3 to raise a breakpoint signal. See the comments in the
  1560  // implementation for the protocol the debugger is expected to
  1561  // follow. InjectDebugCall in the runtime tests demonstrates this protocol.
  1562  //
  1563  // The debugger must ensure that any pointers passed to the function
  1564  // obey escape analysis requirements. Specifically, it must not pass
  1565  // a stack pointer to an escaping argument. debugCallV1 cannot check
  1566  // this invariant.
  1567  //
  1568  // This is ABIInternal because Go code injects its PC directly into new
  1569  // goroutine stacks.
  1570  TEXT runtime·debugCallV1<ABIInternal>(SB),NOSPLIT,$152-0
  1571  	// Save all registers that may contain pointers so they can be
  1572  	// conservatively scanned.
  1573  	//
  1574  	// We can't do anything that might clobber any of these
  1575  	// registers before this.
  1576  	MOVQ	R15, r15-(14*8+8)(SP)
  1577  	MOVQ	R14, r14-(13*8+8)(SP)
  1578  	MOVQ	R13, r13-(12*8+8)(SP)
  1579  	MOVQ	R12, r12-(11*8+8)(SP)
  1580  	MOVQ	R11, r11-(10*8+8)(SP)
  1581  	MOVQ	R10, r10-(9*8+8)(SP)
  1582  	MOVQ	R9, r9-(8*8+8)(SP)
  1583  	MOVQ	R8, r8-(7*8+8)(SP)
  1584  	MOVQ	DI, di-(6*8+8)(SP)
  1585  	MOVQ	SI, si-(5*8+8)(SP)
  1586  	MOVQ	BP, bp-(4*8+8)(SP)
  1587  	MOVQ	BX, bx-(3*8+8)(SP)
  1588  	MOVQ	DX, dx-(2*8+8)(SP)
  1589  	// Save the frame size before we clobber it. Either of the last
  1590  	// saves could clobber this depending on whether there's a saved BP.
  1591  	MOVQ	frameSize-24(FP), DX	// aka -16(RSP) before prologue
  1592  	MOVQ	CX, cx-(1*8+8)(SP)
  1593  	MOVQ	AX, ax-(0*8+8)(SP)
  1594  
  1595  	// Save the argument frame size.
  1596  	MOVQ	DX, frameSize-128(SP)
  1597  
  1598  	// Perform a safe-point check.
  1599  	MOVQ	retpc-8(FP), AX	// Caller's PC
  1600  	MOVQ	AX, 0(SP)
  1601  	CALL	runtime·debugCallCheck(SB)
  1602  	MOVQ	8(SP), AX
  1603  	TESTQ	AX, AX
  1604  	JZ	good
  1605  	// The safety check failed. Put the reason string at the top
  1606  	// of the stack.
  1607  	MOVQ	AX, 0(SP)
  1608  	MOVQ	16(SP), AX
  1609  	MOVQ	AX, 8(SP)
  1610  	// Set AX to 8 and invoke INT3. The debugger should get the
  1611  	// reason a call can't be injected from the top of the stack
  1612  	// and resume execution.
  1613  	MOVQ	$8, AX
  1614  	BYTE	$0xcc
  1615  	JMP	restore
  1616  
  1617  good:
  1618  	// Registers are saved and it's safe to make a call.
  1619  	// Open up a call frame, moving the stack if necessary.
  1620  	//
  1621  	// Once the frame is allocated, this will set AX to 0 and
  1622  	// invoke INT3. The debugger should write the argument
  1623  	// frame for the call at SP, push the trapping PC on the
  1624  	// stack, set the PC to the function to call, set RCX to point
  1625  	// to the closure (if a closure call), and resume execution.
  1626  	//
  1627  	// If the function returns, this will set AX to 1 and invoke
  1628  	// INT3. The debugger can then inspect any return value saved
  1629  	// on the stack at SP and resume execution again.
  1630  	//
  1631  	// If the function panics, this will set AX to 2 and invoke INT3.
  1632  	// The interface{} value of the panic will be at SP. The debugger
  1633  	// can inspect the panic value and resume execution again.
  1634  #define DEBUG_CALL_DISPATCH(NAME,MAXSIZE)	\
  1635  	CMPQ	AX, $MAXSIZE;			\
  1636  	JA	5(PC);				\
  1637  	MOVQ	$NAME(SB), AX;			\
  1638  	MOVQ	AX, 0(SP);			\
  1639  	CALL	runtime·debugCallWrap(SB);	\
  1640  	JMP	restore
  1641  
  1642  	MOVQ	frameSize-128(SP), AX
  1643  	DEBUG_CALL_DISPATCH(debugCall32<>, 32)
  1644  	DEBUG_CALL_DISPATCH(debugCall64<>, 64)
  1645  	DEBUG_CALL_DISPATCH(debugCall128<>, 128)
  1646  	DEBUG_CALL_DISPATCH(debugCall256<>, 256)
  1647  	DEBUG_CALL_DISPATCH(debugCall512<>, 512)
  1648  	DEBUG_CALL_DISPATCH(debugCall1024<>, 1024)
  1649  	DEBUG_CALL_DISPATCH(debugCall2048<>, 2048)
  1650  	DEBUG_CALL_DISPATCH(debugCall4096<>, 4096)
  1651  	DEBUG_CALL_DISPATCH(debugCall8192<>, 8192)
  1652  	DEBUG_CALL_DISPATCH(debugCall16384<>, 16384)
  1653  	DEBUG_CALL_DISPATCH(debugCall32768<>, 32768)
  1654  	DEBUG_CALL_DISPATCH(debugCall65536<>, 65536)
  1655  	// The frame size is too large. Report the error.
  1656  	MOVQ	$debugCallFrameTooLarge<>(SB), AX
  1657  	MOVQ	AX, 0(SP)
  1658  	MOVQ	$20, 8(SP) // length of debugCallFrameTooLarge string
  1659  	MOVQ	$8, AX
  1660  	BYTE	$0xcc
  1661  	JMP	restore
  1662  
  1663  restore:
  1664  	// Calls and failures resume here.
  1665  	//
  1666  	// Set AX to 16 and invoke INT3. The debugger should restore
  1667  	// all registers except RIP and RSP and resume execution.
  1668  	MOVQ	$16, AX
  1669  	BYTE	$0xcc
  1670  	// We must not modify flags after this point.
  1671  
  1672  	// Restore pointer-containing registers, which may have been
  1673  	// modified from the debugger's copy by stack copying.
  1674  	MOVQ	ax-(0*8+8)(SP), AX
  1675  	MOVQ	cx-(1*8+8)(SP), CX
  1676  	MOVQ	dx-(2*8+8)(SP), DX
  1677  	MOVQ	bx-(3*8+8)(SP), BX
  1678  	MOVQ	bp-(4*8+8)(SP), BP
  1679  	MOVQ	si-(5*8+8)(SP), SI
  1680  	MOVQ	di-(6*8+8)(SP), DI
  1681  	MOVQ	r8-(7*8+8)(SP), R8
  1682  	MOVQ	r9-(8*8+8)(SP), R9
  1683  	MOVQ	r10-(9*8+8)(SP), R10
  1684  	MOVQ	r11-(10*8+8)(SP), R11
  1685  	MOVQ	r12-(11*8+8)(SP), R12
  1686  	MOVQ	r13-(12*8+8)(SP), R13
  1687  	MOVQ	r14-(13*8+8)(SP), R14
  1688  	MOVQ	r15-(14*8+8)(SP), R15
  1689  
  1690  	RET
  1691  
  1692  // runtime.debugCallCheck assumes that functions defined with the
  1693  // DEBUG_CALL_FN macro are safe points to inject calls.
  1694  #define DEBUG_CALL_FN(NAME,MAXSIZE)		\
  1695  TEXT NAME(SB),WRAPPER,$MAXSIZE-0;		\
  1696  	NO_LOCAL_POINTERS;			\
  1697  	MOVQ	$0, AX;				\
  1698  	BYTE	$0xcc;				\
  1699  	MOVQ	$1, AX;				\
  1700  	BYTE	$0xcc;				\
  1701  	RET
  1702  DEBUG_CALL_FN(debugCall32<>, 32)
  1703  DEBUG_CALL_FN(debugCall64<>, 64)
  1704  DEBUG_CALL_FN(debugCall128<>, 128)
  1705  DEBUG_CALL_FN(debugCall256<>, 256)
  1706  DEBUG_CALL_FN(debugCall512<>, 512)
  1707  DEBUG_CALL_FN(debugCall1024<>, 1024)
  1708  DEBUG_CALL_FN(debugCall2048<>, 2048)
  1709  DEBUG_CALL_FN(debugCall4096<>, 4096)
  1710  DEBUG_CALL_FN(debugCall8192<>, 8192)
  1711  DEBUG_CALL_FN(debugCall16384<>, 16384)
  1712  DEBUG_CALL_FN(debugCall32768<>, 32768)
  1713  DEBUG_CALL_FN(debugCall65536<>, 65536)
  1714  
  1715  // func debugCallPanicked(val interface{})
  1716  TEXT runtime·debugCallPanicked(SB),NOSPLIT,$16-16
  1717  	// Copy the panic value to the top of stack.
  1718  	MOVQ	val_type+0(FP), AX
  1719  	MOVQ	AX, 0(SP)
  1720  	MOVQ	val_data+8(FP), AX
  1721  	MOVQ	AX, 8(SP)
  1722  	MOVQ	$2, AX
  1723  	BYTE	$0xcc
  1724  	RET
  1725  
  1726  // Note: these functions use a special calling convention to save generated code space.
  1727  // Arguments are passed in registers, but the space for those arguments are allocated
  1728  // in the caller's stack frame. These stubs write the args into that stack space and
  1729  // then tail call to the corresponding runtime handler.
  1730  // The tail call makes these stubs disappear in backtraces.
  1731  // Defined as ABIInternal since they do not use the stack-based Go ABI.
  1732  TEXT runtime·panicIndex<ABIInternal>(SB),NOSPLIT,$0-16
  1733  	MOVQ	AX, x+0(FP)
  1734  	MOVQ	CX, y+8(FP)
  1735  	JMP	runtime·goPanicIndex(SB)
  1736  TEXT runtime·panicIndexU<ABIInternal>(SB),NOSPLIT,$0-16
  1737  	MOVQ	AX, x+0(FP)
  1738  	MOVQ	CX, y+8(FP)
  1739  	JMP	runtime·goPanicIndexU(SB)
  1740  TEXT runtime·panicSliceAlen<ABIInternal>(SB),NOSPLIT,$0-16
  1741  	MOVQ	CX, x+0(FP)
  1742  	MOVQ	DX, y+8(FP)
  1743  	JMP	runtime·goPanicSliceAlen(SB)
  1744  TEXT runtime·panicSliceAlenU<ABIInternal>(SB),NOSPLIT,$0-16
  1745  	MOVQ	CX, x+0(FP)
  1746  	MOVQ	DX, y+8(FP)
  1747  	JMP	runtime·goPanicSliceAlenU(SB)
  1748  TEXT runtime·panicSliceAcap<ABIInternal>(SB),NOSPLIT,$0-16
  1749  	MOVQ	CX, x+0(FP)
  1750  	MOVQ	DX, y+8(FP)
  1751  	JMP	runtime·goPanicSliceAcap(SB)
  1752  TEXT runtime·panicSliceAcapU<ABIInternal>(SB),NOSPLIT,$0-16
  1753  	MOVQ	CX, x+0(FP)
  1754  	MOVQ	DX, y+8(FP)
  1755  	JMP	runtime·goPanicSliceAcapU(SB)
  1756  TEXT runtime·panicSliceB<ABIInternal>(SB),NOSPLIT,$0-16
  1757  	MOVQ	AX, x+0(FP)
  1758  	MOVQ	CX, y+8(FP)
  1759  	JMP	runtime·goPanicSliceB(SB)
  1760  TEXT runtime·panicSliceBU<ABIInternal>(SB),NOSPLIT,$0-16
  1761  	MOVQ	AX, x+0(FP)
  1762  	MOVQ	CX, y+8(FP)
  1763  	JMP	runtime·goPanicSliceBU(SB)
  1764  TEXT runtime·panicSlice3Alen<ABIInternal>(SB),NOSPLIT,$0-16
  1765  	MOVQ	DX, x+0(FP)
  1766  	MOVQ	BX, y+8(FP)
  1767  	JMP	runtime·goPanicSlice3Alen(SB)
  1768  TEXT runtime·panicSlice3AlenU<ABIInternal>(SB),NOSPLIT,$0-16
  1769  	MOVQ	DX, x+0(FP)
  1770  	MOVQ	BX, y+8(FP)
  1771  	JMP	runtime·goPanicSlice3AlenU(SB)
  1772  TEXT runtime·panicSlice3Acap<ABIInternal>(SB),NOSPLIT,$0-16
  1773  	MOVQ	DX, x+0(FP)
  1774  	MOVQ	BX, y+8(FP)
  1775  	JMP	runtime·goPanicSlice3Acap(SB)
  1776  TEXT runtime·panicSlice3AcapU<ABIInternal>(SB),NOSPLIT,$0-16
  1777  	MOVQ	DX, x+0(FP)
  1778  	MOVQ	BX, y+8(FP)
  1779  	JMP	runtime·goPanicSlice3AcapU(SB)
  1780  TEXT runtime·panicSlice3B<ABIInternal>(SB),NOSPLIT,$0-16
  1781  	MOVQ	CX, x+0(FP)
  1782  	MOVQ	DX, y+8(FP)
  1783  	JMP	runtime·goPanicSlice3B(SB)
  1784  TEXT runtime·panicSlice3BU<ABIInternal>(SB),NOSPLIT,$0-16
  1785  	MOVQ	CX, x+0(FP)
  1786  	MOVQ	DX, y+8(FP)
  1787  	JMP	runtime·goPanicSlice3BU(SB)
  1788  TEXT runtime·panicSlice3C<ABIInternal>(SB),NOSPLIT,$0-16
  1789  	MOVQ	AX, x+0(FP)
  1790  	MOVQ	CX, y+8(FP)
  1791  	JMP	runtime·goPanicSlice3C(SB)
  1792  TEXT runtime·panicSlice3CU<ABIInternal>(SB),NOSPLIT,$0-16
  1793  	MOVQ	AX, x+0(FP)
  1794  	MOVQ	CX, y+8(FP)
  1795  	JMP	runtime·goPanicSlice3CU(SB)
  1796  
  1797  #ifdef GOOS_android
  1798  // Use the free TLS_SLOT_APP slot #2 on Android Q.
  1799  // Earlier androids are set up in gcc_android.c.
  1800  DATA runtime·tls_g+0(SB)/8, $16
  1801  GLOBL runtime·tls_g+0(SB), NOPTR, $8
  1802  #endif
  1803  
  1804  // The compiler and assembler's -spectre=ret mode rewrites
  1805  // all indirect CALL AX / JMP AX instructions to be
  1806  // CALL retpolineAX / JMP retpolineAX.
  1807  // See https://support.google.com/faqs/answer/7625886.
  1808  #define RETPOLINE(reg) \
  1809  	/*   CALL setup */     BYTE $0xE8; BYTE $(2+2); BYTE $0; BYTE $0; BYTE $0;	\
  1810  	/* nospec: */									\
  1811  	/*   PAUSE */           BYTE $0xF3; BYTE $0x90;					\
  1812  	/*   JMP nospec */      BYTE $0xEB; BYTE $-(2+2);				\
  1813  	/* setup: */									\
  1814  	/*   MOVQ AX, 0(SP) */  BYTE $0x48|((reg&8)>>1); BYTE $0x89;			\
  1815  	                        BYTE $0x04|((reg&7)<<3); BYTE $0x24;			\
  1816  	/*   RET */             BYTE $0xC3
  1817  
  1818  TEXT runtime·retpolineAX(SB),NOSPLIT,$0; RETPOLINE(0)
  1819  TEXT runtime·retpolineCX(SB),NOSPLIT,$0; RETPOLINE(1)
  1820  TEXT runtime·retpolineDX(SB),NOSPLIT,$0; RETPOLINE(2)
  1821  TEXT runtime·retpolineBX(SB),NOSPLIT,$0; RETPOLINE(3)
  1822  /* SP is 4, can't happen / magic encodings */
  1823  TEXT runtime·retpolineBP(SB),NOSPLIT,$0; RETPOLINE(5)
  1824  TEXT runtime·retpolineSI(SB),NOSPLIT,$0; RETPOLINE(6)
  1825  TEXT runtime·retpolineDI(SB),NOSPLIT,$0; RETPOLINE(7)
  1826  TEXT runtime·retpolineR8(SB),NOSPLIT,$0; RETPOLINE(8)
  1827  TEXT runtime·retpolineR9(SB),NOSPLIT,$0; RETPOLINE(9)
  1828  TEXT runtime·retpolineR10(SB),NOSPLIT,$0; RETPOLINE(10)
  1829  TEXT runtime·retpolineR11(SB),NOSPLIT,$0; RETPOLINE(11)
  1830  TEXT runtime·retpolineR12(SB),NOSPLIT,$0; RETPOLINE(12)
  1831  TEXT runtime·retpolineR13(SB),NOSPLIT,$0; RETPOLINE(13)
  1832  TEXT runtime·retpolineR14(SB),NOSPLIT,$0; RETPOLINE(14)
  1833  TEXT runtime·retpolineR15(SB),NOSPLIT,$0; RETPOLINE(15)
  1834  

View as plain text