...
Run Format

Text file src/crypto/elliptic/p256_asm_amd64.s

Documentation: crypto/elliptic

     1	// Copyright 2015 The Go Authors. All rights reserved.
     2	// Use of this source code is governed by a BSD-style
     3	// license that can be found in the LICENSE file.
     4	
     5	// This file contains constant-time, 64-bit assembly implementation of
     6	// P256. The optimizations performed here are described in detail in:
     7	// S.Gueron and V.Krasnov, "Fast prime field elliptic-curve cryptography with
     8	//                          256-bit primes"
     9	// https://link.springer.com/article/10.1007%2Fs13389-014-0090-x
    10	// https://eprint.iacr.org/2013/816.pdf
    11	
    12	#include "textflag.h"
    13	
    14	#define res_ptr DI
    15	#define x_ptr SI
    16	#define y_ptr CX
    17	
    18	#define acc0 R8
    19	#define acc1 R9
    20	#define acc2 R10
    21	#define acc3 R11
    22	#define acc4 R12
    23	#define acc5 R13
    24	#define t0 R14
    25	#define t1 R15
    26	
    27	DATA p256const0<>+0x00(SB)/8, $0x00000000ffffffff
    28	DATA p256const1<>+0x00(SB)/8, $0xffffffff00000001
    29	DATA p256ordK0<>+0x00(SB)/8, $0xccd1c8aaee00bc4f
    30	DATA p256ord<>+0x00(SB)/8, $0xf3b9cac2fc632551
    31	DATA p256ord<>+0x08(SB)/8, $0xbce6faada7179e84
    32	DATA p256ord<>+0x10(SB)/8, $0xffffffffffffffff
    33	DATA p256ord<>+0x18(SB)/8, $0xffffffff00000000
    34	DATA p256one<>+0x00(SB)/8, $0x0000000000000001
    35	DATA p256one<>+0x08(SB)/8, $0xffffffff00000000
    36	DATA p256one<>+0x10(SB)/8, $0xffffffffffffffff
    37	DATA p256one<>+0x18(SB)/8, $0x00000000fffffffe
    38	GLOBL p256const0<>(SB), 8, $8
    39	GLOBL p256const1<>(SB), 8, $8
    40	GLOBL p256ordK0<>(SB), 8, $8
    41	GLOBL p256ord<>(SB), 8, $32
    42	GLOBL p256one<>(SB), 8, $32
    43	
    44	/* ---------------------------------------*/
    45	// func p256LittleToBig(res []byte, in []uint64)
    46	TEXT ·p256LittleToBig(SB),NOSPLIT,$0
    47		JMP ·p256BigToLittle(SB)
    48	/* ---------------------------------------*/
    49	// func p256BigToLittle(res []uint64, in []byte)
    50	TEXT ·p256BigToLittle(SB),NOSPLIT,$0
    51		MOVQ res+0(FP), res_ptr
    52		MOVQ in+24(FP), x_ptr
    53	
    54		MOVQ (8*0)(x_ptr), acc0
    55		MOVQ (8*1)(x_ptr), acc1
    56		MOVQ (8*2)(x_ptr), acc2
    57		MOVQ (8*3)(x_ptr), acc3
    58	
    59		BSWAPQ acc0
    60		BSWAPQ acc1
    61		BSWAPQ acc2
    62		BSWAPQ acc3
    63	
    64		MOVQ acc3, (8*0)(res_ptr)
    65		MOVQ acc2, (8*1)(res_ptr)
    66		MOVQ acc1, (8*2)(res_ptr)
    67		MOVQ acc0, (8*3)(res_ptr)
    68	
    69		RET
    70	/* ---------------------------------------*/
    71	// func p256MovCond(res, a, b []uint64, cond int)
    72	// If cond == 0 res=b, else res=a
    73	TEXT ·p256MovCond(SB),NOSPLIT,$0
    74		MOVQ res+0(FP), res_ptr
    75		MOVQ a+24(FP), x_ptr
    76		MOVQ b+48(FP), y_ptr
    77		MOVQ cond+72(FP), X12
    78	
    79		PXOR X13, X13
    80		PSHUFD $0, X12, X12
    81		PCMPEQL X13, X12
    82	
    83		MOVOU X12, X0
    84		MOVOU (16*0)(x_ptr), X6
    85		PANDN X6, X0
    86		MOVOU X12, X1
    87		MOVOU (16*1)(x_ptr), X7
    88		PANDN X7, X1
    89		MOVOU X12, X2
    90		MOVOU (16*2)(x_ptr), X8
    91		PANDN X8, X2
    92		MOVOU X12, X3
    93		MOVOU (16*3)(x_ptr), X9
    94		PANDN X9, X3
    95		MOVOU X12, X4
    96		MOVOU (16*4)(x_ptr), X10
    97		PANDN X10, X4
    98		MOVOU X12, X5
    99		MOVOU (16*5)(x_ptr), X11
   100		PANDN X11, X5
   101	
   102		MOVOU (16*0)(y_ptr), X6
   103		MOVOU (16*1)(y_ptr), X7
   104		MOVOU (16*2)(y_ptr), X8
   105		MOVOU (16*3)(y_ptr), X9
   106		MOVOU (16*4)(y_ptr), X10
   107		MOVOU (16*5)(y_ptr), X11
   108	
   109		PAND X12, X6
   110		PAND X12, X7
   111		PAND X12, X8
   112		PAND X12, X9
   113		PAND X12, X10
   114		PAND X12, X11
   115	
   116		PXOR X6, X0
   117		PXOR X7, X1
   118		PXOR X8, X2
   119		PXOR X9, X3
   120		PXOR X10, X4
   121		PXOR X11, X5
   122	
   123		MOVOU X0, (16*0)(res_ptr)
   124		MOVOU X1, (16*1)(res_ptr)
   125		MOVOU X2, (16*2)(res_ptr)
   126		MOVOU X3, (16*3)(res_ptr)
   127		MOVOU X4, (16*4)(res_ptr)
   128		MOVOU X5, (16*5)(res_ptr)
   129	
   130		RET
   131	/* ---------------------------------------*/
   132	// func p256NegCond(val []uint64, cond int)
   133	TEXT ·p256NegCond(SB),NOSPLIT,$0
   134		MOVQ val+0(FP), res_ptr
   135		MOVQ cond+24(FP), t0
   136		// acc = poly
   137		MOVQ $-1, acc0
   138		MOVQ p256const0<>(SB), acc1
   139		MOVQ $0, acc2
   140		MOVQ p256const1<>(SB), acc3
   141		// Load the original value
   142		MOVQ (8*0)(res_ptr), acc5
   143		MOVQ (8*1)(res_ptr), x_ptr
   144		MOVQ (8*2)(res_ptr), y_ptr
   145		MOVQ (8*3)(res_ptr), t1
   146		// Speculatively subtract
   147		SUBQ acc5, acc0
   148		SBBQ x_ptr, acc1
   149		SBBQ y_ptr, acc2
   150		SBBQ t1, acc3
   151		// If condition is 0, keep original value
   152		TESTQ t0, t0
   153		CMOVQEQ acc5, acc0
   154		CMOVQEQ x_ptr, acc1
   155		CMOVQEQ y_ptr, acc2
   156		CMOVQEQ t1, acc3
   157		// Store result
   158		MOVQ acc0, (8*0)(res_ptr)
   159		MOVQ acc1, (8*1)(res_ptr)
   160		MOVQ acc2, (8*2)(res_ptr)
   161		MOVQ acc3, (8*3)(res_ptr)
   162	
   163		RET
   164	/* ---------------------------------------*/
   165	// func p256Sqr(res, in []uint64, n int)
   166	TEXT ·p256Sqr(SB),NOSPLIT,$0
   167		MOVQ res+0(FP), res_ptr
   168		MOVQ in+24(FP), x_ptr
   169		MOVQ n+48(FP), BX
   170	
   171	sqrLoop:
   172	
   173		// y[1:] * y[0]
   174		MOVQ (8*0)(x_ptr), t0
   175	
   176		MOVQ (8*1)(x_ptr), AX
   177		MULQ t0
   178		MOVQ AX, acc1
   179		MOVQ DX, acc2
   180	
   181		MOVQ (8*2)(x_ptr), AX
   182		MULQ t0
   183		ADDQ AX, acc2
   184		ADCQ $0, DX
   185		MOVQ DX, acc3
   186	
   187		MOVQ (8*3)(x_ptr), AX
   188		MULQ t0
   189		ADDQ AX, acc3
   190		ADCQ $0, DX
   191		MOVQ DX, acc4
   192		// y[2:] * y[1]
   193		MOVQ (8*1)(x_ptr), t0
   194	
   195		MOVQ (8*2)(x_ptr), AX
   196		MULQ t0
   197		ADDQ AX, acc3
   198		ADCQ $0, DX
   199		MOVQ DX, t1
   200	
   201		MOVQ (8*3)(x_ptr), AX
   202		MULQ t0
   203		ADDQ t1, acc4
   204		ADCQ $0, DX
   205		ADDQ AX, acc4
   206		ADCQ $0, DX
   207		MOVQ DX, acc5
   208		// y[3] * y[2]
   209		MOVQ (8*2)(x_ptr), t0
   210	
   211		MOVQ (8*3)(x_ptr), AX
   212		MULQ t0
   213		ADDQ AX, acc5
   214		ADCQ $0, DX
   215		MOVQ DX, y_ptr
   216		XORQ t1, t1
   217		// *2
   218		ADDQ acc1, acc1
   219		ADCQ acc2, acc2
   220		ADCQ acc3, acc3
   221		ADCQ acc4, acc4
   222		ADCQ acc5, acc5
   223		ADCQ y_ptr, y_ptr
   224		ADCQ $0, t1
   225		// Missing products
   226		MOVQ (8*0)(x_ptr), AX
   227		MULQ AX
   228		MOVQ AX, acc0
   229		MOVQ DX, t0
   230	
   231		MOVQ (8*1)(x_ptr), AX
   232		MULQ AX
   233		ADDQ t0, acc1
   234		ADCQ AX, acc2
   235		ADCQ $0, DX
   236		MOVQ DX, t0
   237	
   238		MOVQ (8*2)(x_ptr), AX
   239		MULQ AX
   240		ADDQ t0, acc3
   241		ADCQ AX, acc4
   242		ADCQ $0, DX
   243		MOVQ DX, t0
   244	
   245		MOVQ (8*3)(x_ptr), AX
   246		MULQ AX
   247		ADDQ t0, acc5
   248		ADCQ AX, y_ptr
   249		ADCQ DX, t1
   250		MOVQ t1, x_ptr
   251		// First reduction step
   252		MOVQ acc0, AX
   253		MOVQ acc0, t1
   254		SHLQ $32, acc0
   255		MULQ p256const1<>(SB)
   256		SHRQ $32, t1
   257		ADDQ acc0, acc1
   258		ADCQ t1, acc2
   259		ADCQ AX, acc3
   260		ADCQ $0, DX
   261		MOVQ DX, acc0
   262		// Second reduction step
   263		MOVQ acc1, AX
   264		MOVQ acc1, t1
   265		SHLQ $32, acc1
   266		MULQ p256const1<>(SB)
   267		SHRQ $32, t1
   268		ADDQ acc1, acc2
   269		ADCQ t1, acc3
   270		ADCQ AX, acc0
   271		ADCQ $0, DX
   272		MOVQ DX, acc1
   273		// Third reduction step
   274		MOVQ acc2, AX
   275		MOVQ acc2, t1
   276		SHLQ $32, acc2
   277		MULQ p256const1<>(SB)
   278		SHRQ $32, t1
   279		ADDQ acc2, acc3
   280		ADCQ t1, acc0
   281		ADCQ AX, acc1
   282		ADCQ $0, DX
   283		MOVQ DX, acc2
   284		// Last reduction step
   285		XORQ t0, t0
   286		MOVQ acc3, AX
   287		MOVQ acc3, t1
   288		SHLQ $32, acc3
   289		MULQ p256const1<>(SB)
   290		SHRQ $32, t1
   291		ADDQ acc3, acc0
   292		ADCQ t1, acc1
   293		ADCQ AX, acc2
   294		ADCQ $0, DX
   295		MOVQ DX, acc3
   296		// Add bits [511:256] of the sqr result
   297		ADCQ acc4, acc0
   298		ADCQ acc5, acc1
   299		ADCQ y_ptr, acc2
   300		ADCQ x_ptr, acc3
   301		ADCQ $0, t0
   302	
   303		MOVQ acc0, acc4
   304		MOVQ acc1, acc5
   305		MOVQ acc2, y_ptr
   306		MOVQ acc3, t1
   307		// Subtract p256
   308		SUBQ $-1, acc0
   309		SBBQ p256const0<>(SB) ,acc1
   310		SBBQ $0, acc2
   311		SBBQ p256const1<>(SB), acc3
   312		SBBQ $0, t0
   313	
   314		CMOVQCS acc4, acc0
   315		CMOVQCS acc5, acc1
   316		CMOVQCS y_ptr, acc2
   317		CMOVQCS t1, acc3
   318	
   319		MOVQ acc0, (8*0)(res_ptr)
   320		MOVQ acc1, (8*1)(res_ptr)
   321		MOVQ acc2, (8*2)(res_ptr)
   322		MOVQ acc3, (8*3)(res_ptr)
   323		MOVQ res_ptr, x_ptr
   324		DECQ BX
   325		JNE  sqrLoop
   326	
   327		RET
   328	/* ---------------------------------------*/
   329	// func p256Mul(res, in1, in2 []uint64)
   330	TEXT ·p256Mul(SB),NOSPLIT,$0
   331		MOVQ res+0(FP), res_ptr
   332		MOVQ in1+24(FP), x_ptr
   333		MOVQ in2+48(FP), y_ptr
   334		// x * y[0]
   335		MOVQ (8*0)(y_ptr), t0
   336	
   337		MOVQ (8*0)(x_ptr), AX
   338		MULQ t0
   339		MOVQ AX, acc0
   340		MOVQ DX, acc1
   341	
   342		MOVQ (8*1)(x_ptr), AX
   343		MULQ t0
   344		ADDQ AX, acc1
   345		ADCQ $0, DX
   346		MOVQ DX, acc2
   347	
   348		MOVQ (8*2)(x_ptr), AX
   349		MULQ t0
   350		ADDQ AX, acc2
   351		ADCQ $0, DX
   352		MOVQ DX, acc3
   353	
   354		MOVQ (8*3)(x_ptr), AX
   355		MULQ t0
   356		ADDQ AX, acc3
   357		ADCQ $0, DX
   358		MOVQ DX, acc4
   359		XORQ acc5, acc5
   360		// First reduction step
   361		MOVQ acc0, AX
   362		MOVQ acc0, t1
   363		SHLQ $32, acc0
   364		MULQ p256const1<>(SB)
   365		SHRQ $32, t1
   366		ADDQ acc0, acc1
   367		ADCQ t1, acc2
   368		ADCQ AX, acc3
   369		ADCQ DX, acc4
   370		ADCQ $0, acc5
   371		XORQ acc0, acc0
   372		// x * y[1]
   373		MOVQ (8*1)(y_ptr), t0
   374	
   375		MOVQ (8*0)(x_ptr), AX
   376		MULQ t0
   377		ADDQ AX, acc1
   378		ADCQ $0, DX
   379		MOVQ DX, t1
   380	
   381		MOVQ (8*1)(x_ptr), AX
   382		MULQ t0
   383		ADDQ t1, acc2
   384		ADCQ $0, DX
   385		ADDQ AX, acc2
   386		ADCQ $0, DX
   387		MOVQ DX, t1
   388	
   389		MOVQ (8*2)(x_ptr), AX
   390		MULQ t0
   391		ADDQ t1, acc3
   392		ADCQ $0, DX
   393		ADDQ AX, acc3
   394		ADCQ $0, DX
   395		MOVQ DX, t1
   396	
   397		MOVQ (8*3)(x_ptr), AX
   398		MULQ t0
   399		ADDQ t1, acc4
   400		ADCQ $0, DX
   401		ADDQ AX, acc4
   402		ADCQ DX, acc5
   403		ADCQ $0, acc0
   404		// Second reduction step
   405		MOVQ acc1, AX
   406		MOVQ acc1, t1
   407		SHLQ $32, acc1
   408		MULQ p256const1<>(SB)
   409		SHRQ $32, t1
   410		ADDQ acc1, acc2
   411		ADCQ t1, acc3
   412		ADCQ AX, acc4
   413		ADCQ DX, acc5
   414		ADCQ $0, acc0
   415		XORQ acc1, acc1
   416		// x * y[2]
   417		MOVQ (8*2)(y_ptr), t0
   418	
   419		MOVQ (8*0)(x_ptr), AX
   420		MULQ t0
   421		ADDQ AX, acc2
   422		ADCQ $0, DX
   423		MOVQ DX, t1
   424	
   425		MOVQ (8*1)(x_ptr), AX
   426		MULQ t0
   427		ADDQ t1, acc3
   428		ADCQ $0, DX
   429		ADDQ AX, acc3
   430		ADCQ $0, DX
   431		MOVQ DX, t1
   432	
   433		MOVQ (8*2)(x_ptr), AX
   434		MULQ t0
   435		ADDQ t1, acc4
   436		ADCQ $0, DX
   437		ADDQ AX, acc4
   438		ADCQ $0, DX
   439		MOVQ DX, t1
   440	
   441		MOVQ (8*3)(x_ptr), AX
   442		MULQ t0
   443		ADDQ t1, acc5
   444		ADCQ $0, DX
   445		ADDQ AX, acc5
   446		ADCQ DX, acc0
   447		ADCQ $0, acc1
   448		// Third reduction step
   449		MOVQ acc2, AX
   450		MOVQ acc2, t1
   451		SHLQ $32, acc2
   452		MULQ p256const1<>(SB)
   453		SHRQ $32, t1
   454		ADDQ acc2, acc3
   455		ADCQ t1, acc4
   456		ADCQ AX, acc5
   457		ADCQ DX, acc0
   458		ADCQ $0, acc1
   459		XORQ acc2, acc2
   460		// x * y[3]
   461		MOVQ (8*3)(y_ptr), t0
   462	
   463		MOVQ (8*0)(x_ptr), AX
   464		MULQ t0
   465		ADDQ AX, acc3
   466		ADCQ $0, DX
   467		MOVQ DX, t1
   468	
   469		MOVQ (8*1)(x_ptr), AX
   470		MULQ t0
   471		ADDQ t1, acc4
   472		ADCQ $0, DX
   473		ADDQ AX, acc4
   474		ADCQ $0, DX
   475		MOVQ DX, t1
   476	
   477		MOVQ (8*2)(x_ptr), AX
   478		MULQ t0
   479		ADDQ t1, acc5
   480		ADCQ $0, DX
   481		ADDQ AX, acc5
   482		ADCQ $0, DX
   483		MOVQ DX, t1
   484	
   485		MOVQ (8*3)(x_ptr), AX
   486		MULQ t0
   487		ADDQ t1, acc0
   488		ADCQ $0, DX
   489		ADDQ AX, acc0
   490		ADCQ DX, acc1
   491		ADCQ $0, acc2
   492		// Last reduction step
   493		MOVQ acc3, AX
   494		MOVQ acc3, t1
   495		SHLQ $32, acc3
   496		MULQ p256const1<>(SB)
   497		SHRQ $32, t1
   498		ADDQ acc3, acc4
   499		ADCQ t1, acc5
   500		ADCQ AX, acc0
   501		ADCQ DX, acc1
   502		ADCQ $0, acc2
   503		// Copy result [255:0]
   504		MOVQ acc4, x_ptr
   505		MOVQ acc5, acc3
   506		MOVQ acc0, t0
   507		MOVQ acc1, t1
   508		// Subtract p256
   509		SUBQ $-1, acc4
   510		SBBQ p256const0<>(SB) ,acc5
   511		SBBQ $0, acc0
   512		SBBQ p256const1<>(SB), acc1
   513		SBBQ $0, acc2
   514	
   515		CMOVQCS x_ptr, acc4
   516		CMOVQCS acc3, acc5
   517		CMOVQCS t0, acc0
   518		CMOVQCS t1, acc1
   519	
   520		MOVQ acc4, (8*0)(res_ptr)
   521		MOVQ acc5, (8*1)(res_ptr)
   522		MOVQ acc0, (8*2)(res_ptr)
   523		MOVQ acc1, (8*3)(res_ptr)
   524	
   525		RET
   526	/* ---------------------------------------*/
   527	// func p256FromMont(res, in []uint64)
   528	TEXT ·p256FromMont(SB),NOSPLIT,$0
   529		MOVQ res+0(FP), res_ptr
   530		MOVQ in+24(FP), x_ptr
   531	
   532		MOVQ (8*0)(x_ptr), acc0
   533		MOVQ (8*1)(x_ptr), acc1
   534		MOVQ (8*2)(x_ptr), acc2
   535		MOVQ (8*3)(x_ptr), acc3
   536		XORQ acc4, acc4
   537	
   538		// Only reduce, no multiplications are needed
   539		// First stage
   540		MOVQ acc0, AX
   541		MOVQ acc0, t1
   542		SHLQ $32, acc0
   543		MULQ p256const1<>(SB)
   544		SHRQ $32, t1
   545		ADDQ acc0, acc1
   546		ADCQ t1, acc2
   547		ADCQ AX, acc3
   548		ADCQ DX, acc4
   549		XORQ acc5, acc5
   550		// Second stage
   551		MOVQ acc1, AX
   552		MOVQ acc1, t1
   553		SHLQ $32, acc1
   554		MULQ p256const1<>(SB)
   555		SHRQ $32, t1
   556		ADDQ acc1, acc2
   557		ADCQ t1, acc3
   558		ADCQ AX, acc4
   559		ADCQ DX, acc5
   560		XORQ acc0, acc0
   561		// Third stage
   562		MOVQ acc2, AX
   563		MOVQ acc2, t1
   564		SHLQ $32, acc2
   565		MULQ p256const1<>(SB)
   566		SHRQ $32, t1
   567		ADDQ acc2, acc3
   568		ADCQ t1, acc4
   569		ADCQ AX, acc5
   570		ADCQ DX, acc0
   571		XORQ acc1, acc1
   572		// Last stage
   573		MOVQ acc3, AX
   574		MOVQ acc3, t1
   575		SHLQ $32, acc3
   576		MULQ p256const1<>(SB)
   577		SHRQ $32, t1
   578		ADDQ acc3, acc4
   579		ADCQ t1, acc5
   580		ADCQ AX, acc0
   581		ADCQ DX, acc1
   582	
   583		MOVQ acc4, x_ptr
   584		MOVQ acc5, acc3
   585		MOVQ acc0, t0
   586		MOVQ acc1, t1
   587	
   588		SUBQ $-1, acc4
   589		SBBQ p256const0<>(SB), acc5
   590		SBBQ $0, acc0
   591		SBBQ p256const1<>(SB), acc1
   592	
   593		CMOVQCS x_ptr, acc4
   594		CMOVQCS acc3, acc5
   595		CMOVQCS t0, acc0
   596		CMOVQCS t1, acc1
   597	
   598		MOVQ acc4, (8*0)(res_ptr)
   599		MOVQ acc5, (8*1)(res_ptr)
   600		MOVQ acc0, (8*2)(res_ptr)
   601		MOVQ acc1, (8*3)(res_ptr)
   602	
   603		RET
   604	/* ---------------------------------------*/
   605	// Constant time point access to arbitrary point table.
   606	// Indexed from 1 to 15, with -1 offset
   607	// (index 0 is implicitly point at infinity)
   608	// func p256Select(point, table []uint64, idx int)
   609	TEXT ·p256Select(SB),NOSPLIT,$0
   610		MOVQ idx+48(FP),AX
   611		MOVQ table+24(FP),DI
   612		MOVQ point+0(FP),DX
   613	
   614		PXOR X15, X15	// X15 = 0
   615		PCMPEQL X14, X14 // X14 = -1
   616		PSUBL X14, X15   // X15 = 1
   617		MOVL AX, X14
   618		PSHUFD $0, X14, X14
   619	
   620		PXOR X0, X0
   621		PXOR X1, X1
   622		PXOR X2, X2
   623		PXOR X3, X3
   624		PXOR X4, X4
   625		PXOR X5, X5
   626		MOVQ $16, AX
   627	
   628		MOVOU X15, X13
   629	
   630	loop_select:
   631	
   632			MOVOU X13, X12
   633			PADDL X15, X13
   634			PCMPEQL X14, X12
   635	
   636			MOVOU (16*0)(DI), X6
   637			MOVOU (16*1)(DI), X7
   638			MOVOU (16*2)(DI), X8
   639			MOVOU (16*3)(DI), X9
   640			MOVOU (16*4)(DI), X10
   641			MOVOU (16*5)(DI), X11
   642			ADDQ $(16*6), DI
   643	
   644			PAND X12, X6
   645			PAND X12, X7
   646			PAND X12, X8
   647			PAND X12, X9
   648			PAND X12, X10
   649			PAND X12, X11
   650	
   651			PXOR X6, X0
   652			PXOR X7, X1
   653			PXOR X8, X2
   654			PXOR X9, X3
   655			PXOR X10, X4
   656			PXOR X11, X5
   657	
   658			DECQ AX
   659			JNE loop_select
   660	
   661		MOVOU X0, (16*0)(DX)
   662		MOVOU X1, (16*1)(DX)
   663		MOVOU X2, (16*2)(DX)
   664		MOVOU X3, (16*3)(DX)
   665		MOVOU X4, (16*4)(DX)
   666		MOVOU X5, (16*5)(DX)
   667	
   668		RET
   669	/* ---------------------------------------*/
   670	// Constant time point access to base point table.
   671	// func p256SelectBase(point, table []uint64, idx int)
   672	TEXT ·p256SelectBase(SB),NOSPLIT,$0
   673		MOVQ idx+48(FP),AX
   674		MOVQ table+24(FP),DI
   675		MOVQ point+0(FP),DX
   676	
   677		PXOR X15, X15	// X15 = 0
   678		PCMPEQL X14, X14 // X14 = -1
   679		PSUBL X14, X15   // X15 = 1
   680		MOVL AX, X14
   681		PSHUFD $0, X14, X14
   682	
   683		PXOR X0, X0
   684		PXOR X1, X1
   685		PXOR X2, X2
   686		PXOR X3, X3
   687		MOVQ $16, AX
   688	
   689		MOVOU X15, X13
   690	
   691	loop_select_base:
   692	
   693			MOVOU X13, X12
   694			PADDL X15, X13
   695			PCMPEQL X14, X12
   696	
   697			MOVOU (16*0)(DI), X4
   698			MOVOU (16*1)(DI), X5
   699			MOVOU (16*2)(DI), X6
   700			MOVOU (16*3)(DI), X7
   701	
   702			MOVOU (16*4)(DI), X8
   703			MOVOU (16*5)(DI), X9
   704			MOVOU (16*6)(DI), X10
   705			MOVOU (16*7)(DI), X11
   706	
   707			ADDQ $(16*8), DI
   708	
   709			PAND X12, X4
   710			PAND X12, X5
   711			PAND X12, X6
   712			PAND X12, X7
   713	
   714			MOVOU X13, X12
   715			PADDL X15, X13
   716			PCMPEQL X14, X12
   717	
   718			PAND X12, X8
   719			PAND X12, X9
   720			PAND X12, X10
   721			PAND X12, X11
   722	
   723			PXOR X4, X0
   724			PXOR X5, X1
   725			PXOR X6, X2
   726			PXOR X7, X3
   727	
   728			PXOR X8, X0
   729			PXOR X9, X1
   730			PXOR X10, X2
   731			PXOR X11, X3
   732	
   733			DECQ AX
   734			JNE loop_select_base
   735	
   736		MOVOU X0, (16*0)(DX)
   737		MOVOU X1, (16*1)(DX)
   738		MOVOU X2, (16*2)(DX)
   739		MOVOU X3, (16*3)(DX)
   740	
   741		RET
   742	/* ---------------------------------------*/
   743	// func p256OrdMul(res, in1, in2 []uint64)
   744	TEXT ·p256OrdMul(SB),NOSPLIT,$0
   745		MOVQ res+0(FP), res_ptr
   746		MOVQ in1+24(FP), x_ptr
   747		MOVQ in2+48(FP), y_ptr
   748		// x * y[0]
   749		MOVQ (8*0)(y_ptr), t0
   750	
   751		MOVQ (8*0)(x_ptr), AX
   752		MULQ t0
   753		MOVQ AX, acc0
   754		MOVQ DX, acc1
   755	
   756		MOVQ (8*1)(x_ptr), AX
   757		MULQ t0
   758		ADDQ AX, acc1
   759		ADCQ $0, DX
   760		MOVQ DX, acc2
   761	
   762		MOVQ (8*2)(x_ptr), AX
   763		MULQ t0
   764		ADDQ AX, acc2
   765		ADCQ $0, DX
   766		MOVQ DX, acc3
   767	
   768		MOVQ (8*3)(x_ptr), AX
   769		MULQ t0
   770		ADDQ AX, acc3
   771		ADCQ $0, DX
   772		MOVQ DX, acc4
   773		XORQ acc5, acc5
   774		// First reduction step
   775		MOVQ acc0, AX
   776		MULQ p256ordK0<>(SB)
   777		MOVQ AX, t0
   778	
   779		MOVQ p256ord<>+0x00(SB), AX
   780		MULQ t0
   781		ADDQ AX, acc0
   782		ADCQ $0, DX
   783		MOVQ DX, t1
   784	
   785		MOVQ p256ord<>+0x08(SB), AX
   786		MULQ t0
   787		ADDQ t1, acc1
   788		ADCQ $0, DX
   789		ADDQ AX, acc1
   790		ADCQ $0, DX
   791		MOVQ DX, t1
   792	
   793		MOVQ p256ord<>+0x10(SB), AX
   794		MULQ t0
   795		ADDQ t1, acc2
   796		ADCQ $0, DX
   797		ADDQ AX, acc2
   798		ADCQ $0, DX
   799		MOVQ DX, t1
   800	
   801		MOVQ p256ord<>+0x18(SB), AX
   802		MULQ t0
   803		ADDQ t1, acc3
   804		ADCQ $0, DX
   805		ADDQ AX, acc3
   806		ADCQ DX, acc4
   807		ADCQ $0, acc5
   808		// x * y[1]
   809		MOVQ (8*1)(y_ptr), t0
   810	
   811		MOVQ (8*0)(x_ptr), AX
   812		MULQ t0
   813		ADDQ AX, acc1
   814		ADCQ $0, DX
   815		MOVQ DX, t1
   816	
   817		MOVQ (8*1)(x_ptr), AX
   818		MULQ t0
   819		ADDQ t1, acc2
   820		ADCQ $0, DX
   821		ADDQ AX, acc2
   822		ADCQ $0, DX
   823		MOVQ DX, t1
   824	
   825		MOVQ (8*2)(x_ptr), AX
   826		MULQ t0
   827		ADDQ t1, acc3
   828		ADCQ $0, DX
   829		ADDQ AX, acc3
   830		ADCQ $0, DX
   831		MOVQ DX, t1
   832	
   833		MOVQ (8*3)(x_ptr), AX
   834		MULQ t0
   835		ADDQ t1, acc4
   836		ADCQ $0, DX
   837		ADDQ AX, acc4
   838		ADCQ DX, acc5
   839		ADCQ $0, acc0
   840		// Second reduction step
   841		MOVQ acc1, AX
   842		MULQ p256ordK0<>(SB)
   843		MOVQ AX, t0
   844	
   845		MOVQ p256ord<>+0x00(SB), AX
   846		MULQ t0
   847		ADDQ AX, acc1
   848		ADCQ $0, DX
   849		MOVQ DX, t1
   850	
   851		MOVQ p256ord<>+0x08(SB), AX
   852		MULQ t0
   853		ADDQ t1, acc2
   854		ADCQ $0, DX
   855		ADDQ AX, acc2
   856		ADCQ $0, DX
   857		MOVQ DX, t1
   858	
   859		MOVQ p256ord<>+0x10(SB), AX
   860		MULQ t0
   861		ADDQ t1, acc3
   862		ADCQ $0, DX
   863		ADDQ AX, acc3
   864		ADCQ $0, DX
   865		MOVQ DX, t1
   866	
   867		MOVQ p256ord<>+0x18(SB), AX
   868		MULQ t0
   869		ADDQ t1, acc4
   870		ADCQ $0, DX
   871		ADDQ AX, acc4
   872		ADCQ DX, acc5
   873		ADCQ $0, acc0
   874		// x * y[2]
   875		MOVQ (8*2)(y_ptr), t0
   876	
   877		MOVQ (8*0)(x_ptr), AX
   878		MULQ t0
   879		ADDQ AX, acc2
   880		ADCQ $0, DX
   881		MOVQ DX, t1
   882	
   883		MOVQ (8*1)(x_ptr), AX
   884		MULQ t0
   885		ADDQ t1, acc3
   886		ADCQ $0, DX
   887		ADDQ AX, acc3
   888		ADCQ $0, DX
   889		MOVQ DX, t1
   890	
   891		MOVQ (8*2)(x_ptr), AX
   892		MULQ t0
   893		ADDQ t1, acc4
   894		ADCQ $0, DX
   895		ADDQ AX, acc4
   896		ADCQ $0, DX
   897		MOVQ DX, t1
   898	
   899		MOVQ (8*3)(x_ptr), AX
   900		MULQ t0
   901		ADDQ t1, acc5
   902		ADCQ $0, DX
   903		ADDQ AX, acc5
   904		ADCQ DX, acc0
   905		ADCQ $0, acc1
   906		// Third reduction step
   907		MOVQ acc2, AX
   908		MULQ p256ordK0<>(SB)
   909		MOVQ AX, t0
   910	
   911		MOVQ p256ord<>+0x00(SB), AX
   912		MULQ t0
   913		ADDQ AX, acc2
   914		ADCQ $0, DX
   915		MOVQ DX, t1
   916	
   917		MOVQ p256ord<>+0x08(SB), AX
   918		MULQ t0
   919		ADDQ t1, acc3
   920		ADCQ $0, DX
   921		ADDQ AX, acc3
   922		ADCQ $0, DX
   923		MOVQ DX, t1
   924	
   925		MOVQ p256ord<>+0x10(SB), AX
   926		MULQ t0
   927		ADDQ t1, acc4
   928		ADCQ $0, DX
   929		ADDQ AX, acc4
   930		ADCQ $0, DX
   931		MOVQ DX, t1
   932	
   933		MOVQ p256ord<>+0x18(SB), AX
   934		MULQ t0
   935		ADDQ t1, acc5
   936		ADCQ $0, DX
   937		ADDQ AX, acc5
   938		ADCQ DX, acc0
   939		ADCQ $0, acc1
   940		// x * y[3]
   941		MOVQ (8*3)(y_ptr), t0
   942	
   943		MOVQ (8*0)(x_ptr), AX
   944		MULQ t0
   945		ADDQ AX, acc3
   946		ADCQ $0, DX
   947		MOVQ DX, t1
   948	
   949		MOVQ (8*1)(x_ptr), AX
   950		MULQ t0
   951		ADDQ t1, acc4
   952		ADCQ $0, DX
   953		ADDQ AX, acc4
   954		ADCQ $0, DX
   955		MOVQ DX, t1
   956	
   957		MOVQ (8*2)(x_ptr), AX
   958		MULQ t0
   959		ADDQ t1, acc5
   960		ADCQ $0, DX
   961		ADDQ AX, acc5
   962		ADCQ $0, DX
   963		MOVQ DX, t1
   964	
   965		MOVQ (8*3)(x_ptr), AX
   966		MULQ t0
   967		ADDQ t1, acc0
   968		ADCQ $0, DX
   969		ADDQ AX, acc0
   970		ADCQ DX, acc1
   971		ADCQ $0, acc2
   972		// Last reduction step
   973		MOVQ acc3, AX
   974		MULQ p256ordK0<>(SB)
   975		MOVQ AX, t0
   976	
   977		MOVQ p256ord<>+0x00(SB), AX
   978		MULQ t0
   979		ADDQ AX, acc3
   980		ADCQ $0, DX
   981		MOVQ DX, t1
   982	
   983		MOVQ p256ord<>+0x08(SB), AX
   984		MULQ t0
   985		ADDQ t1, acc4
   986		ADCQ $0, DX
   987		ADDQ AX, acc4
   988		ADCQ $0, DX
   989		MOVQ DX, t1
   990	
   991		MOVQ p256ord<>+0x10(SB), AX
   992		MULQ t0
   993		ADDQ t1, acc5
   994		ADCQ $0, DX
   995		ADDQ AX, acc5
   996		ADCQ $0, DX
   997		MOVQ DX, t1
   998	
   999		MOVQ p256ord<>+0x18(SB), AX
  1000		MULQ t0
  1001		ADDQ t1, acc0
  1002		ADCQ $0, DX
  1003		ADDQ AX, acc0
  1004		ADCQ DX, acc1
  1005		ADCQ $0, acc2
  1006		// Copy result [255:0]
  1007		MOVQ acc4, x_ptr
  1008		MOVQ acc5, acc3
  1009		MOVQ acc0, t0
  1010		MOVQ acc1, t1
  1011		// Subtract p256
  1012		SUBQ p256ord<>+0x00(SB), acc4
  1013		SBBQ p256ord<>+0x08(SB) ,acc5
  1014		SBBQ p256ord<>+0x10(SB), acc0
  1015		SBBQ p256ord<>+0x18(SB), acc1
  1016		SBBQ $0, acc2
  1017	
  1018		CMOVQCS x_ptr, acc4
  1019		CMOVQCS acc3, acc5
  1020		CMOVQCS t0, acc0
  1021		CMOVQCS t1, acc1
  1022	
  1023		MOVQ acc4, (8*0)(res_ptr)
  1024		MOVQ acc5, (8*1)(res_ptr)
  1025		MOVQ acc0, (8*2)(res_ptr)
  1026		MOVQ acc1, (8*3)(res_ptr)
  1027	
  1028		RET
  1029	/* ---------------------------------------*/
  1030	// func p256OrdSqr(res, in []uint64, n int)
  1031	TEXT ·p256OrdSqr(SB),NOSPLIT,$0
  1032		MOVQ res+0(FP), res_ptr
  1033		MOVQ in+24(FP), x_ptr
  1034		MOVQ n+48(FP), BX
  1035	
  1036	ordSqrLoop:
  1037	
  1038		// y[1:] * y[0]
  1039		MOVQ (8*0)(x_ptr), t0
  1040	
  1041		MOVQ (8*1)(x_ptr), AX
  1042		MULQ t0
  1043		MOVQ AX, acc1
  1044		MOVQ DX, acc2
  1045	
  1046		MOVQ (8*2)(x_ptr), AX
  1047		MULQ t0
  1048		ADDQ AX, acc2
  1049		ADCQ $0, DX
  1050		MOVQ DX, acc3
  1051	
  1052		MOVQ (8*3)(x_ptr), AX
  1053		MULQ t0
  1054		ADDQ AX, acc3
  1055		ADCQ $0, DX
  1056		MOVQ DX, acc4
  1057		// y[2:] * y[1]
  1058		MOVQ (8*1)(x_ptr), t0
  1059	
  1060		MOVQ (8*2)(x_ptr), AX
  1061		MULQ t0
  1062		ADDQ AX, acc3
  1063		ADCQ $0, DX
  1064		MOVQ DX, t1
  1065	
  1066		MOVQ (8*3)(x_ptr), AX
  1067		MULQ t0
  1068		ADDQ t1, acc4
  1069		ADCQ $0, DX
  1070		ADDQ AX, acc4
  1071		ADCQ $0, DX
  1072		MOVQ DX, acc5
  1073		// y[3] * y[2]
  1074		MOVQ (8*2)(x_ptr), t0
  1075	
  1076		MOVQ (8*3)(x_ptr), AX
  1077		MULQ t0
  1078		ADDQ AX, acc5
  1079		ADCQ $0, DX
  1080		MOVQ DX, y_ptr
  1081		XORQ t1, t1
  1082		// *2
  1083		ADDQ acc1, acc1
  1084		ADCQ acc2, acc2
  1085		ADCQ acc3, acc3
  1086		ADCQ acc4, acc4
  1087		ADCQ acc5, acc5
  1088		ADCQ y_ptr, y_ptr
  1089		ADCQ $0, t1
  1090		// Missing products
  1091		MOVQ (8*0)(x_ptr), AX
  1092		MULQ AX
  1093		MOVQ AX, acc0
  1094		MOVQ DX, t0
  1095	
  1096		MOVQ (8*1)(x_ptr), AX
  1097		MULQ AX
  1098		ADDQ t0, acc1
  1099		ADCQ AX, acc2
  1100		ADCQ $0, DX
  1101		MOVQ DX, t0
  1102	
  1103		MOVQ (8*2)(x_ptr), AX
  1104		MULQ AX
  1105		ADDQ t0, acc3
  1106		ADCQ AX, acc4
  1107		ADCQ $0, DX
  1108		MOVQ DX, t0
  1109	
  1110		MOVQ (8*3)(x_ptr), AX
  1111		MULQ AX
  1112		ADDQ t0, acc5
  1113		ADCQ AX, y_ptr
  1114		ADCQ DX, t1
  1115		MOVQ t1, x_ptr
  1116		// First reduction step
  1117		MOVQ acc0, AX
  1118		MULQ p256ordK0<>(SB)
  1119		MOVQ AX, t0
  1120	
  1121		MOVQ p256ord<>+0x00(SB), AX
  1122		MULQ t0
  1123		ADDQ AX, acc0
  1124		ADCQ $0, DX
  1125		MOVQ DX, t1
  1126	
  1127		MOVQ p256ord<>+0x08(SB), AX
  1128		MULQ t0
  1129		ADDQ t1, acc1
  1130		ADCQ $0, DX
  1131		ADDQ AX, acc1
  1132	
  1133		MOVQ t0, t1
  1134		ADCQ DX, acc2
  1135		ADCQ $0, t1
  1136		SUBQ t0, acc2
  1137		SBBQ $0, t1
  1138	
  1139		MOVQ t0, AX
  1140		MOVQ t0, DX
  1141		MOVQ t0, acc0
  1142		SHLQ $32, AX
  1143		SHRQ $32, DX
  1144	
  1145		ADDQ t1, acc3
  1146		ADCQ $0, acc0
  1147		SUBQ AX, acc3
  1148		SBBQ DX, acc0
  1149		// Second reduction step
  1150		MOVQ acc1, AX
  1151		MULQ p256ordK0<>(SB)
  1152		MOVQ AX, t0
  1153	
  1154		MOVQ p256ord<>+0x00(SB), AX
  1155		MULQ t0
  1156		ADDQ AX, acc1
  1157		ADCQ $0, DX
  1158		MOVQ DX, t1
  1159	
  1160		MOVQ p256ord<>+0x08(SB), AX
  1161		MULQ t0
  1162		ADDQ t1, acc2
  1163		ADCQ $0, DX
  1164		ADDQ AX, acc2
  1165	
  1166		MOVQ t0, t1
  1167		ADCQ DX, acc3
  1168		ADCQ $0, t1
  1169		SUBQ t0, acc3
  1170		SBBQ $0, t1
  1171	
  1172		MOVQ t0, AX
  1173		MOVQ t0, DX
  1174		MOVQ t0, acc1
  1175		SHLQ $32, AX
  1176		SHRQ $32, DX
  1177	
  1178		ADDQ t1, acc0
  1179		ADCQ $0, acc1
  1180		SUBQ AX, acc0
  1181		SBBQ DX, acc1
  1182		// Third reduction step
  1183		MOVQ acc2, AX
  1184		MULQ p256ordK0<>(SB)
  1185		MOVQ AX, t0
  1186	
  1187		MOVQ p256ord<>+0x00(SB), AX
  1188		MULQ t0
  1189		ADDQ AX, acc2
  1190		ADCQ $0, DX
  1191		MOVQ DX, t1
  1192	
  1193		MOVQ p256ord<>+0x08(SB), AX
  1194		MULQ t0
  1195		ADDQ t1, acc3
  1196		ADCQ $0, DX
  1197		ADDQ AX, acc3
  1198	
  1199		MOVQ t0, t1
  1200		ADCQ DX, acc0
  1201		ADCQ $0, t1
  1202		SUBQ t0, acc0
  1203		SBBQ $0, t1
  1204	
  1205		MOVQ t0, AX
  1206		MOVQ t0, DX
  1207		MOVQ t0, acc2
  1208		SHLQ $32, AX
  1209		SHRQ $32, DX
  1210	
  1211		ADDQ t1, acc1
  1212		ADCQ $0, acc2
  1213		SUBQ AX, acc1
  1214		SBBQ DX, acc2
  1215		// Last reduction step
  1216		MOVQ acc3, AX
  1217		MULQ p256ordK0<>(SB)
  1218		MOVQ AX, t0
  1219	
  1220		MOVQ p256ord<>+0x00(SB), AX
  1221		MULQ t0
  1222		ADDQ AX, acc3
  1223		ADCQ $0, DX
  1224		MOVQ DX, t1
  1225	
  1226		MOVQ p256ord<>+0x08(SB), AX
  1227		MULQ t0
  1228		ADDQ t1, acc0
  1229		ADCQ $0, DX
  1230		ADDQ AX, acc0
  1231		ADCQ $0, DX
  1232		MOVQ DX, t1
  1233	
  1234		MOVQ t0, t1
  1235		ADCQ DX, acc1
  1236		ADCQ $0, t1
  1237		SUBQ t0, acc1
  1238		SBBQ $0, t1
  1239	
  1240		MOVQ t0, AX
  1241		MOVQ t0, DX
  1242		MOVQ t0, acc3
  1243		SHLQ $32, AX
  1244		SHRQ $32, DX
  1245	
  1246		ADDQ t1, acc2
  1247		ADCQ $0, acc3
  1248		SUBQ AX, acc2
  1249		SBBQ DX, acc3
  1250		XORQ t0, t0
  1251		// Add bits [511:256] of the sqr result
  1252		ADCQ acc4, acc0
  1253		ADCQ acc5, acc1
  1254		ADCQ y_ptr, acc2
  1255		ADCQ x_ptr, acc3
  1256		ADCQ $0, t0
  1257	
  1258		MOVQ acc0, acc4
  1259		MOVQ acc1, acc5
  1260		MOVQ acc2, y_ptr
  1261		MOVQ acc3, t1
  1262		// Subtract p256
  1263		SUBQ p256ord<>+0x00(SB), acc0
  1264		SBBQ p256ord<>+0x08(SB) ,acc1
  1265		SBBQ p256ord<>+0x10(SB), acc2
  1266		SBBQ p256ord<>+0x18(SB), acc3
  1267		SBBQ $0, t0
  1268	
  1269		CMOVQCS acc4, acc0
  1270		CMOVQCS acc5, acc1
  1271		CMOVQCS y_ptr, acc2
  1272		CMOVQCS t1, acc3
  1273	
  1274		MOVQ acc0, (8*0)(res_ptr)
  1275		MOVQ acc1, (8*1)(res_ptr)
  1276		MOVQ acc2, (8*2)(res_ptr)
  1277		MOVQ acc3, (8*3)(res_ptr)
  1278		MOVQ res_ptr, x_ptr
  1279		DECQ BX
  1280		JNE ordSqrLoop
  1281	
  1282		RET
  1283	/* ---------------------------------------*/
  1284	#undef res_ptr
  1285	#undef x_ptr
  1286	#undef y_ptr
  1287	
  1288	#undef acc0
  1289	#undef acc1
  1290	#undef acc2
  1291	#undef acc3
  1292	#undef acc4
  1293	#undef acc5
  1294	#undef t0
  1295	#undef t1
  1296	/* ---------------------------------------*/
  1297	#define mul0 AX
  1298	#define mul1 DX
  1299	#define acc0 BX
  1300	#define acc1 CX
  1301	#define acc2 R8
  1302	#define acc3 R9
  1303	#define acc4 R10
  1304	#define acc5 R11
  1305	#define acc6 R12
  1306	#define acc7 R13
  1307	#define t0 R14
  1308	#define t1 R15
  1309	#define t2 DI
  1310	#define t3 SI
  1311	#define hlp BP
  1312	/* ---------------------------------------*/
  1313	TEXT p256SubInternal(SB),NOSPLIT,$0
  1314		XORQ mul0, mul0
  1315		SUBQ t0, acc4
  1316		SBBQ t1, acc5
  1317		SBBQ t2, acc6
  1318		SBBQ t3, acc7
  1319		SBBQ $0, mul0
  1320	
  1321		MOVQ acc4, acc0
  1322		MOVQ acc5, acc1
  1323		MOVQ acc6, acc2
  1324		MOVQ acc7, acc3
  1325	
  1326		ADDQ $-1, acc4
  1327		ADCQ p256const0<>(SB), acc5
  1328		ADCQ $0, acc6
  1329		ADCQ p256const1<>(SB), acc7
  1330		ANDQ $1, mul0
  1331	
  1332		CMOVQEQ acc0, acc4
  1333		CMOVQEQ acc1, acc5
  1334		CMOVQEQ acc2, acc6
  1335		CMOVQEQ acc3, acc7
  1336	
  1337		RET
  1338	/* ---------------------------------------*/
  1339	TEXT p256MulInternal(SB),NOSPLIT,$0
  1340		MOVQ acc4, mul0
  1341		MULQ t0
  1342		MOVQ mul0, acc0
  1343		MOVQ mul1, acc1
  1344	
  1345		MOVQ acc4, mul0
  1346		MULQ t1
  1347		ADDQ mul0, acc1
  1348		ADCQ $0, mul1
  1349		MOVQ mul1, acc2
  1350	
  1351		MOVQ acc4, mul0
  1352		MULQ t2
  1353		ADDQ mul0, acc2
  1354		ADCQ $0, mul1
  1355		MOVQ mul1, acc3
  1356	
  1357		MOVQ acc4, mul0
  1358		MULQ t3
  1359		ADDQ mul0, acc3
  1360		ADCQ $0, mul1
  1361		MOVQ mul1, acc4
  1362	
  1363		MOVQ acc5, mul0
  1364		MULQ t0
  1365		ADDQ mul0, acc1
  1366		ADCQ $0, mul1
  1367		MOVQ mul1, hlp
  1368	
  1369		MOVQ acc5, mul0
  1370		MULQ t1
  1371		ADDQ hlp, acc2
  1372		ADCQ $0, mul1
  1373		ADDQ mul0, acc2
  1374		ADCQ $0, mul1
  1375		MOVQ mul1, hlp
  1376	
  1377		MOVQ acc5, mul0
  1378		MULQ t2
  1379		ADDQ hlp, acc3
  1380		ADCQ $0, mul1
  1381		ADDQ mul0, acc3
  1382		ADCQ $0, mul1
  1383		MOVQ mul1, hlp
  1384	
  1385		MOVQ acc5, mul0
  1386		MULQ t3
  1387		ADDQ hlp, acc4
  1388		ADCQ $0, mul1
  1389		ADDQ mul0, acc4
  1390		ADCQ $0, mul1
  1391		MOVQ mul1, acc5
  1392	
  1393		MOVQ acc6, mul0
  1394		MULQ t0
  1395		ADDQ mul0, acc2
  1396		ADCQ $0, mul1
  1397		MOVQ mul1, hlp
  1398	
  1399		MOVQ acc6, mul0
  1400		MULQ t1
  1401		ADDQ hlp, acc3
  1402		ADCQ $0, mul1
  1403		ADDQ mul0, acc3
  1404		ADCQ $0, mul1
  1405		MOVQ mul1, hlp
  1406	
  1407		MOVQ acc6, mul0
  1408		MULQ t2
  1409		ADDQ hlp, acc4
  1410		ADCQ $0, mul1
  1411		ADDQ mul0, acc4
  1412		ADCQ $0, mul1
  1413		MOVQ mul1, hlp
  1414	
  1415		MOVQ acc6, mul0
  1416		MULQ t3
  1417		ADDQ hlp, acc5
  1418		ADCQ $0, mul1
  1419		ADDQ mul0, acc5
  1420		ADCQ $0, mul1
  1421		MOVQ mul1, acc6
  1422	
  1423		MOVQ acc7, mul0
  1424		MULQ t0
  1425		ADDQ mul0, acc3
  1426		ADCQ $0, mul1
  1427		MOVQ mul1, hlp
  1428	
  1429		MOVQ acc7, mul0
  1430		MULQ t1
  1431		ADDQ hlp, acc4
  1432		ADCQ $0, mul1
  1433		ADDQ mul0, acc4
  1434		ADCQ $0, mul1
  1435		MOVQ mul1, hlp
  1436	
  1437		MOVQ acc7, mul0
  1438		MULQ t2
  1439		ADDQ hlp, acc5
  1440		ADCQ $0, mul1
  1441		ADDQ mul0, acc5
  1442		ADCQ $0, mul1
  1443		MOVQ mul1, hlp
  1444	
  1445		MOVQ acc7, mul0
  1446		MULQ t3
  1447		ADDQ hlp, acc6
  1448		ADCQ $0, mul1
  1449		ADDQ mul0, acc6
  1450		ADCQ $0, mul1
  1451		MOVQ mul1, acc7
  1452		// First reduction step
  1453		MOVQ acc0, mul0
  1454		MOVQ acc0, hlp
  1455		SHLQ $32, acc0
  1456		MULQ p256const1<>(SB)
  1457		SHRQ $32, hlp
  1458		ADDQ acc0, acc1
  1459		ADCQ hlp, acc2
  1460		ADCQ mul0, acc3
  1461		ADCQ $0, mul1
  1462		MOVQ mul1, acc0
  1463		// Second reduction step
  1464		MOVQ acc1, mul0
  1465		MOVQ acc1, hlp
  1466		SHLQ $32, acc1
  1467		MULQ p256const1<>(SB)
  1468		SHRQ $32, hlp
  1469		ADDQ acc1, acc2
  1470		ADCQ hlp, acc3
  1471		ADCQ mul0, acc0
  1472		ADCQ $0, mul1
  1473		MOVQ mul1, acc1
  1474		// Third reduction step
  1475		MOVQ acc2, mul0
  1476		MOVQ acc2, hlp
  1477		SHLQ $32, acc2
  1478		MULQ p256const1<>(SB)
  1479		SHRQ $32, hlp
  1480		ADDQ acc2, acc3
  1481		ADCQ hlp, acc0
  1482		ADCQ mul0, acc1
  1483		ADCQ $0, mul1
  1484		MOVQ mul1, acc2
  1485		// Last reduction step
  1486		MOVQ acc3, mul0
  1487		MOVQ acc3, hlp
  1488		SHLQ $32, acc3
  1489		MULQ p256const1<>(SB)
  1490		SHRQ $32, hlp
  1491		ADDQ acc3, acc0
  1492		ADCQ hlp, acc1
  1493		ADCQ mul0, acc2
  1494		ADCQ $0, mul1
  1495		MOVQ mul1, acc3
  1496		MOVQ $0, BP
  1497		// Add bits [511:256] of the result
  1498		ADCQ acc0, acc4
  1499		ADCQ acc1, acc5
  1500		ADCQ acc2, acc6
  1501		ADCQ acc3, acc7
  1502		ADCQ $0, hlp
  1503		// Copy result
  1504		MOVQ acc4, acc0
  1505		MOVQ acc5, acc1
  1506		MOVQ acc6, acc2
  1507		MOVQ acc7, acc3
  1508		// Subtract p256
  1509		SUBQ $-1, acc4
  1510		SBBQ p256const0<>(SB) ,acc5
  1511		SBBQ $0, acc6
  1512		SBBQ p256const1<>(SB), acc7
  1513		SBBQ $0, hlp
  1514		// If the result of the subtraction is negative, restore the previous result
  1515		CMOVQCS acc0, acc4
  1516		CMOVQCS acc1, acc5
  1517		CMOVQCS acc2, acc6
  1518		CMOVQCS acc3, acc7
  1519	
  1520		RET
  1521	/* ---------------------------------------*/
  1522	TEXT p256SqrInternal(SB),NOSPLIT,$0
  1523	
  1524		MOVQ acc4, mul0
  1525		MULQ acc5
  1526		MOVQ mul0, acc1
  1527		MOVQ mul1, acc2
  1528	
  1529		MOVQ acc4, mul0
  1530		MULQ acc6
  1531		ADDQ mul0, acc2
  1532		ADCQ $0, mul1
  1533		MOVQ mul1, acc3
  1534	
  1535		MOVQ acc4, mul0
  1536		MULQ acc7
  1537		ADDQ mul0, acc3
  1538		ADCQ $0, mul1
  1539		MOVQ mul1, t0
  1540	
  1541		MOVQ acc5, mul0
  1542		MULQ acc6
  1543		ADDQ mul0, acc3
  1544		ADCQ $0, mul1
  1545		MOVQ mul1, hlp
  1546	
  1547		MOVQ acc5, mul0
  1548		MULQ acc7
  1549		ADDQ hlp, t0
  1550		ADCQ $0, mul1
  1551		ADDQ mul0, t0
  1552		ADCQ $0, mul1
  1553		MOVQ mul1, t1
  1554	
  1555		MOVQ acc6, mul0
  1556		MULQ acc7
  1557		ADDQ mul0, t1
  1558		ADCQ $0, mul1
  1559		MOVQ mul1, t2
  1560		XORQ t3, t3
  1561		// *2
  1562		ADDQ acc1, acc1
  1563		ADCQ acc2, acc2
  1564		ADCQ acc3, acc3
  1565		ADCQ t0, t0
  1566		ADCQ t1, t1
  1567		ADCQ t2, t2
  1568		ADCQ $0, t3
  1569		// Missing products
  1570		MOVQ acc4, mul0
  1571		MULQ mul0
  1572		MOVQ mul0, acc0
  1573		MOVQ DX, acc4
  1574	
  1575		MOVQ acc5, mul0
  1576		MULQ mul0
  1577		ADDQ acc4, acc1
  1578		ADCQ mul0, acc2
  1579		ADCQ $0, DX
  1580		MOVQ DX, acc4
  1581	
  1582		MOVQ acc6, mul0
  1583		MULQ mul0
  1584		ADDQ acc4, acc3
  1585		ADCQ mul0, t0
  1586		ADCQ $0, DX
  1587		MOVQ DX, acc4
  1588	
  1589		MOVQ acc7, mul0
  1590		MULQ mul0
  1591		ADDQ acc4, t1
  1592		ADCQ mul0, t2
  1593		ADCQ DX, t3
  1594		// First reduction step
  1595		MOVQ acc0, mul0
  1596		MOVQ acc0, hlp
  1597		SHLQ $32, acc0
  1598		MULQ p256const1<>(SB)
  1599		SHRQ $32, hlp
  1600		ADDQ acc0, acc1
  1601		ADCQ hlp, acc2
  1602		ADCQ mul0, acc3
  1603		ADCQ $0, mul1
  1604		MOVQ mul1, acc0
  1605		// Second reduction step
  1606		MOVQ acc1, mul0
  1607		MOVQ acc1, hlp
  1608		SHLQ $32, acc1
  1609		MULQ p256const1<>(SB)
  1610		SHRQ $32, hlp
  1611		ADDQ acc1, acc2
  1612		ADCQ hlp, acc3
  1613		ADCQ mul0, acc0
  1614		ADCQ $0, mul1
  1615		MOVQ mul1, acc1
  1616		// Third reduction step
  1617		MOVQ acc2, mul0
  1618		MOVQ acc2, hlp
  1619		SHLQ $32, acc2
  1620		MULQ p256const1<>(SB)
  1621		SHRQ $32, hlp
  1622		ADDQ acc2, acc3
  1623		ADCQ hlp, acc0
  1624		ADCQ mul0, acc1
  1625		ADCQ $0, mul1
  1626		MOVQ mul1, acc2
  1627		// Last reduction step
  1628		MOVQ acc3, mul0
  1629		MOVQ acc3, hlp
  1630		SHLQ $32, acc3
  1631		MULQ p256const1<>(SB)
  1632		SHRQ $32, hlp
  1633		ADDQ acc3, acc0
  1634		ADCQ hlp, acc1
  1635		ADCQ mul0, acc2
  1636		ADCQ $0, mul1
  1637		MOVQ mul1, acc3
  1638		MOVQ $0, BP
  1639		// Add bits [511:256] of the result
  1640		ADCQ acc0, t0
  1641		ADCQ acc1, t1
  1642		ADCQ acc2, t2
  1643		ADCQ acc3, t3
  1644		ADCQ $0, hlp
  1645		// Copy result
  1646		MOVQ t0, acc4
  1647		MOVQ t1, acc5
  1648		MOVQ t2, acc6
  1649		MOVQ t3, acc7
  1650		// Subtract p256
  1651		SUBQ $-1, acc4
  1652		SBBQ p256const0<>(SB) ,acc5
  1653		SBBQ $0, acc6
  1654		SBBQ p256const1<>(SB), acc7
  1655		SBBQ $0, hlp
  1656		// If the result of the subtraction is negative, restore the previous result
  1657		CMOVQCS t0, acc4
  1658		CMOVQCS t1, acc5
  1659		CMOVQCS t2, acc6
  1660		CMOVQCS t3, acc7
  1661	
  1662		RET
  1663	/* ---------------------------------------*/
  1664	#define p256MulBy2Inline\
  1665		XORQ mul0, mul0;\
  1666		ADDQ acc4, acc4;\
  1667		ADCQ acc5, acc5;\
  1668		ADCQ acc6, acc6;\
  1669		ADCQ acc7, acc7;\
  1670		ADCQ $0, mul0;\
  1671		MOVQ acc4, t0;\
  1672		MOVQ acc5, t1;\
  1673		MOVQ acc6, t2;\
  1674		MOVQ acc7, t3;\
  1675		SUBQ $-1, t0;\
  1676		SBBQ p256const0<>(SB), t1;\
  1677		SBBQ $0, t2;\
  1678		SBBQ p256const1<>(SB), t3;\
  1679		SBBQ $0, mul0;\
  1680		CMOVQCS acc4, t0;\
  1681		CMOVQCS acc5, t1;\
  1682		CMOVQCS acc6, t2;\
  1683		CMOVQCS acc7, t3;
  1684	/* ---------------------------------------*/
  1685	#define p256AddInline \
  1686		XORQ mul0, mul0;\
  1687		ADDQ t0, acc4;\
  1688		ADCQ t1, acc5;\
  1689		ADCQ t2, acc6;\
  1690		ADCQ t3, acc7;\
  1691		ADCQ $0, mul0;\
  1692		MOVQ acc4, t0;\
  1693		MOVQ acc5, t1;\
  1694		MOVQ acc6, t2;\
  1695		MOVQ acc7, t3;\
  1696		SUBQ $-1, t0;\
  1697		SBBQ p256const0<>(SB), t1;\
  1698		SBBQ $0, t2;\
  1699		SBBQ p256const1<>(SB), t3;\
  1700		SBBQ $0, mul0;\
  1701		CMOVQCS acc4, t0;\
  1702		CMOVQCS acc5, t1;\
  1703		CMOVQCS acc6, t2;\
  1704		CMOVQCS acc7, t3;
  1705	/* ---------------------------------------*/
  1706	#define LDacc(src) MOVQ src(8*0), acc4; MOVQ src(8*1), acc5; MOVQ src(8*2), acc6; MOVQ src(8*3), acc7
  1707	#define LDt(src)   MOVQ src(8*0), t0; MOVQ src(8*1), t1; MOVQ src(8*2), t2; MOVQ src(8*3), t3
  1708	#define ST(dst)    MOVQ acc4, dst(8*0); MOVQ acc5, dst(8*1); MOVQ acc6, dst(8*2); MOVQ acc7, dst(8*3)
  1709	#define STt(dst)   MOVQ t0, dst(8*0); MOVQ t1, dst(8*1); MOVQ t2, dst(8*2); MOVQ t3, dst(8*3)
  1710	#define acc2t      MOVQ acc4, t0; MOVQ acc5, t1; MOVQ acc6, t2; MOVQ acc7, t3
  1711	#define t2acc      MOVQ t0, acc4; MOVQ t1, acc5; MOVQ t2, acc6; MOVQ t3, acc7
  1712	/* ---------------------------------------*/
  1713	#define x1in(off) (32*0 + off)(SP)
  1714	#define y1in(off) (32*1 + off)(SP)
  1715	#define z1in(off) (32*2 + off)(SP)
  1716	#define x2in(off) (32*3 + off)(SP)
  1717	#define y2in(off) (32*4 + off)(SP)
  1718	#define xout(off) (32*5 + off)(SP)
  1719	#define yout(off) (32*6 + off)(SP)
  1720	#define zout(off) (32*7 + off)(SP)
  1721	#define s2(off)   (32*8 + off)(SP)
  1722	#define z1sqr(off) (32*9 + off)(SP)
  1723	#define h(off)	  (32*10 + off)(SP)
  1724	#define r(off)	  (32*11 + off)(SP)
  1725	#define hsqr(off) (32*12 + off)(SP)
  1726	#define rsqr(off) (32*13 + off)(SP)
  1727	#define hcub(off) (32*14 + off)(SP)
  1728	#define rptr	  (32*15)(SP)
  1729	#define sel_save  (32*15 + 8)(SP)
  1730	#define zero_save (32*15 + 8 + 4)(SP)
  1731	
  1732	// func p256PointAddAffineAsm(res, in1, in2 []uint64, sign, sel, zero int)
  1733	TEXT ·p256PointAddAffineAsm(SB),0,$512-96
  1734		// Move input to stack in order to free registers
  1735		MOVQ res+0(FP), AX
  1736		MOVQ in1+24(FP), BX
  1737		MOVQ in2+48(FP), CX
  1738		MOVQ sign+72(FP), DX
  1739		MOVQ sel+80(FP), t1
  1740		MOVQ zero+88(FP), t2
  1741	
  1742		MOVOU (16*0)(BX), X0
  1743		MOVOU (16*1)(BX), X1
  1744		MOVOU (16*2)(BX), X2
  1745		MOVOU (16*3)(BX), X3
  1746		MOVOU (16*4)(BX), X4
  1747		MOVOU (16*5)(BX), X5
  1748	
  1749		MOVOU X0, x1in(16*0)
  1750		MOVOU X1, x1in(16*1)
  1751		MOVOU X2, y1in(16*0)
  1752		MOVOU X3, y1in(16*1)
  1753		MOVOU X4, z1in(16*0)
  1754		MOVOU X5, z1in(16*1)
  1755	
  1756		MOVOU (16*0)(CX), X0
  1757		MOVOU (16*1)(CX), X1
  1758	
  1759		MOVOU X0, x2in(16*0)
  1760		MOVOU X1, x2in(16*1)
  1761		// Store pointer to result
  1762		MOVQ mul0, rptr
  1763		MOVL t1, sel_save
  1764		MOVL t2, zero_save
  1765		// Negate y2in based on sign
  1766		MOVQ (16*2 + 8*0)(CX), acc4
  1767		MOVQ (16*2 + 8*1)(CX), acc5
  1768		MOVQ (16*2 + 8*2)(CX), acc6
  1769		MOVQ (16*2 + 8*3)(CX), acc7
  1770		MOVQ $-1, acc0
  1771		MOVQ p256const0<>(SB), acc1
  1772		MOVQ $0, acc2
  1773		MOVQ p256const1<>(SB), acc3
  1774		XORQ mul0, mul0
  1775		// Speculatively subtract
  1776		SUBQ acc4, acc0
  1777		SBBQ acc5, acc1
  1778		SBBQ acc6, acc2
  1779		SBBQ acc7, acc3
  1780		SBBQ $0, mul0
  1781		MOVQ acc0, t0
  1782		MOVQ acc1, t1
  1783		MOVQ acc2, t2
  1784		MOVQ acc3, t3
  1785		// Add in case the operand was > p256
  1786		ADDQ $-1, acc0
  1787		ADCQ p256const0<>(SB), acc1
  1788		ADCQ $0, acc2
  1789		ADCQ p256const1<>(SB), acc3
  1790		ADCQ $0, mul0
  1791		CMOVQNE t0, acc0
  1792		CMOVQNE t1, acc1
  1793		CMOVQNE t2, acc2
  1794		CMOVQNE t3, acc3
  1795		// If condition is 0, keep original value
  1796		TESTQ DX, DX
  1797		CMOVQEQ acc4, acc0
  1798		CMOVQEQ acc5, acc1
  1799		CMOVQEQ acc6, acc2
  1800		CMOVQEQ acc7, acc3
  1801		// Store result
  1802		MOVQ acc0, y2in(8*0)
  1803		MOVQ acc1, y2in(8*1)
  1804		MOVQ acc2, y2in(8*2)
  1805		MOVQ acc3, y2in(8*3)
  1806		// Begin point add
  1807		LDacc (z1in)
  1808		CALL p256SqrInternal(SB)	// z1ˆ2
  1809		ST (z1sqr)
  1810	
  1811		LDt (x2in)
  1812		CALL p256MulInternal(SB)	// x2 * z1ˆ2
  1813	
  1814		LDt (x1in)
  1815		CALL p256SubInternal(SB)	// h = u2 - u1
  1816		ST (h)
  1817	
  1818		LDt (z1in)
  1819		CALL p256MulInternal(SB)	// z3 = h * z1
  1820		ST (zout)
  1821	
  1822		LDacc (z1sqr)
  1823		CALL p256MulInternal(SB)	// z1ˆ3
  1824	
  1825		LDt (y2in)
  1826		CALL p256MulInternal(SB)	// s2 = y2 * z1ˆ3
  1827		ST (s2)
  1828	
  1829		LDt (y1in)
  1830		CALL p256SubInternal(SB)	// r = s2 - s1
  1831		ST (r)
  1832	
  1833		CALL p256SqrInternal(SB)	// rsqr = rˆ2
  1834		ST (rsqr)
  1835	
  1836		LDacc (h)
  1837		CALL p256SqrInternal(SB)	// hsqr = hˆ2
  1838		ST (hsqr)
  1839	
  1840		LDt (h)
  1841		CALL p256MulInternal(SB)	// hcub = hˆ3
  1842		ST (hcub)
  1843	
  1844		LDt (y1in)
  1845		CALL p256MulInternal(SB)	// y1 * hˆ3
  1846		ST (s2)
  1847	
  1848		LDacc (x1in)
  1849		LDt (hsqr)
  1850		CALL p256MulInternal(SB)	// u1 * hˆ2
  1851		ST (h)
  1852	
  1853		p256MulBy2Inline			// u1 * hˆ2 * 2, inline
  1854		LDacc (rsqr)
  1855		CALL p256SubInternal(SB)	// rˆ2 - u1 * hˆ2 * 2
  1856	
  1857		LDt (hcub)
  1858		CALL p256SubInternal(SB)
  1859		ST (xout)
  1860	
  1861		MOVQ acc4, t0
  1862		MOVQ acc5, t1
  1863		MOVQ acc6, t2
  1864		MOVQ acc7, t3
  1865		LDacc (h)
  1866		CALL p256SubInternal(SB)
  1867	
  1868		LDt (r)
  1869		CALL p256MulInternal(SB)
  1870	
  1871		LDt (s2)
  1872		CALL p256SubInternal(SB)
  1873		ST (yout)
  1874		// Load stored values from stack
  1875		MOVQ rptr, AX
  1876		MOVL sel_save, BX
  1877		MOVL zero_save, CX
  1878		// The result is not valid if (sel == 0), conditional choose
  1879		MOVOU xout(16*0), X0
  1880		MOVOU xout(16*1), X1
  1881		MOVOU yout(16*0), X2
  1882		MOVOU yout(16*1), X3
  1883		MOVOU zout(16*0), X4
  1884		MOVOU zout(16*1), X5
  1885	
  1886		MOVL BX, X6
  1887		MOVL CX, X7
  1888	
  1889		PXOR X8, X8
  1890		PCMPEQL X9, X9
  1891	
  1892		PSHUFD $0, X6, X6
  1893		PSHUFD $0, X7, X7
  1894	
  1895		PCMPEQL X8, X6
  1896		PCMPEQL X8, X7
  1897	
  1898		MOVOU X6, X15
  1899		PANDN X9, X15
  1900	
  1901		MOVOU x1in(16*0), X9
  1902		MOVOU x1in(16*1), X10
  1903		MOVOU y1in(16*0), X11
  1904		MOVOU y1in(16*1), X12
  1905		MOVOU z1in(16*0), X13
  1906		MOVOU z1in(16*1), X14
  1907	
  1908		PAND X15, X0
  1909		PAND X15, X1
  1910		PAND X15, X2
  1911		PAND X15, X3
  1912		PAND X15, X4
  1913		PAND X15, X5
  1914	
  1915		PAND X6, X9
  1916		PAND X6, X10
  1917		PAND X6, X11
  1918		PAND X6, X12
  1919		PAND X6, X13
  1920		PAND X6, X14
  1921	
  1922		PXOR X9, X0
  1923		PXOR X10, X1
  1924		PXOR X11, X2
  1925		PXOR X12, X3
  1926		PXOR X13, X4
  1927		PXOR X14, X5
  1928		// Similarly if zero == 0
  1929		PCMPEQL X9, X9
  1930		MOVOU X7, X15
  1931		PANDN X9, X15
  1932	
  1933		MOVOU x2in(16*0), X9
  1934		MOVOU x2in(16*1), X10
  1935		MOVOU y2in(16*0), X11
  1936		MOVOU y2in(16*1), X12
  1937		MOVOU p256one<>+0x00(SB), X13
  1938		MOVOU p256one<>+0x10(SB), X14
  1939	
  1940		PAND X15, X0
  1941		PAND X15, X1
  1942		PAND X15, X2
  1943		PAND X15, X3
  1944		PAND X15, X4
  1945		PAND X15, X5
  1946	
  1947		PAND X7, X9
  1948		PAND X7, X10
  1949		PAND X7, X11
  1950		PAND X7, X12
  1951		PAND X7, X13
  1952		PAND X7, X14
  1953	
  1954		PXOR X9, X0
  1955		PXOR X10, X1
  1956		PXOR X11, X2
  1957		PXOR X12, X3
  1958		PXOR X13, X4
  1959		PXOR X14, X5
  1960		// Finally output the result
  1961		MOVOU X0, (16*0)(AX)
  1962		MOVOU X1, (16*1)(AX)
  1963		MOVOU X2, (16*2)(AX)
  1964		MOVOU X3, (16*3)(AX)
  1965		MOVOU X4, (16*4)(AX)
  1966		MOVOU X5, (16*5)(AX)
  1967		MOVQ $0, rptr
  1968	
  1969		RET
  1970	#undef x1in
  1971	#undef y1in
  1972	#undef z1in
  1973	#undef x2in
  1974	#undef y2in
  1975	#undef xout
  1976	#undef yout
  1977	#undef zout
  1978	#undef s2
  1979	#undef z1sqr
  1980	#undef h
  1981	#undef r
  1982	#undef hsqr
  1983	#undef rsqr
  1984	#undef hcub
  1985	#undef rptr
  1986	#undef sel_save
  1987	#undef zero_save
  1988	
  1989	// p256IsZero returns 1 in AX if [acc4..acc7] represents zero and zero
  1990	// otherwise. It writes to [acc4..acc7], t0 and t1.
  1991	TEXT p256IsZero(SB),NOSPLIT,$0
  1992		// AX contains a flag that is set if the input is zero.
  1993		XORQ AX, AX
  1994		MOVQ $1, t1
  1995	
  1996		// Check whether [acc4..acc7] are all zero.
  1997		MOVQ acc4, t0
  1998		ORQ acc5, t0
  1999		ORQ acc6, t0
  2000		ORQ acc7, t0
  2001	
  2002		// Set the zero flag if so. (CMOV of a constant to a register doesn't
  2003		// appear to be supported in Go. Thus t1 = 1.)
  2004		CMOVQEQ t1, AX
  2005	
  2006		// XOR [acc4..acc7] with P and compare with zero again.
  2007		XORQ $-1, acc4
  2008		XORQ p256const0<>(SB), acc5
  2009		XORQ p256const1<>(SB), acc7
  2010		ORQ acc5, acc4
  2011		ORQ acc6, acc4
  2012		ORQ acc7, acc4
  2013	
  2014		// Set the zero flag if so.
  2015		CMOVQEQ t1, AX
  2016		RET
  2017	
  2018	/* ---------------------------------------*/
  2019	#define x1in(off) (32*0 + off)(SP)
  2020	#define y1in(off) (32*1 + off)(SP)
  2021	#define z1in(off) (32*2 + off)(SP)
  2022	#define x2in(off) (32*3 + off)(SP)
  2023	#define y2in(off) (32*4 + off)(SP)
  2024	#define z2in(off) (32*5 + off)(SP)
  2025	
  2026	#define xout(off) (32*6 + off)(SP)
  2027	#define yout(off) (32*7 + off)(SP)
  2028	#define zout(off) (32*8 + off)(SP)
  2029	
  2030	#define u1(off)    (32*9 + off)(SP)
  2031	#define u2(off)    (32*10 + off)(SP)
  2032	#define s1(off)    (32*11 + off)(SP)
  2033	#define s2(off)    (32*12 + off)(SP)
  2034	#define z1sqr(off) (32*13 + off)(SP)
  2035	#define z2sqr(off) (32*14 + off)(SP)
  2036	#define h(off)     (32*15 + off)(SP)
  2037	#define r(off)     (32*16 + off)(SP)
  2038	#define hsqr(off)  (32*17 + off)(SP)
  2039	#define rsqr(off)  (32*18 + off)(SP)
  2040	#define hcub(off)  (32*19 + off)(SP)
  2041	#define rptr       (32*20)(SP)
  2042	#define points_eq  (32*20+8)(SP)
  2043	
  2044	//func p256PointAddAsm(res, in1, in2 []uint64) int
  2045	TEXT ·p256PointAddAsm(SB),0,$680-80
  2046		// See https://hyperelliptic.org/EFD/g1p/auto-shortw-jacobian-3.html#addition-add-2007-bl
  2047		// Move input to stack in order to free registers
  2048		MOVQ res+0(FP), AX
  2049		MOVQ in1+24(FP), BX
  2050		MOVQ in2+48(FP), CX
  2051	
  2052		MOVOU (16*0)(BX), X0
  2053		MOVOU (16*1)(BX), X1
  2054		MOVOU (16*2)(BX), X2
  2055		MOVOU (16*3)(BX), X3
  2056		MOVOU (16*4)(BX), X4
  2057		MOVOU (16*5)(BX), X5
  2058	
  2059		MOVOU X0, x1in(16*0)
  2060		MOVOU X1, x1in(16*1)
  2061		MOVOU X2, y1in(16*0)
  2062		MOVOU X3, y1in(16*1)
  2063		MOVOU X4, z1in(16*0)
  2064		MOVOU X5, z1in(16*1)
  2065	
  2066		MOVOU (16*0)(CX), X0
  2067		MOVOU (16*1)(CX), X1
  2068		MOVOU (16*2)(CX), X2
  2069		MOVOU (16*3)(CX), X3
  2070		MOVOU (16*4)(CX), X4
  2071		MOVOU (16*5)(CX), X5
  2072	
  2073		MOVOU X0, x2in(16*0)
  2074		MOVOU X1, x2in(16*1)
  2075		MOVOU X2, y2in(16*0)
  2076		MOVOU X3, y2in(16*1)
  2077		MOVOU X4, z2in(16*0)
  2078		MOVOU X5, z2in(16*1)
  2079		// Store pointer to result
  2080		MOVQ AX, rptr
  2081		// Begin point add
  2082		LDacc (z2in)
  2083		CALL p256SqrInternal(SB)	// z2ˆ2
  2084		ST (z2sqr)
  2085		LDt (z2in)
  2086		CALL p256MulInternal(SB)	// z2ˆ3
  2087		LDt (y1in)
  2088		CALL p256MulInternal(SB)	// s1 = z2ˆ3*y1
  2089		ST (s1)
  2090	
  2091		LDacc (z1in)
  2092		CALL p256SqrInternal(SB)	// z1ˆ2
  2093		ST (z1sqr)
  2094		LDt (z1in)
  2095		CALL p256MulInternal(SB)	// z1ˆ3
  2096		LDt (y2in)
  2097		CALL p256MulInternal(SB)	// s2 = z1ˆ3*y2
  2098		ST (s2)
  2099	
  2100		LDt (s1)
  2101		CALL p256SubInternal(SB)	// r = s2 - s1
  2102		ST (r)
  2103		CALL p256IsZero(SB)
  2104		MOVQ AX, points_eq
  2105	
  2106		LDacc (z2sqr)
  2107		LDt (x1in)
  2108		CALL p256MulInternal(SB)	// u1 = x1 * z2ˆ2
  2109		ST (u1)
  2110		LDacc (z1sqr)
  2111		LDt (x2in)
  2112		CALL p256MulInternal(SB)	// u2 = x2 * z1ˆ2
  2113		ST (u2)
  2114	
  2115		LDt (u1)
  2116		CALL p256SubInternal(SB)	// h = u2 - u1
  2117		ST (h)
  2118		CALL p256IsZero(SB)
  2119		ANDQ points_eq, AX
  2120		MOVQ AX, points_eq
  2121	
  2122		LDacc (r)
  2123		CALL p256SqrInternal(SB)	// rsqr = rˆ2
  2124		ST (rsqr)
  2125	
  2126		LDacc (h)
  2127		CALL p256SqrInternal(SB)	// hsqr = hˆ2
  2128		ST (hsqr)
  2129	
  2130		LDt (h)
  2131		CALL p256MulInternal(SB)	// hcub = hˆ3
  2132		ST (hcub)
  2133	
  2134		LDt (s1)
  2135		CALL p256MulInternal(SB)
  2136		ST (s2)
  2137	
  2138		LDacc (z1in)
  2139		LDt (z2in)
  2140		CALL p256MulInternal(SB)	// z1 * z2
  2141		LDt (h)
  2142		CALL p256MulInternal(SB)	// z1 * z2 * h
  2143		ST (zout)
  2144	
  2145		LDacc (hsqr)
  2146		LDt (u1)
  2147		CALL p256MulInternal(SB)	// hˆ2 * u1
  2148		ST (u2)
  2149	
  2150		p256MulBy2Inline	// u1 * hˆ2 * 2, inline
  2151		LDacc (rsqr)
  2152		CALL p256SubInternal(SB)	// rˆ2 - u1 * hˆ2 * 2
  2153	
  2154		LDt (hcub)
  2155		CALL p256SubInternal(SB)
  2156		ST (xout)
  2157	
  2158		MOVQ acc4, t0
  2159		MOVQ acc5, t1
  2160		MOVQ acc6, t2
  2161		MOVQ acc7, t3
  2162		LDacc (u2)
  2163		CALL p256SubInternal(SB)
  2164	
  2165		LDt (r)
  2166		CALL p256MulInternal(SB)
  2167	
  2168		LDt (s2)
  2169		CALL p256SubInternal(SB)
  2170		ST (yout)
  2171	
  2172		MOVOU xout(16*0), X0
  2173		MOVOU xout(16*1), X1
  2174		MOVOU yout(16*0), X2
  2175		MOVOU yout(16*1), X3
  2176		MOVOU zout(16*0), X4
  2177		MOVOU zout(16*1), X5
  2178		// Finally output the result
  2179		MOVQ rptr, AX
  2180		MOVQ $0, rptr
  2181		MOVOU X0, (16*0)(AX)
  2182		MOVOU X1, (16*1)(AX)
  2183		MOVOU X2, (16*2)(AX)
  2184		MOVOU X3, (16*3)(AX)
  2185		MOVOU X4, (16*4)(AX)
  2186		MOVOU X5, (16*5)(AX)
  2187	
  2188		MOVQ points_eq, AX
  2189		MOVQ AX, ret+72(FP)
  2190	
  2191		RET
  2192	#undef x1in
  2193	#undef y1in
  2194	#undef z1in
  2195	#undef x2in
  2196	#undef y2in
  2197	#undef z2in
  2198	#undef xout
  2199	#undef yout
  2200	#undef zout
  2201	#undef s1
  2202	#undef s2
  2203	#undef u1
  2204	#undef u2
  2205	#undef z1sqr
  2206	#undef z2sqr
  2207	#undef h
  2208	#undef r
  2209	#undef hsqr
  2210	#undef rsqr
  2211	#undef hcub
  2212	#undef rptr
  2213	/* ---------------------------------------*/
  2214	#define x(off) (32*0 + off)(SP)
  2215	#define y(off) (32*1 + off)(SP)
  2216	#define z(off) (32*2 + off)(SP)
  2217	
  2218	#define s(off)	(32*3 + off)(SP)
  2219	#define m(off)	(32*4 + off)(SP)
  2220	#define zsqr(off) (32*5 + off)(SP)
  2221	#define tmp(off)  (32*6 + off)(SP)
  2222	#define rptr	  (32*7)(SP)
  2223	
  2224	//func p256PointDoubleAsm(res, in []uint64)
  2225	TEXT ·p256PointDoubleAsm(SB),NOSPLIT,$256-48
  2226		// Move input to stack in order to free registers
  2227		MOVQ res+0(FP), AX
  2228		MOVQ in+24(FP), BX
  2229	
  2230		MOVOU (16*0)(BX), X0
  2231		MOVOU (16*1)(BX), X1
  2232		MOVOU (16*2)(BX), X2
  2233		MOVOU (16*3)(BX), X3
  2234		MOVOU (16*4)(BX), X4
  2235		MOVOU (16*5)(BX), X5
  2236	
  2237		MOVOU X0, x(16*0)
  2238		MOVOU X1, x(16*1)
  2239		MOVOU X2, y(16*0)
  2240		MOVOU X3, y(16*1)
  2241		MOVOU X4, z(16*0)
  2242		MOVOU X5, z(16*1)
  2243		// Store pointer to result
  2244		MOVQ AX, rptr
  2245		// Begin point double
  2246		LDacc (z)
  2247		CALL p256SqrInternal(SB)
  2248		ST (zsqr)
  2249	
  2250		LDt (x)
  2251		p256AddInline
  2252		STt (m)
  2253	
  2254		LDacc (z)
  2255		LDt (y)
  2256		CALL p256MulInternal(SB)
  2257		p256MulBy2Inline
  2258		MOVQ rptr, AX
  2259		// Store z
  2260		MOVQ t0, (16*4 + 8*0)(AX)
  2261		MOVQ t1, (16*4 + 8*1)(AX)
  2262		MOVQ t2, (16*4 + 8*2)(AX)
  2263		MOVQ t3, (16*4 + 8*3)(AX)
  2264	
  2265		LDacc (x)
  2266		LDt (zsqr)
  2267		CALL p256SubInternal(SB)
  2268		LDt (m)
  2269		CALL p256MulInternal(SB)
  2270		ST (m)
  2271		// Multiply by 3
  2272		p256MulBy2Inline
  2273		LDacc (m)
  2274		p256AddInline
  2275		STt (m)
  2276		////////////////////////
  2277		LDacc (y)
  2278		p256MulBy2Inline
  2279		t2acc
  2280		CALL p256SqrInternal(SB)
  2281		ST (s)
  2282		CALL p256SqrInternal(SB)
  2283		// Divide by 2
  2284		XORQ mul0, mul0
  2285		MOVQ acc4, t0
  2286		MOVQ acc5, t1
  2287		MOVQ acc6, t2
  2288		MOVQ acc7, t3
  2289	
  2290		ADDQ $-1, acc4
  2291		ADCQ p256const0<>(SB), acc5
  2292		ADCQ $0, acc6
  2293		ADCQ p256const1<>(SB), acc7
  2294		ADCQ $0, mul0
  2295		TESTQ $1, t0
  2296	
  2297		CMOVQEQ t0, acc4
  2298		CMOVQEQ t1, acc5
  2299		CMOVQEQ t2, acc6
  2300		CMOVQEQ t3, acc7
  2301		ANDQ t0, mul0
  2302	
  2303		SHRQ $1, acc4:acc5
  2304		SHRQ $1, acc5:acc6
  2305		SHRQ $1, acc6:acc7
  2306		SHRQ $1, acc7:mul0
  2307		ST (y)
  2308		/////////////////////////
  2309		LDacc (x)
  2310		LDt (s)
  2311		CALL p256MulInternal(SB)
  2312		ST (s)
  2313		p256MulBy2Inline
  2314		STt (tmp)
  2315	
  2316		LDacc (m)
  2317		CALL p256SqrInternal(SB)
  2318		LDt (tmp)
  2319		CALL p256SubInternal(SB)
  2320	
  2321		MOVQ rptr, AX
  2322		// Store x
  2323		MOVQ acc4, (16*0 + 8*0)(AX)
  2324		MOVQ acc5, (16*0 + 8*1)(AX)
  2325		MOVQ acc6, (16*0 + 8*2)(AX)
  2326		MOVQ acc7, (16*0 + 8*3)(AX)
  2327	
  2328		acc2t
  2329		LDacc (s)
  2330		CALL p256SubInternal(SB)
  2331	
  2332		LDt (m)
  2333		CALL p256MulInternal(SB)
  2334	
  2335		LDt (y)
  2336		CALL p256SubInternal(SB)
  2337		MOVQ rptr, AX
  2338		// Store y
  2339		MOVQ acc4, (16*2 + 8*0)(AX)
  2340		MOVQ acc5, (16*2 + 8*1)(AX)
  2341		MOVQ acc6, (16*2 + 8*2)(AX)
  2342		MOVQ acc7, (16*2 + 8*3)(AX)
  2343		///////////////////////
  2344		MOVQ $0, rptr
  2345	
  2346		RET
  2347	/* ---------------------------------------*/
  2348	

View as plain text