...
Run Format

Text file src/crypto/aes/gcm_amd64.s

Documentation: crypto/aes

     1	// Copyright 2015 The Go Authors. All rights reserved.
     2	// Use of this source code is governed by a BSD-style
     3	// license that can be found in the LICENSE file.
     4	
     5	// This is an optimized implementation of AES-GCM using AES-NI and CLMUL-NI
     6	// The implementation uses some optimization as described in:
     7	// [1] Gueron, S., Kounavis, M.E.: Intel® Carry-Less Multiplication
     8	//     Instruction and its Usage for Computing the GCM Mode rev. 2.02
     9	// [2] Gueron, S., Krasnov, V.: Speeding up Counter Mode in Software and
    10	//     Hardware
    11	
    12	#include "textflag.h"
    13	
    14	#define B0 X0
    15	#define B1 X1
    16	#define B2 X2
    17	#define B3 X3
    18	#define B4 X4
    19	#define B5 X5
    20	#define B6 X6
    21	#define B7 X7
    22	
    23	#define ACC0 X8
    24	#define ACC1 X9
    25	#define ACCM X10
    26	
    27	#define T0 X11
    28	#define T1 X12
    29	#define T2 X13
    30	#define POLY X14
    31	#define BSWAP X15
    32	
    33	DATA bswapMask<>+0x00(SB)/8, $0x08090a0b0c0d0e0f
    34	DATA bswapMask<>+0x08(SB)/8, $0x0001020304050607
    35	
    36	DATA gcmPoly<>+0x00(SB)/8, $0x0000000000000001
    37	DATA gcmPoly<>+0x08(SB)/8, $0xc200000000000000
    38	
    39	DATA andMask<>+0x00(SB)/8, $0x00000000000000ff
    40	DATA andMask<>+0x08(SB)/8, $0x0000000000000000
    41	DATA andMask<>+0x10(SB)/8, $0x000000000000ffff
    42	DATA andMask<>+0x18(SB)/8, $0x0000000000000000
    43	DATA andMask<>+0x20(SB)/8, $0x0000000000ffffff
    44	DATA andMask<>+0x28(SB)/8, $0x0000000000000000
    45	DATA andMask<>+0x30(SB)/8, $0x00000000ffffffff
    46	DATA andMask<>+0x38(SB)/8, $0x0000000000000000
    47	DATA andMask<>+0x40(SB)/8, $0x000000ffffffffff
    48	DATA andMask<>+0x48(SB)/8, $0x0000000000000000
    49	DATA andMask<>+0x50(SB)/8, $0x0000ffffffffffff
    50	DATA andMask<>+0x58(SB)/8, $0x0000000000000000
    51	DATA andMask<>+0x60(SB)/8, $0x00ffffffffffffff
    52	DATA andMask<>+0x68(SB)/8, $0x0000000000000000
    53	DATA andMask<>+0x70(SB)/8, $0xffffffffffffffff
    54	DATA andMask<>+0x78(SB)/8, $0x0000000000000000
    55	DATA andMask<>+0x80(SB)/8, $0xffffffffffffffff
    56	DATA andMask<>+0x88(SB)/8, $0x00000000000000ff
    57	DATA andMask<>+0x90(SB)/8, $0xffffffffffffffff
    58	DATA andMask<>+0x98(SB)/8, $0x000000000000ffff
    59	DATA andMask<>+0xa0(SB)/8, $0xffffffffffffffff
    60	DATA andMask<>+0xa8(SB)/8, $0x0000000000ffffff
    61	DATA andMask<>+0xb0(SB)/8, $0xffffffffffffffff
    62	DATA andMask<>+0xb8(SB)/8, $0x00000000ffffffff
    63	DATA andMask<>+0xc0(SB)/8, $0xffffffffffffffff
    64	DATA andMask<>+0xc8(SB)/8, $0x000000ffffffffff
    65	DATA andMask<>+0xd0(SB)/8, $0xffffffffffffffff
    66	DATA andMask<>+0xd8(SB)/8, $0x0000ffffffffffff
    67	DATA andMask<>+0xe0(SB)/8, $0xffffffffffffffff
    68	DATA andMask<>+0xe8(SB)/8, $0x00ffffffffffffff
    69	
    70	GLOBL bswapMask<>(SB), (NOPTR+RODATA), $16
    71	GLOBL gcmPoly<>(SB), (NOPTR+RODATA), $16
    72	GLOBL andMask<>(SB), (NOPTR+RODATA), $240
    73	
    74	// func gcmAesFinish(productTable *[256]byte, tagMask, T *[16]byte, pLen, dLen uint64)
    75	TEXT ·gcmAesFinish(SB),NOSPLIT,$0
    76	#define pTbl DI
    77	#define tMsk SI
    78	#define tPtr DX
    79	#define plen AX
    80	#define dlen CX
    81	
    82		MOVQ productTable+0(FP), pTbl
    83		MOVQ tagMask+8(FP), tMsk
    84		MOVQ T+16(FP), tPtr
    85		MOVQ pLen+24(FP), plen
    86		MOVQ dLen+32(FP), dlen
    87	
    88		MOVOU (tPtr), ACC0
    89		MOVOU (tMsk), T2
    90	
    91		MOVOU bswapMask<>(SB), BSWAP
    92		MOVOU gcmPoly<>(SB), POLY
    93	
    94		SHLQ $3, plen
    95		SHLQ $3, dlen
    96	
    97		MOVQ plen, B0
    98		PINSRQ $1, dlen, B0
    99	
   100		PXOR ACC0, B0
   101	
   102		MOVOU (16*14)(pTbl), ACC0
   103		MOVOU (16*15)(pTbl), ACCM
   104		MOVOU ACC0, ACC1
   105	
   106		PCLMULQDQ $0x00, B0, ACC0
   107		PCLMULQDQ $0x11, B0, ACC1
   108		PSHUFD $78, B0, T0
   109		PXOR B0, T0
   110		PCLMULQDQ $0x00, T0, ACCM
   111	
   112		PXOR ACC0, ACCM
   113		PXOR ACC1, ACCM
   114		MOVOU ACCM, T0
   115		PSRLDQ $8, ACCM
   116		PSLLDQ $8, T0
   117		PXOR ACCM, ACC1
   118		PXOR T0, ACC0
   119	
   120		MOVOU POLY, T0
   121		PCLMULQDQ $0x01, ACC0, T0
   122		PSHUFD $78, ACC0, ACC0
   123		PXOR T0, ACC0
   124	
   125		MOVOU POLY, T0
   126		PCLMULQDQ $0x01, ACC0, T0
   127		PSHUFD $78, ACC0, ACC0
   128		PXOR T0, ACC0
   129	
   130		PXOR ACC1, ACC0
   131	
   132		PSHUFB BSWAP, ACC0
   133		PXOR T2, ACC0
   134		MOVOU ACC0, (tPtr)
   135	
   136		RET
   137	#undef pTbl
   138	#undef tMsk
   139	#undef tPtr
   140	#undef plen
   141	#undef dlen
   142	
   143	// func gcmAesInit(productTable *[256]byte, ks []uint32)
   144	TEXT ·gcmAesInit(SB),NOSPLIT,$0
   145	#define dst DI
   146	#define KS SI
   147	#define NR DX
   148	
   149		MOVQ productTable+0(FP), dst
   150		MOVQ ks_base+8(FP), KS
   151		MOVQ ks_len+16(FP), NR
   152	
   153		SHRQ $2, NR
   154		DECQ NR
   155	
   156		MOVOU bswapMask<>(SB), BSWAP
   157		MOVOU gcmPoly<>(SB), POLY
   158	
   159		// Encrypt block 0, with the AES key to generate the hash key H
   160		MOVOU (16*0)(KS), B0
   161		MOVOU (16*1)(KS), T0
   162		AESENC T0, B0
   163		MOVOU (16*2)(KS), T0
   164		AESENC T0, B0
   165		MOVOU (16*3)(KS), T0
   166		AESENC T0, B0
   167		MOVOU (16*4)(KS), T0
   168		AESENC T0, B0
   169		MOVOU (16*5)(KS), T0
   170		AESENC T0, B0
   171		MOVOU (16*6)(KS), T0
   172		AESENC T0, B0
   173		MOVOU (16*7)(KS), T0
   174		AESENC T0, B0
   175		MOVOU (16*8)(KS), T0
   176		AESENC T0, B0
   177		MOVOU (16*9)(KS), T0
   178		AESENC T0, B0
   179		MOVOU (16*10)(KS), T0
   180		CMPQ NR, $12
   181		JB initEncLast
   182		AESENC T0, B0
   183		MOVOU (16*11)(KS), T0
   184		AESENC T0, B0
   185		MOVOU (16*12)(KS), T0
   186		JE initEncLast
   187		AESENC T0, B0
   188		MOVOU (16*13)(KS), T0
   189		AESENC T0, B0
   190		MOVOU (16*14)(KS), T0
   191	initEncLast:
   192		AESENCLAST T0, B0
   193	
   194		PSHUFB BSWAP, B0
   195		// H * 2
   196		PSHUFD $0xff, B0, T0
   197		MOVOU B0, T1
   198		PSRAL $31, T0
   199		PAND POLY, T0
   200		PSRLL $31, T1
   201		PSLLDQ $4, T1
   202		PSLLL $1, B0
   203		PXOR T0, B0
   204		PXOR T1, B0
   205		// Karatsuba pre-computations
   206		MOVOU B0, (16*14)(dst)
   207		PSHUFD $78, B0, B1
   208		PXOR B0, B1
   209		MOVOU B1, (16*15)(dst)
   210	
   211		MOVOU B0, B2
   212		MOVOU B1, B3
   213		// Now prepare powers of H and pre-computations for them
   214		MOVQ $7, AX
   215	
   216	initLoop:
   217			MOVOU B2, T0
   218			MOVOU B2, T1
   219			MOVOU B3, T2
   220			PCLMULQDQ $0x00, B0, T0
   221			PCLMULQDQ $0x11, B0, T1
   222			PCLMULQDQ $0x00, B1, T2
   223	
   224			PXOR T0, T2
   225			PXOR T1, T2
   226			MOVOU T2, B4
   227			PSLLDQ $8, B4
   228			PSRLDQ $8, T2
   229			PXOR B4, T0
   230			PXOR T2, T1
   231	
   232			MOVOU POLY, B2
   233			PCLMULQDQ $0x01, T0, B2
   234			PSHUFD $78, T0, T0
   235			PXOR B2, T0
   236			MOVOU POLY, B2
   237			PCLMULQDQ $0x01, T0, B2
   238			PSHUFD $78, T0, T0
   239			PXOR T0, B2
   240			PXOR T1, B2
   241	
   242			MOVOU B2, (16*12)(dst)
   243			PSHUFD $78, B2, B3
   244			PXOR B2, B3
   245			MOVOU B3, (16*13)(dst)
   246	
   247			DECQ AX
   248			LEAQ (-16*2)(dst), dst
   249		JNE initLoop
   250	
   251		RET
   252	#undef NR
   253	#undef KS
   254	#undef dst
   255	
   256	// func gcmAesData(productTable *[256]byte, data []byte, T *[16]byte)
   257	TEXT ·gcmAesData(SB),NOSPLIT,$0
   258	#define pTbl DI
   259	#define aut SI
   260	#define tPtr CX
   261	#define autLen DX
   262	
   263	#define reduceRound(a) 	MOVOU POLY, T0;	PCLMULQDQ $0x01, a, T0; PSHUFD $78, a, a; PXOR T0, a
   264	#define mulRoundAAD(X ,i) \
   265		MOVOU (16*(i*2))(pTbl), T1;\
   266		MOVOU T1, T2;\
   267		PCLMULQDQ $0x00, X, T1;\
   268		PXOR T1, ACC0;\
   269		PCLMULQDQ $0x11, X, T2;\
   270		PXOR T2, ACC1;\
   271		PSHUFD $78, X, T1;\
   272		PXOR T1, X;\
   273		MOVOU (16*(i*2+1))(pTbl), T1;\
   274		PCLMULQDQ $0x00, X, T1;\
   275		PXOR T1, ACCM
   276	
   277		MOVQ productTable+0(FP), pTbl
   278		MOVQ data_base+8(FP), aut
   279		MOVQ data_len+16(FP), autLen
   280		MOVQ T+32(FP), tPtr
   281	
   282		PXOR ACC0, ACC0
   283		MOVOU bswapMask<>(SB), BSWAP
   284		MOVOU gcmPoly<>(SB), POLY
   285	
   286		TESTQ autLen, autLen
   287		JEQ dataBail
   288	
   289		CMPQ autLen, $13	// optimize the TLS case
   290		JE dataTLS
   291		CMPQ autLen, $128
   292		JB startSinglesLoop
   293		JMP dataOctaLoop
   294	
   295	dataTLS:
   296		MOVOU (16*14)(pTbl), T1
   297		MOVOU (16*15)(pTbl), T2
   298		PXOR B0, B0
   299		MOVQ (aut), B0
   300		PINSRD $2, 8(aut), B0
   301		PINSRB $12, 12(aut), B0
   302		XORQ autLen, autLen
   303		JMP dataMul
   304	
   305	dataOctaLoop:
   306			CMPQ autLen, $128
   307			JB startSinglesLoop
   308			SUBQ $128, autLen
   309	
   310			MOVOU (16*0)(aut), X0
   311			MOVOU (16*1)(aut), X1
   312			MOVOU (16*2)(aut), X2
   313			MOVOU (16*3)(aut), X3
   314			MOVOU (16*4)(aut), X4
   315			MOVOU (16*5)(aut), X5
   316			MOVOU (16*6)(aut), X6
   317			MOVOU (16*7)(aut), X7
   318			LEAQ (16*8)(aut), aut
   319			PSHUFB BSWAP, X0
   320			PSHUFB BSWAP, X1
   321			PSHUFB BSWAP, X2
   322			PSHUFB BSWAP, X3
   323			PSHUFB BSWAP, X4
   324			PSHUFB BSWAP, X5
   325			PSHUFB BSWAP, X6
   326			PSHUFB BSWAP, X7
   327			PXOR ACC0, X0
   328	
   329			MOVOU (16*0)(pTbl), ACC0
   330			MOVOU (16*1)(pTbl), ACCM
   331			MOVOU ACC0, ACC1
   332			PSHUFD $78, X0, T1
   333			PXOR X0, T1
   334			PCLMULQDQ $0x00, X0, ACC0
   335			PCLMULQDQ $0x11, X0, ACC1
   336			PCLMULQDQ $0x00, T1, ACCM
   337	
   338			mulRoundAAD(X1, 1)
   339			mulRoundAAD(X2, 2)
   340			mulRoundAAD(X3, 3)
   341			mulRoundAAD(X4, 4)
   342			mulRoundAAD(X5, 5)
   343			mulRoundAAD(X6, 6)
   344			mulRoundAAD(X7, 7)
   345	
   346			PXOR ACC0, ACCM
   347			PXOR ACC1, ACCM
   348			MOVOU ACCM, T0
   349			PSRLDQ $8, ACCM
   350			PSLLDQ $8, T0
   351			PXOR ACCM, ACC1
   352			PXOR T0, ACC0
   353			reduceRound(ACC0)
   354			reduceRound(ACC0)
   355			PXOR ACC1, ACC0
   356		JMP dataOctaLoop
   357	
   358	startSinglesLoop:
   359		MOVOU (16*14)(pTbl), T1
   360		MOVOU (16*15)(pTbl), T2
   361	
   362	dataSinglesLoop:
   363	
   364			CMPQ autLen, $16
   365			JB dataEnd
   366			SUBQ $16, autLen
   367	
   368			MOVOU (aut), B0
   369	dataMul:
   370			PSHUFB BSWAP, B0
   371			PXOR ACC0, B0
   372	
   373			MOVOU T1, ACC0
   374			MOVOU T2, ACCM
   375			MOVOU T1, ACC1
   376	
   377			PSHUFD $78, B0, T0
   378			PXOR B0, T0
   379			PCLMULQDQ $0x00, B0, ACC0
   380			PCLMULQDQ $0x11, B0, ACC1
   381			PCLMULQDQ $0x00, T0, ACCM
   382	
   383			PXOR ACC0, ACCM
   384			PXOR ACC1, ACCM
   385			MOVOU ACCM, T0
   386			PSRLDQ $8, ACCM
   387			PSLLDQ $8, T0
   388			PXOR ACCM, ACC1
   389			PXOR T0, ACC0
   390	
   391			MOVOU POLY, T0
   392			PCLMULQDQ $0x01, ACC0, T0
   393			PSHUFD $78, ACC0, ACC0
   394			PXOR T0, ACC0
   395	
   396			MOVOU POLY, T0
   397			PCLMULQDQ $0x01, ACC0, T0
   398			PSHUFD $78, ACC0, ACC0
   399			PXOR T0, ACC0
   400			PXOR ACC1, ACC0
   401	
   402			LEAQ 16(aut), aut
   403	
   404		JMP dataSinglesLoop
   405	
   406	dataEnd:
   407	
   408		TESTQ autLen, autLen
   409		JEQ dataBail
   410	
   411		PXOR B0, B0
   412		LEAQ -1(aut)(autLen*1), aut
   413	
   414	dataLoadLoop:
   415	
   416			PSLLDQ $1, B0
   417			PINSRB $0, (aut), B0
   418	
   419			LEAQ -1(aut), aut
   420			DECQ autLen
   421			JNE dataLoadLoop
   422	
   423		JMP dataMul
   424	
   425	dataBail:
   426		MOVOU ACC0, (tPtr)
   427		RET
   428	#undef pTbl
   429	#undef aut
   430	#undef tPtr
   431	#undef autLen
   432	
   433	// func gcmAesEnc(productTable *[256]byte, dst, src []byte, ctr, T *[16]byte, ks []uint32)
   434	TEXT ·gcmAesEnc(SB),0,$256-96
   435	#define pTbl DI
   436	#define ctx DX
   437	#define ctrPtr CX
   438	#define ptx SI
   439	#define ks AX
   440	#define tPtr R8
   441	#define ptxLen R9
   442	#define aluCTR R10
   443	#define aluTMP R11
   444	#define aluK R12
   445	#define NR R13
   446	
   447	#define increment(i) ADDL $1, aluCTR; MOVL aluCTR, aluTMP; XORL aluK, aluTMP; BSWAPL aluTMP; MOVL aluTMP, (3*4 + 8*16 + i*16)(SP)
   448	#define aesRnd(k) AESENC k, B0; AESENC k, B1; AESENC k, B2; AESENC k, B3; AESENC k, B4; AESENC k, B5; AESENC k, B6; AESENC k, B7
   449	#define aesRound(i) MOVOU (16*i)(ks), T0;AESENC T0, B0; AESENC T0, B1; AESENC T0, B2; AESENC T0, B3; AESENC T0, B4; AESENC T0, B5; AESENC T0, B6; AESENC T0, B7
   450	#define aesRndLast(k) AESENCLAST k, B0; AESENCLAST k, B1; AESENCLAST k, B2; AESENCLAST k, B3; AESENCLAST k, B4; AESENCLAST k, B5; AESENCLAST k, B6; AESENCLAST k, B7
   451	#define combinedRound(i) \
   452		MOVOU (16*i)(ks), T0;\
   453		AESENC T0, B0;\
   454		AESENC T0, B1;\
   455		AESENC T0, B2;\
   456		AESENC T0, B3;\
   457		 MOVOU (16*(i*2))(pTbl), T1;\
   458		 MOVOU T1, T2;\
   459		AESENC T0, B4;\
   460		AESENC T0, B5;\
   461		AESENC T0, B6;\
   462		AESENC T0, B7;\
   463		 MOVOU (16*i)(SP), T0;\
   464		 PCLMULQDQ $0x00, T0, T1;\
   465		 PXOR T1, ACC0;\
   466		 PSHUFD $78, T0, T1;\
   467		 PCLMULQDQ $0x11, T0, T2;\
   468		 PXOR T1, T0;\
   469		 PXOR T2, ACC1;\
   470		 MOVOU (16*(i*2+1))(pTbl), T2;\
   471		 PCLMULQDQ $0x00, T2, T0;\
   472		 PXOR T0, ACCM
   473	#define mulRound(i) \
   474		MOVOU (16*i)(SP), T0;\
   475		MOVOU (16*(i*2))(pTbl), T1;\
   476		MOVOU T1, T2;\
   477		PCLMULQDQ $0x00, T0, T1;\
   478		PXOR T1, ACC0;\
   479		PCLMULQDQ $0x11, T0, T2;\
   480		PXOR T2, ACC1;\
   481		PSHUFD $78, T0, T1;\
   482		PXOR T1, T0;\
   483		MOVOU (16*(i*2+1))(pTbl), T1;\
   484		PCLMULQDQ $0x00, T0, T1;\
   485		PXOR T1, ACCM
   486	
   487		MOVQ productTable+0(FP), pTbl
   488		MOVQ dst+8(FP), ctx
   489		MOVQ src_base+32(FP), ptx
   490		MOVQ src_len+40(FP), ptxLen
   491		MOVQ ctr+56(FP), ctrPtr
   492		MOVQ T+64(FP), tPtr
   493		MOVQ ks_base+72(FP), ks
   494		MOVQ ks_len+80(FP), NR
   495	
   496		SHRQ $2, NR
   497		DECQ NR
   498	
   499		MOVOU bswapMask<>(SB), BSWAP
   500		MOVOU gcmPoly<>(SB), POLY
   501	
   502		MOVOU (tPtr), ACC0
   503		PXOR ACC1, ACC1
   504		PXOR ACCM, ACCM
   505		MOVOU (ctrPtr), B0
   506		MOVL (3*4)(ctrPtr), aluCTR
   507		MOVOU (ks), T0
   508		MOVL (3*4)(ks), aluK
   509		BSWAPL aluCTR
   510		BSWAPL aluK
   511	
   512		PXOR B0, T0
   513		MOVOU T0, (8*16 + 0*16)(SP)
   514		increment(0)
   515	
   516		CMPQ ptxLen, $128
   517		JB gcmAesEncSingles
   518		SUBQ $128, ptxLen
   519	
   520		// We have at least 8 blocks to encrypt, prepare the rest of the counters
   521		MOVOU T0, (8*16 + 1*16)(SP)
   522		increment(1)
   523		MOVOU T0, (8*16 + 2*16)(SP)
   524		increment(2)
   525		MOVOU T0, (8*16 + 3*16)(SP)
   526		increment(3)
   527		MOVOU T0, (8*16 + 4*16)(SP)
   528		increment(4)
   529		MOVOU T0, (8*16 + 5*16)(SP)
   530		increment(5)
   531		MOVOU T0, (8*16 + 6*16)(SP)
   532		increment(6)
   533		MOVOU T0, (8*16 + 7*16)(SP)
   534		increment(7)
   535	
   536		MOVOU (8*16 + 0*16)(SP), B0
   537		MOVOU (8*16 + 1*16)(SP), B1
   538		MOVOU (8*16 + 2*16)(SP), B2
   539		MOVOU (8*16 + 3*16)(SP), B3
   540		MOVOU (8*16 + 4*16)(SP), B4
   541		MOVOU (8*16 + 5*16)(SP), B5
   542		MOVOU (8*16 + 6*16)(SP), B6
   543		MOVOU (8*16 + 7*16)(SP), B7
   544	
   545		aesRound(1)
   546		increment(0)
   547		aesRound(2)
   548		increment(1)
   549		aesRound(3)
   550		increment(2)
   551		aesRound(4)
   552		increment(3)
   553		aesRound(5)
   554		increment(4)
   555		aesRound(6)
   556		increment(5)
   557		aesRound(7)
   558		increment(6)
   559		aesRound(8)
   560		increment(7)
   561		aesRound(9)
   562		MOVOU (16*10)(ks), T0
   563		CMPQ NR, $12
   564		JB encLast1
   565		aesRnd(T0)
   566		aesRound(11)
   567		MOVOU (16*12)(ks), T0
   568		JE encLast1
   569		aesRnd(T0)
   570		aesRound(13)
   571		MOVOU (16*14)(ks), T0
   572	encLast1:
   573		aesRndLast(T0)
   574	
   575		MOVOU (16*0)(ptx), T0
   576		PXOR T0, B0
   577		MOVOU (16*1)(ptx), T0
   578		PXOR T0, B1
   579		MOVOU (16*2)(ptx), T0
   580		PXOR T0, B2
   581		MOVOU (16*3)(ptx), T0
   582		PXOR T0, B3
   583		MOVOU (16*4)(ptx), T0
   584		PXOR T0, B4
   585		MOVOU (16*5)(ptx), T0
   586		PXOR T0, B5
   587		MOVOU (16*6)(ptx), T0
   588		PXOR T0, B6
   589		MOVOU (16*7)(ptx), T0
   590		PXOR T0, B7
   591	
   592		MOVOU B0, (16*0)(ctx)
   593		PSHUFB BSWAP, B0
   594		PXOR ACC0, B0
   595		MOVOU B1, (16*1)(ctx)
   596		PSHUFB BSWAP, B1
   597		MOVOU B2, (16*2)(ctx)
   598		PSHUFB BSWAP, B2
   599		MOVOU B3, (16*3)(ctx)
   600		PSHUFB BSWAP, B3
   601		MOVOU B4, (16*4)(ctx)
   602		PSHUFB BSWAP, B4
   603		MOVOU B5, (16*5)(ctx)
   604		PSHUFB BSWAP, B5
   605		MOVOU B6, (16*6)(ctx)
   606		PSHUFB BSWAP, B6
   607		MOVOU B7, (16*7)(ctx)
   608		PSHUFB BSWAP, B7
   609	
   610		MOVOU B0, (16*0)(SP)
   611		MOVOU B1, (16*1)(SP)
   612		MOVOU B2, (16*2)(SP)
   613		MOVOU B3, (16*3)(SP)
   614		MOVOU B4, (16*4)(SP)
   615		MOVOU B5, (16*5)(SP)
   616		MOVOU B6, (16*6)(SP)
   617		MOVOU B7, (16*7)(SP)
   618	
   619		LEAQ 128(ptx), ptx
   620		LEAQ 128(ctx), ctx
   621	
   622	gcmAesEncOctetsLoop:
   623	
   624			CMPQ ptxLen, $128
   625			JB gcmAesEncOctetsEnd
   626			SUBQ $128, ptxLen
   627	
   628			MOVOU (8*16 + 0*16)(SP), B0
   629			MOVOU (8*16 + 1*16)(SP), B1
   630			MOVOU (8*16 + 2*16)(SP), B2
   631			MOVOU (8*16 + 3*16)(SP), B3
   632			MOVOU (8*16 + 4*16)(SP), B4
   633			MOVOU (8*16 + 5*16)(SP), B5
   634			MOVOU (8*16 + 6*16)(SP), B6
   635			MOVOU (8*16 + 7*16)(SP), B7
   636	
   637			MOVOU (16*0)(SP), T0
   638			PSHUFD $78, T0, T1
   639			PXOR T0, T1
   640	
   641			MOVOU (16*0)(pTbl), ACC0
   642			MOVOU (16*1)(pTbl), ACCM
   643			MOVOU ACC0, ACC1
   644	
   645			PCLMULQDQ $0x00, T1, ACCM
   646			PCLMULQDQ $0x00, T0, ACC0
   647			PCLMULQDQ $0x11, T0, ACC1
   648	
   649			combinedRound(1)
   650			increment(0)
   651			combinedRound(2)
   652			increment(1)
   653			combinedRound(3)
   654			increment(2)
   655			combinedRound(4)
   656			increment(3)
   657			combinedRound(5)
   658			increment(4)
   659			combinedRound(6)
   660			increment(5)
   661			combinedRound(7)
   662			increment(6)
   663	
   664			aesRound(8)
   665			increment(7)
   666	
   667			PXOR ACC0, ACCM
   668			PXOR ACC1, ACCM
   669			MOVOU ACCM, T0
   670			PSRLDQ $8, ACCM
   671			PSLLDQ $8, T0
   672			PXOR ACCM, ACC1
   673			PXOR T0, ACC0
   674	
   675			reduceRound(ACC0)
   676			aesRound(9)
   677	
   678			reduceRound(ACC0)
   679			PXOR ACC1, ACC0
   680	
   681			MOVOU (16*10)(ks), T0
   682			CMPQ NR, $12
   683			JB encLast2
   684			aesRnd(T0)
   685			aesRound(11)
   686			MOVOU (16*12)(ks), T0
   687			JE encLast2
   688			aesRnd(T0)
   689			aesRound(13)
   690			MOVOU (16*14)(ks), T0
   691	encLast2:
   692			aesRndLast(T0)
   693	
   694			MOVOU (16*0)(ptx), T0
   695			PXOR T0, B0
   696			MOVOU (16*1)(ptx), T0
   697			PXOR T0, B1
   698			MOVOU (16*2)(ptx), T0
   699			PXOR T0, B2
   700			MOVOU (16*3)(ptx), T0
   701			PXOR T0, B3
   702			MOVOU (16*4)(ptx), T0
   703			PXOR T0, B4
   704			MOVOU (16*5)(ptx), T0
   705			PXOR T0, B5
   706			MOVOU (16*6)(ptx), T0
   707			PXOR T0, B6
   708			MOVOU (16*7)(ptx), T0
   709			PXOR T0, B7
   710	
   711			MOVOU B0, (16*0)(ctx)
   712			PSHUFB BSWAP, B0
   713			PXOR ACC0, B0
   714			MOVOU B1, (16*1)(ctx)
   715			PSHUFB BSWAP, B1
   716			MOVOU B2, (16*2)(ctx)
   717			PSHUFB BSWAP, B2
   718			MOVOU B3, (16*3)(ctx)
   719			PSHUFB BSWAP, B3
   720			MOVOU B4, (16*4)(ctx)
   721			PSHUFB BSWAP, B4
   722			MOVOU B5, (16*5)(ctx)
   723			PSHUFB BSWAP, B5
   724			MOVOU B6, (16*6)(ctx)
   725			PSHUFB BSWAP, B6
   726			MOVOU B7, (16*7)(ctx)
   727			PSHUFB BSWAP, B7
   728	
   729			MOVOU B0, (16*0)(SP)
   730			MOVOU B1, (16*1)(SP)
   731			MOVOU B2, (16*2)(SP)
   732			MOVOU B3, (16*3)(SP)
   733			MOVOU B4, (16*4)(SP)
   734			MOVOU B5, (16*5)(SP)
   735			MOVOU B6, (16*6)(SP)
   736			MOVOU B7, (16*7)(SP)
   737	
   738			LEAQ 128(ptx), ptx
   739			LEAQ 128(ctx), ctx
   740	
   741			JMP gcmAesEncOctetsLoop
   742	
   743	gcmAesEncOctetsEnd:
   744	
   745		MOVOU (16*0)(SP), T0
   746		MOVOU (16*0)(pTbl), ACC0
   747		MOVOU (16*1)(pTbl), ACCM
   748		MOVOU ACC0, ACC1
   749		PSHUFD $78, T0, T1
   750		PXOR T0, T1
   751		PCLMULQDQ $0x00, T0, ACC0
   752		PCLMULQDQ $0x11, T0, ACC1
   753		PCLMULQDQ $0x00, T1, ACCM
   754	
   755		mulRound(1)
   756		mulRound(2)
   757		mulRound(3)
   758		mulRound(4)
   759		mulRound(5)
   760		mulRound(6)
   761		mulRound(7)
   762	
   763		PXOR ACC0, ACCM
   764		PXOR ACC1, ACCM
   765		MOVOU ACCM, T0
   766		PSRLDQ $8, ACCM
   767		PSLLDQ $8, T0
   768		PXOR ACCM, ACC1
   769		PXOR T0, ACC0
   770	
   771		reduceRound(ACC0)
   772		reduceRound(ACC0)
   773		PXOR ACC1, ACC0
   774	
   775		TESTQ ptxLen, ptxLen
   776		JE gcmAesEncDone
   777	
   778		SUBQ $7, aluCTR
   779	
   780	gcmAesEncSingles:
   781	
   782		MOVOU (16*1)(ks), B1
   783		MOVOU (16*2)(ks), B2
   784		MOVOU (16*3)(ks), B3
   785		MOVOU (16*4)(ks), B4
   786		MOVOU (16*5)(ks), B5
   787		MOVOU (16*6)(ks), B6
   788		MOVOU (16*7)(ks), B7
   789	
   790		MOVOU (16*14)(pTbl), T2
   791	
   792	gcmAesEncSinglesLoop:
   793	
   794			CMPQ ptxLen, $16
   795			JB gcmAesEncTail
   796			SUBQ $16, ptxLen
   797	
   798			MOVOU (8*16 + 0*16)(SP), B0
   799			increment(0)
   800	
   801			AESENC B1, B0
   802			AESENC B2, B0
   803			AESENC B3, B0
   804			AESENC B4, B0
   805			AESENC B5, B0
   806			AESENC B6, B0
   807			AESENC B7, B0
   808			MOVOU (16*8)(ks), T0
   809			AESENC T0, B0
   810			MOVOU (16*9)(ks), T0
   811			AESENC T0, B0
   812			MOVOU (16*10)(ks), T0
   813			CMPQ NR, $12
   814			JB encLast3
   815			AESENC T0, B0
   816			MOVOU (16*11)(ks), T0
   817			AESENC T0, B0
   818			MOVOU (16*12)(ks), T0
   819			JE encLast3
   820			AESENC T0, B0
   821			MOVOU (16*13)(ks), T0
   822			AESENC T0, B0
   823			MOVOU (16*14)(ks), T0
   824	encLast3:
   825			AESENCLAST T0, B0
   826	
   827			MOVOU (ptx), T0
   828			PXOR T0, B0
   829			MOVOU B0, (ctx)
   830	
   831			PSHUFB BSWAP, B0
   832			PXOR ACC0, B0
   833	
   834			MOVOU T2, ACC0
   835			MOVOU T2, ACC1
   836			MOVOU (16*15)(pTbl), ACCM
   837	
   838			PSHUFD $78, B0, T0
   839			PXOR B0, T0
   840			PCLMULQDQ $0x00, B0, ACC0
   841			PCLMULQDQ $0x11, B0, ACC1
   842			PCLMULQDQ $0x00, T0, ACCM
   843	
   844			PXOR ACC0, ACCM
   845			PXOR ACC1, ACCM
   846			MOVOU ACCM, T0
   847			PSRLDQ $8, ACCM
   848			PSLLDQ $8, T0
   849			PXOR ACCM, ACC1
   850			PXOR T0, ACC0
   851	
   852			reduceRound(ACC0)
   853			reduceRound(ACC0)
   854			PXOR ACC1, ACC0
   855	
   856			LEAQ (16*1)(ptx), ptx
   857			LEAQ (16*1)(ctx), ctx
   858	
   859		JMP gcmAesEncSinglesLoop
   860	
   861	gcmAesEncTail:
   862		TESTQ ptxLen, ptxLen
   863		JE gcmAesEncDone
   864	
   865		MOVOU (8*16 + 0*16)(SP), B0
   866		AESENC B1, B0
   867		AESENC B2, B0
   868		AESENC B3, B0
   869		AESENC B4, B0
   870		AESENC B5, B0
   871		AESENC B6, B0
   872		AESENC B7, B0
   873		MOVOU (16*8)(ks), T0
   874		AESENC T0, B0
   875		MOVOU (16*9)(ks), T0
   876		AESENC T0, B0
   877		MOVOU (16*10)(ks), T0
   878		CMPQ NR, $12
   879		JB encLast4
   880		AESENC T0, B0
   881		MOVOU (16*11)(ks), T0
   882		AESENC T0, B0
   883		MOVOU (16*12)(ks), T0
   884		JE encLast4
   885		AESENC T0, B0
   886		MOVOU (16*13)(ks), T0
   887		AESENC T0, B0
   888		MOVOU (16*14)(ks), T0
   889	encLast4:
   890		AESENCLAST T0, B0
   891		MOVOU B0, T0
   892	
   893		LEAQ -1(ptx)(ptxLen*1), ptx
   894	
   895		MOVQ ptxLen, aluTMP
   896		SHLQ $4, aluTMP
   897	
   898		LEAQ andMask<>(SB), aluCTR
   899		MOVOU -16(aluCTR)(aluTMP*1), T1
   900	
   901		PXOR B0, B0
   902	ptxLoadLoop:
   903			PSLLDQ $1, B0
   904			PINSRB $0, (ptx), B0
   905			LEAQ -1(ptx), ptx
   906			DECQ ptxLen
   907		JNE ptxLoadLoop
   908	
   909		PXOR T0, B0
   910		PAND T1, B0
   911		MOVOU B0, (ctx)	// I assume there is always space, due to TAG in the end of the CT
   912	
   913		PSHUFB BSWAP, B0
   914		PXOR ACC0, B0
   915	
   916		MOVOU T2, ACC0
   917		MOVOU T2, ACC1
   918		MOVOU (16*15)(pTbl), ACCM
   919	
   920		PSHUFD $78, B0, T0
   921		PXOR B0, T0
   922		PCLMULQDQ $0x00, B0, ACC0
   923		PCLMULQDQ $0x11, B0, ACC1
   924		PCLMULQDQ $0x00, T0, ACCM
   925	
   926		PXOR ACC0, ACCM
   927		PXOR ACC1, ACCM
   928		MOVOU ACCM, T0
   929		PSRLDQ $8, ACCM
   930		PSLLDQ $8, T0
   931		PXOR ACCM, ACC1
   932		PXOR T0, ACC0
   933	
   934		reduceRound(ACC0)
   935		reduceRound(ACC0)
   936		PXOR ACC1, ACC0
   937	
   938	gcmAesEncDone:
   939		MOVOU ACC0, (tPtr)
   940		RET
   941	#undef increment
   942	
   943	// func gcmAesDec(productTable *[256]byte, dst, src []byte, ctr, T *[16]byte, ks []uint32)
   944	TEXT ·gcmAesDec(SB),0,$128-96
   945	#define increment(i) ADDL $1, aluCTR; MOVL aluCTR, aluTMP; XORL aluK, aluTMP; BSWAPL aluTMP; MOVL aluTMP, (3*4 + i*16)(SP)
   946	#define combinedDecRound(i) \
   947		MOVOU (16*i)(ks), T0;\
   948		AESENC T0, B0;\
   949		AESENC T0, B1;\
   950		AESENC T0, B2;\
   951		AESENC T0, B3;\
   952		MOVOU (16*(i*2))(pTbl), T1;\
   953		MOVOU T1, T2;\
   954		AESENC T0, B4;\
   955		AESENC T0, B5;\
   956		AESENC T0, B6;\
   957		AESENC T0, B7;\
   958		MOVOU (16*i)(ctx), T0;\
   959		PSHUFB BSWAP, T0;\
   960		PCLMULQDQ $0x00, T0, T1;\
   961		PXOR T1, ACC0;\
   962		PSHUFD $78, T0, T1;\
   963		PCLMULQDQ $0x11, T0, T2;\
   964		PXOR T1, T0;\
   965		PXOR T2, ACC1;\
   966		MOVOU (16*(i*2+1))(pTbl), T2;\
   967		PCLMULQDQ $0x00, T2, T0;\
   968		PXOR T0, ACCM
   969	
   970		MOVQ productTable+0(FP), pTbl
   971		MOVQ dst+8(FP), ptx
   972		MOVQ src_base+32(FP), ctx
   973		MOVQ src_len+40(FP), ptxLen
   974		MOVQ ctr+56(FP), ctrPtr
   975		MOVQ T+64(FP), tPtr
   976		MOVQ ks_base+72(FP), ks
   977		MOVQ ks_len+80(FP), NR
   978	
   979		SHRQ $2, NR
   980		DECQ NR
   981	
   982		MOVOU bswapMask<>(SB), BSWAP
   983		MOVOU gcmPoly<>(SB), POLY
   984	
   985		MOVOU (tPtr), ACC0
   986		PXOR ACC1, ACC1
   987		PXOR ACCM, ACCM
   988		MOVOU (ctrPtr), B0
   989		MOVL (3*4)(ctrPtr), aluCTR
   990		MOVOU (ks), T0
   991		MOVL (3*4)(ks), aluK
   992		BSWAPL aluCTR
   993		BSWAPL aluK
   994	
   995		PXOR B0, T0
   996		MOVOU T0, (0*16)(SP)
   997		increment(0)
   998	
   999		CMPQ ptxLen, $128
  1000		JB gcmAesDecSingles
  1001	
  1002		MOVOU T0, (1*16)(SP)
  1003		increment(1)
  1004		MOVOU T0, (2*16)(SP)
  1005		increment(2)
  1006		MOVOU T0, (3*16)(SP)
  1007		increment(3)
  1008		MOVOU T0, (4*16)(SP)
  1009		increment(4)
  1010		MOVOU T0, (5*16)(SP)
  1011		increment(5)
  1012		MOVOU T0, (6*16)(SP)
  1013		increment(6)
  1014		MOVOU T0, (7*16)(SP)
  1015		increment(7)
  1016	
  1017	gcmAesDecOctetsLoop:
  1018	
  1019			CMPQ ptxLen, $128
  1020			JB gcmAesDecEndOctets
  1021			SUBQ $128, ptxLen
  1022	
  1023			MOVOU (0*16)(SP), B0
  1024			MOVOU (1*16)(SP), B1
  1025			MOVOU (2*16)(SP), B2
  1026			MOVOU (3*16)(SP), B3
  1027			MOVOU (4*16)(SP), B4
  1028			MOVOU (5*16)(SP), B5
  1029			MOVOU (6*16)(SP), B6
  1030			MOVOU (7*16)(SP), B7
  1031	
  1032			MOVOU (16*0)(ctx), T0
  1033			PSHUFB BSWAP, T0
  1034			PXOR ACC0, T0
  1035			PSHUFD $78, T0, T1
  1036			PXOR T0, T1
  1037	
  1038			MOVOU (16*0)(pTbl), ACC0
  1039			MOVOU (16*1)(pTbl), ACCM
  1040			MOVOU ACC0, ACC1
  1041	
  1042			PCLMULQDQ $0x00, T1, ACCM
  1043			PCLMULQDQ $0x00, T0, ACC0
  1044			PCLMULQDQ $0x11, T0, ACC1
  1045	
  1046			combinedDecRound(1)
  1047			increment(0)
  1048			combinedDecRound(2)
  1049			increment(1)
  1050			combinedDecRound(3)
  1051			increment(2)
  1052			combinedDecRound(4)
  1053			increment(3)
  1054			combinedDecRound(5)
  1055			increment(4)
  1056			combinedDecRound(6)
  1057			increment(5)
  1058			combinedDecRound(7)
  1059			increment(6)
  1060	
  1061			aesRound(8)
  1062			increment(7)
  1063	
  1064			PXOR ACC0, ACCM
  1065			PXOR ACC1, ACCM
  1066			MOVOU ACCM, T0
  1067			PSRLDQ $8, ACCM
  1068			PSLLDQ $8, T0
  1069			PXOR ACCM, ACC1
  1070			PXOR T0, ACC0
  1071	
  1072			reduceRound(ACC0)
  1073			aesRound(9)
  1074	
  1075			reduceRound(ACC0)
  1076			PXOR ACC1, ACC0
  1077	
  1078			MOVOU (16*10)(ks), T0
  1079			CMPQ NR, $12
  1080			JB decLast1
  1081			aesRnd(T0)
  1082			aesRound(11)
  1083			MOVOU (16*12)(ks), T0
  1084			JE decLast1
  1085			aesRnd(T0)
  1086			aesRound(13)
  1087			MOVOU (16*14)(ks), T0
  1088	decLast1:
  1089			aesRndLast(T0)
  1090	
  1091			MOVOU (16*0)(ctx), T0
  1092			PXOR T0, B0
  1093			MOVOU (16*1)(ctx), T0
  1094			PXOR T0, B1
  1095			MOVOU (16*2)(ctx), T0
  1096			PXOR T0, B2
  1097			MOVOU (16*3)(ctx), T0
  1098			PXOR T0, B3
  1099			MOVOU (16*4)(ctx), T0
  1100			PXOR T0, B4
  1101			MOVOU (16*5)(ctx), T0
  1102			PXOR T0, B5
  1103			MOVOU (16*6)(ctx), T0
  1104			PXOR T0, B6
  1105			MOVOU (16*7)(ctx), T0
  1106			PXOR T0, B7
  1107	
  1108			MOVOU B0, (16*0)(ptx)
  1109			MOVOU B1, (16*1)(ptx)
  1110			MOVOU B2, (16*2)(ptx)
  1111			MOVOU B3, (16*3)(ptx)
  1112			MOVOU B4, (16*4)(ptx)
  1113			MOVOU B5, (16*5)(ptx)
  1114			MOVOU B6, (16*6)(ptx)
  1115			MOVOU B7, (16*7)(ptx)
  1116	
  1117			LEAQ 128(ptx), ptx
  1118			LEAQ 128(ctx), ctx
  1119	
  1120			JMP gcmAesDecOctetsLoop
  1121	
  1122	gcmAesDecEndOctets:
  1123	
  1124		SUBQ $7, aluCTR
  1125	
  1126	gcmAesDecSingles:
  1127	
  1128		MOVOU (16*1)(ks), B1
  1129		MOVOU (16*2)(ks), B2
  1130		MOVOU (16*3)(ks), B3
  1131		MOVOU (16*4)(ks), B4
  1132		MOVOU (16*5)(ks), B5
  1133		MOVOU (16*6)(ks), B6
  1134		MOVOU (16*7)(ks), B7
  1135	
  1136		MOVOU (16*14)(pTbl), T2
  1137	
  1138	gcmAesDecSinglesLoop:
  1139	
  1140			CMPQ ptxLen, $16
  1141			JB gcmAesDecTail
  1142			SUBQ $16, ptxLen
  1143	
  1144			MOVOU (ctx), B0
  1145			MOVOU B0, T1
  1146			PSHUFB BSWAP, B0
  1147			PXOR ACC0, B0
  1148	
  1149			MOVOU T2, ACC0
  1150			MOVOU T2, ACC1
  1151			MOVOU (16*15)(pTbl), ACCM
  1152	
  1153			PCLMULQDQ $0x00, B0, ACC0
  1154			PCLMULQDQ $0x11, B0, ACC1
  1155			PSHUFD $78, B0, T0
  1156			PXOR B0, T0
  1157			PCLMULQDQ $0x00, T0, ACCM
  1158	
  1159			PXOR ACC0, ACCM
  1160			PXOR ACC1, ACCM
  1161			MOVOU ACCM, T0
  1162			PSRLDQ $8, ACCM
  1163			PSLLDQ $8, T0
  1164			PXOR ACCM, ACC1
  1165			PXOR T0, ACC0
  1166	
  1167			reduceRound(ACC0)
  1168			reduceRound(ACC0)
  1169			PXOR ACC1, ACC0
  1170	
  1171			MOVOU (0*16)(SP), B0
  1172			increment(0)
  1173			AESENC B1, B0
  1174			AESENC B2, B0
  1175			AESENC B3, B0
  1176			AESENC B4, B0
  1177			AESENC B5, B0
  1178			AESENC B6, B0
  1179			AESENC B7, B0
  1180			MOVOU (16*8)(ks), T0
  1181			AESENC T0, B0
  1182			MOVOU (16*9)(ks), T0
  1183			AESENC T0, B0
  1184			MOVOU (16*10)(ks), T0
  1185			CMPQ NR, $12
  1186			JB decLast2
  1187			AESENC T0, B0
  1188			MOVOU (16*11)(ks), T0
  1189			AESENC T0, B0
  1190			MOVOU (16*12)(ks), T0
  1191			JE decLast2
  1192			AESENC T0, B0
  1193			MOVOU (16*13)(ks), T0
  1194			AESENC T0, B0
  1195			MOVOU (16*14)(ks), T0
  1196	decLast2:
  1197			AESENCLAST T0, B0
  1198	
  1199			PXOR T1, B0
  1200			MOVOU B0, (ptx)
  1201	
  1202			LEAQ (16*1)(ptx), ptx
  1203			LEAQ (16*1)(ctx), ctx
  1204	
  1205		JMP gcmAesDecSinglesLoop
  1206	
  1207	gcmAesDecTail:
  1208	
  1209		TESTQ ptxLen, ptxLen
  1210		JE gcmAesDecDone
  1211	
  1212		MOVQ ptxLen, aluTMP
  1213		SHLQ $4, aluTMP
  1214		LEAQ andMask<>(SB), aluCTR
  1215		MOVOU -16(aluCTR)(aluTMP*1), T1
  1216	
  1217		MOVOU (ctx), B0	// I assume there is TAG attached to the ctx, and there is no read overflow
  1218		PAND T1, B0
  1219	
  1220		MOVOU B0, T1
  1221		PSHUFB BSWAP, B0
  1222		PXOR ACC0, B0
  1223	
  1224		MOVOU (16*14)(pTbl), ACC0
  1225		MOVOU (16*15)(pTbl), ACCM
  1226		MOVOU ACC0, ACC1
  1227	
  1228		PCLMULQDQ $0x00, B0, ACC0
  1229		PCLMULQDQ $0x11, B0, ACC1
  1230		PSHUFD $78, B0, T0
  1231		PXOR B0, T0
  1232		PCLMULQDQ $0x00, T0, ACCM
  1233	
  1234		PXOR ACC0, ACCM
  1235		PXOR ACC1, ACCM
  1236		MOVOU ACCM, T0
  1237		PSRLDQ $8, ACCM
  1238		PSLLDQ $8, T0
  1239		PXOR ACCM, ACC1
  1240		PXOR T0, ACC0
  1241	
  1242		reduceRound(ACC0)
  1243		reduceRound(ACC0)
  1244		PXOR ACC1, ACC0
  1245	
  1246		MOVOU (0*16)(SP), B0
  1247		increment(0)
  1248		AESENC B1, B0
  1249		AESENC B2, B0
  1250		AESENC B3, B0
  1251		AESENC B4, B0
  1252		AESENC B5, B0
  1253		AESENC B6, B0
  1254		AESENC B7, B0
  1255		MOVOU (16*8)(ks), T0
  1256		AESENC T0, B0
  1257		MOVOU (16*9)(ks), T0
  1258		AESENC T0, B0
  1259		MOVOU (16*10)(ks), T0
  1260		CMPQ NR, $12
  1261		JB decLast3
  1262		AESENC T0, B0
  1263		MOVOU (16*11)(ks), T0
  1264		AESENC T0, B0
  1265		MOVOU (16*12)(ks), T0
  1266		JE decLast3
  1267		AESENC T0, B0
  1268		MOVOU (16*13)(ks), T0
  1269		AESENC T0, B0
  1270		MOVOU (16*14)(ks), T0
  1271	decLast3:
  1272		AESENCLAST T0, B0
  1273		PXOR T1, B0
  1274	
  1275	ptxStoreLoop:
  1276			PEXTRB $0, B0, (ptx)
  1277			PSRLDQ $1, B0
  1278			LEAQ 1(ptx), ptx
  1279			DECQ ptxLen
  1280	
  1281		JNE ptxStoreLoop
  1282	
  1283	gcmAesDecDone:
  1284	
  1285		MOVOU ACC0, (tPtr)
  1286		RET

View as plain text