; /* poly1305_asm.asm */
; /*
; * Copyright (C) 2006-2026 wolfSSL Inc.
; *
; * This file is part of wolfSSL.
; *
; * wolfSSL is free software; you can redistribute it and/or modify
; * it under the terms of the GNU General Public License as published by
; * the Free Software Foundation; either version 3 of the License, or
; * (at your option) any later version.
; *
; * wolfSSL is distributed in the hope that it will be useful,
; * but WITHOUT ANY WARRANTY; without even the implied warranty of
; * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
; * GNU General Public License for more details.
; *
; * You should have received a copy of the GNU General Public License
; * along with this program; if not, write to the Free Software
; * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA
; */
IF @Version LT 1200
; AVX2 instructions not recognized by old versions of MASM
IFNDEF NO_AVX2_SUPPORT
NO_AVX2_SUPPORT = 1
ENDIF
; MOVBE instruction not recognized by old versions of MASM
IFNDEF NO_MOVBE_SUPPORT
NO_MOVBE_SUPPORT = 1
ENDIF
ENDIF
IFNDEF HAVE_INTEL_AVX1
HAVE_INTEL_AVX1 = 1
ENDIF
IFNDEF NO_AVX2_SUPPORT
HAVE_INTEL_AVX2 = 1
ENDIF
IFNDEF _WIN64
_WIN64 = 1
ENDIF
IFDEF HAVE_INTEL_AVX1
_text SEGMENT READONLY PARA
poly1305_setkey_avx PROC
push r12
push r13
mov r12, 1152921487695413247
mov r13, 1152921487695413244
mov rax, QWORD PTR [rdx]
mov r8, QWORD PTR [rdx+8]
mov r9, QWORD PTR [rdx+16]
mov r10, QWORD PTR [rdx+24]
and rax, r12
and r8, r13
mov r12, rax
mov r13, r8
xor r11, r11
mov QWORD PTR [rcx], rax
mov QWORD PTR [rcx+8], r8
mov QWORD PTR [rcx+24], r11
mov QWORD PTR [rcx+32], r11
mov QWORD PTR [rcx+40], r11
mov QWORD PTR [rcx+48], r9
mov QWORD PTR [rcx+56], r10
mov QWORD PTR [rcx+352], r11
mov QWORD PTR [rcx+408], r11
mov QWORD PTR [rcx+360], rax
mov QWORD PTR [rcx+416], r8
add r12, rax
add r13, r8
mov QWORD PTR [rcx+368], r12
mov QWORD PTR [rcx+424], r13
add r12, rax
add r13, r8
mov QWORD PTR [rcx+376], r12
mov QWORD PTR [rcx+432], r13
add r12, rax
add r13, r8
mov QWORD PTR [rcx+384], r12
mov QWORD PTR [rcx+440], r13
add r12, rax
add r13, r8
mov QWORD PTR [rcx+392], r12
mov QWORD PTR [rcx+448], r13
add r12, rax
add r13, r8
mov QWORD PTR [rcx+400], r12
mov QWORD PTR [rcx+456], r13
mov QWORD PTR [rcx+608], r11
mov BYTE PTR [rcx+616], 1
pop r13
pop r12
ret
poly1305_setkey_avx ENDP
_text ENDS
_text SEGMENT READONLY PARA
poly1305_block_avx PROC
push r15
push rbx
push r12
push r13
push r14
mov r15, QWORD PTR [rcx]
mov rbx, QWORD PTR [rcx+8]
mov r8, QWORD PTR [rcx+24]
mov r9, QWORD PTR [rcx+32]
mov r10, QWORD PTR [rcx+40]
xor r14, r14
mov r14b, BYTE PTR [rcx+616]
; h += m
mov r11, QWORD PTR [rdx]
mov r12, QWORD PTR [rdx+8]
add r8, r11
adc r9, r12
mov rax, rbx
adc r10, r14
; r[1] * h[0] => rdx, rax ==> t2, t1
mul r8
mov r12, rax
mov r13, rdx
; r[0] * h[1] => rdx, rax ++> t2, t1
mov rax, r15
mul r9
add r12, rax
mov rax, r15
adc r13, rdx
; r[0] * h[0] => rdx, rax ==> t4, t0
mul r8
mov r11, rax
mov r8, rdx
; r[1] * h[1] => rdx, rax =+> t3, t2
mov rax, rbx
mul r9
; r[0] * h[2] +> t2
add r13, QWORD PTR [rcx+8*r10+352]
mov r14, rdx
add r12, r8
adc r13, rax
; r[1] * h[2] +> t3
adc r14, QWORD PTR [rcx+8*r10+408]
; r * h in r14, r13, r12, r11
; h = (r * h) mod 2^130 - 5
mov r10, r13
and r13, -4
and r10, 3
add r11, r13
mov r8, r13
adc r12, r14
adc r10, 0
shrd r8, r14, 2
shr r14, 2
add r8, r11
adc r12, r14
mov r9, r12
adc r10, 0
; h in r10, r9, r8
; Store h to ctx
mov QWORD PTR [rcx+24], r8
mov QWORD PTR [rcx+32], r9
mov QWORD PTR [rcx+40], r10
pop r14
pop r13
pop r12
pop rbx
pop r15
ret
poly1305_block_avx ENDP
_text ENDS
_text SEGMENT READONLY PARA
poly1305_blocks_avx PROC
push rdi
push rsi
push r15
push rbx
push r12
push r13
push r14
mov rdi, rcx
mov rsi, rdx
mov rcx, r8
mov r15, QWORD PTR [rdi]
mov rbx, QWORD PTR [rdi+8]
mov r8, QWORD PTR [rdi+24]
mov r9, QWORD PTR [rdi+32]
mov r10, QWORD PTR [rdi+40]
L_poly1305_avx_blocks_start:
; h += m
mov r11, QWORD PTR [rsi]
mov r12, QWORD PTR [rsi+8]
add r8, r11
adc r9, r12
mov rax, rbx
adc r10, 0
; r[1] * h[0] => rdx, rax ==> t2, t1
mul r8
mov r12, rax
mov r13, rdx
; r[0] * h[1] => rdx, rax ++> t2, t1
mov rax, r15
mul r9
add r12, rax
mov rax, r15
adc r13, rdx
; r[0] * h[0] => rdx, rax ==> t4, t0
mul r8
mov r11, rax
mov r8, rdx
; r[1] * h[1] => rdx, rax =+> t3, t2
mov rax, rbx
mul r9
; r[0] * h[2] +> t2
add r13, QWORD PTR [rdi+8*r10+360]
mov r14, rdx
add r12, r8
adc r13, rax
; r[1] * h[2] +> t3
adc r14, QWORD PTR [rdi+8*r10+416]
; r * h in r14, r13, r12, r11
; h = (r * h) mod 2^130 - 5
mov r10, r13
and r13, -4
and r10, 3
add r11, r13
mov r8, r13
adc r12, r14
adc r10, 0
shrd r8, r14, 2
shr r14, 2
add r8, r11
adc r12, r14
mov r9, r12
adc r10, 0
; h in r10, r9, r8
; Next block from message
add rsi, 16
sub rcx, 16
jg L_poly1305_avx_blocks_start
; Store h to ctx
mov QWORD PTR [rdi+24], r8
mov QWORD PTR [rdi+32], r9
mov QWORD PTR [rdi+40], r10
pop r14
pop r13
pop r12
pop rbx
pop r15
pop rsi
pop rdi
ret
poly1305_blocks_avx ENDP
_text ENDS
_text SEGMENT READONLY PARA
poly1305_final_avx PROC
push rdi
push rbx
push r12
mov rdi, rcx
mov rbx, rdx
mov rax, QWORD PTR [rdi+608]
test rax, rax
je L_poly1305_avx_final_no_more
mov BYTE PTR [rdi+rax+480], 1
jmp L_poly1305_avx_final_cmp_rem
L_poly1305_avx_final_zero_rem:
mov BYTE PTR [rdi+rax+480], 0
L_poly1305_avx_final_cmp_rem:
inc al
cmp rax, 16
jl L_poly1305_avx_final_zero_rem
mov BYTE PTR [rdi+616], 0
lea rdx, QWORD PTR [rdi+480]
call poly1305_block_avx
L_poly1305_avx_final_no_more:
mov rax, QWORD PTR [rdi+24]
mov rdx, QWORD PTR [rdi+32]
mov rcx, QWORD PTR [rdi+40]
mov r11, QWORD PTR [rdi+48]
mov r12, QWORD PTR [rdi+56]
; h %= p
; h = (h + pad)
; mod 2^130 - 5
mov r8, rcx
and rcx, 3
shr r8, 2
; Multiply by 5
lea r8, QWORD PTR [r8+4*r8+0]
add rax, r8
adc rdx, 0
adc rcx, 0
; Fixup when between (1 << 130) - 1 and (1 << 130) - 5
mov r8, rax
mov r9, rdx
mov r10, rcx
add r8, 5
adc r9, 0
adc r10, 0
cmp r10, 4
cmove rax, r8
cmove rdx, r9
; h += pad
add rax, r11
adc rdx, r12
mov QWORD PTR [rbx], rax
mov QWORD PTR [rbx+8], rdx
; Zero out r
mov QWORD PTR [rdi], 0
mov QWORD PTR [rdi+8], 0
; Zero out h
mov QWORD PTR [rdi+24], 0
mov QWORD PTR [rdi+32], 0
mov QWORD PTR [rdi+40], 0
; Zero out pad
mov QWORD PTR [rdi+48], 0
mov QWORD PTR [rdi+56], 0
pop r12
pop rbx
pop rdi
ret
poly1305_final_avx ENDP
_text ENDS
ENDIF
IFDEF HAVE_INTEL_AVX2
_text SEGMENT READONLY PARA
poly1305_calc_powers_avx2 PROC
push r12
push r13
push r14
push r15
push rdi
push rsi
push rbx
push rbp
mov r8, QWORD PTR [rcx]
mov r9, QWORD PTR [rcx+8]
xor r10, r10
; Convert to 26 bits in 32
mov rax, r8
mov rdx, r8
mov rsi, r8
mov rbx, r9
mov rbp, r9
shr rdx, 26
shrd rsi, r9, 52
shr rbx, 14
shrd rbp, r10, 40
and rax, 67108863
and rdx, 67108863
and rsi, 67108863
and rbx, 67108863
and rbp, 67108863
mov DWORD PTR [rcx+224], eax
mov DWORD PTR [rcx+228], edx
mov DWORD PTR [rcx+232], esi
mov DWORD PTR [rcx+236], ebx
mov DWORD PTR [rcx+240], ebp
mov DWORD PTR [rcx+244], 0
; Square 128-bit
mov rax, r9
mul r8
xor r14, r14
mov r12, rax
mov r13, rdx
add r12, rax
adc r13, rdx
adc r14, 0
mov rax, r8
mul rax
mov r11, rax
mov rdi, rdx
mov rax, r9
mul rax
add r12, rdi
adc r13, rax
adc r14, rdx
; Reduce 256-bit to 130-bit
mov rax, r13
mov rdx, r14
and rax, -4
and r13, 3
add r11, rax
adc r12, rdx
adc r13, 0
shrd rax, rdx, 2
shr rdx, 2
add r11, rax
adc r12, rdx
adc r13, 0
mov rax, r13
shr rax, 2
lea rax, QWORD PTR [rax+4*rax+0]
and r13, 3
add r11, rax
adc r12, 0
adc r13, 0
; Convert to 26 bits in 32
mov rax, r11
mov rdx, r11
mov rsi, r11
mov rbx, r12
mov rbp, r12
shr rdx, 26
shrd rsi, r12, 52
shr rbx, 14
shrd rbp, r13, 40
and rax, 67108863
and rdx, 67108863
and rsi, 67108863
and rbx, 67108863
and rbp, 67108863
mov DWORD PTR [rcx+256], eax
mov DWORD PTR [rcx+260], edx
mov DWORD PTR [rcx+264], esi
mov DWORD PTR [rcx+268], ebx
mov DWORD PTR [rcx+272], ebp
mov DWORD PTR [rcx+276], 0
; Multiply 128-bit by 130-bit
; r1[0] * r2[0]
mov rax, r8
mul r11
mov r14, rax
mov r15, rdx
; r1[0] * r2[1]
mov rax, r8
mul r12
mov rdi, 0
add r15, rax
adc rdi, rdx
; r1[1] * r2[0]
mov rax, r9
mul r11
mov rsi, 0
add r15, rax
adc rdi, rdx
adc rsi, 0
; r1[0] * r2[2]
mov rax, r8
mul r13
add rdi, rax
adc rsi, rdx
; r1[1] * r2[1]
mov rax, r9
mul r12
mov rbx, 0
add rdi, rax
adc rsi, rdx
adc rbx, 0
; r1[1] * r2[2]
mov rax, r9
mul r13
add rsi, rax
adc rbx, rdx
; Reduce 260-bit to 130-bit
mov rax, rdi
mov rdx, rsi
mov rbx, rbx
and rax, -4
and rdi, 3
add r14, rax
adc r15, rdx
adc rdi, rbx
shrd rax, rdx, 2
shrd rdx, rbx, 2
shr rbx, 2
add r14, rax
adc r15, rdx
adc rdi, rbx
mov rax, rdi
and rdi, 3
shr rax, 2
lea rax, QWORD PTR [rax+4*rax+0]
add r14, rax
adc r15, 0
adc rdi, 0
; Convert to 26 bits in 32
mov rax, r14
mov rdx, r14
mov rsi, r14
mov rbx, r15
mov rbp, r15
shr rdx, 26
shrd rsi, r15, 52
shr rbx, 14
shrd rbp, rdi, 40
and rax, 67108863
and rdx, 67108863
and rsi, 67108863
and rbx, 67108863
and rbp, 67108863
mov DWORD PTR [rcx+288], eax
mov DWORD PTR [rcx+292], edx
mov DWORD PTR [rcx+296], esi
mov DWORD PTR [rcx+300], ebx
mov DWORD PTR [rcx+304], ebp
mov DWORD PTR [rcx+308], 0
; Square 130-bit
mov rax, r12
mul r11
xor r14, r14
mov r9, rax
mov r10, rdx
add r9, rax
adc r10, rdx
adc r14, 0
mov rax, r11
mul rax
mov r8, rax
mov rdi, rdx
mov rax, r12
mul rax
add r9, rdi
adc r10, rax
adc r14, rdx
mov rax, r13
mul rax
mov r15, rax
mov rax, r13
mul r11
add r10, rax
adc r14, rdx
adc r15, 0
add r10, rax
adc r14, rdx
adc r15, 0
mov rax, r13
mul r12
add r14, rax
adc r15, rdx
add r14, rax
adc r15, rdx
; Reduce 260-bit to 130-bit
mov rax, r10
mov rdx, r14
mov rdi, r15
and rax, -4
and r10, 3
add r8, rax
adc r9, rdx
adc r10, rdi
shrd rax, rdx, 2
shrd rdx, rdi, 2
shr rdi, 2
add r8, rax
adc r9, rdx
adc r10, rdi
mov rax, r10
and r10, 3
shr rax, 2
lea rax, QWORD PTR [rax+4*rax+0]
add r8, rax
adc r9, 0
adc r10, 0
; Convert to 26 bits in 32
mov rax, r8
mov rdx, r8
mov rsi, r8
mov rbx, r9
mov rbp, r9
shr rdx, 26
shrd rsi, r9, 52
shr rbx, 14
shrd rbp, r10, 40
and rax, 67108863
and rdx, 67108863
and rsi, 67108863
and rbx, 67108863
and rbp, 67108863
mov DWORD PTR [rcx+320], eax
mov DWORD PTR [rcx+324], edx
mov DWORD PTR [rcx+328], esi
mov DWORD PTR [rcx+332], ebx
mov DWORD PTR [rcx+336], ebp
mov DWORD PTR [rcx+340], 0
pop rbp
pop rbx
pop rsi
pop rdi
pop r15
pop r14
pop r13
pop r12
ret
poly1305_calc_powers_avx2 ENDP
_text ENDS
_text SEGMENT READONLY PARA
poly1305_setkey_avx2 PROC
call poly1305_setkey_avx
vpxor ymm0, ymm0, ymm0
vmovdqu YMMWORD PTR [rcx+64], ymm0
vmovdqu YMMWORD PTR [rcx+96], ymm0
vmovdqu YMMWORD PTR [rcx+128], ymm0
vmovdqu YMMWORD PTR [rcx+160], ymm0
vmovdqu YMMWORD PTR [rcx+192], ymm0
mov QWORD PTR [rcx+608], 0
mov WORD PTR [rcx+616], 0
ret
poly1305_setkey_avx2 ENDP
_text ENDS
_DATA SEGMENT
ALIGN 16
L_poly1305_avx2_blocks_mask QWORD 67108863, 67108863,
67108863, 67108863
ptr_L_poly1305_avx2_blocks_mask QWORD L_poly1305_avx2_blocks_mask
_DATA ENDS
_DATA SEGMENT
ALIGN 16
L_poly1305_avx2_blocks_hibit QWORD 16777216, 16777216,
16777216, 16777216
ptr_L_poly1305_avx2_blocks_hibit QWORD L_poly1305_avx2_blocks_hibit
_DATA ENDS
_text SEGMENT READONLY PARA
poly1305_blocks_avx2 PROC
push r12
push rdi
push rsi
push rbx
push r13
push r14
mov rdi, rcx
mov rsi, rdx
mov rdx, r8
sub rsp, 480
vmovdqu OWORD PTR [rsp+320], xmm6
vmovdqu OWORD PTR [rsp+336], xmm7
vmovdqu OWORD PTR [rsp+352], xmm8
vmovdqu OWORD PTR [rsp+368], xmm9
vmovdqu OWORD PTR [rsp+384], xmm10
vmovdqu OWORD PTR [rsp+400], xmm11
vmovdqu OWORD PTR [rsp+416], xmm12
vmovdqu OWORD PTR [rsp+432], xmm13
vmovdqu OWORD PTR [rsp+448], xmm14
vmovdqu OWORD PTR [rsp+464], xmm15
mov r13, QWORD PTR [ptr_L_poly1305_avx2_blocks_mask]
mov r14, QWORD PTR [ptr_L_poly1305_avx2_blocks_hibit]
mov rcx, rsp
and rcx, -32
add rcx, 32
vpxor ymm15, ymm15, ymm15
mov rbx, rcx
lea rax, QWORD PTR [rdi+64]
add rbx, 160
cmp WORD PTR [rdi+616], 0
jne L_poly1305_avx2_blocks_begin_h
; Load the message data
vmovdqu ymm0, YMMWORD PTR [rsi]
vmovdqu ymm1, YMMWORD PTR [rsi+32]
vperm2i128 ymm2, ymm0, ymm1, 32
vperm2i128 ymm0, ymm0, ymm1, 49
vpunpckldq ymm1, ymm2, ymm0
vpunpckhdq ymm3, ymm2, ymm0
vpunpckldq ymm0, ymm1, ymm15
vpunpckhdq ymm1, ymm1, ymm15
vpunpckldq ymm2, ymm3, ymm15
vpunpckhdq ymm3, ymm3, ymm15
vmovdqu ymm4, YMMWORD PTR [r14]
vpsllq ymm1, ymm1, 6
vpsllq ymm2, ymm2, 12
vpsllq ymm3, ymm3, 18
vmovdqu ymm14, YMMWORD PTR [r13]
; Reduce, in place, the message data
vpsrlq ymm10, ymm0, 26
vpsrlq ymm11, ymm3, 26
vpand ymm0, ymm0, ymm14
vpand ymm3, ymm3, ymm14
vpaddq ymm1, ymm10, ymm1
vpaddq ymm4, ymm11, ymm4
vpsrlq ymm10, ymm1, 26
vpsrlq ymm11, ymm4, 26
vpand ymm1, ymm1, ymm14
vpand ymm4, ymm4, ymm14
vpaddq ymm2, ymm10, ymm2
vpslld ymm12, ymm11, 2
vpaddd ymm12, ymm11, ymm12
vpsrlq ymm10, ymm2, 26
vpaddq ymm0, ymm12, ymm0
vpsrlq ymm11, ymm0, 26
vpand ymm2, ymm2, ymm14
vpand ymm0, ymm0, ymm14
vpaddq ymm3, ymm10, ymm3
vpaddq ymm1, ymm11, ymm1
vpsrlq ymm10, ymm3, 26
vpand ymm3, ymm3, ymm14
vpaddq ymm4, ymm10, ymm4
add rsi, 64
sub rdx, 64
jz L_poly1305_avx2_blocks_store
jmp L_poly1305_avx2_blocks_load_r4
L_poly1305_avx2_blocks_begin_h:
; Load the H values.
vmovdqu ymm0, YMMWORD PTR [rax]
vmovdqu ymm1, YMMWORD PTR [rax+32]
vmovdqu ymm2, YMMWORD PTR [rax+64]
vmovdqu ymm3, YMMWORD PTR [rax+96]
vmovdqu ymm4, YMMWORD PTR [rax+128]
; Check if there is a power of r to load - otherwise use r^4.
cmp BYTE PTR [rdi+616], 0
je L_poly1305_avx2_blocks_load_r4
; Load the 4 powers of r - r^4, r^3, r^2, r^1.
vmovdqu ymm8, YMMWORD PTR [rdi+224]
vmovdqu ymm7, YMMWORD PTR [rdi+256]
vmovdqu ymm6, YMMWORD PTR [rdi+288]
vmovdqu ymm5, YMMWORD PTR [rdi+320]
vpermq ymm5, ymm5, 216
vpermq ymm6, ymm6, 216
vpermq ymm7, ymm7, 216
vpermq ymm8, ymm8, 216
vpunpcklqdq ymm10, ymm5, ymm6
vpunpckhqdq ymm11, ymm5, ymm6
vpunpcklqdq ymm12, ymm7, ymm8
vpunpckhqdq ymm13, ymm7, ymm8
vperm2i128 ymm5, ymm10, ymm12, 32
vperm2i128 ymm7, ymm10, ymm12, 49
vperm2i128 ymm9, ymm11, ymm13, 32
vpsrlq ymm6, ymm5, 32
vpsrlq ymm8, ymm7, 32
jmp L_poly1305_avx2_blocks_mul_5
L_poly1305_avx2_blocks_load_r4:
; Load r^4 into all four positions.
vmovdqu ymm13, YMMWORD PTR [rdi+320]
vpermq ymm5, ymm13, 0
vpsrlq ymm14, ymm13, 32
vpermq ymm7, ymm13, 85
vpermq ymm9, ymm13, 170
vpermq ymm6, ymm14, 0
vpermq ymm8, ymm14, 85
L_poly1305_avx2_blocks_mul_5:
; Multiply top 4 26-bit values of all four H by 5
vpslld ymm10, ymm6, 2
vpslld ymm11, ymm7, 2
vpslld ymm12, ymm8, 2
vpslld ymm13, ymm9, 2
vpaddq ymm10, ymm6, ymm10
vpaddq ymm11, ymm7, ymm11
vpaddq ymm12, ymm8, ymm12
vpaddq ymm13, ymm9, ymm13
; Store powers of r and multiple of 5 for use in multiply.
vmovdqa YMMWORD PTR [rbx], ymm10
vmovdqa YMMWORD PTR [rbx+32], ymm11
vmovdqa YMMWORD PTR [rbx+64], ymm12
vmovdqa YMMWORD PTR [rbx+96], ymm13
vmovdqa YMMWORD PTR [rcx], ymm5
vmovdqa YMMWORD PTR [rcx+32], ymm6
vmovdqa YMMWORD PTR [rcx+64], ymm7
vmovdqa YMMWORD PTR [rcx+96], ymm8
vmovdqa YMMWORD PTR [rcx+128], ymm9
vmovdqu ymm14, YMMWORD PTR [r13]
; If not finished then loop over data
cmp BYTE PTR [rdi+616], 1
jne L_poly1305_avx2_blocks_start
; Do last multiply, reduce, add the four H together and move to
; 32-bit registers
vpmuludq ymm5, ymm4, [rbx]
vpmuludq ymm10, ymm3, [rbx+32]
vpmuludq ymm6, ymm4, [rbx+32]
vpmuludq ymm11, ymm3, [rbx+64]
vpmuludq ymm7, ymm4, [rbx+64]
vpaddq ymm5, ymm10, ymm5
vpmuludq ymm12, ymm2, [rbx+64]
vpmuludq ymm8, ymm4, [rbx+96]
vpaddq ymm6, ymm11, ymm6
vpmuludq ymm13, ymm1, [rbx+96]
vpmuludq ymm10, ymm2, [rbx+96]
vpaddq ymm5, ymm12, ymm5
vpmuludq ymm11, ymm3, [rbx+96]
vpmuludq ymm12, ymm3, [rcx]
vpaddq ymm5, ymm13, ymm5
vpmuludq ymm9, ymm4, [rcx]
vpaddq ymm6, ymm10, ymm6
vpmuludq ymm13, ymm0, [rcx]
vpaddq ymm7, ymm11, ymm7
vpmuludq ymm10, ymm1, [rcx]
vpaddq ymm8, ymm12, ymm8
vpmuludq ymm11, ymm2, [rcx]
vpmuludq ymm12, ymm2, [rcx+32]
vpaddq ymm5, ymm13, ymm5
vpmuludq ymm13, ymm3, [rcx+32]
vpaddq ymm6, ymm10, ymm6
vpmuludq ymm10, ymm0, [rcx+32]
vpaddq ymm7, ymm11, ymm7
vpmuludq ymm11, ymm1, [rcx+32]
vpaddq ymm8, ymm12, ymm8
vpmuludq ymm12, ymm1, [rcx+64]
vpaddq ymm9, ymm13, ymm9
vpmuludq ymm13, ymm2, [rcx+64]
vpaddq ymm6, ymm10, ymm6
vpmuludq ymm10, ymm0, [rcx+64]
vpaddq ymm7, ymm11, ymm7
vpmuludq ymm11, ymm0, [rcx+96]
vpaddq ymm8, ymm12, ymm8
vpmuludq ymm12, ymm1, [rcx+96]
vpaddq ymm9, ymm13, ymm9
vpaddq ymm7, ymm10, ymm7
vpmuludq ymm13, ymm0, [rcx+128]
vpaddq ymm8, ymm11, ymm8
vpaddq ymm9, ymm12, ymm9
vpaddq ymm9, ymm13, ymm9
vpsrlq ymm10, ymm5, 26
vpsrlq ymm11, ymm8, 26
vpand ymm5, ymm5, ymm14
vpand ymm8, ymm8, ymm14
vpaddq ymm6, ymm10, ymm6
vpaddq ymm9, ymm11, ymm9
vpsrlq ymm10, ymm6, 26
vpsrlq ymm11, ymm9, 26
vpand ymm1, ymm6, ymm14
vpand ymm4, ymm9, ymm14
vpaddq ymm7, ymm10, ymm7
vpslld ymm12, ymm11, 2
vpaddd ymm12, ymm11, ymm12
vpsrlq ymm10, ymm7, 26
vpaddq ymm5, ymm12, ymm5
vpsrlq ymm11, ymm5, 26
vpand ymm2, ymm7, ymm14
vpand ymm0, ymm5, ymm14
vpaddq ymm8, ymm10, ymm8
vpaddq ymm1, ymm11, ymm1
vpsrlq ymm10, ymm8, 26
vpand ymm3, ymm8, ymm14
vpaddq ymm4, ymm10, ymm4
vpsrldq ymm5, ymm0, 8
vpsrldq ymm6, ymm1, 8
vpsrldq ymm7, ymm2, 8
vpsrldq ymm8, ymm3, 8
vpsrldq ymm9, ymm4, 8
vpaddq ymm0, ymm5, ymm0
vpaddq ymm1, ymm6, ymm1
vpaddq ymm2, ymm7, ymm2
vpaddq ymm3, ymm8, ymm3
vpaddq ymm4, ymm9, ymm4
vpermq ymm5, ymm0, 2
vpermq ymm6, ymm1, 2
vpermq ymm7, ymm2, 2
vpermq ymm8, ymm3, 2
vpermq ymm9, ymm4, 2
vpaddq ymm0, ymm5, ymm0
vpaddq ymm1, ymm6, ymm1
vpaddq ymm2, ymm7, ymm2
vpaddq ymm3, ymm8, ymm3
vpaddq ymm4, ymm9, ymm4
vmovd r8d, xmm0
vmovd r9d, xmm1
vmovd r10d, xmm2
vmovd r11d, xmm3
vmovd r12d, xmm4
jmp L_poly1305_avx2_blocks_end_calc
L_poly1305_avx2_blocks_start:
vmovdqu ymm5, YMMWORD PTR [rsi]
vmovdqu ymm6, YMMWORD PTR [rsi+32]
vperm2i128 ymm7, ymm5, ymm6, 32
vperm2i128 ymm5, ymm5, ymm6, 49
vpunpckldq ymm6, ymm7, ymm5
vpunpckhdq ymm8, ymm7, ymm5
vpunpckldq ymm5, ymm6, ymm15
vpunpckhdq ymm6, ymm6, ymm15
vpunpckldq ymm7, ymm8, ymm15
vpunpckhdq ymm8, ymm8, ymm15
vmovdqu ymm9, YMMWORD PTR [r14]
vpsllq ymm6, ymm6, 6
vpsllq ymm7, ymm7, 12
vpsllq ymm8, ymm8, 18
vpmuludq ymm10, ymm4, [rbx]
vpaddq ymm5, ymm10, ymm5
vpmuludq ymm10, ymm3, [rbx+32]
vpmuludq ymm11, ymm4, [rbx+32]
vpaddq ymm6, ymm11, ymm6
vpmuludq ymm11, ymm3, [rbx+64]
vpmuludq ymm12, ymm4, [rbx+64]
vpaddq ymm7, ymm12, ymm7
vpaddq ymm5, ymm10, ymm5
vpmuludq ymm12, ymm2, [rbx+64]
vpmuludq ymm13, ymm4, [rbx+96]
vpaddq ymm8, ymm13, ymm8
vpaddq ymm6, ymm11, ymm6
vpmuludq ymm13, ymm1, [rbx+96]
vpmuludq ymm10, ymm2, [rbx+96]
vpaddq ymm5, ymm12, ymm5
vpmuludq ymm11, ymm3, [rbx+96]
vpmuludq ymm12, ymm3, [rcx]
vpaddq ymm5, ymm13, ymm5
vpmuludq ymm13, ymm4, [rcx]
vpaddq ymm9, ymm13, ymm9
vpaddq ymm6, ymm10, ymm6
vpmuludq ymm13, ymm0, [rcx]
vpaddq ymm7, ymm11, ymm7
vpmuludq ymm10, ymm1, [rcx]
vpaddq ymm8, ymm12, ymm8
vpmuludq ymm11, ymm2, [rcx]
vpmuludq ymm12, ymm2, [rcx+32]
vpaddq ymm5, ymm13, ymm5
vpmuludq ymm13, ymm3, [rcx+32]
vpaddq ymm6, ymm10, ymm6
vpmuludq ymm10, ymm0, [rcx+32]
vpaddq ymm7, ymm11, ymm7
vpmuludq ymm11, ymm1, [rcx+32]
vpaddq ymm8, ymm12, ymm8
vpmuludq ymm12, ymm1, [rcx+64]
vpaddq ymm9, ymm13, ymm9
vpmuludq ymm13, ymm2, [rcx+64]
vpaddq ymm6, ymm10, ymm6
vpmuludq ymm10, ymm0, [rcx+64]
vpaddq ymm7, ymm11, ymm7
vpmuludq ymm11, ymm0, [rcx+96]
vpaddq ymm8, ymm12, ymm8
vpmuludq ymm12, ymm1, [rcx+96]
vpaddq ymm9, ymm13, ymm9
vpaddq ymm7, ymm10, ymm7
vpmuludq ymm13, ymm0, [rcx+128]
vpaddq ymm8, ymm11, ymm8
vpaddq ymm9, ymm12, ymm9
vpaddq ymm9, ymm13, ymm9
vpsrlq ymm10, ymm5, 26
vpsrlq ymm11, ymm8, 26
vpand ymm5, ymm5, ymm14
vpand ymm8, ymm8, ymm14
vpaddq ymm6, ymm10, ymm6
vpaddq ymm9, ymm11, ymm9
vpsrlq ymm10, ymm6, 26
vpsrlq ymm11, ymm9, 26
vpand ymm1, ymm6, ymm14
vpand ymm4, ymm9, ymm14
vpaddq ymm7, ymm10, ymm7
vpslld ymm12, ymm11, 2
vpaddd ymm12, ymm11, ymm12
vpsrlq ymm10, ymm7, 26
vpaddq ymm5, ymm12, ymm5
vpsrlq ymm11, ymm5, 26
vpand ymm2, ymm7, ymm14
vpand ymm0, ymm5, ymm14
vpaddq ymm8, ymm10, ymm8
vpaddq ymm1, ymm11, ymm1
vpsrlq ymm10, ymm8, 26
vpand ymm3, ymm8, ymm14
vpaddq ymm4, ymm10, ymm4
add rsi, 64
sub rdx, 64
jnz L_poly1305_avx2_blocks_start
L_poly1305_avx2_blocks_store:
; Store four H values - state
vmovdqu YMMWORD PTR [rax], ymm0
vmovdqu YMMWORD PTR [rax+32], ymm1
vmovdqu YMMWORD PTR [rax+64], ymm2
vmovdqu YMMWORD PTR [rax+96], ymm3
vmovdqu YMMWORD PTR [rax+128], ymm4
L_poly1305_avx2_blocks_end_calc:
cmp BYTE PTR [rdi+616], 0
je L_poly1305_avx2_blocks_complete
mov rax, r8
mov rdx, r10
mov rcx, r12
shr rdx, 12
shr rcx, 24
shl r9, 26
shl r10, 52
shl r11, 14
shl r12, 40
add rax, r9
adc rax, r10
adc rdx, r11
adc rdx, r12
adc rcx, 0
mov r8, rcx
and rcx, 3
shr r8, 2
lea r8, QWORD PTR [r8+4*r8+0]
add rax, r8
adc rdx, 0
adc rcx, 0
mov QWORD PTR [rdi+24], rax
mov QWORD PTR [rdi+32], rdx
mov QWORD PTR [rdi+40], rcx
L_poly1305_avx2_blocks_complete:
mov BYTE PTR [rdi+617], 1
vzeroupper
vmovdqu xmm6, OWORD PTR [rsp+320]
vmovdqu xmm7, OWORD PTR [rsp+336]
vmovdqu xmm8, OWORD PTR [rsp+352]
vmovdqu xmm9, OWORD PTR [rsp+368]
vmovdqu xmm10, OWORD PTR [rsp+384]
vmovdqu xmm11, OWORD PTR [rsp+400]
vmovdqu xmm12, OWORD PTR [rsp+416]
vmovdqu xmm13, OWORD PTR [rsp+432]
vmovdqu xmm14, OWORD PTR [rsp+448]
vmovdqu xmm15, OWORD PTR [rsp+464]
add rsp, 480
pop r14
pop r13
pop rbx
pop rsi
pop rdi
pop r12
ret
poly1305_blocks_avx2 ENDP
_text ENDS
_text SEGMENT READONLY PARA
poly1305_final_avx2 PROC
push rdi
push rsi
mov rdi, rcx
mov rsi, rdx
mov BYTE PTR [rdi+616], 1
mov cl, BYTE PTR [rdi+617]
cmp cl, 0
je L_poly1305_avx2_final_done_blocks_X4
push rsi
mov r8, 64
xor rdx, rdx
mov rcx, rdi
call poly1305_blocks_avx2
pop rsi
L_poly1305_avx2_final_done_blocks_X4:
mov rax, QWORD PTR [rdi+608]
mov rcx, rax
and rcx, -16
cmp cl, 0
je L_poly1305_avx2_final_done_blocks
push rcx
push rax
push rsi
mov r8, rcx
lea rdx, QWORD PTR [rdi+480]
mov rcx, rdi
call poly1305_blocks_avx
pop rsi
pop rax
pop rcx
L_poly1305_avx2_final_done_blocks:
sub QWORD PTR [rdi+608], rcx
xor rdx, rdx
jmp L_poly1305_avx2_final_cmp_copy
L_poly1305_avx2_final_start_copy:
mov r8b, BYTE PTR [rdi+rcx+480]
mov BYTE PTR [rdi+rdx+480], r8b
inc cl
inc dl
L_poly1305_avx2_final_cmp_copy:
cmp al, cl
jne L_poly1305_avx2_final_start_copy
mov rcx, rdi
mov rdx, rsi
call poly1305_final_avx
vpxor ymm0, ymm0, ymm0
vmovdqu YMMWORD PTR [rdi+64], ymm0
vmovdqu YMMWORD PTR [rdi+96], ymm0
vmovdqu YMMWORD PTR [rdi+128], ymm0
vmovdqu YMMWORD PTR [rdi+160], ymm0
vmovdqu YMMWORD PTR [rdi+192], ymm0
vmovdqu YMMWORD PTR [rdi+224], ymm0
vmovdqu YMMWORD PTR [rdi+256], ymm0
vmovdqu YMMWORD PTR [rdi+288], ymm0
vmovdqu YMMWORD PTR [rdi+320], ymm0
mov QWORD PTR [rdi+608], 0
mov WORD PTR [rdi+616], 0
vzeroupper
pop rsi
pop rdi
ret
poly1305_final_avx2 ENDP
_text ENDS
ENDIF
END