; /* chacha_asm.asm */
; /*
; * Copyright (C) 2006-2026 wolfSSL Inc.
; *
; * This file is part of wolfSSL.
; *
; * wolfSSL is free software; you can redistribute it and/or modify
; * it under the terms of the GNU General Public License as published by
; * the Free Software Foundation; either version 3 of the License, or
; * (at your option) any later version.
; *
; * wolfSSL is distributed in the hope that it will be useful,
; * but WITHOUT ANY WARRANTY; without even the implied warranty of
; * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
; * GNU General Public License for more details.
; *
; * You should have received a copy of the GNU General Public License
; * along with this program; if not, write to the Free Software
; * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA
; */
IF @Version LT 1200
; AVX2 instructions not recognized by old versions of MASM
IFNDEF NO_AVX2_SUPPORT
NO_AVX2_SUPPORT = 1
ENDIF
; MOVBE instruction not recognized by old versions of MASM
IFNDEF NO_MOVBE_SUPPORT
NO_MOVBE_SUPPORT = 1
ENDIF
ENDIF
IFNDEF HAVE_INTEL_AVX1
HAVE_INTEL_AVX1 = 1
ENDIF
IFNDEF NO_AVX2_SUPPORT
HAVE_INTEL_AVX2 = 1
ENDIF
IFNDEF _WIN64
_WIN64 = 1
ENDIF
_text SEGMENT READONLY PARA
chacha_encrypt_x64 PROC
push rbx
push rbp
push r12
push r13
push r14
push r15
sub rsp, 64
cmp r9d, 64
jl L_chacha_x64_small
L_chacha_x64_start:
sub rsp, 48
mov QWORD PTR [rsp+24], r8
mov QWORD PTR [rsp+32], rdx
mov QWORD PTR [rsp+40], r9
mov rax, QWORD PTR [rcx+32]
mov rbx, QWORD PTR [rcx+40]
mov QWORD PTR [rsp+8], rax
mov QWORD PTR [rsp+16], rbx
mov eax, DWORD PTR [rcx]
mov ebx, DWORD PTR [rcx+4]
mov r9d, DWORD PTR [rcx+8]
mov r8d, DWORD PTR [rcx+12]
mov r8d, DWORD PTR [rcx+16]
mov r9d, DWORD PTR [rcx+20]
mov r10d, DWORD PTR [rcx+24]
mov r11d, DWORD PTR [rcx+28]
mov r12d, DWORD PTR [rcx+48]
mov r13d, DWORD PTR [rcx+52]
mov r14d, DWORD PTR [rcx+56]
mov r15d, DWORD PTR [rcx+60]
mov BYTE PTR [rsp], 10
mov edx, DWORD PTR [rsp+8]
mov ebp, DWORD PTR [rsp+12]
L_chacha_x64_block_crypt_start:
add eax, r8d
add ebx, r9d
xor r12d, eax
xor r13d, ebx
rol r12d, 16
rol r13d, 16
add edx, r12d
add ebp, r13d
xor r8d, edx
xor r9d, ebp
rol r8d, 12
rol r9d, 12
add eax, r8d
add ebx, r9d
xor r12d, eax
xor r13d, ebx
rol r12d, 8
rol r13d, 8
add edx, r12d
add ebp, r13d
xor r8d, edx
xor r9d, ebp
rol r8d, 7
rol r9d, 7
mov DWORD PTR [rsp+8], edx
mov DWORD PTR [rsp+12], ebp
mov edx, DWORD PTR [rsp+16]
mov ebp, DWORD PTR [rsp+20]
add r9d, r10d
add r8d, r11d
xor r14d, r9d
xor r15d, r8d
rol r14d, 16
rol r15d, 16
add edx, r14d
add ebp, r15d
xor r10d, edx
xor r11d, ebp
rol r10d, 12
rol r11d, 12
add r9d, r10d
add r8d, r11d
xor r14d, r9d
xor r15d, r8d
rol r14d, 8
rol r15d, 8
add edx, r14d
add ebp, r15d
xor r10d, edx
xor r11d, ebp
rol r10d, 7
rol r11d, 7
add eax, r9d
add ebx, r10d
xor r15d, eax
xor r12d, ebx
rol r15d, 16
rol r12d, 16
add edx, r15d
add ebp, r12d
xor r9d, edx
xor r10d, ebp
rol r9d, 12
rol r10d, 12
add eax, r9d
add ebx, r10d
xor r15d, eax
xor r12d, ebx
rol r15d, 8
rol r12d, 8
add edx, r15d
add ebp, r12d
xor r9d, edx
xor r10d, ebp
rol r9d, 7
rol r10d, 7
mov DWORD PTR [rsp+16], edx
mov DWORD PTR [rsp+20], ebp
mov edx, DWORD PTR [rsp+8]
mov ebp, DWORD PTR [rsp+12]
add r9d, r11d
add r8d, r8d
xor r13d, r9d
xor r14d, r8d
rol r13d, 16
rol r14d, 16
add edx, r13d
add ebp, r14d
xor r11d, edx
xor r8d, ebp
rol r11d, 12
rol r8d, 12
add r9d, r11d
add r8d, r8d
xor r13d, r9d
xor r14d, r8d
rol r13d, 8
rol r14d, 8
add edx, r13d
add ebp, r14d
xor r11d, edx
xor r8d, ebp
rol r11d, 7
rol r8d, 7
dec BYTE PTR [rsp]
jnz L_chacha_x64_block_crypt_start
mov DWORD PTR [rsp+8], edx
mov DWORD PTR [rsp+12], ebp
mov rdx, QWORD PTR [rsp+32]
mov rbp, QWORD PTR [rsp+24]
add eax, DWORD PTR [rcx]
add ebx, DWORD PTR [rcx+4]
add r9d, DWORD PTR [rcx+8]
add r8d, DWORD PTR [rcx+12]
add r8d, DWORD PTR [rcx+16]
add r9d, DWORD PTR [rcx+20]
add r10d, DWORD PTR [rcx+24]
add r11d, DWORD PTR [rcx+28]
add r12d, DWORD PTR [rcx+48]
add r13d, DWORD PTR [rcx+52]
add r14d, DWORD PTR [rcx+56]
add r15d, DWORD PTR [rcx+60]
xor eax, DWORD PTR [rdx]
xor ebx, DWORD PTR [rdx+4]
xor r9d, DWORD PTR [rdx+8]
xor r8d, DWORD PTR [rdx+12]
xor r8d, DWORD PTR [rdx+16]
xor r9d, DWORD PTR [rdx+20]
xor r10d, DWORD PTR [rdx+24]
xor r11d, DWORD PTR [rdx+28]
xor r12d, DWORD PTR [rdx+48]
xor r13d, DWORD PTR [rdx+52]
xor r14d, DWORD PTR [rdx+56]
xor r15d, DWORD PTR [rdx+60]
mov DWORD PTR [rbp], eax
mov DWORD PTR [rbp+4], ebx
mov DWORD PTR [rbp+8], r9d
mov DWORD PTR [rbp+12], r8d
mov DWORD PTR [rbp+16], r8d
mov DWORD PTR [rbp+20], r9d
mov DWORD PTR [rbp+24], r10d
mov DWORD PTR [rbp+28], r11d
mov DWORD PTR [rbp+48], r12d
mov DWORD PTR [rbp+52], r13d
mov DWORD PTR [rbp+56], r14d
mov DWORD PTR [rbp+60], r15d
mov eax, DWORD PTR [rsp+8]
mov ebx, DWORD PTR [rsp+12]
mov r9d, DWORD PTR [rsp+16]
mov r8d, DWORD PTR [rsp+20]
add eax, DWORD PTR [rcx+32]
add ebx, DWORD PTR [rcx+36]
add r9d, DWORD PTR [rcx+40]
add r8d, DWORD PTR [rcx+44]
xor eax, DWORD PTR [rdx+32]
xor ebx, DWORD PTR [rdx+36]
xor r9d, DWORD PTR [rdx+40]
xor r8d, DWORD PTR [rdx+44]
mov DWORD PTR [rbp+32], eax
mov DWORD PTR [rbp+36], ebx
mov DWORD PTR [rbp+40], r9d
mov DWORD PTR [rbp+44], r8d
mov r8, QWORD PTR [rsp+24]
mov r9, QWORD PTR [rsp+40]
add DWORD PTR [rcx+48], 1
add rsp, 48
sub r9d, 64
add rdx, 64
add r8, 64
cmp r9d, 64
jge L_chacha_x64_start
L_chacha_x64_small:
cmp r9d, 0
je L_chacha_x64_done
sub rsp, 48
mov QWORD PTR [rsp+24], r8
mov QWORD PTR [rsp+32], rdx
mov QWORD PTR [rsp+40], r9
mov rax, QWORD PTR [rcx+32]
mov rbx, QWORD PTR [rcx+40]
mov QWORD PTR [rsp+8], rax
mov QWORD PTR [rsp+16], rbx
mov eax, DWORD PTR [rcx]
mov ebx, DWORD PTR [rcx+4]
mov r9d, DWORD PTR [rcx+8]
mov r8d, DWORD PTR [rcx+12]
mov r8d, DWORD PTR [rcx+16]
mov r9d, DWORD PTR [rcx+20]
mov r10d, DWORD PTR [rcx+24]
mov r11d, DWORD PTR [rcx+28]
mov r12d, DWORD PTR [rcx+48]
mov r13d, DWORD PTR [rcx+52]
mov r14d, DWORD PTR [rcx+56]
mov r15d, DWORD PTR [rcx+60]
mov BYTE PTR [rsp], 10
mov edx, DWORD PTR [rsp+8]
mov ebp, DWORD PTR [rsp+12]
L_chacha_x64_partial_crypt_start:
add eax, r8d
add ebx, r9d
xor r12d, eax
xor r13d, ebx
rol r12d, 16
rol r13d, 16
add edx, r12d
add ebp, r13d
xor r8d, edx
xor r9d, ebp
rol r8d, 12
rol r9d, 12
add eax, r8d
add ebx, r9d
xor r12d, eax
xor r13d, ebx
rol r12d, 8
rol r13d, 8
add edx, r12d
add ebp, r13d
xor r8d, edx
xor r9d, ebp
rol r8d, 7
rol r9d, 7
mov DWORD PTR [rsp+8], edx
mov DWORD PTR [rsp+12], ebp
mov edx, DWORD PTR [rsp+16]
mov ebp, DWORD PTR [rsp+20]
add r9d, r10d
add r8d, r11d
xor r14d, r9d
xor r15d, r8d
rol r14d, 16
rol r15d, 16
add edx, r14d
add ebp, r15d
xor r10d, edx
xor r11d, ebp
rol r10d, 12
rol r11d, 12
add r9d, r10d
add r8d, r11d
xor r14d, r9d
xor r15d, r8d
rol r14d, 8
rol r15d, 8
add edx, r14d
add ebp, r15d
xor r10d, edx
xor r11d, ebp
rol r10d, 7
rol r11d, 7
add eax, r9d
add ebx, r10d
xor r15d, eax
xor r12d, ebx
rol r15d, 16
rol r12d, 16
add edx, r15d
add ebp, r12d
xor r9d, edx
xor r10d, ebp
rol r9d, 12
rol r10d, 12
add eax, r9d
add ebx, r10d
xor r15d, eax
xor r12d, ebx
rol r15d, 8
rol r12d, 8
add edx, r15d
add ebp, r12d
xor r9d, edx
xor r10d, ebp
rol r9d, 7
rol r10d, 7
mov DWORD PTR [rsp+16], edx
mov DWORD PTR [rsp+20], ebp
mov edx, DWORD PTR [rsp+8]
mov ebp, DWORD PTR [rsp+12]
add r9d, r11d
add r8d, r8d
xor r13d, r9d
xor r14d, r8d
rol r13d, 16
rol r14d, 16
add edx, r13d
add ebp, r14d
xor r11d, edx
xor r8d, ebp
rol r11d, 12
rol r8d, 12
add r9d, r11d
add r8d, r8d
xor r13d, r9d
xor r14d, r8d
rol r13d, 8
rol r14d, 8
add edx, r13d
add ebp, r14d
xor r11d, edx
xor r8d, ebp
rol r11d, 7
rol r8d, 7
dec BYTE PTR [rsp]
jnz L_chacha_x64_partial_crypt_start
mov DWORD PTR [rsp+8], edx
mov DWORD PTR [rsp+12], ebp
mov rdx, QWORD PTR [rsp+32]
add eax, DWORD PTR [rcx]
add ebx, DWORD PTR [rcx+4]
add r9d, DWORD PTR [rcx+8]
add r8d, DWORD PTR [rcx+12]
add r8d, DWORD PTR [rcx+16]
add r9d, DWORD PTR [rcx+20]
add r10d, DWORD PTR [rcx+24]
add r11d, DWORD PTR [rcx+28]
add r12d, DWORD PTR [rcx+48]
add r13d, DWORD PTR [rcx+52]
add r14d, DWORD PTR [rcx+56]
add r15d, DWORD PTR [rcx+60]
lea rbp, QWORD PTR [rcx+80]
mov DWORD PTR [rbp], eax
mov DWORD PTR [rbp+4], ebx
mov DWORD PTR [rbp+8], r9d
mov DWORD PTR [rbp+12], r8d
mov DWORD PTR [rbp+16], r8d
mov DWORD PTR [rbp+20], r9d
mov DWORD PTR [rbp+24], r10d
mov DWORD PTR [rbp+28], r11d
mov DWORD PTR [rbp+48], r12d
mov DWORD PTR [rbp+52], r13d
mov DWORD PTR [rbp+56], r14d
mov DWORD PTR [rbp+60], r15d
mov eax, DWORD PTR [rsp+8]
mov ebx, DWORD PTR [rsp+12]
mov r9d, DWORD PTR [rsp+16]
mov r8d, DWORD PTR [rsp+20]
add eax, DWORD PTR [rcx+32]
add ebx, DWORD PTR [rcx+36]
add r9d, DWORD PTR [rcx+40]
add r8d, DWORD PTR [rcx+44]
mov DWORD PTR [rbp+32], eax
mov DWORD PTR [rbp+36], ebx
mov DWORD PTR [rbp+40], r9d
mov DWORD PTR [rbp+44], r8d
mov r8, QWORD PTR [rsp+24]
mov r9, QWORD PTR [rsp+40]
add DWORD PTR [rcx+48], 1
add rsp, 48
mov r8d, r9d
xor rbx, rbx
and r8d, 7
jz L_chacha_x64_partial_start64
L_chacha_x64_partial_start8:
movzx eax, BYTE PTR [rbp+rbx]
xor al, BYTE PTR [rdx+rbx]
mov BYTE PTR [r8+rbx], al
inc ebx
cmp ebx, r8d
jne L_chacha_x64_partial_start8
je L_chacha_x64_partial_end64
L_chacha_x64_partial_start64:
mov rax, QWORD PTR [rbp+rbx]
xor rax, QWORD PTR [rdx+rbx]
mov QWORD PTR [r8+rbx], rax
add ebx, 8
L_chacha_x64_partial_end64:
cmp ebx, r9d
jne L_chacha_x64_partial_start64
mov r9d, 64
sub r9d, ebx
mov DWORD PTR [rcx+76], r9d
L_chacha_x64_done:
add rsp, 64
pop r15
pop r14
pop r13
pop r12
pop rbp
pop rbx
ret
chacha_encrypt_x64 ENDP
_text ENDS
IFDEF HAVE_INTEL_AVX1
_DATA SEGMENT
ALIGN 16
L_chacha20_avx1_rotl8 QWORD 433757367256023043, 1012478749960636427
ptr_L_chacha20_avx1_rotl8 QWORD L_chacha20_avx1_rotl8
_DATA ENDS
_DATA SEGMENT
ALIGN 16
L_chacha20_avx1_rotl16 QWORD 361421592464458498, 940142975169071882
ptr_L_chacha20_avx1_rotl16 QWORD L_chacha20_avx1_rotl16
_DATA ENDS
_DATA SEGMENT
ALIGN 16
L_chacha20_avx1_add QWORD 4294967296, 12884901890
ptr_L_chacha20_avx1_add QWORD L_chacha20_avx1_add
_DATA ENDS
_DATA SEGMENT
ALIGN 16
L_chacha20_avx1_four QWORD 17179869188, 17179869188
ptr_L_chacha20_avx1_four QWORD L_chacha20_avx1_four
_DATA ENDS
_text SEGMENT READONLY PARA
chacha_encrypt_avx1 PROC
push r12
push r13
push r14
push r15
push rdi
push rsi
sub rsp, 560
vmovdqu OWORD PTR [rsp+400], xmm6
vmovdqu OWORD PTR [rsp+416], xmm7
vmovdqu OWORD PTR [rsp+432], xmm8
vmovdqu OWORD PTR [rsp+448], xmm9
vmovdqu OWORD PTR [rsp+464], xmm10
vmovdqu OWORD PTR [rsp+480], xmm11
vmovdqu OWORD PTR [rsp+496], xmm12
vmovdqu OWORD PTR [rsp+512], xmm13
vmovdqu OWORD PTR [rsp+528], xmm14
vmovdqu OWORD PTR [rsp+544], xmm15
mov r11, rsp
lea r12, QWORD PTR [rsp+256]
mov r14, QWORD PTR [ptr_L_chacha20_avx1_rotl8]
mov r15, QWORD PTR [ptr_L_chacha20_avx1_rotl16]
mov rdi, QWORD PTR [ptr_L_chacha20_avx1_add]
mov rsi, QWORD PTR [ptr_L_chacha20_avx1_four]
add r11, 15
add r12, 15
and r11, -16
and r12, -16
mov eax, r9d
shr eax, 8
jz L_chacha20_avx1_end128
vpshufd xmm0, [rcx], 0
vpshufd xmm1, [rcx+4], 0
vpshufd xmm2, [rcx+8], 0
vpshufd xmm3, [rcx+12], 0
vpshufd xmm4, [rcx+16], 0
vpshufd xmm5, [rcx+20], 0
vpshufd xmm6, [rcx+24], 0
vpshufd xmm7, [rcx+28], 0
vpshufd xmm8, [rcx+32], 0
vpshufd xmm9, [rcx+36], 0
vpshufd xmm10, [rcx+40], 0
vpshufd xmm11, [rcx+44], 0
vpshufd xmm12, [rcx+48], 0
vpshufd xmm13, [rcx+52], 0
vpshufd xmm14, [rcx+56], 0
vpshufd xmm15, [rcx+60], 0
vpaddd xmm12, xmm12, OWORD PTR [rdi]
vmovdqa OWORD PTR [r11], xmm0
vmovdqa OWORD PTR [r11+16], xmm1
vmovdqa OWORD PTR [r11+32], xmm2
vmovdqa OWORD PTR [r11+48], xmm3
vmovdqa OWORD PTR [r11+64], xmm4
vmovdqa OWORD PTR [r11+80], xmm5
vmovdqa OWORD PTR [r11+96], xmm6
vmovdqa OWORD PTR [r11+112], xmm7
vmovdqa OWORD PTR [r11+128], xmm8
vmovdqa OWORD PTR [r11+144], xmm9
vmovdqa OWORD PTR [r11+160], xmm10
vmovdqa OWORD PTR [r11+176], xmm11
vmovdqa OWORD PTR [r11+192], xmm12
vmovdqa OWORD PTR [r11+208], xmm13
vmovdqa OWORD PTR [r11+224], xmm14
vmovdqa OWORD PTR [r11+240], xmm15
L_chacha20_avx1_start128:
vmovdqa OWORD PTR [r12+48], xmm11
mov r10b, 10
L_chacha20_avx1_loop128:
vpaddd xmm0, xmm0, xmm4
vpxor xmm12, xmm12, xmm0
vmovdqa xmm11, OWORD PTR [r12+48]
vpshufb xmm12, xmm12, OWORD PTR [r15]
vpaddd xmm8, xmm8, xmm12
vpxor xmm4, xmm4, xmm8
vpaddd xmm1, xmm1, xmm5
vpxor xmm13, xmm13, xmm1
vpshufb xmm13, xmm13, OWORD PTR [r15]
vpaddd xmm9, xmm9, xmm13
vpxor xmm5, xmm5, xmm9
vpaddd xmm2, xmm2, xmm6
vpxor xmm14, xmm14, xmm2
vpshufb xmm14, xmm14, OWORD PTR [r15]
vpaddd xmm10, xmm10, xmm14
vpxor xmm6, xmm6, xmm10
vpaddd xmm3, xmm3, xmm7
vpxor xmm15, xmm15, xmm3
vpshufb xmm15, xmm15, OWORD PTR [r15]
vpaddd xmm11, xmm11, xmm15
vpxor xmm7, xmm7, xmm11
vmovdqa OWORD PTR [r12+48], xmm11
vpsrld xmm11, xmm4, 20
vpslld xmm4, xmm4, 12
vpxor xmm4, xmm4, xmm11
vpsrld xmm11, xmm5, 20
vpslld xmm5, xmm5, 12
vpxor xmm5, xmm5, xmm11
vpsrld xmm11, xmm6, 20
vpslld xmm6, xmm6, 12
vpxor xmm6, xmm6, xmm11
vpsrld xmm11, xmm7, 20
vpslld xmm7, xmm7, 12
vpxor xmm7, xmm7, xmm11
vpaddd xmm0, xmm0, xmm4
vpxor xmm12, xmm12, xmm0
vmovdqa xmm11, OWORD PTR [r12+48]
vpshufb xmm12, xmm12, OWORD PTR [r14]
vpaddd xmm8, xmm8, xmm12
vpxor xmm4, xmm4, xmm8
vpaddd xmm1, xmm1, xmm5
vpxor xmm13, xmm13, xmm1
vpshufb xmm13, xmm13, OWORD PTR [r14]
vpaddd xmm9, xmm9, xmm13
vpxor xmm5, xmm5, xmm9
vpaddd xmm2, xmm2, xmm6
vpxor xmm14, xmm14, xmm2
vpshufb xmm14, xmm14, OWORD PTR [r14]
vpaddd xmm10, xmm10, xmm14
vpxor xmm6, xmm6, xmm10
vpaddd xmm3, xmm3, xmm7
vpxor xmm15, xmm15, xmm3
vpshufb xmm15, xmm15, OWORD PTR [r14]
vpaddd xmm11, xmm11, xmm15
vpxor xmm7, xmm7, xmm11
vmovdqa OWORD PTR [r12+48], xmm11
vpsrld xmm11, xmm4, 25
vpslld xmm4, xmm4, 7
vpxor xmm4, xmm4, xmm11
vpsrld xmm11, xmm5, 25
vpslld xmm5, xmm5, 7
vpxor xmm5, xmm5, xmm11
vpsrld xmm11, xmm6, 25
vpslld xmm6, xmm6, 7
vpxor xmm6, xmm6, xmm11
vpsrld xmm11, xmm7, 25
vpslld xmm7, xmm7, 7
vpxor xmm7, xmm7, xmm11
vpaddd xmm0, xmm0, xmm5
vpxor xmm15, xmm15, xmm0
vmovdqa xmm11, OWORD PTR [r12+48]
vpshufb xmm15, xmm15, OWORD PTR [r15]
vpaddd xmm10, xmm10, xmm15
vpxor xmm5, xmm5, xmm10
vpaddd xmm1, xmm1, xmm6
vpxor xmm12, xmm12, xmm1
vpshufb xmm12, xmm12, OWORD PTR [r15]
vpaddd xmm11, xmm11, xmm12
vpxor xmm6, xmm6, xmm11
vpaddd xmm2, xmm2, xmm7
vpxor xmm13, xmm13, xmm2
vpshufb xmm13, xmm13, OWORD PTR [r15]
vpaddd xmm8, xmm8, xmm13
vpxor xmm7, xmm7, xmm8
vpaddd xmm3, xmm3, xmm4
vpxor xmm14, xmm14, xmm3
vpshufb xmm14, xmm14, OWORD PTR [r15]
vpaddd xmm9, xmm9, xmm14
vpxor xmm4, xmm4, xmm9
vmovdqa OWORD PTR [r12+48], xmm11
vpsrld xmm11, xmm5, 20
vpslld xmm5, xmm5, 12
vpxor xmm5, xmm5, xmm11
vpsrld xmm11, xmm6, 20
vpslld xmm6, xmm6, 12
vpxor xmm6, xmm6, xmm11
vpsrld xmm11, xmm7, 20
vpslld xmm7, xmm7, 12
vpxor xmm7, xmm7, xmm11
vpsrld xmm11, xmm4, 20
vpslld xmm4, xmm4, 12
vpxor xmm4, xmm4, xmm11
vpaddd xmm0, xmm0, xmm5
vpxor xmm15, xmm15, xmm0
vmovdqa xmm11, OWORD PTR [r12+48]
vpshufb xmm15, xmm15, OWORD PTR [r14]
vpaddd xmm10, xmm10, xmm15
vpxor xmm5, xmm5, xmm10
vpaddd xmm1, xmm1, xmm6
vpxor xmm12, xmm12, xmm1
vpshufb xmm12, xmm12, OWORD PTR [r14]
vpaddd xmm11, xmm11, xmm12
vpxor xmm6, xmm6, xmm11
vpaddd xmm2, xmm2, xmm7
vpxor xmm13, xmm13, xmm2
vpshufb xmm13, xmm13, OWORD PTR [r14]
vpaddd xmm8, xmm8, xmm13
vpxor xmm7, xmm7, xmm8
vpaddd xmm3, xmm3, xmm4
vpxor xmm14, xmm14, xmm3
vpshufb xmm14, xmm14, OWORD PTR [r14]
vpaddd xmm9, xmm9, xmm14
vpxor xmm4, xmm4, xmm9
vmovdqa OWORD PTR [r12+48], xmm11
vpsrld xmm11, xmm5, 25
vpslld xmm5, xmm5, 7
vpxor xmm5, xmm5, xmm11
vpsrld xmm11, xmm6, 25
vpslld xmm6, xmm6, 7
vpxor xmm6, xmm6, xmm11
vpsrld xmm11, xmm7, 25
vpslld xmm7, xmm7, 7
vpxor xmm7, xmm7, xmm11
vpsrld xmm11, xmm4, 25
vpslld xmm4, xmm4, 7
vpxor xmm4, xmm4, xmm11
dec r10b
jnz L_chacha20_avx1_loop128
vmovdqa xmm11, OWORD PTR [r12+48]
vpaddd xmm0, xmm0, OWORD PTR [r11]
vpaddd xmm1, xmm1, OWORD PTR [r11+16]
vpaddd xmm2, xmm2, OWORD PTR [r11+32]
vpaddd xmm3, xmm3, OWORD PTR [r11+48]
vpaddd xmm4, xmm4, OWORD PTR [r11+64]
vpaddd xmm5, xmm5, OWORD PTR [r11+80]
vpaddd xmm6, xmm6, OWORD PTR [r11+96]
vpaddd xmm7, xmm7, OWORD PTR [r11+112]
vpaddd xmm8, xmm8, OWORD PTR [r11+128]
vpaddd xmm9, xmm9, OWORD PTR [r11+144]
vpaddd xmm10, xmm10, OWORD PTR [r11+160]
vpaddd xmm11, xmm11, OWORD PTR [r11+176]
vpaddd xmm12, xmm12, OWORD PTR [r11+192]
vpaddd xmm13, xmm13, OWORD PTR [r11+208]
vpaddd xmm14, xmm14, OWORD PTR [r11+224]
vpaddd xmm15, xmm15, OWORD PTR [r11+240]
vmovdqa OWORD PTR [r12], xmm8
vmovdqa OWORD PTR [r12+16], xmm9
vmovdqa OWORD PTR [r12+32], xmm10
vmovdqa OWORD PTR [r12+48], xmm11
vmovdqa OWORD PTR [r12+64], xmm12
vmovdqa OWORD PTR [r12+80], xmm13
vmovdqa OWORD PTR [r12+96], xmm14
vmovdqa OWORD PTR [r12+112], xmm15
vpunpckldq xmm8, xmm0, xmm1
vpunpckldq xmm9, xmm2, xmm3
vpunpckhdq xmm12, xmm0, xmm1
vpunpckhdq xmm13, xmm2, xmm3
vpunpckldq xmm10, xmm4, xmm5
vpunpckldq xmm11, xmm6, xmm7
vpunpckhdq xmm14, xmm4, xmm5
vpunpckhdq xmm15, xmm6, xmm7
vpunpcklqdq xmm0, xmm8, xmm9
vpunpcklqdq xmm1, xmm10, xmm11
vpunpckhqdq xmm2, xmm8, xmm9
vpunpckhqdq xmm3, xmm10, xmm11
vpunpcklqdq xmm4, xmm12, xmm13
vpunpcklqdq xmm5, xmm14, xmm15
vpunpckhqdq xmm6, xmm12, xmm13
vpunpckhqdq xmm7, xmm14, xmm15
vmovdqu xmm8, OWORD PTR [rdx]
vmovdqu xmm9, OWORD PTR [rdx+16]
vmovdqu xmm10, OWORD PTR [rdx+64]
vmovdqu xmm11, OWORD PTR [rdx+80]
vmovdqu xmm12, OWORD PTR [rdx+128]
vmovdqu xmm13, OWORD PTR [rdx+144]
vmovdqu xmm14, OWORD PTR [rdx+192]
vmovdqu xmm15, OWORD PTR [rdx+208]
vpxor xmm0, xmm0, xmm8
vpxor xmm1, xmm1, xmm9
vpxor xmm2, xmm2, xmm10
vpxor xmm3, xmm3, xmm11
vpxor xmm4, xmm4, xmm12
vpxor xmm5, xmm5, xmm13
vpxor xmm6, xmm6, xmm14
vpxor xmm7, xmm7, xmm15
vmovdqu OWORD PTR [r8], xmm0
vmovdqu OWORD PTR [r8+16], xmm1
vmovdqu OWORD PTR [r8+64], xmm2
vmovdqu OWORD PTR [r8+80], xmm3
vmovdqu OWORD PTR [r8+128], xmm4
vmovdqu OWORD PTR [r8+144], xmm5
vmovdqu OWORD PTR [r8+192], xmm6
vmovdqu OWORD PTR [r8+208], xmm7
vmovdqa xmm0, OWORD PTR [r12]
vmovdqa xmm1, OWORD PTR [r12+16]
vmovdqa xmm2, OWORD PTR [r12+32]
vmovdqa xmm3, OWORD PTR [r12+48]
vmovdqa xmm4, OWORD PTR [r12+64]
vmovdqa xmm5, OWORD PTR [r12+80]
vmovdqa xmm6, OWORD PTR [r12+96]
vmovdqa xmm7, OWORD PTR [r12+112]
vpunpckldq xmm8, xmm0, xmm1
vpunpckldq xmm9, xmm2, xmm3
vpunpckhdq xmm12, xmm0, xmm1
vpunpckhdq xmm13, xmm2, xmm3
vpunpckldq xmm10, xmm4, xmm5
vpunpckldq xmm11, xmm6, xmm7
vpunpckhdq xmm14, xmm4, xmm5
vpunpckhdq xmm15, xmm6, xmm7
vpunpcklqdq xmm0, xmm8, xmm9
vpunpcklqdq xmm1, xmm10, xmm11
vpunpckhqdq xmm2, xmm8, xmm9
vpunpckhqdq xmm3, xmm10, xmm11
vpunpcklqdq xmm4, xmm12, xmm13
vpunpcklqdq xmm5, xmm14, xmm15
vpunpckhqdq xmm6, xmm12, xmm13
vpunpckhqdq xmm7, xmm14, xmm15
vmovdqu xmm8, OWORD PTR [rdx+32]
vmovdqu xmm9, OWORD PTR [rdx+48]
vmovdqu xmm10, OWORD PTR [rdx+96]
vmovdqu xmm11, OWORD PTR [rdx+112]
vmovdqu xmm12, OWORD PTR [rdx+160]
vmovdqu xmm13, OWORD PTR [rdx+176]
vmovdqu xmm14, OWORD PTR [rdx+224]
vmovdqu xmm15, OWORD PTR [rdx+240]
vpxor xmm0, xmm0, xmm8
vpxor xmm1, xmm1, xmm9
vpxor xmm2, xmm2, xmm10
vpxor xmm3, xmm3, xmm11
vpxor xmm4, xmm4, xmm12
vpxor xmm5, xmm5, xmm13
vpxor xmm6, xmm6, xmm14
vpxor xmm7, xmm7, xmm15
vmovdqu OWORD PTR [r8+32], xmm0
vmovdqu OWORD PTR [r8+48], xmm1
vmovdqu OWORD PTR [r8+96], xmm2
vmovdqu OWORD PTR [r8+112], xmm3
vmovdqu OWORD PTR [r8+160], xmm4
vmovdqu OWORD PTR [r8+176], xmm5
vmovdqu OWORD PTR [r8+224], xmm6
vmovdqu OWORD PTR [r8+240], xmm7
vmovdqa xmm12, OWORD PTR [r11+192]
add rdx, 256
add r8, 256
vpaddd xmm12, xmm12, OWORD PTR [rsi]
sub r9d, 256
vmovdqa OWORD PTR [r11+192], xmm12
cmp r9d, 256
jl L_chacha20_avx1_done128
vmovdqa xmm0, OWORD PTR [r11]
vmovdqa xmm1, OWORD PTR [r11+16]
vmovdqa xmm2, OWORD PTR [r11+32]
vmovdqa xmm3, OWORD PTR [r11+48]
vmovdqa xmm4, OWORD PTR [r11+64]
vmovdqa xmm5, OWORD PTR [r11+80]
vmovdqa xmm6, OWORD PTR [r11+96]
vmovdqa xmm7, OWORD PTR [r11+112]
vmovdqa xmm8, OWORD PTR [r11+128]
vmovdqa xmm9, OWORD PTR [r11+144]
vmovdqa xmm10, OWORD PTR [r11+160]
vmovdqa xmm11, OWORD PTR [r11+176]
vmovdqa xmm12, OWORD PTR [r11+192]
vmovdqa xmm13, OWORD PTR [r11+208]
vmovdqa xmm14, OWORD PTR [r11+224]
vmovdqa xmm15, OWORD PTR [r11+240]
jmp L_chacha20_avx1_start128
L_chacha20_avx1_done128:
shl eax, 2
add DWORD PTR [rcx+48], eax
L_chacha20_avx1_end128:
cmp r9d, 64
jl L_chacha20_avx1_block_done
L_chacha20_avx1_block_start:
vmovdqu xmm0, OWORD PTR [rcx]
vmovdqu xmm1, OWORD PTR [rcx+16]
vmovdqu xmm2, OWORD PTR [rcx+32]
vmovdqu xmm3, OWORD PTR [rcx+48]
vmovdqa xmm5, xmm0
vmovdqa xmm6, xmm1
vmovdqa xmm7, xmm2
vmovdqa xmm8, xmm3
mov al, 10
L_chacha20_avx1_block_crypt_start:
vpaddd xmm0, xmm0, xmm1
vpxor xmm3, xmm3, xmm0
vpshufb xmm3, xmm3, OWORD PTR [r15]
vpaddd xmm2, xmm2, xmm3
vpxor xmm1, xmm1, xmm2
vpsrld xmm4, xmm1, 20
vpslld xmm1, xmm1, 12
vpxor xmm1, xmm1, xmm4
vpaddd xmm0, xmm0, xmm1
vpxor xmm3, xmm3, xmm0
vpshufb xmm3, xmm3, OWORD PTR [r14]
vpaddd xmm2, xmm2, xmm3
vpxor xmm1, xmm1, xmm2
vpsrld xmm4, xmm1, 25
vpslld xmm1, xmm1, 7
vpxor xmm1, xmm1, xmm4
vpshufd xmm1, xmm1, 57
vpshufd xmm2, xmm2, 78
vpshufd xmm3, xmm3, 147
vpaddd xmm0, xmm0, xmm1
vpxor xmm3, xmm3, xmm0
vpshufb xmm3, xmm3, OWORD PTR [r15]
vpaddd xmm2, xmm2, xmm3
vpxor xmm1, xmm1, xmm2
vpsrld xmm4, xmm1, 20
vpslld xmm1, xmm1, 12
vpxor xmm1, xmm1, xmm4
vpaddd xmm0, xmm0, xmm1
vpxor xmm3, xmm3, xmm0
vpshufb xmm3, xmm3, OWORD PTR [r14]
vpaddd xmm2, xmm2, xmm3
vpxor xmm1, xmm1, xmm2
vpsrld xmm4, xmm1, 25
vpslld xmm1, xmm1, 7
vpxor xmm1, xmm1, xmm4
vpshufd xmm1, xmm1, 147
vpshufd xmm2, xmm2, 78
vpshufd xmm3, xmm3, 57
dec al
jnz L_chacha20_avx1_block_crypt_start
vpaddd xmm0, xmm0, xmm5
vpaddd xmm1, xmm1, xmm6
vpaddd xmm2, xmm2, xmm7
vpaddd xmm3, xmm3, xmm8
vmovdqu xmm5, OWORD PTR [rdx]
vmovdqu xmm6, OWORD PTR [rdx+16]
vmovdqu xmm7, OWORD PTR [rdx+32]
vmovdqu xmm8, OWORD PTR [rdx+48]
vpxor xmm0, xmm0, xmm5
vpxor xmm1, xmm1, xmm6
vpxor xmm2, xmm2, xmm7
vpxor xmm3, xmm3, xmm8
vmovdqu OWORD PTR [r8], xmm0
vmovdqu OWORD PTR [r8+16], xmm1
vmovdqu OWORD PTR [r8+32], xmm2
vmovdqu OWORD PTR [r8+48], xmm3
add DWORD PTR [rcx+48], 1
sub r9d, 64
add rdx, 64
add r8, 64
cmp r9d, 64
jge L_chacha20_avx1_block_start
L_chacha20_avx1_block_done:
cmp r9d, 0
je L_chacha20_avx1_partial_done
lea r12, QWORD PTR [rcx+80]
vmovdqu xmm0, OWORD PTR [rcx]
vmovdqu xmm1, OWORD PTR [rcx+16]
vmovdqu xmm2, OWORD PTR [rcx+32]
vmovdqu xmm3, OWORD PTR [rcx+48]
vmovdqa xmm5, xmm0
vmovdqa xmm6, xmm1
vmovdqa xmm7, xmm2
vmovdqa xmm8, xmm3
mov al, 10
L_chacha20_avx1_partial_crypt_start:
vpaddd xmm0, xmm0, xmm1
vpxor xmm3, xmm3, xmm0
vpshufb xmm3, xmm3, OWORD PTR [r15]
vpaddd xmm2, xmm2, xmm3
vpxor xmm1, xmm1, xmm2
vpsrld xmm4, xmm1, 20
vpslld xmm1, xmm1, 12
vpxor xmm1, xmm1, xmm4
vpaddd xmm0, xmm0, xmm1
vpxor xmm3, xmm3, xmm0
vpshufb xmm3, xmm3, OWORD PTR [r14]
vpaddd xmm2, xmm2, xmm3
vpxor xmm1, xmm1, xmm2
vpsrld xmm4, xmm1, 25
vpslld xmm1, xmm1, 7
vpxor xmm1, xmm1, xmm4
vpshufd xmm1, xmm1, 57
vpshufd xmm2, xmm2, 78
vpshufd xmm3, xmm3, 147
vpaddd xmm0, xmm0, xmm1
vpxor xmm3, xmm3, xmm0
vpshufb xmm3, xmm3, OWORD PTR [r15]
vpaddd xmm2, xmm2, xmm3
vpxor xmm1, xmm1, xmm2
vpsrld xmm4, xmm1, 20
vpslld xmm1, xmm1, 12
vpxor xmm1, xmm1, xmm4
vpaddd xmm0, xmm0, xmm1
vpxor xmm3, xmm3, xmm0
vpshufb xmm3, xmm3, OWORD PTR [r14]
vpaddd xmm2, xmm2, xmm3
vpxor xmm1, xmm1, xmm2
vpsrld xmm4, xmm1, 25
vpslld xmm1, xmm1, 7
vpxor xmm1, xmm1, xmm4
vpshufd xmm1, xmm1, 147
vpshufd xmm2, xmm2, 78
vpshufd xmm3, xmm3, 57
dec al
jnz L_chacha20_avx1_partial_crypt_start
vpaddd xmm0, xmm0, xmm5
vpaddd xmm1, xmm1, xmm6
vpaddd xmm2, xmm2, xmm7
vpaddd xmm3, xmm3, xmm8
vmovdqu OWORD PTR [r12], xmm0
vmovdqu OWORD PTR [r12+16], xmm1
vmovdqu OWORD PTR [r12+32], xmm2
vmovdqu OWORD PTR [r12+48], xmm3
add DWORD PTR [rcx+48], 1
mov r10d, r9d
xor r13, r13
and r10d, 7
jz L_chacha20_avx1_partial_start64
L_chacha20_avx1_partial_start8:
movzx eax, BYTE PTR [r12+r13]
xor al, BYTE PTR [rdx+r13]
mov BYTE PTR [r8+r13], al
inc r13d
cmp r13d, r10d
jne L_chacha20_avx1_partial_start8
je L_chacha20_avx1_partial_end64
L_chacha20_avx1_partial_start64:
mov rax, QWORD PTR [r12+r13]
xor rax, QWORD PTR [rdx+r13]
mov QWORD PTR [r8+r13], rax
add r13d, 8
L_chacha20_avx1_partial_end64:
cmp r13d, r9d
jne L_chacha20_avx1_partial_start64
mov r10d, 64
sub r10d, r13d
mov DWORD PTR [rcx+76], r10d
L_chacha20_avx1_partial_done:
vmovdqu xmm6, OWORD PTR [rsp+400]
vmovdqu xmm7, OWORD PTR [rsp+416]
vmovdqu xmm8, OWORD PTR [rsp+432]
vmovdqu xmm9, OWORD PTR [rsp+448]
vmovdqu xmm10, OWORD PTR [rsp+464]
vmovdqu xmm11, OWORD PTR [rsp+480]
vmovdqu xmm12, OWORD PTR [rsp+496]
vmovdqu xmm13, OWORD PTR [rsp+512]
vmovdqu xmm14, OWORD PTR [rsp+528]
vmovdqu xmm15, OWORD PTR [rsp+544]
add rsp, 560
pop rsi
pop rdi
pop r15
pop r14
pop r13
pop r12
ret
chacha_encrypt_avx1 ENDP
_text ENDS
ENDIF
IFDEF HAVE_INTEL_AVX2
_DATA SEGMENT
ALIGN 16
L_chacha20_avx2_rotl8 QWORD 433757367256023043, 1012478749960636427,
433757367256023043, 1012478749960636427
ptr_L_chacha20_avx2_rotl8 QWORD L_chacha20_avx2_rotl8
_DATA ENDS
_DATA SEGMENT
ALIGN 16
L_chacha20_avx2_rotl16 QWORD 361421592464458498, 940142975169071882,
361421592464458498, 940142975169071882
ptr_L_chacha20_avx2_rotl16 QWORD L_chacha20_avx2_rotl16
_DATA ENDS
_DATA SEGMENT
ALIGN 16
L_chacha20_avx2_add QWORD 4294967296, 12884901890,
21474836484, 30064771078
ptr_L_chacha20_avx2_add QWORD L_chacha20_avx2_add
_DATA ENDS
_DATA SEGMENT
ALIGN 16
L_chacha20_avx2_eight QWORD 34359738376, 34359738376,
34359738376, 34359738376
ptr_L_chacha20_avx2_eight QWORD L_chacha20_avx2_eight
_DATA ENDS
_text SEGMENT READONLY PARA
chacha_encrypt_avx2 PROC
push r12
push r13
push r14
push r15
push rdi
sub rsp, 960
vmovdqu OWORD PTR [rsp+800], xmm6
vmovdqu OWORD PTR [rsp+816], xmm7
vmovdqu OWORD PTR [rsp+832], xmm8
vmovdqu OWORD PTR [rsp+848], xmm9
vmovdqu OWORD PTR [rsp+864], xmm10
vmovdqu OWORD PTR [rsp+880], xmm11
vmovdqu OWORD PTR [rsp+896], xmm12
vmovdqu OWORD PTR [rsp+912], xmm13
vmovdqu OWORD PTR [rsp+928], xmm14
vmovdqu OWORD PTR [rsp+944], xmm15
mov r11, rsp
mov r13, QWORD PTR [ptr_L_chacha20_avx2_rotl8]
mov r14, QWORD PTR [ptr_L_chacha20_avx2_rotl16]
mov r15, QWORD PTR [ptr_L_chacha20_avx2_add]
mov rdi, QWORD PTR [ptr_L_chacha20_avx2_eight]
lea r12, QWORD PTR [rsp+512]
add r11, 31
add r12, 31
and r11, -32
and r12, -32
mov eax, r9d
shr eax, 9
jz L_chacha20_avx2_end256
vpbroadcastd ymm0, DWORD PTR [rcx]
vpbroadcastd ymm1, DWORD PTR [rcx+4]
vpbroadcastd ymm2, DWORD PTR [rcx+8]
vpbroadcastd ymm3, DWORD PTR [rcx+12]
vpbroadcastd ymm4, DWORD PTR [rcx+16]
vpbroadcastd ymm5, DWORD PTR [rcx+20]
vpbroadcastd ymm6, DWORD PTR [rcx+24]
vpbroadcastd ymm7, DWORD PTR [rcx+28]
vpbroadcastd ymm8, DWORD PTR [rcx+32]
vpbroadcastd ymm9, DWORD PTR [rcx+36]
vpbroadcastd ymm10, DWORD PTR [rcx+40]
vpbroadcastd ymm11, DWORD PTR [rcx+44]
vpbroadcastd ymm12, DWORD PTR [rcx+48]
vpbroadcastd ymm13, DWORD PTR [rcx+52]
vpbroadcastd ymm14, DWORD PTR [rcx+56]
vpbroadcastd ymm15, DWORD PTR [rcx+60]
vpaddd ymm12, ymm12, YMMWORD PTR [r15]
vmovdqa YMMWORD PTR [r11], ymm0
vmovdqa YMMWORD PTR [r11+32], ymm1
vmovdqa YMMWORD PTR [r11+64], ymm2
vmovdqa YMMWORD PTR [r11+96], ymm3
vmovdqa YMMWORD PTR [r11+128], ymm4
vmovdqa YMMWORD PTR [r11+160], ymm5
vmovdqa YMMWORD PTR [r11+192], ymm6
vmovdqa YMMWORD PTR [r11+224], ymm7
vmovdqa YMMWORD PTR [r11+256], ymm8
vmovdqa YMMWORD PTR [r11+288], ymm9
vmovdqa YMMWORD PTR [r11+320], ymm10
vmovdqa YMMWORD PTR [r11+352], ymm11
vmovdqa YMMWORD PTR [r11+384], ymm12
vmovdqa YMMWORD PTR [r11+416], ymm13
vmovdqa YMMWORD PTR [r11+448], ymm14
vmovdqa YMMWORD PTR [r11+480], ymm15
L_chacha20_avx2_start256:
mov r10b, 10
vmovdqa YMMWORD PTR [r12+96], ymm11
L_chacha20_avx2_loop256:
vpaddd ymm0, ymm0, ymm4
vpxor ymm12, ymm12, ymm0
vmovdqa ymm11, YMMWORD PTR [r12+96]
vpshufb ymm12, ymm12, YMMWORD PTR [r14]
vpaddd ymm8, ymm8, ymm12
vpxor ymm4, ymm4, ymm8
vpaddd ymm1, ymm1, ymm5
vpxor ymm13, ymm13, ymm1
vpshufb ymm13, ymm13, YMMWORD PTR [r14]
vpaddd ymm9, ymm9, ymm13
vpxor ymm5, ymm5, ymm9
vpaddd ymm2, ymm2, ymm6
vpxor ymm14, ymm14, ymm2
vpshufb ymm14, ymm14, YMMWORD PTR [r14]
vpaddd ymm10, ymm10, ymm14
vpxor ymm6, ymm6, ymm10
vpaddd ymm3, ymm3, ymm7
vpxor ymm15, ymm15, ymm3
vpshufb ymm15, ymm15, YMMWORD PTR [r14]
vpaddd ymm11, ymm11, ymm15
vpxor ymm7, ymm7, ymm11
vmovdqa YMMWORD PTR [r12+96], ymm11
vpsrld ymm11, ymm4, 20
vpslld ymm4, ymm4, 12
vpxor ymm4, ymm4, ymm11
vpsrld ymm11, ymm5, 20
vpslld ymm5, ymm5, 12
vpxor ymm5, ymm5, ymm11
vpsrld ymm11, ymm6, 20
vpslld ymm6, ymm6, 12
vpxor ymm6, ymm6, ymm11
vpsrld ymm11, ymm7, 20
vpslld ymm7, ymm7, 12
vpxor ymm7, ymm7, ymm11
vpaddd ymm0, ymm0, ymm4
vpxor ymm12, ymm12, ymm0
vmovdqa ymm11, YMMWORD PTR [r12+96]
vpshufb ymm12, ymm12, YMMWORD PTR [r13]
vpaddd ymm8, ymm8, ymm12
vpxor ymm4, ymm4, ymm8
vpaddd ymm1, ymm1, ymm5
vpxor ymm13, ymm13, ymm1
vpshufb ymm13, ymm13, YMMWORD PTR [r13]
vpaddd ymm9, ymm9, ymm13
vpxor ymm5, ymm5, ymm9
vpaddd ymm2, ymm2, ymm6
vpxor ymm14, ymm14, ymm2
vpshufb ymm14, ymm14, YMMWORD PTR [r13]
vpaddd ymm10, ymm10, ymm14
vpxor ymm6, ymm6, ymm10
vpaddd ymm3, ymm3, ymm7
vpxor ymm15, ymm15, ymm3
vpshufb ymm15, ymm15, YMMWORD PTR [r13]
vpaddd ymm11, ymm11, ymm15
vpxor ymm7, ymm7, ymm11
vmovdqa YMMWORD PTR [r12+96], ymm11
vpsrld ymm11, ymm4, 25
vpslld ymm4, ymm4, 7
vpxor ymm4, ymm4, ymm11
vpsrld ymm11, ymm5, 25
vpslld ymm5, ymm5, 7
vpxor ymm5, ymm5, ymm11
vpsrld ymm11, ymm6, 25
vpslld ymm6, ymm6, 7
vpxor ymm6, ymm6, ymm11
vpsrld ymm11, ymm7, 25
vpslld ymm7, ymm7, 7
vpxor ymm7, ymm7, ymm11
vpaddd ymm0, ymm0, ymm5
vpxor ymm15, ymm15, ymm0
vmovdqa ymm11, YMMWORD PTR [r12+96]
vpshufb ymm15, ymm15, YMMWORD PTR [r14]
vpaddd ymm10, ymm10, ymm15
vpxor ymm5, ymm5, ymm10
vpaddd ymm1, ymm1, ymm6
vpxor ymm12, ymm12, ymm1
vpshufb ymm12, ymm12, YMMWORD PTR [r14]
vpaddd ymm11, ymm11, ymm12
vpxor ymm6, ymm6, ymm11
vpaddd ymm2, ymm2, ymm7
vpxor ymm13, ymm13, ymm2
vpshufb ymm13, ymm13, YMMWORD PTR [r14]
vpaddd ymm8, ymm8, ymm13
vpxor ymm7, ymm7, ymm8
vpaddd ymm3, ymm3, ymm4
vpxor ymm14, ymm14, ymm3
vpshufb ymm14, ymm14, YMMWORD PTR [r14]
vpaddd ymm9, ymm9, ymm14
vpxor ymm4, ymm4, ymm9
vmovdqa YMMWORD PTR [r12+96], ymm11
vpsrld ymm11, ymm5, 20
vpslld ymm5, ymm5, 12
vpxor ymm5, ymm5, ymm11
vpsrld ymm11, ymm6, 20
vpslld ymm6, ymm6, 12
vpxor ymm6, ymm6, ymm11
vpsrld ymm11, ymm7, 20
vpslld ymm7, ymm7, 12
vpxor ymm7, ymm7, ymm11
vpsrld ymm11, ymm4, 20
vpslld ymm4, ymm4, 12
vpxor ymm4, ymm4, ymm11
vpaddd ymm0, ymm0, ymm5
vpxor ymm15, ymm15, ymm0
vmovdqa ymm11, YMMWORD PTR [r12+96]
vpshufb ymm15, ymm15, YMMWORD PTR [r13]
vpaddd ymm10, ymm10, ymm15
vpxor ymm5, ymm5, ymm10
vpaddd ymm1, ymm1, ymm6
vpxor ymm12, ymm12, ymm1
vpshufb ymm12, ymm12, YMMWORD PTR [r13]
vpaddd ymm11, ymm11, ymm12
vpxor ymm6, ymm6, ymm11
vpaddd ymm2, ymm2, ymm7
vpxor ymm13, ymm13, ymm2
vpshufb ymm13, ymm13, YMMWORD PTR [r13]
vpaddd ymm8, ymm8, ymm13
vpxor ymm7, ymm7, ymm8
vpaddd ymm3, ymm3, ymm4
vpxor ymm14, ymm14, ymm3
vpshufb ymm14, ymm14, YMMWORD PTR [r13]
vpaddd ymm9, ymm9, ymm14
vpxor ymm4, ymm4, ymm9
vmovdqa YMMWORD PTR [r12+96], ymm11
vpsrld ymm11, ymm5, 25
vpslld ymm5, ymm5, 7
vpxor ymm5, ymm5, ymm11
vpsrld ymm11, ymm6, 25
vpslld ymm6, ymm6, 7
vpxor ymm6, ymm6, ymm11
vpsrld ymm11, ymm7, 25
vpslld ymm7, ymm7, 7
vpxor ymm7, ymm7, ymm11
vpsrld ymm11, ymm4, 25
vpslld ymm4, ymm4, 7
vpxor ymm4, ymm4, ymm11
dec r10b
jnz L_chacha20_avx2_loop256
vmovdqa ymm11, YMMWORD PTR [r12+96]
vpaddd ymm0, ymm0, YMMWORD PTR [r11]
vpaddd ymm1, ymm1, YMMWORD PTR [r11+32]
vpaddd ymm2, ymm2, YMMWORD PTR [r11+64]
vpaddd ymm3, ymm3, YMMWORD PTR [r11+96]
vpaddd ymm4, ymm4, YMMWORD PTR [r11+128]
vpaddd ymm5, ymm5, YMMWORD PTR [r11+160]
vpaddd ymm6, ymm6, YMMWORD PTR [r11+192]
vpaddd ymm7, ymm7, YMMWORD PTR [r11+224]
vpaddd ymm8, ymm8, YMMWORD PTR [r11+256]
vpaddd ymm9, ymm9, YMMWORD PTR [r11+288]
vpaddd ymm10, ymm10, YMMWORD PTR [r11+320]
vpaddd ymm11, ymm11, YMMWORD PTR [r11+352]
vpaddd ymm12, ymm12, YMMWORD PTR [r11+384]
vpaddd ymm13, ymm13, YMMWORD PTR [r11+416]
vpaddd ymm14, ymm14, YMMWORD PTR [r11+448]
vpaddd ymm15, ymm15, YMMWORD PTR [r11+480]
vmovdqa YMMWORD PTR [r12], ymm8
vmovdqa YMMWORD PTR [r12+32], ymm9
vmovdqa YMMWORD PTR [r12+64], ymm10
vmovdqa YMMWORD PTR [r12+96], ymm11
vmovdqa YMMWORD PTR [r12+128], ymm12
vmovdqa YMMWORD PTR [r12+160], ymm13
vmovdqa YMMWORD PTR [r12+192], ymm14
vmovdqa YMMWORD PTR [r12+224], ymm15
vpunpckldq ymm8, ymm0, ymm1
vpunpckldq ymm9, ymm2, ymm3
vpunpckhdq ymm12, ymm0, ymm1
vpunpckhdq ymm13, ymm2, ymm3
vpunpckldq ymm10, ymm4, ymm5
vpunpckldq ymm11, ymm6, ymm7
vpunpckhdq ymm14, ymm4, ymm5
vpunpckhdq ymm15, ymm6, ymm7
vpunpcklqdq ymm0, ymm8, ymm9
vpunpcklqdq ymm1, ymm10, ymm11
vpunpckhqdq ymm2, ymm8, ymm9
vpunpckhqdq ymm3, ymm10, ymm11
vpunpcklqdq ymm4, ymm12, ymm13
vpunpcklqdq ymm5, ymm14, ymm15
vpunpckhqdq ymm6, ymm12, ymm13
vpunpckhqdq ymm7, ymm14, ymm15
vperm2i128 ymm8, ymm0, ymm1, 32
vperm2i128 ymm9, ymm2, ymm3, 32
vperm2i128 ymm12, ymm0, ymm1, 49
vperm2i128 ymm13, ymm2, ymm3, 49
vperm2i128 ymm10, ymm4, ymm5, 32
vperm2i128 ymm11, ymm6, ymm7, 32
vperm2i128 ymm14, ymm4, ymm5, 49
vperm2i128 ymm15, ymm6, ymm7, 49
vmovdqu ymm0, YMMWORD PTR [rdx]
vmovdqu ymm1, YMMWORD PTR [rdx+64]
vmovdqu ymm2, YMMWORD PTR [rdx+128]
vmovdqu ymm3, YMMWORD PTR [rdx+192]
vmovdqu ymm4, YMMWORD PTR [rdx+256]
vmovdqu ymm5, YMMWORD PTR [rdx+320]
vmovdqu ymm6, YMMWORD PTR [rdx+384]
vmovdqu ymm7, YMMWORD PTR [rdx+448]
vpxor ymm8, ymm8, ymm0
vpxor ymm9, ymm9, ymm1
vpxor ymm10, ymm10, ymm2
vpxor ymm11, ymm11, ymm3
vpxor ymm12, ymm12, ymm4
vpxor ymm13, ymm13, ymm5
vpxor ymm14, ymm14, ymm6
vpxor ymm15, ymm15, ymm7
vmovdqu YMMWORD PTR [r8], ymm8
vmovdqu YMMWORD PTR [r8+64], ymm9
vmovdqu YMMWORD PTR [r8+128], ymm10
vmovdqu YMMWORD PTR [r8+192], ymm11
vmovdqu YMMWORD PTR [r8+256], ymm12
vmovdqu YMMWORD PTR [r8+320], ymm13
vmovdqu YMMWORD PTR [r8+384], ymm14
vmovdqu YMMWORD PTR [r8+448], ymm15
vmovdqa ymm0, YMMWORD PTR [r12]
vmovdqa ymm1, YMMWORD PTR [r12+32]
vmovdqa ymm2, YMMWORD PTR [r12+64]
vmovdqa ymm3, YMMWORD PTR [r12+96]
vmovdqa ymm4, YMMWORD PTR [r12+128]
vmovdqa ymm5, YMMWORD PTR [r12+160]
vmovdqa ymm6, YMMWORD PTR [r12+192]
vmovdqa ymm7, YMMWORD PTR [r12+224]
vpunpckldq ymm8, ymm0, ymm1
vpunpckldq ymm9, ymm2, ymm3
vpunpckhdq ymm12, ymm0, ymm1
vpunpckhdq ymm13, ymm2, ymm3
vpunpckldq ymm10, ymm4, ymm5
vpunpckldq ymm11, ymm6, ymm7
vpunpckhdq ymm14, ymm4, ymm5
vpunpckhdq ymm15, ymm6, ymm7
vpunpcklqdq ymm0, ymm8, ymm9
vpunpcklqdq ymm1, ymm10, ymm11
vpunpckhqdq ymm2, ymm8, ymm9
vpunpckhqdq ymm3, ymm10, ymm11
vpunpcklqdq ymm4, ymm12, ymm13
vpunpcklqdq ymm5, ymm14, ymm15
vpunpckhqdq ymm6, ymm12, ymm13
vpunpckhqdq ymm7, ymm14, ymm15
vperm2i128 ymm8, ymm0, ymm1, 32
vperm2i128 ymm9, ymm2, ymm3, 32
vperm2i128 ymm12, ymm0, ymm1, 49
vperm2i128 ymm13, ymm2, ymm3, 49
vperm2i128 ymm10, ymm4, ymm5, 32
vperm2i128 ymm11, ymm6, ymm7, 32
vperm2i128 ymm14, ymm4, ymm5, 49
vperm2i128 ymm15, ymm6, ymm7, 49
vmovdqu ymm0, YMMWORD PTR [rdx+32]
vmovdqu ymm1, YMMWORD PTR [rdx+96]
vmovdqu ymm2, YMMWORD PTR [rdx+160]
vmovdqu ymm3, YMMWORD PTR [rdx+224]
vmovdqu ymm4, YMMWORD PTR [rdx+288]
vmovdqu ymm5, YMMWORD PTR [rdx+352]
vmovdqu ymm6, YMMWORD PTR [rdx+416]
vmovdqu ymm7, YMMWORD PTR [rdx+480]
vpxor ymm8, ymm8, ymm0
vpxor ymm9, ymm9, ymm1
vpxor ymm10, ymm10, ymm2
vpxor ymm11, ymm11, ymm3
vpxor ymm12, ymm12, ymm4
vpxor ymm13, ymm13, ymm5
vpxor ymm14, ymm14, ymm6
vpxor ymm15, ymm15, ymm7
vmovdqu YMMWORD PTR [r8+32], ymm8
vmovdqu YMMWORD PTR [r8+96], ymm9
vmovdqu YMMWORD PTR [r8+160], ymm10
vmovdqu YMMWORD PTR [r8+224], ymm11
vmovdqu YMMWORD PTR [r8+288], ymm12
vmovdqu YMMWORD PTR [r8+352], ymm13
vmovdqu YMMWORD PTR [r8+416], ymm14
vmovdqu YMMWORD PTR [r8+480], ymm15
vmovdqa ymm12, YMMWORD PTR [r11+384]
add rdx, 512
add r8, 512
vpaddd ymm12, ymm12, YMMWORD PTR [rdi]
sub r9d, 512
vmovdqa YMMWORD PTR [r11+384], ymm12
cmp r9d, 512
jl L_chacha20_avx2_done256
vmovdqa ymm0, YMMWORD PTR [r11]
vmovdqa ymm1, YMMWORD PTR [r11+32]
vmovdqa ymm2, YMMWORD PTR [r11+64]
vmovdqa ymm3, YMMWORD PTR [r11+96]
vmovdqa ymm4, YMMWORD PTR [r11+128]
vmovdqa ymm5, YMMWORD PTR [r11+160]
vmovdqa ymm6, YMMWORD PTR [r11+192]
vmovdqa ymm7, YMMWORD PTR [r11+224]
vmovdqa ymm8, YMMWORD PTR [r11+256]
vmovdqa ymm9, YMMWORD PTR [r11+288]
vmovdqa ymm10, YMMWORD PTR [r11+320]
vmovdqa ymm11, YMMWORD PTR [r11+352]
vmovdqa ymm12, YMMWORD PTR [r11+384]
vmovdqa ymm13, YMMWORD PTR [r11+416]
vmovdqa ymm14, YMMWORD PTR [r11+448]
vmovdqa ymm15, YMMWORD PTR [r11+480]
jmp L_chacha20_avx2_start256
L_chacha20_avx2_done256:
shl eax, 3
add DWORD PTR [rcx+48], eax
L_chacha20_avx2_end256:
call chacha_encrypt_avx1
vzeroupper
vmovdqu xmm6, OWORD PTR [rsp+800]
vmovdqu xmm7, OWORD PTR [rsp+816]
vmovdqu xmm8, OWORD PTR [rsp+832]
vmovdqu xmm9, OWORD PTR [rsp+848]
vmovdqu xmm10, OWORD PTR [rsp+864]
vmovdqu xmm11, OWORD PTR [rsp+880]
vmovdqu xmm12, OWORD PTR [rsp+896]
vmovdqu xmm13, OWORD PTR [rsp+912]
vmovdqu xmm14, OWORD PTR [rsp+928]
vmovdqu xmm15, OWORD PTR [rsp+944]
add rsp, 960
pop rdi
pop r15
pop r14
pop r13
pop r12
ret
chacha_encrypt_avx2 ENDP
_text ENDS
ENDIF
END