rind 0.1.12

// Written for Graviola by Joe Birr-Pixton, 2024.
// SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT-0
// AVX2 by-8 implementation inspired by YuriMyakotin/ChaCha20-SIMD

//! ChaCha20: rind's AVX2 kernels where the CPU has AVX2, and otherwise
//! aws-lc's `chacha-x86_64` routines, imported by `admin/import-aws-lc`, at
//! the lengths aws-lc runs each: four blocks at a time with SSSE3 above 128
//! bytes, and integer code up to that, or at every length without SSSE3.
//!
//! The aws-lc assembly is not formally verified: its correctness rests on its
//! use and testing in OpenSSL, BoringSSL, and aws-lc, and on rind's tests of
//! it against RFC 8439, Wycheproof, aws-lc-rs, and the AVX2 kernels.

use core::arch::x86_64::*;

use super::cpu::{HaveAvx2, HaveSsse3};

pub(crate) struct ChaCha20 {
    key: [u32; 8],
    /// The block counter, then the 96-bit nonce.
    counter_nonce: [u32; 4],
    avx2: Option<HaveAvx2>,
    ssse3: Option<HaveSsse3>,
}

impl ChaCha20 {
    /// `nonce` is the little-endian block counter followed by the 96-bit nonce.
    pub(crate) fn new(key: &[u8; 32], nonce: &[u8; 16]) -> Self {
        Self {
            key: words(key),
            counter_nonce: words(nonce),
            avx2: HaveAvx2::check(),
            ssse3: HaveSsse3::check(),
        }
    }

    /// XORs the keystream into `buffer`, starting at the current block, then
    /// moves the counter past every block it touched.
    pub(crate) fn cipher(&mut self, buffer: &mut [u8]) {
        match self.avx2 {
            // SAFETY: the token proves the `avx2` cpu feature
            Some(_) => unsafe { cipher_avx2(&self.key, &self.counter_nonce, buffer) },
            None => chacha20_ctr32(&self.key, &self.counter_nonce, buffer, self.ssse3),
        }
        self.counter_nonce[0] =
            self.counter_nonce[0].wrapping_add(buffer.len().div_ceil(64) as u32);
    }
}

/// `bytes` as little-endian words.
fn words<const B: usize, const W: usize>(bytes: &[u8; B]) -> [u32; W] {
    const { assert!(B == 4 * W) };
    let mut words = [0; W];
    for (w, b) in words.iter_mut().zip(bytes.as_chunks::<4>().0) {
        *w = u32::from_le_bytes(*b);
    }
    words
}

/// Below this length aws-lc runs its integer routine, which computes one
/// block at a time.
const SSSE3_4X_MIN_LEN: usize = 129;

/// XORs the keystream for `key` and `counter_nonce` into `buffer`. The
/// routines' 32-bit counter wraps unspecified past 2^32 blocks, which callers
/// rule out with `mid::chacha20::check_counter`.
fn chacha20_ctr32(
    key: &[u32; 8],
    counter_nonce: &[u32; 4],
    buffer: &mut [u8],
    ssse3: Option<HaveSsse3>,
) {
    let len = buffer.len();
    let ptr = buffer.as_mut_ptr();
    match (len, ssse3) {
        // Both routines require at least one byte.
        (0, _) => {}
        // SAFETY: `buffer` is `len` readable and writable bytes, which the
        // routine allows to alias exactly, and `key` and `counter_nonce` are
        // 8 and 4 words. The token proves the `ssse3` cpu feature.
        (SSSE3_4X_MIN_LEN.., Some(_)) => unsafe {
            chacha20_ctr32_ssse3_4x(ptr, ptr, len, key, counter_nonce)
        },
        // SAFETY: as above.
        _ => unsafe { chacha20_ctr32_nohw(ptr, ptr, len, key, counter_nonce) },
    }
}

/// Counters, a counter increment, and four of them, as the routines load
/// them.
#[repr(C, align(64))]
struct Words([u32; 32]);

/// The rotate-by-16 and -24 shuffle masks, then the ChaCha20 constant, as
/// the routines load them.
#[repr(C, align(64))]
struct Bytes([u8; 49]);

static ZERO: Words = Words(include!("aws_lc/chacha-x86_64.zero.rs"));
static ROT16: Bytes = Bytes(include!("aws_lc/chacha-x86_64.rot16.rs"));

// Both routines take the System V ABI on every OS, since their text is
// aws-lc's ELF and Mach-O flavors, which read their arguments from its
// registers.

#[unsafe(naked)]
unsafe extern "sysv64" fn chacha20_ctr32_nohw(
    out: *mut u8,
    input: *const u8,
    len: usize,
    key: *const [u32; 8],
    counter_nonce: *const [u32; 4],
) {
    #[cfg(not(target_vendor = "apple"))]
    core::arch::naked_asm!(
        include_str!("aws_lc/ChaCha20_ctr32_nohw.linux.s"),
        zero = sym ZERO,
        options(att_syntax),
    );
    #[cfg(target_vendor = "apple")]
    core::arch::naked_asm!(
        include_str!("aws_lc/ChaCha20_ctr32_nohw.apple.s"),
        zero = sym ZERO,
        options(att_syntax),
    );
}

#[unsafe(naked)]
unsafe extern "sysv64" fn chacha20_ctr32_ssse3_4x(
    out: *mut u8,
    input: *const u8,
    len: usize,
    key: *const [u32; 8],
    counter_nonce: *const [u32; 4],
) {
    #[cfg(not(target_vendor = "apple"))]
    core::arch::naked_asm!(
        include_str!("aws_lc/ChaCha20_ctr32_ssse3_4x.linux.s"),
        zero = sym ZERO,
        rot16 = sym ROT16,
        options(att_syntax),
    );
    #[cfg(target_vendor = "apple")]
    core::arch::naked_asm!(
        include_str!("aws_lc/ChaCha20_ctr32_ssse3_4x.apple.s"),
        zero = sym ZERO,
        rot16 = sym ROT16,
        options(att_syntax),
    );
}

macro_rules! rotate_left {
    ($reg:expr, 8) => {
        // this is a byte shuffle leftwards, except little-endian
        _mm256_shuffle_epi8(
            $reg,
            _mm256_set_epi8(
                14, 13, 12, 15, 10, 9, 8, 11, 6, 5, 4, 7, 2, 1, 0, 3, 14, 13, 12, 15, 10, 9, 8, 11,
                6, 5, 4, 7, 2, 1, 0, 3,
            ),
        )
    };
    ($reg:expr, 16) => {
        // this is a two-byte shuffle leftwards, except little-endian
        _mm256_shuffle_epi8(
            $reg,
            _mm256_set_epi8(
                13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2, 13, 12, 15, 14, 9, 8, 11, 10,
                5, 4, 7, 6, 1, 0, 3, 2,
            ),
        )
    };
    ($reg:expr, $rot:literal) => {
        _mm256_or_si256(
            _mm256_slli_epi32($reg, $rot),
            _mm256_srli_epi32($reg, 32 - $rot),
        )
    };
}

#[target_feature(enable = "ssse3,avx2")]
fn cipher_avx2(key: &[u32; 8], counter_nonce: &[u32; 4], buffer: &mut [u8]) {
    // SAFETY: `SIGMA`, `key`, and `counter_nonce` are all readable
    let (z07, mut z8f) = unsafe {
        (
            _mm256_set_m128i(
                _mm_lddqu_si128(SIGMA.as_ptr().cast()),
                _mm_lddqu_si128(key[0..4].as_ptr().cast()),
            ),
            _mm256_set_m128i(
                _mm_lddqu_si128(key[4..8].as_ptr().cast()),
                _mm_lddqu_si128(counter_nonce.as_ptr().cast()),
            ),
        )
    };

    let (by8, remainder) = buffer.as_chunks_mut::<512>();
    for block in by8 {
        core_8x(z07, &mut z8f, block);
    }
    for block in remainder.chunks_mut(128) {
        core_2x(z07, &mut z8f, block);
    }
}

/// Computes 8 blocks.  Does _NOT_ handle ragged output.
#[target_feature(enable = "avx2")]
fn core_8x(t07: __m256i, z8f: &mut __m256i, xor_out_512: &mut [u8]) {
    let t8f = *z8f;
    *z8f = _mm256_add_epi32(*z8f, _mm256_set_epi32(0, 0, 0, 0, 0, 0, 0, 8));

    let z03_z03 = _mm256_broadcastsi128_si256(_mm256_extracti128_si256(t07, 1));
    let z47_z47 = _mm256_broadcastsi128_si256(_mm256_extracti128_si256(t07, 0));
    let z8b_z8b = _mm256_broadcastsi128_si256(_mm256_extracti128_si256(t8f, 1));
    let zcf_zcf = _mm256_broadcastsi128_si256(_mm256_extracti128_si256(t8f, 0));

    let save_z03 = z03_z03;
    let save_z47 = z47_z47;
    let save_z8b = z8b_z8b;
    let save_zcf = zcf_zcf;

    let mut z03_z03 = [z03_z03; 4];
    let mut z47_z47 = [z47_z47; 4];
    let mut z8b_z8b = [z8b_z8b; 4];
    let mut zcf_zcf = [zcf_zcf; 4];

    zcf_zcf[0] = _mm256_add_epi32(zcf_zcf[0], _mm256_set_epi32(0, 0, 0, 0, 0, 0, 0, 4));
    zcf_zcf[1] = _mm256_add_epi32(zcf_zcf[1], _mm256_set_epi32(0, 0, 0, 1, 0, 0, 0, 5));
    zcf_zcf[2] = _mm256_add_epi32(zcf_zcf[2], _mm256_set_epi32(0, 0, 0, 2, 0, 0, 0, 6));
    zcf_zcf[3] = _mm256_add_epi32(zcf_zcf[3], _mm256_set_epi32(0, 0, 0, 3, 0, 0, 0, 7));

    for _ in 0..10 {
        for i in 0..4 {
            z03_z03[i] = _mm256_add_epi32(z03_z03[i], z47_z47[i]);
        }
        for i in 0..4 {
            zcf_zcf[i] = _mm256_xor_si256(zcf_zcf[i], z03_z03[i]);
        }
        for z in &mut zcf_zcf {
            *z = rotate_left!(*z, 16);
        }

        for i in 0..4 {
            z8b_z8b[i] = _mm256_add_epi32(z8b_z8b[i], zcf_zcf[i]);
        }
        for i in 0..4 {
            z47_z47[i] = _mm256_xor_si256(z47_z47[i], z8b_z8b[i]);
        }
        for z in &mut z47_z47 {
            *z = rotate_left!(*z, 12);
        }

        for i in 0..4 {
            z03_z03[i] = _mm256_add_epi32(z03_z03[i], z47_z47[i]);
        }
        for i in 0..4 {
            zcf_zcf[i] = _mm256_xor_si256(zcf_zcf[i], z03_z03[i]);
        }
        for z in &mut zcf_zcf {
            *z = rotate_left!(*z, 8);
        }

        for i in 0..4 {
            z8b_z8b[i] = _mm256_add_epi32(z8b_z8b[i], zcf_zcf[i]);
        }
        for i in 0..4 {
            z47_z47[i] = _mm256_xor_si256(z47_z47[i], z8b_z8b[i]);
        }
        for z in &mut z47_z47 {
            *z = rotate_left!(*z, 7);
        }

        for z in &mut z47_z47 {
            *z = _mm256_shuffle_epi32(*z, 0b00_11_10_01);
        }
        for z in &mut z8b_z8b {
            *z = _mm256_shuffle_epi32(*z, 0b01_00_11_10);
        }
        for z in &mut zcf_zcf {
            *z = _mm256_shuffle_epi32(*z, 0b10_01_00_11);
        }

        for i in 0..4 {
            z03_z03[i] = _mm256_add_epi32(z03_z03[i], z47_z47[i]);
        }
        for i in 0..4 {
            zcf_zcf[i] = _mm256_xor_si256(zcf_zcf[i], z03_z03[i]);
        }
        for z in &mut zcf_zcf {
            *z = rotate_left!(*z, 16);
        }

        for i in 0..4 {
            z8b_z8b[i] = _mm256_add_epi32(z8b_z8b[i], zcf_zcf[i]);
        }
        for i in 0..4 {
            z47_z47[i] = _mm256_xor_si256(z47_z47[i], z8b_z8b[i]);
        }
        for z in &mut z47_z47 {
            *z = rotate_left!(*z, 12);
        }

        for i in 0..4 {
            z03_z03[i] = _mm256_add_epi32(z03_z03[i], z47_z47[i]);
        }
        for i in 0..4 {
            zcf_zcf[i] = _mm256_xor_si256(zcf_zcf[i], z03_z03[i]);
        }
        for z in &mut zcf_zcf {
            *z = rotate_left!(*z, 8);
        }

        for i in 0..4 {
            z8b_z8b[i] = _mm256_add_epi32(z8b_z8b[i], zcf_zcf[i]);
        }
        for i in 0..4 {
            z47_z47[i] = _mm256_xor_si256(z47_z47[i], z8b_z8b[i]);
        }
        for z in &mut z47_z47 {
            *z = rotate_left!(*z, 7);
        }

        for z in &mut z47_z47 {
            *z = _mm256_shuffle_epi32(*z, 0b10_01_00_11);
        }
        for z in &mut z8b_z8b {
            *z = _mm256_shuffle_epi32(*z, 0b01_00_11_10);
        }
        for z in &mut zcf_zcf {
            *z = _mm256_shuffle_epi32(*z, 0b00_11_10_01);
        }
    }

    for i in 0..4 {
        z03_z03[i] = _mm256_add_epi32(z03_z03[i], save_z03);
        z47_z47[i] = _mm256_add_epi32(z47_z47[i], save_z47);
        z8b_z8b[i] = _mm256_add_epi32(z8b_z8b[i], save_z8b);
        zcf_zcf[i] = _mm256_add_epi32(zcf_zcf[i], save_zcf);
    }

    // reapply counter adjustments as save_zcf did not include them
    zcf_zcf[0] = _mm256_add_epi32(zcf_zcf[0], _mm256_set_epi32(0, 0, 0, 0, 0, 0, 0, 4));
    zcf_zcf[1] = _mm256_add_epi32(zcf_zcf[1], _mm256_set_epi32(0, 0, 0, 1, 0, 0, 0, 5));
    zcf_zcf[2] = _mm256_add_epi32(zcf_zcf[2], _mm256_set_epi32(0, 0, 0, 2, 0, 0, 0, 6));
    zcf_zcf[3] = _mm256_add_epi32(zcf_zcf[3], _mm256_set_epi32(0, 0, 0, 3, 0, 0, 0, 7));

    // our eight output keystream blocks
    let a0 = _mm256_permute2x128_si256(z03_z03[0], z47_z47[0], 0b0011_0001);
    let a1 = _mm256_permute2x128_si256(z8b_z8b[0], zcf_zcf[0], 0b0011_0001);
    let b0 = _mm256_permute2x128_si256(z03_z03[1], z47_z47[1], 0b0011_0001);
    let b1 = _mm256_permute2x128_si256(z8b_z8b[1], zcf_zcf[1], 0b0011_0001);

    let c0 = _mm256_permute2x128_si256(z03_z03[2], z47_z47[2], 0b0011_0001);
    let c1 = _mm256_permute2x128_si256(z8b_z8b[2], zcf_zcf[2], 0b0011_0001);
    let d0 = _mm256_permute2x128_si256(z03_z03[3], z47_z47[3], 0b0011_0001);
    let d1 = _mm256_permute2x128_si256(z8b_z8b[3], zcf_zcf[3], 0b0011_0001);

    let e0 = _mm256_permute2x128_si256(z03_z03[0], z47_z47[0], 0b0010_0000);
    let e1 = _mm256_permute2x128_si256(z8b_z8b[0], zcf_zcf[0], 0b0010_0000);
    let f0 = _mm256_permute2x128_si256(z03_z03[1], z47_z47[1], 0b0010_0000);
    let f1 = _mm256_permute2x128_si256(z8b_z8b[1], zcf_zcf[1], 0b0010_0000);

    let g0 = _mm256_permute2x128_si256(z03_z03[2], z47_z47[2], 0b0010_0000);
    let g1 = _mm256_permute2x128_si256(z8b_z8b[2], zcf_zcf[2], 0b0010_0000);
    let h0 = _mm256_permute2x128_si256(z03_z03[3], z47_z47[3], 0b0010_0000);
    let h1 = _mm256_permute2x128_si256(z8b_z8b[3], zcf_zcf[3], 0b0010_0000);

    let ptr: *mut __m256i = xor_out_512.as_mut_ptr().cast();

    // SAFETY: `xor_out_512` is 512 bytes due to chunking in caller. Observe that
    // `ptr` is unaligned which is allowed as we only use it here with unaligned
    // stores/loads.
    unsafe {
        _mm256_storeu_si256(
            ptr.add(0),
            _mm256_xor_si256(_mm256_loadu_si256(ptr.add(0)), a0),
        );
        _mm256_storeu_si256(
            ptr.add(1),
            _mm256_xor_si256(_mm256_loadu_si256(ptr.add(1)), a1),
        );

        _mm256_storeu_si256(
            ptr.add(2),
            _mm256_xor_si256(_mm256_loadu_si256(ptr.add(2)), b0),
        );
        _mm256_storeu_si256(
            ptr.add(3),
            _mm256_xor_si256(_mm256_loadu_si256(ptr.add(3)), b1),
        );

        _mm256_storeu_si256(
            ptr.add(4),
            _mm256_xor_si256(_mm256_loadu_si256(ptr.add(4)), c0),
        );
        _mm256_storeu_si256(
            ptr.add(5),
            _mm256_xor_si256(_mm256_loadu_si256(ptr.add(5)), c1),
        );

        _mm256_storeu_si256(
            ptr.add(6),
            _mm256_xor_si256(_mm256_loadu_si256(ptr.add(6)), d0),
        );
        _mm256_storeu_si256(
            ptr.add(7),
            _mm256_xor_si256(_mm256_loadu_si256(ptr.add(7)), d1),
        );

        _mm256_storeu_si256(
            ptr.add(8),
            _mm256_xor_si256(_mm256_loadu_si256(ptr.add(8)), e0),
        );
        _mm256_storeu_si256(
            ptr.add(9),
            _mm256_xor_si256(_mm256_loadu_si256(ptr.add(9)), e1),
        );

        _mm256_storeu_si256(
            ptr.add(10),
            _mm256_xor_si256(_mm256_loadu_si256(ptr.add(10)), f0),
        );
        _mm256_storeu_si256(
            ptr.add(11),
            _mm256_xor_si256(_mm256_loadu_si256(ptr.add(11)), f1),
        );

        _mm256_storeu_si256(
            ptr.add(12),
            _mm256_xor_si256(_mm256_loadu_si256(ptr.add(12)), g0),
        );
        _mm256_storeu_si256(
            ptr.add(13),
            _mm256_xor_si256(_mm256_loadu_si256(ptr.add(13)), g1),
        );

        _mm256_storeu_si256(
            ptr.add(14),
            _mm256_xor_si256(_mm256_loadu_si256(ptr.add(14)), h0),
        );
        _mm256_storeu_si256(
            ptr.add(15),
            _mm256_xor_si256(_mm256_loadu_si256(ptr.add(15)), h1),
        );
    }
}

/// Computes 2 blocks, but also handles ragged output (ie, xor_out may
/// be 0..64 bytes).
#[target_feature(enable = "avx2")]
fn core_2x(t07: __m256i, z8f: &mut __m256i, xor_out: &mut [u8]) {
    let t8f = *z8f;
    let blocks_used = if xor_out.len() > 32 { 2 } else { 1 };
    *z8f = _mm256_add_epi32(*z8f, _mm256_set_epi32(0, 0, 0, 0, 0, 0, 0, blocks_used));

    let mut z03_z03 = _mm256_broadcastsi128_si256(_mm256_extracti128_si256(t07, 1));
    let mut z47_z47 = _mm256_broadcastsi128_si256(_mm256_extracti128_si256(t07, 0));
    let mut z8b_z8b = _mm256_broadcastsi128_si256(_mm256_extracti128_si256(t8f, 1));
    let mut zcf_zcf = _mm256_broadcastsi128_si256(_mm256_extracti128_si256(t8f, 0));

    // we will calculate two blocks, so increment the counter of the second block
    zcf_zcf = _mm256_add_epi32(zcf_zcf, _mm256_set_epi32(0, 0, 0, 0, 0, 0, 0, 1));
    let save_z03 = z03_z03;
    let save_z47 = z47_z47;
    let save_z8b = z8b_z8b;
    let save_zcf = zcf_zcf;

    for _ in 0..10 {
        z03_z03 = _mm256_add_epi32(z03_z03, z47_z47);
        zcf_zcf = _mm256_xor_si256(zcf_zcf, z03_z03);
        zcf_zcf = rotate_left!(zcf_zcf, 16);

        z8b_z8b = _mm256_add_epi32(z8b_z8b, zcf_zcf);
        z47_z47 = _mm256_xor_si256(z47_z47, z8b_z8b);
        z47_z47 = rotate_left!(z47_z47, 12);

        z03_z03 = _mm256_add_epi32(z03_z03, z47_z47);
        zcf_zcf = _mm256_xor_si256(zcf_zcf, z03_z03);
        zcf_zcf = rotate_left!(zcf_zcf, 8);

        z8b_z8b = _mm256_add_epi32(z8b_z8b, zcf_zcf);
        z47_z47 = _mm256_xor_si256(z47_z47, z8b_z8b);
        z47_z47 = rotate_left!(z47_z47, 7);

        z47_z47 = _mm256_shuffle_epi32(z47_z47, 0b00_11_10_01);
        z8b_z8b = _mm256_shuffle_epi32(z8b_z8b, 0b01_00_11_10);
        zcf_zcf = _mm256_shuffle_epi32(zcf_zcf, 0b10_01_00_11);

        z03_z03 = _mm256_add_epi32(z03_z03, z47_z47);
        zcf_zcf = _mm256_xor_si256(zcf_zcf, z03_z03);
        zcf_zcf = rotate_left!(zcf_zcf, 16);

        z8b_z8b = _mm256_add_epi32(z8b_z8b, zcf_zcf);
        z47_z47 = _mm256_xor_si256(z47_z47, z8b_z8b);
        z47_z47 = rotate_left!(z47_z47, 12);

        z03_z03 = _mm256_add_epi32(z03_z03, z47_z47);
        zcf_zcf = _mm256_xor_si256(zcf_zcf, z03_z03);
        zcf_zcf = rotate_left!(zcf_zcf, 8);

        z8b_z8b = _mm256_add_epi32(z8b_z8b, zcf_zcf);
        z47_z47 = _mm256_xor_si256(z47_z47, z8b_z8b);
        z47_z47 = rotate_left!(z47_z47, 7);

        z47_z47 = _mm256_shuffle_epi32(z47_z47, 0b10_01_00_11);
        z8b_z8b = _mm256_shuffle_epi32(z8b_z8b, 0b01_00_11_10);
        zcf_zcf = _mm256_shuffle_epi32(zcf_zcf, 0b00_11_10_01);
    }

    let z03_z03 = _mm256_add_epi32(z03_z03, save_z03);
    let z47_z47 = _mm256_add_epi32(z47_z47, save_z47);
    let z8b_z8b = _mm256_add_epi32(z8b_z8b, save_z8b);
    let zcf_zcf = _mm256_add_epi32(zcf_zcf, save_zcf);

    let a0 = _mm256_permute2x128_si256(z03_z03, z47_z47, 0b0011_0001);
    let a1 = _mm256_permute2x128_si256(z8b_z8b, zcf_zcf, 0b0011_0001);
    let b0 = _mm256_permute2x128_si256(z03_z03, z47_z47, 0b0010_0000);
    let b1 = _mm256_permute2x128_si256(z8b_z8b, zcf_zcf, 0b0010_0000);

    let ptr: *mut __m256i = xor_out.as_mut_ptr().cast();

    if xor_out.len() == 128 {
        // SAFETY: `ptr` is 128 bytes, ie 4 x 256-bit ymm registers
        unsafe {
            let ia0 = _mm256_loadu_si256(ptr.add(0));
            let ia1 = _mm256_loadu_si256(ptr.add(1));
            let ib0 = _mm256_loadu_si256(ptr.add(2));
            let ib1 = _mm256_loadu_si256(ptr.add(3));

            _mm256_storeu_si256(ptr.add(0), _mm256_xor_si256(a0, ia0));
            _mm256_storeu_si256(ptr.add(1), _mm256_xor_si256(a1, ia1));
            _mm256_storeu_si256(ptr.add(2), _mm256_xor_si256(b0, ib0));
            _mm256_storeu_si256(ptr.add(3), _mm256_xor_si256(b1, ib1));
        }
    } else if xor_out.len() == 32 {
        // SAFETY: `ptr` is 32 bytes, ie one 256-bit ymm register
        unsafe {
            let ia0 = _mm256_loadu_si256(ptr.add(0));
            _mm256_storeu_si256(ptr.add(0), _mm256_xor_si256(a0, ia0));
        }
    } else {
        // slow path
        let mut ks = [0u8; 128];

        // SAFETY: `ks` is 128 bytes and writable.
        unsafe {
            _mm256_storeu_si256(ks[0..32].as_mut_ptr().cast(), a0);
            _mm256_storeu_si256(ks[32..64].as_mut_ptr().cast(), a1);
            _mm256_storeu_si256(ks[64..96].as_mut_ptr().cast(), b0);
            _mm256_storeu_si256(ks[96..128].as_mut_ptr().cast(), b1);
        }

        for (o, k) in xor_out.iter_mut().zip(ks.iter()) {
            *o ^= *k;
        }
    }
}

const SIGMA: [u8; 16] = *b"expand 32-byte k";

#[cfg(test)]
#[cfg_attr(coverage_nightly, coverage(off))]
mod tests {
    use super::*;
    use crate::low::hchacha20;

    // The CPU's fastest path hides the others from every other test, so
    // aws-lc's routines, with SSSE3 and without, are checked against it
    // directly: every length through both routines' boundaries and two
    // four-block batches, each continued by a second call, which a whole
    // number of blocks leaves contiguous.
    #[test]
    fn paths_match() {
        let key = core::array::from_fn(|i| i as u8);
        let nonce = core::array::from_fn(|i| (i * 7) as u8);
        for len in 0..=600 {
            let mut want = vec![0x5a; len + 200];
            let mut fastest = ChaCha20::new(&key, &nonce);
            let (first, second) = want.split_at_mut(len);
            fastest.cipher(first);
            fastest.cipher(second);

            for (path, ssse3) in [("ssse3", HaveSsse3::check()), ("nohw", None)] {
                let mut aws_lc = ChaCha20 {
                    avx2: None,
                    ssse3,
                    ..ChaCha20::new(&key, &nonce)
                };
                let mut got = vec![0x5a; len + 200];
                let (first, second) = got.split_at_mut(len);
                aws_lc.cipher(first);
                aws_lc.cipher(second);
                assert_eq!(got, want, "{path}, {len} bytes, then 200");
            }

            if len % 64 == 0 {
                let mut whole = vec![0x5a; len + 200];
                ChaCha20::new(&key, &nonce).cipher(&mut whole);
                assert_eq!(whole, want, "{len} bytes, then 200, at once");
            }
        }
    }

    #[test]
    fn test_vectors() {
        // From draft-agl-tls-chacha20poly1305-04 section 7
        let mut c = ChaCha20::new(&[0u8; 32], &[0u8; 16]);
        let mut block = [0u8; 64];
        c.cipher(&mut block);
        assert_eq!(
            block,
            [
                0x76, 0xb8, 0xe0, 0xad, 0xa0, 0xf1, 0x3d, 0x90, 0x40, 0x5d, 0x6a, 0xe5, 0x53, 0x86,
                0xbd, 0x28, 0xbd, 0xd2, 0x19, 0xb8, 0xa0, 0x8d, 0xed, 0x1a, 0xa8, 0x36, 0xef, 0xcc,
                0x8b, 0x77, 0x0d, 0xc7, 0xda, 0x41, 0x59, 0x7c, 0x51, 0x57, 0x48, 0x8d, 0x77, 0x24,
                0xe0, 0x3f, 0xb8, 0xd8, 0x4a, 0x37, 0x6a, 0x43, 0xb8, 0xf4, 0x15, 0x18, 0xa1, 0x1c,
                0xc3, 0x87, 0xb6, 0x69, 0xb2, 0xee, 0x65, 0x86
            ]
        );

        let mut key = [0u8; 32];
        key[31] = 0x01;
        let mut c = ChaCha20::new(&key, &[0u8; 16]);
        let mut block = [0u8; 64];
        c.cipher(&mut block);
        assert_eq!(
            block,
            [
                0x45, 0x40, 0xf0, 0x5a, 0x9f, 0x1f, 0xb2, 0x96, 0xd7, 0x73, 0x6e, 0x7b, 0x20, 0x8e,
                0x3c, 0x96, 0xeb, 0x4f, 0xe1, 0x83, 0x46, 0x88, 0xd2, 0x60, 0x4f, 0x45, 0x09, 0x52,
                0xed, 0x43, 0x2d, 0x41, 0xbb, 0xe2, 0xa0, 0xb6, 0xea, 0x75, 0x66, 0xd2, 0xa5, 0xd1,
                0xe7, 0xe2, 0x0d, 0x42, 0xaf, 0x2c, 0x53, 0xd7, 0x92, 0xb1, 0xc4, 0x3f, 0xea, 0x81,
                0x7e, 0x9a, 0xd2, 0x75, 0xae, 0x54, 0x69, 0x63
            ]
        );

        let mut nonce = [0u8; 16];
        nonce[15] = 0x01;
        let mut c = ChaCha20::new(&[0u8; 32], &nonce);
        let mut block = [0u8; 64];
        c.cipher(&mut block);
        assert_eq!(
            block[..60],
            [
                0xde, 0x9c, 0xba, 0x7b, 0xf3, 0xd6, 0x9e, 0xf5, 0xe7, 0x86, 0xdc, 0x63, 0x97, 0x3f,
                0x65, 0x3a, 0x0b, 0x49, 0xe0, 0x15, 0xad, 0xbf, 0xf7, 0x13, 0x4f, 0xcb, 0x7d, 0xf1,
                0x37, 0x82, 0x10, 0x31, 0xe8, 0x5a, 0x05, 0x02, 0x78, 0xa7, 0x08, 0x45, 0x27, 0x21,
                0x4f, 0x73, 0xef, 0xc7, 0xfa, 0x5b, 0x52, 0x77, 0x06, 0x2e, 0xb7, 0xa0, 0x43, 0x3e,
                0x44, 0x5f, 0x41, 0xe3
            ]
        );

        let mut nonce = [0u8; 16];
        nonce[8] = 0x01;
        let mut c = ChaCha20::new(&[0u8; 32], &nonce);
        let mut block = [0u8; 64];
        c.cipher(&mut block);
        assert_eq!(
            block,
            [
                0xef, 0x3f, 0xdf, 0xd6, 0xc6, 0x15, 0x78, 0xfb, 0xf5, 0xcf, 0x35, 0xbd, 0x3d, 0xd3,
                0x3b, 0x80, 0x09, 0x63, 0x16, 0x34, 0xd2, 0x1e, 0x42, 0xac, 0x33, 0x96, 0x0b, 0xd1,
                0x38, 0xe5, 0x0d, 0x32, 0x11, 0x1e, 0x4c, 0xaf, 0x23, 0x7e, 0xe5, 0x3c, 0xa8, 0xad,
                0x64, 0x26, 0x19, 0x4a, 0x88, 0x54, 0x5d, 0xdc, 0x49, 0x7a, 0x0b, 0x46, 0x6e, 0x7d,
                0x6b, 0xbd, 0xb0, 0x04, 0x1b, 0x2f, 0x58, 0x6b
            ]
        );

        let mut c = ChaCha20::new(
            &[
                0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d,
                0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b,
                0x1c, 0x1d, 0x1e, 0x1f,
            ],
            &[
                0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05,
                0x06, 0x07,
            ],
        );

        let mut block = [0u8; 256];
        c.cipher(&mut block);

        assert_eq!(
            block,
            [
                0xf7, 0x98, 0xa1, 0x89, 0xf1, 0x95, 0xe6, 0x69, 0x82, 0x10, 0x5f, 0xfb, 0x64, 0x0b,
                0xb7, 0x75, 0x7f, 0x57, 0x9d, 0xa3, 0x16, 0x02, 0xfc, 0x93, 0xec, 0x01, 0xac, 0x56,
                0xf8, 0x5a, 0xc3, 0xc1, 0x34, 0xa4, 0x54, 0x7b, 0x73, 0x3b, 0x46, 0x41, 0x30, 0x42,
                0xc9, 0x44, 0x00, 0x49, 0x17, 0x69, 0x05, 0xd3, 0xbe, 0x59, 0xea, 0x1c, 0x53, 0xf1,
                0x59, 0x16, 0x15, 0x5c, 0x2b, 0xe8, 0x24, 0x1a, 0x38, 0x00, 0x8b, 0x9a, 0x26, 0xbc,
                0x35, 0x94, 0x1e, 0x24, 0x44, 0x17, 0x7c, 0x8a, 0xde, 0x66, 0x89, 0xde, 0x95, 0x26,
                0x49, 0x86, 0xd9, 0x58, 0x89, 0xfb, 0x60, 0xe8, 0x46, 0x29, 0xc9, 0xbd, 0x9a, 0x5a,
                0xcb, 0x1c, 0xc1, 0x18, 0xbe, 0x56, 0x3e, 0xb9, 0xb3, 0xa4, 0xa4, 0x72, 0xf8, 0x2e,
                0x09, 0xa7, 0xe7, 0x78, 0x49, 0x2b, 0x56, 0x2e, 0xf7, 0x13, 0x0e, 0x88, 0xdf, 0xe0,
                0x31, 0xc7, 0x9d, 0xb9, 0xd4, 0xf7, 0xc7, 0xa8, 0x99, 0x15, 0x1b, 0x9a, 0x47, 0x50,
                0x32, 0xb6, 0x3f, 0xc3, 0x85, 0x24, 0x5f, 0xe0, 0x54, 0xe3, 0xdd, 0x5a, 0x97, 0xa5,
                0xf5, 0x76, 0xfe, 0x06, 0x40, 0x25, 0xd3, 0xce, 0x04, 0x2c, 0x56, 0x6a, 0xb2, 0xc5,
                0x07, 0xb1, 0x38, 0xdb, 0x85, 0x3e, 0x3d, 0x69, 0x59, 0x66, 0x09, 0x96, 0x54, 0x6c,
                0xc9, 0xc4, 0xa6, 0xea, 0xfd, 0xc7, 0x77, 0xc0, 0x40, 0xd7, 0x0e, 0xaf, 0x46, 0xf7,
                0x6d, 0xad, 0x39, 0x79, 0xe5, 0xc5, 0x36, 0x0c, 0x33, 0x17, 0x16, 0x6a, 0x1c, 0x89,
                0x4c, 0x94, 0xa3, 0x71, 0x87, 0x6a, 0x94, 0xdf, 0x76, 0x28, 0xfe, 0x4e, 0xaa, 0xf2,
                0xcc, 0xb2, 0x7d, 0x5a, 0xaa, 0xe0, 0xad, 0x7a, 0xd0, 0xf9, 0xd4, 0xb6, 0xad, 0x3b,
                0x54, 0x09, 0x87, 0x46, 0xd4, 0x52, 0x4d, 0x38, 0x40, 0x7a, 0x6d, 0xeb, 0x3a, 0xb7,
                0x8f, 0xab, 0x78, 0xc9
            ]
        );
    }

    #[test]
    fn xchacha_test_vectors() {
        // From draft-irtf-cfrg-xchacha-03, A.2
        // https://datatracker.ietf.org/doc/html/draft-irtf-cfrg-xchacha-03#appendix-A.2

        let key = [
            0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d,
            0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b,
            0x9c, 0x9d, 0x9e, 0x9f,
        ];
        let nonce = *b"@ABCDEFGHIJKLMNOPQRSTUVX";
        let mut counter_nonce = [0u8; 16];
        counter_nonce[8..].copy_from_slice(&nonce[16..]);
        let mut c = ChaCha20::new(
            &hchacha20(&key, nonce[..16].try_into().unwrap()),
            &counter_nonce,
        );

        let mut buffer = [
            0x54, 0x68, 0x65, 0x20, 0x64, 0x68, 0x6f, 0x6c, 0x65, 0x20, 0x28, 0x70, 0x72, 0x6f,
            0x6e, 0x6f, 0x75, 0x6e, 0x63, 0x65, 0x64, 0x20, 0x22, 0x64, 0x6f, 0x6c, 0x65, 0x22,
            0x29, 0x20, 0x69, 0x73, 0x20, 0x61, 0x6c, 0x73, 0x6f, 0x20, 0x6b, 0x6e, 0x6f, 0x77,
            0x6e, 0x20, 0x61, 0x73, 0x20, 0x74, 0x68, 0x65, 0x20, 0x41, 0x73, 0x69, 0x61, 0x74,
            0x69, 0x63, 0x20, 0x77, 0x69, 0x6c, 0x64, 0x20, 0x64, 0x6f, 0x67, 0x2c, 0x20, 0x72,
            0x65, 0x64, 0x20, 0x64, 0x6f, 0x67, 0x2c, 0x20, 0x61, 0x6e, 0x64, 0x20, 0x77, 0x68,
            0x69, 0x73, 0x74, 0x6c, 0x69, 0x6e, 0x67, 0x20, 0x64, 0x6f, 0x67, 0x2e, 0x20, 0x49,
            0x74, 0x20, 0x69, 0x73, 0x20, 0x61, 0x62, 0x6f, 0x75, 0x74, 0x20, 0x74, 0x68, 0x65,
            0x20, 0x73, 0x69, 0x7a, 0x65, 0x20, 0x6f, 0x66, 0x20, 0x61, 0x20, 0x47, 0x65, 0x72,
            0x6d, 0x61, 0x6e, 0x20, 0x73, 0x68, 0x65, 0x70, 0x68, 0x65, 0x72, 0x64, 0x20, 0x62,
            0x75, 0x74, 0x20, 0x6c, 0x6f, 0x6f, 0x6b, 0x73, 0x20, 0x6d, 0x6f, 0x72, 0x65, 0x20,
            0x6c, 0x69, 0x6b, 0x65, 0x20, 0x61, 0x20, 0x6c, 0x6f, 0x6e, 0x67, 0x2d, 0x6c, 0x65,
            0x67, 0x67, 0x65, 0x64, 0x20, 0x66, 0x6f, 0x78, 0x2e, 0x20, 0x54, 0x68, 0x69, 0x73,
            0x20, 0x68, 0x69, 0x67, 0x68, 0x6c, 0x79, 0x20, 0x65, 0x6c, 0x75, 0x73, 0x69, 0x76,
            0x65, 0x20, 0x61, 0x6e, 0x64, 0x20, 0x73, 0x6b, 0x69, 0x6c, 0x6c, 0x65, 0x64, 0x20,
            0x6a, 0x75, 0x6d, 0x70, 0x65, 0x72, 0x20, 0x69, 0x73, 0x20, 0x63, 0x6c, 0x61, 0x73,
            0x73, 0x69, 0x66, 0x69, 0x65, 0x64, 0x20, 0x77, 0x69, 0x74, 0x68, 0x20, 0x77, 0x6f,
            0x6c, 0x76, 0x65, 0x73, 0x2c, 0x20, 0x63, 0x6f, 0x79, 0x6f, 0x74, 0x65, 0x73, 0x2c,
            0x20, 0x6a, 0x61, 0x63, 0x6b, 0x61, 0x6c, 0x73, 0x2c, 0x20, 0x61, 0x6e, 0x64, 0x20,
            0x66, 0x6f, 0x78, 0x65, 0x73, 0x20, 0x69, 0x6e, 0x20, 0x74, 0x68, 0x65, 0x20, 0x74,
            0x61, 0x78, 0x6f, 0x6e, 0x6f, 0x6d, 0x69, 0x63, 0x20, 0x66, 0x61, 0x6d, 0x69, 0x6c,
            0x79, 0x20, 0x43, 0x61, 0x6e, 0x69, 0x64, 0x61, 0x65, 0x2e,
        ];
        c.cipher(&mut buffer);

        let expected = [
            0x45, 0x59, 0xab, 0xba, 0x4e, 0x48, 0xc1, 0x61, 0x02, 0xe8, 0xbb, 0x2c, 0x05, 0xe6,
            0x94, 0x7f, 0x50, 0xa7, 0x86, 0xde, 0x16, 0x2f, 0x9b, 0x0b, 0x7e, 0x59, 0x2a, 0x9b,
            0x53, 0xd0, 0xd4, 0xe9, 0x8d, 0x8d, 0x64, 0x10, 0xd5, 0x40, 0xa1, 0xa6, 0x37, 0x5b,
            0x26, 0xd8, 0x0d, 0xac, 0xe4, 0xfa, 0xb5, 0x23, 0x84, 0xc7, 0x31, 0xac, 0xbf, 0x16,
            0xa5, 0x92, 0x3c, 0x0c, 0x48, 0xd3, 0x57, 0x5d, 0x4d, 0x0d, 0x2c, 0x67, 0x3b, 0x66,
            0x6f, 0xaa, 0x73, 0x10, 0x61, 0x27, 0x77, 0x01, 0x09, 0x3a, 0x6b, 0xf7, 0xa1, 0x58,
            0xa8, 0x86, 0x42, 0x92, 0xa4, 0x1c, 0x48, 0xe3, 0xa9, 0xb4, 0xc0, 0xda, 0xec, 0xe0,
            0xf8, 0xd9, 0x8d, 0x0d, 0x7e, 0x05, 0xb3, 0x7a, 0x30, 0x7b, 0xbb, 0x66, 0x33, 0x31,
            0x64, 0xec, 0x9e, 0x1b, 0x24, 0xea, 0x0d, 0x6c, 0x3f, 0xfd, 0xdc, 0xec, 0x4f, 0x68,
            0xe7, 0x44, 0x30, 0x56, 0x19, 0x3a, 0x03, 0xc8, 0x10, 0xe1, 0x13, 0x44, 0xca, 0x06,
            0xd8, 0xed, 0x8a, 0x2b, 0xfb, 0x1e, 0x8d, 0x48, 0xcf, 0xa6, 0xbc, 0x0e, 0xb4, 0xe2,
            0x46, 0x4b, 0x74, 0x81, 0x42, 0x40, 0x7c, 0x9f, 0x43, 0x1a, 0xee, 0x76, 0x99, 0x60,
            0xe1, 0x5b, 0xa8, 0xb9, 0x68, 0x90, 0x46, 0x6e, 0xf2, 0x45, 0x75, 0x99, 0x85, 0x23,
            0x85, 0xc6, 0x61, 0xf7, 0x52, 0xce, 0x20, 0xf9, 0xda, 0x0c, 0x09, 0xab, 0x6b, 0x19,
            0xdf, 0x74, 0xe7, 0x6a, 0x95, 0x96, 0x74, 0x46, 0xf8, 0xd0, 0xfd, 0x41, 0x5e, 0x7b,
            0xee, 0x2a, 0x12, 0xa1, 0x14, 0xc2, 0x0e, 0xb5, 0x29, 0x2a, 0xe7, 0xa3, 0x49, 0xae,
            0x57, 0x78, 0x20, 0xd5, 0x52, 0x0a, 0x1f, 0x3f, 0xb6, 0x2a, 0x17, 0xce, 0x6a, 0x7e,
            0x68, 0xfa, 0x7c, 0x79, 0x11, 0x1d, 0x88, 0x60, 0x92, 0x0b, 0xc0, 0x48, 0xef, 0x43,
            0xfe, 0x84, 0x48, 0x6c, 0xcb, 0x87, 0xc2, 0x5f, 0x0a, 0xe0, 0x45, 0xf0, 0xcc, 0xe1,
            0xe7, 0x98, 0x9a, 0x9a, 0xa2, 0x20, 0xa2, 0x8b, 0xdd, 0x48, 0x27, 0xe7, 0x51, 0xa2,
            0x4a, 0x6d, 0x5c, 0x62, 0xd7, 0x90, 0xa6, 0x63, 0x93, 0xb9, 0x31, 0x11, 0xc1, 0xa5,
            0x5d, 0xd7, 0x42, 0x1a, 0x10, 0x18, 0x49, 0x74, 0xc7, 0xc5,
        ];

        assert_eq!(buffer, expected);
    }
}