rind 0.1.12

// Written for Graviola by Joe Birr-Pixton, 2026.
// SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT-0

// Plastering over the differences between (eg) mlkem_ntt() between x86-64 and aarch64.
//
// Each picks s2n-bignum's AVX2 routine where the CPU has AVX2, and otherwise
// the portable port of mlkem-native's C, which measured faster there than
// libcrux's portable ML-KEM. The two keep coefficients in different orders,
// so a process never mixes them: the CPU's features are read once.

use super::cpu::HaveAvx2;
use crate::low::generic::{mlkem as generic, mlkem_poly as portable};

/// Forward number-theoretic transform from ML-KEM
pub(crate) fn mlkem_ntt(a: &mut [i16; 256]) {
    match HaveAvx2::check() {
        Some(_) => super::mlkem_ntt::mlkem_ntt(a, &Q_DATA.0),
        None => portable::mlkem_ntt(a),
    }
}

/// Inverse number-theoretic transform from ML-KEM
pub(crate) fn mlkem_intt(a: &mut [i16; 256]) {
    match HaveAvx2::check() {
        Some(_) => super::mlkem_intt::mlkem_intt(a, &Q_DATA.0),
        None => portable::mlkem_intt(a),
    }
}

/// Precompute the mulcache data for a polynomial in the NTT domain
pub(crate) fn mlkem_mulcache_compute(x: &mut [i16; 128], a: &[i16; 256]) {
    match HaveAvx2::check() {
        Some(_) => super::mlkem_mulcache_compute::mlkem_mulcache_compute(x, a, &Q_DATA.0),
        None => portable::mlkem_mulcache_compute(x, a),
    }
}

/// Do rejection sampling, where `input` is SHAKE128 bytes, and `r` is a
/// set of polynomial coefficients to write.
///
/// Returns the number of items of `r` written.
pub(crate) fn mlkem_rej_uniform_vartime(r: &mut [i16; 256], input: &[u8]) -> u64 {
    match HaveAvx2::check() {
        Some(_) => super::mlkem_rej_uniform_vartime::mlkem_rej_uniform_vartime(
            r,
            input,
            &MLKEM_REJ_UNIFORM_TABLE,
        ),
        None => portable::mlkem_rej_uniform_vartime(r, input),
    }
}

/// Scalar product of 3-element polynomial vectors in the NTT domain, with
/// the mulcache `bt` of `b`
pub(crate) fn mlkem_basemul_k3(
    r: &mut [i16; 256],
    a: &[i16; 768],
    b: &[i16; 768],
    bt: &[i16; 384],
) {
    match HaveAvx2::check() {
        Some(_) => super::mlkem_basemul_k3::mlkem_basemul_k3(r, a, b, bt),
        None => portable::mlkem_basemul_k3(r, a, b, bt),
    }
}

/// Canonical reduction of polynomial coefficients for ML-KEM
pub(crate) fn mlkem_reduce(a: &mut [i16; 256]) {
    match HaveAvx2::check() {
        Some(_) => super::mlkem_reduce::mlkem_reduce(a),
        None => portable::mlkem_reduce(a),
    }
}

/// Conversion of ML-KEM polynomial coefficients to Montgomery form
pub(crate) fn mlkem_tomont(a: &mut [i16; 256]) {
    match HaveAvx2::check() {
        Some(_) => super::mlkem_tomont::mlkem_tomont(a),
        None => portable::mlkem_tomont(a),
    }
}

/// Unpack ML-KEM polynomial coefficients as 12-bit numbers
pub(crate) fn mlkem_frombytes(r: &mut [i16; 256], a: &[u8; 384]) {
    match HaveAvx2::check() {
        Some(_) => super::mlkem_frombytes::mlkem_frombytes(r, a),
        None => generic::mlkem_frombytes(r, a),
    }
}

/// Pack ML-KEM polynomial coefficients as 12-bit numbers
pub(crate) fn mlkem_tobytes(r: &mut [u8; 384], a: &[i16; 256]) {
    match HaveAvx2::check() {
        Some(_) => super::mlkem_tobytes::mlkem_tobytes(r, a),
        None => portable::mlkem_tobytes(r, a),
    }
}

/// Compress ML-KEM polynomial coefficients in `[0, Q)` to 10 bits each, and pack them
pub(crate) fn mlkem_compress_d10(r: &mut [u8; 320], a: &[i16; 256]) {
    match HaveAvx2::check() {
        Some(_) => super::mlkem_compress::mlkem_compress_d10(r, a),
        None => generic::mlkem_compress_d10(r, a),
    }
}

/// Compress ML-KEM polynomial coefficients in `[0, Q)` to 4 bits each, and pack them
pub(crate) fn mlkem_compress_d4(r: &mut [u8; 128], a: &[i16; 256]) {
    match HaveAvx2::check() {
        Some(_) => super::mlkem_compress::mlkem_compress_d4(r, a),
        None => generic::mlkem_compress_d4(r, a),
    }
}

/// Reorder ML-KEM polynomial coefficients from their natural order into the
/// order the other routines use
pub(crate) fn mlkem_unpack(a: &mut [i16; 256]) {
    match HaveAvx2::check() {
        Some(_) => super::mlkem_unpack::mlkem_unpack(a),
        None => generic::mlkem_unpack(a),
    }
}

#[repr(align(32))]
struct Coeffs<const N: usize>([i16; N]);

const Q_DATA: Coeffs<624> = Coeffs([
    3854, 3340, 2826, 2312, 1798, 1284, 770, 256, 3854, 3340, 2826, 2312, 1798, 1284, 770, 256, 7,
    0, 6, 0, 5, 0, 4, 0, 3, 0, 2, 0, 1, 0, 0, 0, 31498, 31498, 31498, 31498, -758, -758, -758,
    -758, 0, 0, 0, 0, 0, 0, 0, 0, 14745, 14745, 14745, 14745, 14745, 14745, 14745, 14745, 14745,
    14745, 14745, 14745, 14745, 14745, 14745, 14745, -359, -359, -359, -359, -359, -359, -359,
    -359, -359, -359, -359, -359, -359, -359, -359, -359, 13525, 13525, 13525, 13525, 13525, 13525,
    13525, 13525, -12402, -12402, -12402, -12402, -12402, -12402, -12402, -12402, 1493, 1493, 1493,
    1493, 1493, 1493, 1493, 1493, 1422, 1422, 1422, 1422, 1422, 1422, 1422, 1422, -20907, -20907,
    -20907, -20907, 27758, 27758, 27758, 27758, -3799, -3799, -3799, -3799, -15690, -15690, -15690,
    -15690, -171, -171, -171, -171, 622, 622, 622, 622, 1577, 1577, 1577, 1577, 182, 182, 182, 182,
    -5827, -5827, 17363, 17363, -26360, -26360, -29057, -29057, 5571, 5571, -1102, -1102, 21438,
    21438, -26242, -26242, 573, 573, -1325, -1325, 264, 264, 383, 383, -829, -829, 1458, 1458,
    -1602, -1602, -130, -130, -5689, -6516, 1496, 30967, -23565, 20179, 20710, 25080, -12796,
    26616, 16064, -12442, 9134, -650, -25986, 27837, 1223, 652, -552, 1015, -1293, 1491, -282,
    -1544, 516, -8, -320, -666, -1618, -1162, 126, 1469, -335, -11477, -32227, 20494, -27738, 945,
    -14883, 6182, 32010, 10631, 29175, -28762, -18486, 17560, -14430, -5276, -1103, 555, -1251,
    1550, 422, 177, -291, 1574, -246, 1159, -777, -602, -1590, -872, 418, -156, 11182, 13387,
    -14233, -21655, 13131, -4587, 23092, 5493, -32502, 30317, -18741, 12639, 20100, 18525, 19529,
    -12619, 430, 843, 871, 105, 587, -235, -460, 1653, 778, -147, 1483, 1119, 644, 349, 329, -75,
    787, 787, 787, 787, 787, 787, 787, 787, 787, 787, 787, 787, 787, 787, 787, 787, -1517, -1517,
    -1517, -1517, -1517, -1517, -1517, -1517, -1517, -1517, -1517, -1517, -1517, -1517, -1517,
    -1517, 28191, 28191, 28191, 28191, 28191, 28191, 28191, 28191, -16694, -16694, -16694, -16694,
    -16694, -16694, -16694, -16694, 287, 287, 287, 287, 287, 287, 287, 287, 202, 202, 202, 202,
    202, 202, 202, 202, 10690, 10690, 10690, 10690, 1358, 1358, 1358, 1358, -11202, -11202, -11202,
    -11202, 31164, 31164, 31164, 31164, 962, 962, 962, 962, -1202, -1202, -1202, -1202, -1474,
    -1474, -1474, -1474, 1468, 1468, 1468, 1468, -28073, -28073, 24313, 24313, -10532, -10532,
    8800, 8800, 18426, 18426, 8859, 8859, 26675, 26675, -16163, -16163, -681, -681, 1017, 1017,
    732, 732, 608, 608, -1542, -1542, 411, 411, -205, -205, -1571, -1571, 19883, -28250, -15887,
    -8898, -28309, 9075, -30199, 18249, 13426, 14017, -29156, -12757, 16832, 4311, -24155, -17915,
    -853, -90, -271, 830, 107, -1421, -247, -951, -398, 961, -1508, -725, 448, -1065, 677, -1275,
    -31183, 25435, -7382, 24391, -20927, 10946, 24214, 16989, 10335, -7934, -22502, 10906, 31636,
    28644, 23998, -17422, 817, 603, 1322, -1465, -1215, 1218, -874, -1187, -1185, -1278, -1510,
    -870, -108, 996, 958, 1522, 20297, 2146, 15355, -32384, -6280, -14903, -11044, 14469, -21498,
    -20198, 23210, -17442, -23860, -20257, 7756, 23132, 1097, 610, -1285, 384, -136, -1335, 220,
    -1659, -1530, 794, -854, 478, -308, 991, -1460, 1628, -1103, 555, -1251, 1550, 422, 177, -291,
    1574, -246, 1159, -777, -602, -1590, -872, 418, -156, 430, 843, 871, 105, 587, -235, -460,
    1653, 778, -147, 1483, 1119, 644, 349, 329, -75, 817, 603, 1322, -1465, -1215, 1218, -874,
    -1187, -1185, -1278, -1510, -870, -108, 996, 958, 1522, 1097, 610, -1285, 384, -136, -1335,
    220, -1659, -1530, 794, -854, 478, -308, 991, -1460, 1628, -335, -11477, -32227, 20494, -27738,
    945, -14883, 6182, 32010, 10631, 29175, -28762, -18486, 17560, -14430, -5276, 11182, 13387,
    -14233, -21655, 13131, -4587, 23092, 5493, -32502, 30317, -18741, 12639, 20100, 18525, 19529,
    -12619, -31183, 25435, -7382, 24391, -20927, 10946, 24214, 16989, 10335, -7934, -22502, 10906,
    31636, 28644, 23998, -17422, 20297, 2146, 15355, -32384, -6280, -14903, -11044, 14469, -21498,
    -20198, 23210, -17442, -23860, -20257, 7756, 23132,
]);

const MLKEM_REJ_UNIFORM_TABLE: [i8; 4096] = [
    -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 0
    0, 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 1
    2, 3, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 2
    0, 1, 2, 3, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 3
    4, 5, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 4
    0, 1, 4, 5, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 5
    2, 3, 4, 5, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 6
    0, 1, 2, 3, 4, 5, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 7
    6, 7, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 8
    0, 1, 6, 7, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 9
    2, 3, 6, 7, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 10
    0, 1, 2, 3, 6, 7, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 11
    4, 5, 6, 7, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 12
    0, 1, 4, 5, 6, 7, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 13
    2, 3, 4, 5, 6, 7, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 14
    0, 1, 2, 3, 4, 5, 6, 7, -1, -1, -1, -1, -1, -1, -1, -1, // 15
    8, 9, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 16
    0, 1, 8, 9, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 17
    2, 3, 8, 9, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 18
    0, 1, 2, 3, 8, 9, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 19
    4, 5, 8, 9, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 20
    0, 1, 4, 5, 8, 9, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 21
    2, 3, 4, 5, 8, 9, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 22
    0, 1, 2, 3, 4, 5, 8, 9, -1, -1, -1, -1, -1, -1, -1, -1, // 23
    6, 7, 8, 9, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 24
    0, 1, 6, 7, 8, 9, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 25
    2, 3, 6, 7, 8, 9, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 26
    0, 1, 2, 3, 6, 7, 8, 9, -1, -1, -1, -1, -1, -1, -1, -1, // 27
    4, 5, 6, 7, 8, 9, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 28
    0, 1, 4, 5, 6, 7, 8, 9, -1, -1, -1, -1, -1, -1, -1, -1, // 29
    2, 3, 4, 5, 6, 7, 8, 9, -1, -1, -1, -1, -1, -1, -1, -1, // 30
    0, 1, 2, 3, 4, 5, 6, 7, 8, 9, -1, -1, -1, -1, -1, -1, // 31
    10, 11, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 32
    0, 1, 10, 11, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 33
    2, 3, 10, 11, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 34
    0, 1, 2, 3, 10, 11, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 35
    4, 5, 10, 11, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 36
    0, 1, 4, 5, 10, 11, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 37
    2, 3, 4, 5, 10, 11, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 38
    0, 1, 2, 3, 4, 5, 10, 11, -1, -1, -1, -1, -1, -1, -1, -1, // 39
    6, 7, 10, 11, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 40
    0, 1, 6, 7, 10, 11, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 41
    2, 3, 6, 7, 10, 11, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 42
    0, 1, 2, 3, 6, 7, 10, 11, -1, -1, -1, -1, -1, -1, -1, -1, // 43
    4, 5, 6, 7, 10, 11, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 44
    0, 1, 4, 5, 6, 7, 10, 11, -1, -1, -1, -1, -1, -1, -1, -1, // 45
    2, 3, 4, 5, 6, 7, 10, 11, -1, -1, -1, -1, -1, -1, -1, -1, // 46
    0, 1, 2, 3, 4, 5, 6, 7, 10, 11, -1, -1, -1, -1, -1, -1, // 47
    8, 9, 10, 11, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 48
    0, 1, 8, 9, 10, 11, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 49
    2, 3, 8, 9, 10, 11, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 50
    0, 1, 2, 3, 8, 9, 10, 11, -1, -1, -1, -1, -1, -1, -1, -1, // 51
    4, 5, 8, 9, 10, 11, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 52
    0, 1, 4, 5, 8, 9, 10, 11, -1, -1, -1, -1, -1, -1, -1, -1, // 53
    2, 3, 4, 5, 8, 9, 10, 11, -1, -1, -1, -1, -1, -1, -1, -1, // 54
    0, 1, 2, 3, 4, 5, 8, 9, 10, 11, -1, -1, -1, -1, -1, -1, // 55
    6, 7, 8, 9, 10, 11, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 56
    0, 1, 6, 7, 8, 9, 10, 11, -1, -1, -1, -1, -1, -1, -1, -1, // 57
    2, 3, 6, 7, 8, 9, 10, 11, -1, -1, -1, -1, -1, -1, -1, -1, // 58
    0, 1, 2, 3, 6, 7, 8, 9, 10, 11, -1, -1, -1, -1, -1, -1, // 59
    4, 5, 6, 7, 8, 9, 10, 11, -1, -1, -1, -1, -1, -1, -1, -1, // 60
    0, 1, 4, 5, 6, 7, 8, 9, 10, 11, -1, -1, -1, -1, -1, -1, // 61
    2, 3, 4, 5, 6, 7, 8, 9, 10, 11, -1, -1, -1, -1, -1, -1, // 62
    0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, -1, -1, -1, -1, // 63
    12, 13, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 64
    0, 1, 12, 13, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 65
    2, 3, 12, 13, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 66
    0, 1, 2, 3, 12, 13, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 67
    4, 5, 12, 13, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 68
    0, 1, 4, 5, 12, 13, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 69
    2, 3, 4, 5, 12, 13, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 70
    0, 1, 2, 3, 4, 5, 12, 13, -1, -1, -1, -1, -1, -1, -1, -1, // 71
    6, 7, 12, 13, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 72
    0, 1, 6, 7, 12, 13, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 73
    2, 3, 6, 7, 12, 13, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 74
    0, 1, 2, 3, 6, 7, 12, 13, -1, -1, -1, -1, -1, -1, -1, -1, // 75
    4, 5, 6, 7, 12, 13, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 76
    0, 1, 4, 5, 6, 7, 12, 13, -1, -1, -1, -1, -1, -1, -1, -1, // 77
    2, 3, 4, 5, 6, 7, 12, 13, -1, -1, -1, -1, -1, -1, -1, -1, // 78
    0, 1, 2, 3, 4, 5, 6, 7, 12, 13, -1, -1, -1, -1, -1, -1, // 79
    8, 9, 12, 13, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 80
    0, 1, 8, 9, 12, 13, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 81
    2, 3, 8, 9, 12, 13, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 82
    0, 1, 2, 3, 8, 9, 12, 13, -1, -1, -1, -1, -1, -1, -1, -1, // 83
    4, 5, 8, 9, 12, 13, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 84
    0, 1, 4, 5, 8, 9, 12, 13, -1, -1, -1, -1, -1, -1, -1, -1, // 85
    2, 3, 4, 5, 8, 9, 12, 13, -1, -1, -1, -1, -1, -1, -1, -1, // 86
    0, 1, 2, 3, 4, 5, 8, 9, 12, 13, -1, -1, -1, -1, -1, -1, // 87
    6, 7, 8, 9, 12, 13, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 88
    0, 1, 6, 7, 8, 9, 12, 13, -1, -1, -1, -1, -1, -1, -1, -1, // 89
    2, 3, 6, 7, 8, 9, 12, 13, -1, -1, -1, -1, -1, -1, -1, -1, // 90
    0, 1, 2, 3, 6, 7, 8, 9, 12, 13, -1, -1, -1, -1, -1, -1, // 91
    4, 5, 6, 7, 8, 9, 12, 13, -1, -1, -1, -1, -1, -1, -1, -1, // 92
    0, 1, 4, 5, 6, 7, 8, 9, 12, 13, -1, -1, -1, -1, -1, -1, // 93
    2, 3, 4, 5, 6, 7, 8, 9, 12, 13, -1, -1, -1, -1, -1, -1, // 94
    0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 12, 13, -1, -1, -1, -1, // 95
    10, 11, 12, 13, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 96
    0, 1, 10, 11, 12, 13, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 97
    2, 3, 10, 11, 12, 13, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 98
    0, 1, 2, 3, 10, 11, 12, 13, -1, -1, -1, -1, -1, -1, -1, -1, // 99
    4, 5, 10, 11, 12, 13, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 100
    0, 1, 4, 5, 10, 11, 12, 13, -1, -1, -1, -1, -1, -1, -1, -1, // 101
    2, 3, 4, 5, 10, 11, 12, 13, -1, -1, -1, -1, -1, -1, -1, -1, // 102
    0, 1, 2, 3, 4, 5, 10, 11, 12, 13, -1, -1, -1, -1, -1, -1, // 103
    6, 7, 10, 11, 12, 13, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 104
    0, 1, 6, 7, 10, 11, 12, 13, -1, -1, -1, -1, -1, -1, -1, -1, // 105
    2, 3, 6, 7, 10, 11, 12, 13, -1, -1, -1, -1, -1, -1, -1, -1, // 106
    0, 1, 2, 3, 6, 7, 10, 11, 12, 13, -1, -1, -1, -1, -1, -1, // 107
    4, 5, 6, 7, 10, 11, 12, 13, -1, -1, -1, -1, -1, -1, -1, -1, // 108
    0, 1, 4, 5, 6, 7, 10, 11, 12, 13, -1, -1, -1, -1, -1, -1, // 109
    2, 3, 4, 5, 6, 7, 10, 11, 12, 13, -1, -1, -1, -1, -1, -1, // 110
    0, 1, 2, 3, 4, 5, 6, 7, 10, 11, 12, 13, -1, -1, -1, -1, // 111
    8, 9, 10, 11, 12, 13, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 112
    0, 1, 8, 9, 10, 11, 12, 13, -1, -1, -1, -1, -1, -1, -1, -1, // 113
    2, 3, 8, 9, 10, 11, 12, 13, -1, -1, -1, -1, -1, -1, -1, -1, // 114
    0, 1, 2, 3, 8, 9, 10, 11, 12, 13, -1, -1, -1, -1, -1, -1, // 115
    4, 5, 8, 9, 10, 11, 12, 13, -1, -1, -1, -1, -1, -1, -1, -1, // 116
    0, 1, 4, 5, 8, 9, 10, 11, 12, 13, -1, -1, -1, -1, -1, -1, // 117
    2, 3, 4, 5, 8, 9, 10, 11, 12, 13, -1, -1, -1, -1, -1, -1, // 118
    0, 1, 2, 3, 4, 5, 8, 9, 10, 11, 12, 13, -1, -1, -1, -1, // 119
    6, 7, 8, 9, 10, 11, 12, 13, -1, -1, -1, -1, -1, -1, -1, -1, // 120
    0, 1, 6, 7, 8, 9, 10, 11, 12, 13, -1, -1, -1, -1, -1, -1, // 121
    2, 3, 6, 7, 8, 9, 10, 11, 12, 13, -1, -1, -1, -1, -1, -1, // 122
    0, 1, 2, 3, 6, 7, 8, 9, 10, 11, 12, 13, -1, -1, -1, -1, // 123
    4, 5, 6, 7, 8, 9, 10, 11, 12, 13, -1, -1, -1, -1, -1, -1, // 124
    0, 1, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, -1, -1, -1, -1, // 125
    2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, -1, -1, -1, -1, // 126
    0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, -1, -1, // 127
    14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 128
    0, 1, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 129
    2, 3, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 130
    0, 1, 2, 3, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 131
    4, 5, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 132
    0, 1, 4, 5, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 133
    2, 3, 4, 5, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 134
    0, 1, 2, 3, 4, 5, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, // 135
    6, 7, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 136
    0, 1, 6, 7, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 137
    2, 3, 6, 7, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 138
    0, 1, 2, 3, 6, 7, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, // 139
    4, 5, 6, 7, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 140
    0, 1, 4, 5, 6, 7, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, // 141
    2, 3, 4, 5, 6, 7, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, // 142
    0, 1, 2, 3, 4, 5, 6, 7, 14, 15, -1, -1, -1, -1, -1, -1, // 143
    8, 9, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 144
    0, 1, 8, 9, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 145
    2, 3, 8, 9, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 146
    0, 1, 2, 3, 8, 9, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, // 147
    4, 5, 8, 9, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 148
    0, 1, 4, 5, 8, 9, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, // 149
    2, 3, 4, 5, 8, 9, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, // 150
    0, 1, 2, 3, 4, 5, 8, 9, 14, 15, -1, -1, -1, -1, -1, -1, // 151
    6, 7, 8, 9, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 152
    0, 1, 6, 7, 8, 9, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, // 153
    2, 3, 6, 7, 8, 9, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, // 154
    0, 1, 2, 3, 6, 7, 8, 9, 14, 15, -1, -1, -1, -1, -1, -1, // 155
    4, 5, 6, 7, 8, 9, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, // 156
    0, 1, 4, 5, 6, 7, 8, 9, 14, 15, -1, -1, -1, -1, -1, -1, // 157
    2, 3, 4, 5, 6, 7, 8, 9, 14, 15, -1, -1, -1, -1, -1, -1, // 158
    0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 14, 15, -1, -1, -1, -1, // 159
    10, 11, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 160
    0, 1, 10, 11, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 161
    2, 3, 10, 11, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 162
    0, 1, 2, 3, 10, 11, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, // 163
    4, 5, 10, 11, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 164
    0, 1, 4, 5, 10, 11, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, // 165
    2, 3, 4, 5, 10, 11, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, // 166
    0, 1, 2, 3, 4, 5, 10, 11, 14, 15, -1, -1, -1, -1, -1, -1, // 167
    6, 7, 10, 11, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 168
    0, 1, 6, 7, 10, 11, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, // 169
    2, 3, 6, 7, 10, 11, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, // 170
    0, 1, 2, 3, 6, 7, 10, 11, 14, 15, -1, -1, -1, -1, -1, -1, // 171
    4, 5, 6, 7, 10, 11, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, // 172
    0, 1, 4, 5, 6, 7, 10, 11, 14, 15, -1, -1, -1, -1, -1, -1, // 173
    2, 3, 4, 5, 6, 7, 10, 11, 14, 15, -1, -1, -1, -1, -1, -1, // 174
    0, 1, 2, 3, 4, 5, 6, 7, 10, 11, 14, 15, -1, -1, -1, -1, // 175
    8, 9, 10, 11, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 176
    0, 1, 8, 9, 10, 11, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, // 177
    2, 3, 8, 9, 10, 11, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, // 178
    0, 1, 2, 3, 8, 9, 10, 11, 14, 15, -1, -1, -1, -1, -1, -1, // 179
    4, 5, 8, 9, 10, 11, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, // 180
    0, 1, 4, 5, 8, 9, 10, 11, 14, 15, -1, -1, -1, -1, -1, -1, // 181
    2, 3, 4, 5, 8, 9, 10, 11, 14, 15, -1, -1, -1, -1, -1, -1, // 182
    0, 1, 2, 3, 4, 5, 8, 9, 10, 11, 14, 15, -1, -1, -1, -1, // 183
    6, 7, 8, 9, 10, 11, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, // 184
    0, 1, 6, 7, 8, 9, 10, 11, 14, 15, -1, -1, -1, -1, -1, -1, // 185
    2, 3, 6, 7, 8, 9, 10, 11, 14, 15, -1, -1, -1, -1, -1, -1, // 186
    0, 1, 2, 3, 6, 7, 8, 9, 10, 11, 14, 15, -1, -1, -1, -1, // 187
    4, 5, 6, 7, 8, 9, 10, 11, 14, 15, -1, -1, -1, -1, -1, -1, // 188
    0, 1, 4, 5, 6, 7, 8, 9, 10, 11, 14, 15, -1, -1, -1, -1, // 189
    2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 14, 15, -1, -1, -1, -1, // 190
    0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 14, 15, -1, -1, // 191
    12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 192
    0, 1, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 193
    2, 3, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 194
    0, 1, 2, 3, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, // 195
    4, 5, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 196
    0, 1, 4, 5, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, // 197
    2, 3, 4, 5, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, // 198
    0, 1, 2, 3, 4, 5, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, // 199
    6, 7, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 200
    0, 1, 6, 7, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, // 201
    2, 3, 6, 7, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, // 202
    0, 1, 2, 3, 6, 7, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, // 203
    4, 5, 6, 7, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, // 204
    0, 1, 4, 5, 6, 7, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, // 205
    2, 3, 4, 5, 6, 7, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, // 206
    0, 1, 2, 3, 4, 5, 6, 7, 12, 13, 14, 15, -1, -1, -1, -1, // 207
    8, 9, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 208
    0, 1, 8, 9, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, // 209
    2, 3, 8, 9, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, // 210
    0, 1, 2, 3, 8, 9, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, // 211
    4, 5, 8, 9, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, // 212
    0, 1, 4, 5, 8, 9, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, // 213
    2, 3, 4, 5, 8, 9, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, // 214
    0, 1, 2, 3, 4, 5, 8, 9, 12, 13, 14, 15, -1, -1, -1, -1, // 215
    6, 7, 8, 9, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, // 216
    0, 1, 6, 7, 8, 9, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, // 217
    2, 3, 6, 7, 8, 9, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, // 218
    0, 1, 2, 3, 6, 7, 8, 9, 12, 13, 14, 15, -1, -1, -1, -1, // 219
    4, 5, 6, 7, 8, 9, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, // 220
    0, 1, 4, 5, 6, 7, 8, 9, 12, 13, 14, 15, -1, -1, -1, -1, // 221
    2, 3, 4, 5, 6, 7, 8, 9, 12, 13, 14, 15, -1, -1, -1, -1, // 222
    0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 12, 13, 14, 15, -1, -1, // 223
    10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 224
    0, 1, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, // 225
    2, 3, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, // 226
    0, 1, 2, 3, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, // 227
    4, 5, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, // 228
    0, 1, 4, 5, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, // 229
    2, 3, 4, 5, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, // 230
    0, 1, 2, 3, 4, 5, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, // 231
    6, 7, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, // 232
    0, 1, 6, 7, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, // 233
    2, 3, 6, 7, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, // 234
    0, 1, 2, 3, 6, 7, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, // 235
    4, 5, 6, 7, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, // 236
    0, 1, 4, 5, 6, 7, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, // 237
    2, 3, 4, 5, 6, 7, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, // 238
    0, 1, 2, 3, 4, 5, 6, 7, 10, 11, 12, 13, 14, 15, -1, -1, // 239
    8, 9, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, // 240
    0, 1, 8, 9, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, // 241
    2, 3, 8, 9, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, // 242
    0, 1, 2, 3, 8, 9, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, // 243
    4, 5, 8, 9, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, // 244
    0, 1, 4, 5, 8, 9, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, // 245
    2, 3, 4, 5, 8, 9, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, // 246
    0, 1, 2, 3, 4, 5, 8, 9, 10, 11, 12, 13, 14, 15, -1, -1, // 247
    6, 7, 8, 9, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, // 248
    0, 1, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, // 249
    2, 3, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, // 250
    0, 1, 2, 3, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, -1, -1, // 251
    4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, // 252
    0, 1, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, -1, -1, // 253
    2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, -1, -1, // 254
    0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, // 255
];

#[cfg(test)]
#[cfg_attr(coverage_nightly, coverage(off))]
mod tests {
    use super::*;

    // AVX2 hides the portable compression from every other test too, so it
    // is checked against mlkem-native's here.
    #[test]
    fn compress_matches_portable() {
        if HaveAvx2::check().is_none() {
            return;
        }
        // Every coefficient value in [0, Q).
        for start in (0..3329 + 7).step_by(256 - 7) {
            let a = Coeffs::<256>(core::array::from_fn(|i| ((start + i) % 3329) as i16));

            let (mut want, mut got) = ([0; 320], [0; 320]);
            generic::mlkem_compress_d10(&mut want, &a.0);
            super::super::mlkem_compress::mlkem_compress_d10(&mut got, &a.0);
            assert_eq!(want, got, "d10, from {start}");

            let (mut want, mut got) = ([0; 128], [0; 128]);
            generic::mlkem_compress_d4(&mut want, &a.0);
            super::super::mlkem_compress::mlkem_compress_d4(&mut got, &a.0);
            assert_eq!(want, got, "d4, from {start}");
        }
    }

    // AVX2, where present, hides the portable routines from every other
    // test, so each is checked against s2n-bignum's. Those keep NTT-domain
    // coefficients in their own order: `mlkem_unpack` converts natural order
    // to it, and `mlkem_tobytes` converts back as it packs.
    #[test]
    fn portable_matches_avx2() {
        if HaveAvx2::check().is_none() {
            return;
        }
        let mut seed = 0x9e37_79b9_7f4a_7c15u64;
        let mut next = move |bound: i16| {
            seed ^= seed << 13;
            seed ^= seed >> 7;
            seed ^= seed << 17;
            (seed % (2 * bound as u64 - 1)) as i16 - (bound - 1)
        };
        // The AVX2 routines load aligned vectors.
        let to_avx2 = |x: &[i16; 256]| {
            let mut y = Coeffs(*x);
            super::super::mlkem_unpack::mlkem_unpack(&mut y.0);
            y
        };
        let bytes_avx2 = |mut y: Coeffs<256>| {
            super::super::mlkem_reduce::mlkem_reduce(&mut y.0);
            let mut out = [0; 384];
            super::super::mlkem_tobytes::mlkem_tobytes(&mut out, &y.0);
            out
        };
        let bytes_portable = |mut x: [i16; 256]| {
            portable::mlkem_reduce(&mut x);
            let mut out = [0; 384];
            portable::mlkem_tobytes(&mut out, &x);
            out
        };

        for round in 0..16 {
            let x: [i16; 256] = core::array::from_fn(|_| next(3329));
            // The forward NTT takes natural order to the AVX2 routines' own,
            // and the inverse takes it back.
            let mut want = Coeffs(x);
            super::super::mlkem_ntt::mlkem_ntt(&mut want.0, &Q_DATA.0);
            let mut got = x;
            portable::mlkem_ntt(&mut got);
            assert_eq!(bytes_portable(got), bytes_avx2(want), "ntt, round {round}");

            let mut want = to_avx2(&x);
            super::super::mlkem_intt::mlkem_intt(&mut want.0, &Q_DATA.0);
            let mut got = x;
            portable::mlkem_intt(&mut got);
            assert_eq!(
                bytes_portable(got),
                bytes_portable(want.0),
                "intt, round {round}"
            );

            type Op = (&'static str, fn(&mut [i16; 256]), fn(&mut [i16; 256]));
            let ops: [Op; 2] = [
                (
                    "tomont",
                    super::super::mlkem_tomont::mlkem_tomont,
                    portable::mlkem_tomont,
                ),
                (
                    "reduce",
                    super::super::mlkem_reduce::mlkem_reduce,
                    portable::mlkem_reduce,
                ),
            ];
            for (name, avx2, port) in ops {
                let mut want = to_avx2(&x);
                avx2(&mut want.0);
                let mut got = x;
                port(&mut got);
                assert_eq!(
                    bytes_portable(got),
                    bytes_avx2(want),
                    "{name}, round {round}"
                );
            }

            // Base multiplication, whose inputs are at most 2^12.
            let a: [i16; 768] = core::array::from_fn(|_| next(4096));
            let b: [i16; 768] = core::array::from_fn(|_| next(4096));
            let (mut a_avx2, mut b_avx2) = (Coeffs([0; 768]), Coeffs([0; 768]));
            let (mut bt, mut bt_avx2) = ([0; 384], Coeffs([0; 384]));
            for k in 0..3 {
                let poly = |v: &[i16; 768]| -> [i16; 256] {
                    v[256 * k..256 * (k + 1)].try_into().unwrap()
                };
                a_avx2.0[256 * k..256 * (k + 1)].copy_from_slice(&to_avx2(&poly(&a)).0);
                b_avx2.0[256 * k..256 * (k + 1)].copy_from_slice(&to_avx2(&poly(&b)).0);
                portable::mlkem_mulcache_compute(
                    (&mut bt[128 * k..128 * (k + 1)]).try_into().unwrap(),
                    &poly(&b),
                );
                super::super::mlkem_mulcache_compute::mlkem_mulcache_compute(
                    (&mut bt_avx2.0[128 * k..128 * (k + 1)]).try_into().unwrap(),
                    b_avx2.0[256 * k..256 * (k + 1)].try_into().unwrap(),
                    &Q_DATA.0,
                );
            }
            let mut want = Coeffs([0; 256]);
            super::super::mlkem_basemul_k3::mlkem_basemul_k3(
                &mut want.0,
                &a_avx2.0,
                &b_avx2.0,
                &bt_avx2.0,
            );
            let mut got = [0; 256];
            portable::mlkem_basemul_k3(&mut got, &a, &b, &bt);
            assert_eq!(
                bytes_portable(got),
                bytes_avx2(want),
                "basemul, round {round}"
            );

            // Packing, from canonical coefficients.
            let mut canonical = x;
            portable::mlkem_reduce(&mut canonical);
            let packed = bytes_portable(canonical);
            let mut want = Coeffs([0; 256]);
            super::super::mlkem_frombytes::mlkem_frombytes(&mut want.0, &packed);
            let mut got = [0; 256];
            generic::mlkem_frombytes(&mut got, &packed);
            assert_eq!(to_avx2(&got).0, want.0, "frombytes, round {round}");
            assert_eq!(
                bytes_avx2(to_avx2(&canonical)),
                packed,
                "tobytes, round {round}"
            );

            // Rejection sampling: short input, and enough to fill `r`.
            let input: Vec<u8> = (0..840).map(|_| next(256) as u8).collect();
            for len in [12, 48, 504, 840] {
                let (mut want, mut got) = (Coeffs([0; 256]), [0; 256]);
                let want_n = super::super::mlkem_rej_uniform_vartime::mlkem_rej_uniform_vartime(
                    &mut want.0,
                    &input[..len],
                    &MLKEM_REJ_UNIFORM_TABLE,
                );
                let got_n = portable::mlkem_rej_uniform_vartime(&mut got, &input[..len]);
                assert_eq!(
                    (got_n, got),
                    (want_n, want.0),
                    "rej_uniform, {len} bytes, round {round}"
                );
            }
        }
    }
}