use crate::keccak::{PI, RHO};
use std::arch::x86_64::*;
const ROT_ROW: [[u64; 8]; 5] = [
[
RHO[0] as u64,
RHO[1] as u64,
RHO[2] as u64,
RHO[3] as u64,
RHO[4] as u64,
0,
0,
0,
],
[
RHO[5] as u64,
RHO[6] as u64,
RHO[7] as u64,
RHO[8] as u64,
RHO[9] as u64,
0,
0,
0,
],
[
RHO[10] as u64,
RHO[11] as u64,
RHO[12] as u64,
RHO[13] as u64,
RHO[14] as u64,
0,
0,
0,
],
[
RHO[15] as u64,
RHO[16] as u64,
RHO[17] as u64,
RHO[18] as u64,
RHO[19] as u64,
0,
0,
0,
],
[
RHO[20] as u64,
RHO[21] as u64,
RHO[22] as u64,
RHO[23] as u64,
RHO[24] as u64,
0,
0,
0,
],
];
const THETA_MINUS1_IDX: [u64; 8] = [4, 0, 1, 2, 3, 0, 0, 0];
const THETA_PLUS1_IDX: [u64; 8] = [1, 2, 3, 4, 0, 0, 0, 0];
const CHI_SHIFT1_IDX: [u64; 8] = [1, 2, 3, 4, 0, 0, 0, 0];
const CHI_SHIFT2_IDX: [u64; 8] = [2, 3, 4, 0, 1, 0, 0, 0];
const CHI_TERNARY_IMM: i32 = 0xD2;
const fn source_row(j: usize) -> usize {
PI[j] / 5
}
const fn source_lane(j: usize) -> usize {
PI[j] % 5
}
const fn pi_mask(z: usize, y: usize) -> u8 {
let mut mask = 0u8;
let mut x = 0;
while x < 5 {
if source_row(5 * z + x) == y {
mask |= 1 << x;
}
x += 1;
}
mask
}
const fn pi_idx(z: usize, y: usize) -> [u64; 8] {
let mut idx = [0u64; 8];
let mut x = 0;
while x < 5 {
if source_row(5 * z + x) == y {
idx[x] = source_lane(5 * z + x) as u64;
}
x += 1;
}
idx
}
const fn build_pi_masks() -> [[u8; 5]; 5] {
let mut out = [[0u8; 5]; 5];
let mut z = 0;
while z < 5 {
let mut y = 0;
while y < 5 {
out[z][y] = pi_mask(z, y);
y += 1;
}
z += 1;
}
out
}
const fn build_pi_idxs() -> [[[u64; 8]; 5]; 5] {
let mut out = [[[0u64; 8]; 5]; 5];
let mut z = 0;
while z < 5 {
let mut y = 0;
while y < 5 {
out[z][y] = pi_idx(z, y);
y += 1;
}
z += 1;
}
out
}
const PI_MASKS: [[u8; 5]; 5] = build_pi_masks();
const PI_IDXS: [[[u64; 8]; 5]; 5] = build_pi_idxs();
use crate::keccak::round_constants;
#[inline]
unsafe fn load(v: &[u64; 8]) -> __m512i {
_mm512_loadu_si512(v.as_ptr() as *const i32 as *const _)
}
#[target_feature(enable = "avx512f")]
pub unsafe fn keccak_f1600_avx512(state: &mut [u64; 25]) {
let mut padded_in = [0u64; 40];
for y in 0..5 {
padded_in[y * 8..y * 8 + 5].copy_from_slice(&state[y * 5..y * 5 + 5]);
}
let mut row: [__m512i; 5] = [
load(padded_in[0..8].try_into().unwrap()),
load(padded_in[8..16].try_into().unwrap()),
load(padded_in[16..24].try_into().unwrap()),
load(padded_in[24..32].try_into().unwrap()),
load(padded_in[32..40].try_into().unwrap()),
];
let theta_minus1 = load(&THETA_MINUS1_IDX);
let theta_plus1 = load(&THETA_PLUS1_IDX);
let chi_shift1 = load(&CHI_SHIFT1_IDX);
let chi_shift2 = load(&CHI_SHIFT2_IDX);
let rot: [__m512i; 5] = [
load(&ROT_ROW[0]),
load(&ROT_ROW[1]),
load(&ROT_ROW[2]),
load(&ROT_ROW[3]),
load(&ROT_ROW[4]),
];
let mut pi_idx_vecs: [__m512i; 25] = [_mm512_setzero_si512(); 25];
for z in 0..5 {
for y in 0..5 {
pi_idx_vecs[z * 5 + y] = load(&PI_IDXS[z][y]);
}
}
for &rc in round_constants().iter() {
let mut c = row[0];
c = _mm512_xor_si512(c, row[1]);
c = _mm512_xor_si512(c, row[2]);
c = _mm512_xor_si512(c, row[3]);
c = _mm512_xor_si512(c, row[4]);
let c_minus1 = _mm512_permutexvar_epi64(theta_minus1, c);
let c_plus1 = _mm512_permutexvar_epi64(theta_plus1, c);
let c_plus1_rot1 = _mm512_rol_epi64::<1>(c_plus1);
let d = _mm512_xor_si512(c_minus1, c_plus1_rot1);
for r in row.iter_mut() {
*r = _mm512_xor_si512(*r, d);
}
let mut rotated = [_mm512_setzero_si512(); 5];
for y in 0..5 {
rotated[y] = _mm512_rolv_epi64(row[y], rot[y]);
}
let mut new_row = [_mm512_setzero_si512(); 5];
for z in 0..5 {
let mut acc = _mm512_setzero_si512();
for y in 0..5 {
let candidate = _mm512_permutexvar_epi64(pi_idx_vecs[z * 5 + y], rotated[y]);
acc = _mm512_mask_blend_epi64(PI_MASKS[z][y], acc, candidate);
}
new_row[z] = acc;
}
for z in 0..5 {
let a = new_row[z];
let b = _mm512_permutexvar_epi64(chi_shift1, a);
let c = _mm512_permutexvar_epi64(chi_shift2, a);
row[z] = _mm512_ternarylogic_epi64::<CHI_TERNARY_IMM>(a, b, c);
}
let mut rc_lanes = [0u64; 8];
rc_lanes[0] = rc;
let rc_vec = load(&rc_lanes);
row[0] = _mm512_xor_si512(row[0], rc_vec);
}
let mut padded_out = [0u64; 40];
for y in 0..5 {
_mm512_storeu_si512(
padded_out[y * 8..y * 8 + 8].as_mut_ptr() as *mut i32 as *mut _,
row[y],
);
state[y * 5..y * 5 + 5].copy_from_slice(&padded_out[y * 8..y * 8 + 5]);
}
}
#[inline]
pub fn keccak_f1600_dispatch(state: &mut [u64; 25], use_avx512: bool) {
if use_avx512 && is_x86_feature_detected!("avx512f") {
unsafe { keccak_f1600_avx512(state) }
} else {
crate::keccak::keccak_f1600(state);
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::keccak::{keccak_f1600, PI, RC, RHO};
fn avx512_available() -> bool {
is_x86_feature_detected!("avx512f")
}
#[test]
fn ternarylogic_truth_table_is_chi() {
if !avx512_available() {
eprintln!("skipping: no avx512f on this host");
return;
}
unsafe {
for a_bit in 0u64..2 {
for b_bit in 0u64..2 {
for c_bit in 0u64..2 {
let a = if a_bit == 1 { u64::MAX } else { 0 };
let b = if b_bit == 1 { u64::MAX } else { 0 };
let c = if c_bit == 1 { u64::MAX } else { 0 };
let expected = a ^ (!b & c);
let av = _mm512_set1_epi64(a as i64);
let bv = _mm512_set1_epi64(b as i64);
let cv = _mm512_set1_epi64(c as i64);
let rv = _mm512_ternarylogic_epi64::<CHI_TERNARY_IMM>(av, bv, cv);
let mut out = [0u64; 8];
_mm512_storeu_si512(out.as_mut_ptr() as *mut i32 as *mut _, rv);
assert_eq!(
out[0], expected,
"a={a_bit} b={b_bit} c={c_bit}: ternarylogic(0xD2) mismatch"
);
}
}
}
}
}
#[test]
fn table_consistency() {
for y in 0..5 {
for x in 0..5 {
assert_eq!(
ROT_ROW[y][x],
RHO[5 * y + x] as u64,
"ROT_ROW[{y}][{x}] must equal RHO[{}]",
5 * y + x
);
}
}
for z in 0..5 {
for x in 0..5 {
let j = PI[5 * z + x];
let expected_row = j / 5;
let expected_lane = (j % 5) as u64;
let mut rows_claiming_this_lane = 0;
for y in 0..5 {
let bit_set = (PI_MASKS[z][y] >> x) & 1 == 1;
if bit_set {
rows_claiming_this_lane += 1;
assert_eq!(
y,
expected_row,
"PI_MASKS[{z}][{y}] claims lane {x}, but PI[{}] says the \
source row is {expected_row}",
5 * z + x
);
assert_eq!(
PI_IDXS[z][y][x],
expected_lane,
"PI_IDXS[{z}][{y}][{x}] must equal source_lane(PI[{}]) = {expected_lane}",
5 * z + x
);
}
}
assert_eq!(
rows_claiming_this_lane, 1,
"PI_MASKS[{z}][*] must have exactly one row claiming destination lane {x}"
);
}
}
assert_eq!(
round_constants(),
RC,
"round_constants() must equal keccak::RC"
);
}
#[test]
fn matches_scalar_on_zero_state() {
if !avx512_available() {
eprintln!("skipping: no avx512f on this host");
return;
}
let mut scalar_state = [0u64; 25];
keccak_f1600(&mut scalar_state);
let mut vector_state = [0u64; 25];
unsafe { keccak_f1600_avx512(&mut vector_state) };
assert_eq!(
vector_state, scalar_state,
"AVX-512 permutation of the zero state must match the scalar implementation"
);
assert_eq!(vector_state[0], 0xF1258F7940E1DDE7);
assert_eq!(vector_state[1], 0x84D5CCF933C0478A);
assert_eq!(vector_state[2], 0xD598261EA65AA9EE);
assert_eq!(vector_state[3], 0xBD1547306F80494D);
assert_eq!(vector_state[4], 0x8B284E056253D057);
}
#[test]
fn matches_scalar_on_random_states() {
if !avx512_available() {
eprintln!("skipping: no avx512f on this host");
return;
}
let mut seed: u64 = 0x9E3779B97F4A7C15;
let mut next = || {
seed ^= seed << 13;
seed ^= seed >> 7;
seed ^= seed << 17;
seed
};
for trial in 0..2000 {
let mut state = [0u64; 25];
for lane in state.iter_mut() {
*lane = next();
}
let mut scalar_state = state;
keccak_f1600(&mut scalar_state);
let mut vector_state = state;
unsafe { keccak_f1600_avx512(&mut vector_state) };
assert_eq!(
vector_state, scalar_state,
"trial {trial}: AVX-512 and scalar permutations diverged on random input {state:?}"
);
}
}
#[test]
fn matches_scalar_after_multiple_permutations() {
if !avx512_available() {
eprintln!("skipping: no avx512f on this host");
return;
}
let mut scalar_state = [0u64; 25];
let mut vector_state = [0u64; 25];
for iteration in 0..50 {
keccak_f1600(&mut scalar_state);
unsafe { keccak_f1600_avx512(&mut vector_state) };
assert_eq!(
vector_state, scalar_state,
"diverged after {iteration} chained permutations"
);
}
}
#[test]
fn dispatch_respects_use_avx512_flag() {
let mut direct_scalar = [1u64; 25];
keccak_f1600(&mut direct_scalar);
let mut via_dispatch_off = [1u64; 25];
keccak_f1600_dispatch(&mut via_dispatch_off, false);
assert_eq!(direct_scalar, via_dispatch_off);
}
#[test]
fn dispatch_avx512_true_matches_scalar_output() {
let mut expected = [7u64; 25];
keccak_f1600(&mut expected);
let mut actual = [7u64; 25];
keccak_f1600_dispatch(&mut actual, true);
assert_eq!(expected, actual);
}
}