use core::slice::from_raw_parts;
use fearless_simd::{Simd, SimdBase, SimdCombine, u8x32, u8x64};
use super::sbox::{SimdSbox4, VpermOp, dispatch_sbox, splat_x4};
use crate::{
arch::soft::{self, Soft128, WordState, state_to_word, word_to_state},
backend::{
aes128::Aes128,
group::MidGroup,
tree::{compress_mid, expand_seed, finalize_merge4_pre, len_key_wide, pack_root_v},
},
const_fn::lane_ptr,
state::{BLOCK_LEN, MID_MAX, SHORT_MAX, State},
};
struct Vperm4Ctx<S: Simd, E: SimdSbox4<S>> {
simd: S,
sbox: E,
}
impl<S: Simd, E: SimdSbox4<S>> Vperm4Ctx<S, E> {
#[inline(always)]
fn new(simd: S) -> Self {
Self {
simd,
sbox: E::new(simd),
}
}
}
#[inline(always)]
unsafe fn read_x2<S: Simd>(simd: S, ptr: *const u8) -> u8x32<S> {
u8x32::from_slice(simd, unsafe { from_raw_parts(ptr, 32) })
}
#[inline(always)]
unsafe fn read_x4<S: Simd>(simd: S, ptr: *const u8) -> u8x64<S> {
u8x64::from_slice(simd, unsafe { from_raw_parts(ptr, 64) })
}
#[inline(always)]
unsafe fn read_x4_pair<S: Simd>(simd: S, ptr: *const u8) -> [u8x64<S>; 2] {
unsafe { [read_x4(simd, ptr), read_x4(simd, ptr.add(64))] }
}
#[inline(always)]
unsafe fn load_lane_pair<S: Simd, const SEED0: bool>(
simd: S,
sv: State,
lane_base: usize,
) -> [u8x64<S>; 2] {
debug_assert!(lane_base == 0 || lane_base == 8);
let [k0, k1] = unsafe { read_x4_pair(simd, lane_ptr::<SEED0>().add(lane_base * BLOCK_LEN)) };
if SEED0 {
[k0, k1]
} else {
let sv4 = splat_x4(simd, sv);
[sv4 ^ k0, sv4 ^ k1]
}
}
#[inline(always)]
unsafe fn read_swap32_x4<S: Simd>(simd: S, ptr: *const u8) -> u8x64<S> {
unsafe {
let lo = read_x2(simd, ptr);
let hi = read_x2(simd, ptr.add(32));
hi.combine(lo)
}
}
#[inline(always)]
unsafe fn read_swapped_half<S: Simd, const IS_HI: bool>(
simd: S,
p: *const u8,
) -> (u8x64<S>, u8x64<S>) {
unsafe {
if IS_HI {
let r1 = read_x4(simd, p);
let r0 = read_x4(simd, p.add(192));
(r0, r1)
} else {
(read_x4(simd, p.add(64)), read_x4(simd, p.add(128)))
}
}
}
#[inline(always)]
fn round_pair<S: Simd, E: SimdSbox4<S>>(
sbox: &E,
vec_li: [u8x64<S>; 2],
r0: u8x64<S>,
r1: u8x64<S>,
) -> [u8x64<S>; 2] {
[
sbox.aes_round(vec_li[0] ^ r0),
sbox.aes_round(vec_li[1] ^ r1),
]
}
#[inline(always)]
fn split_x4<S: Simd>(v: u8x64<S>) -> [WordState; 4] {
let raw_li: [u8; 64] = v.into();
let chunk_li = raw_li.as_chunks::<16>().0;
[
state_to_word(chunk_li[0]),
state_to_word(chunk_li[1]),
state_to_word(chunk_li[2]),
state_to_word(chunk_li[3]),
]
}
#[inline(always)]
fn finalize_x4<S: Simd, const HI: bool>(p_x4: u8x64<S>, lk: WordState) -> (WordState, WordState) {
unsafe { finalize_merge4_pre::<Soft128, HI>(Soft128, split_x4(p_x4), lk) }
}
impl<S: Simd, E: SimdSbox4<S>> MidGroup for &Vperm4Ctx<S, E> {
type G4 = u8x64<S>;
type G8 = [u8x64<S>; 2];
type L = Soft128;
#[inline(always)]
fn lane(self) -> Soft128 {
Soft128
}
#[inline(always)]
unsafe fn init4<const SEED0: bool>(self, sv: WordState) -> Self::G4 {
unsafe {
let k = read_x4(self.simd, lane_ptr::<SEED0>());
if SEED0 {
k
} else {
splat_x4(self.simd, word_to_state(sv)) ^ k
}
}
}
#[inline(always)]
unsafe fn absorb4_init(self, key_li: Self::G4, ptr: *const u8) -> Self::G4 {
unsafe { self.absorb4_linear(key_li, ptr) }
}
#[inline(always)]
unsafe fn absorb4_linear(self, acc: Self::G4, ptr: *const u8) -> Self::G4 {
self
.sbox
.aes_round(acc ^ unsafe { read_x4(self.simd, ptr) })
}
#[inline(always)]
unsafe fn absorb4_swap32(self, acc: Self::G4, ptr: *const u8) -> Self::G4 {
self
.sbox
.aes_round(acc ^ unsafe { read_swap32_x4(self.simd, ptr) })
}
#[inline(always)]
unsafe fn merge4<const HI: bool>(self, acc: Self::G4, lk: WordState) -> (WordState, WordState) {
finalize_x4::<S, HI>(acc, lk)
}
#[inline(always)]
unsafe fn init8<const SEED0: bool>(self, sv: WordState, lane_base: usize) -> Self::G8 {
unsafe { load_lane_pair::<S, SEED0>(self.simd, word_to_state(sv), lane_base) }
}
#[inline(always)]
unsafe fn absorb8_init(self, key_li: Self::G8, ptr: *const u8) -> Self::G8 {
unsafe { self.absorb8_pair64(key_li, ptr, ptr.add(64)) }
}
#[inline(always)]
unsafe fn absorb8_pair64(self, acc: Self::G8, p_lo: *const u8, p_hi: *const u8) -> Self::G8 {
unsafe {
[
self.sbox.aes_round(acc[0] ^ read_x4(self.simd, p_lo)),
self.sbox.aes_round(acc[1] ^ read_x4(self.simd, p_hi)),
]
}
}
#[inline(always)]
unsafe fn absorb8_swap32(self, acc: Self::G8, last128: *const u8) -> Self::G8 {
unsafe {
[
self
.sbox
.aes_round(acc[0] ^ read_swap32_x4(self.simd, last128)),
self
.sbox
.aes_round(acc[1] ^ read_swap32_x4(self.simd, last128.add(64))),
]
}
}
#[inline(always)]
unsafe fn merge8<const HI: bool>(self, acc: Self::G8, lk: WordState) -> (WordState, WordState) {
finalize_x4::<S, HI>(self.sbox.aes_round(acc[0]) ^ acc[1], lk)
}
#[inline(always)]
unsafe fn merge16<const HI: bool>(
self,
lo: Self::G8,
hi: Self::G8,
lk: WordState,
) -> (WordState, WordState) {
let m0 = self.sbox.aes_round(lo[0]) ^ hi[0];
let m1 = self.sbox.aes_round(lo[1]) ^ hi[1];
finalize_x4::<S, HI>(self.sbox.aes_round(m0) ^ m1, lk)
}
}
struct HalfTile {
sv_state: State,
init_ptr: *const u8,
pair_ptr: *const u8,
pair_cnt: usize,
last: *const u8,
flag: u8,
}
impl HalfTile {
const INIT: u8 = 1;
const ODD: u8 = 2;
const FINISH: u8 = 4;
}
#[inline(never)]
unsafe fn bulk_half_tile<S: Simd, E: SimdSbox4<S>, const IS_HI: bool, const SEED0: bool>(
simd: S,
acc_li: &mut [u8; 128],
tile: &HalfTile,
) {
simd.vectorize(
#[inline(always)]
|| {
let sbox = E::new(simd);
let do_init = (tile.flag & HalfTile::INIT) != 0;
let do_odd = (tile.flag & HalfTile::ODD) != 0;
let do_finish = (tile.flag & HalfTile::FINISH) != 0;
unsafe {
let mut vec_li = if do_init {
let lane_base = if IS_HI { 8 } else { 0 };
let dp = if IS_HI {
tile.init_ptr.add(128)
} else {
tile.init_ptr
};
let k_li = load_lane_pair::<S, SEED0>(simd, tile.sv_state, lane_base);
let [d0, d1] = read_x4_pair(simd, dp);
round_pair(&sbox, k_li, d0, d1)
} else {
read_x4_pair(simd, acc_li.as_ptr())
};
let mut p = tile.pair_ptr;
let mut rem = tile.pair_cnt;
while rem > 0 {
let (r0, r1) = read_swapped_half::<S, IS_HI>(simd, p);
vec_li = round_pair(&sbox, vec_li, r0, r1);
let [q0, q1] = read_x4_pair(simd, p.add(if IS_HI { 384 } else { 256 }));
vec_li = round_pair(&sbox, vec_li, q0, q1);
p = p.add(512);
rem -= 1;
}
if do_odd {
let (r0, r1) = read_swapped_half::<S, IS_HI>(simd, p);
vec_li = round_pair(&sbox, vec_li, r0, r1);
}
if do_finish {
let [f0, f1] = read_x4_pair(simd, tile.last.add(if IS_HI { 0 } else { 128 }));
vec_li = round_pair(&sbox, vec_li, f0, f1);
}
vec_li[0].store_slice(&mut acc_li[..64]);
vec_li[1].store_slice(&mut acc_li[64..]);
}
},
);
}
#[inline(always)]
unsafe fn compress_long_vperm<S: Simd, E: SimdSbox4<S>, const HI: bool, const SEED0: bool>(
simd: S,
data: &[u8],
seed: u64,
) -> (WordState, WordState) {
let len = data.len();
debug_assert!(len > SHORT_MAX);
if len <= MID_MAX {
let ctx = Vperm4Ctx::<S, E>::new(simd);
return unsafe { compress_mid::<&Vperm4Ctx<S, E>, HI, SEED0>(&ctx, data, seed) };
}
const TILE_PAIRS: usize = 16;
let data_ptr = data.as_ptr();
unsafe {
let rem_stripes = ((len - 1) >> 8) - 1;
let mut rem_pairs = rem_stripes >> 1;
let has_odd = (rem_stripes & 1) != 0;
let last = data_ptr.add(len - 256);
let sv_state = if SEED0 {
[0u8; 16]
} else {
word_to_state(expand_seed::<Soft128, false>(Soft128, seed))
};
let mut lo_acc_li = [0u8; 128];
let mut hi_acc_li = [0u8; 128];
let mut tile_ptr = data_ptr.add(256);
let mut is_first = true;
loop {
let take = rem_pairs.min(TILE_PAIRS);
let is_last = take == rem_pairs;
let mut flag = 0u8;
if is_first {
flag |= HalfTile::INIT;
}
if is_last && has_odd {
flag |= HalfTile::ODD;
}
if is_last {
flag |= HalfTile::FINISH;
}
let tile = HalfTile {
sv_state,
init_ptr: data_ptr,
pair_ptr: tile_ptr,
pair_cnt: take,
last,
flag,
};
bulk_half_tile::<S, E, false, SEED0>(simd, &mut lo_acc_li, &tile);
bulk_half_tile::<S, E, true, SEED0>(simd, &mut hi_acc_li, &tile);
rem_pairs -= take;
if rem_pairs == 0 {
break;
}
tile_ptr = tile_ptr.add(take * 512);
is_first = false;
}
let lo_li = read_x4_pair(simd, lo_acc_li.as_ptr());
let hi_li = read_x4_pair(simd, hi_acc_li.as_ptr());
let lk = len_key_wide(Soft128, len);
let ctx = Vperm4Ctx::<S, E>::new(simd);
(&ctx).merge16::<HI>(lo_li, hi_li, lk)
}
}
struct HashLoOp<'a, const SEED0: bool> {
data: &'a [u8],
seed: u64,
}
impl<S: Simd, const SEED0: bool> VpermOp<S> for HashLoOp<'_, SEED0> {
type Out = u128;
#[inline(always)]
fn run<E: SimdSbox4<S>>(self, simd: S) -> u128 {
unsafe {
Soft128.to_u128(compress_long_vperm::<S, E, false, SEED0>(simd, self.data, self.seed).0)
}
}
#[inline(always)]
fn fallback(self) -> u128 {
soft::compress_128::<SEED0>(self.data, self.seed)
}
}
struct HashHiOp<'a, const SEED0: bool> {
data: &'a [u8],
seed: u64,
}
impl<S: Simd, const SEED0: bool> VpermOp<S> for HashHiOp<'_, SEED0> {
type Out = [u8; 32];
#[inline(always)]
fn run<E: SimdSbox4<S>>(self, simd: S) -> [u8; 32] {
unsafe {
let (lo, hi) = compress_long_vperm::<S, E, true, SEED0>(simd, self.data, self.seed);
pack_root_v(Soft128, lo, hi)
}
}
#[inline(always)]
fn fallback(self) -> [u8; 32] {
soft::compress_256::<SEED0>(self.data, self.seed)
}
}
#[inline(always)]
pub(super) fn vperm_hash_lo<S: Simd, const SEED0: bool>(simd: S, data: &[u8], seed: u64) -> u128 {
dispatch_sbox(simd, HashLoOp::<SEED0> { data, seed })
}
#[inline(always)]
pub(super) fn vperm_hash_hi<S: Simd, const SEED0: bool>(
simd: S,
data: &[u8],
seed: u64,
) -> [u8; 32] {
dispatch_sbox(simd, HashHiOp::<SEED0> { data, seed })
}