use core::array::from_fn;
use crate::{
backend::{
aes128::Aes128,
group::{GroupV, LaneGroup, MidGroup},
mode::PostXorAes,
tree::finalize_merge4_lk,
},
const_fn::{CTR_STEP_LI, M_CTR, lane_ptr},
state::BLOCK_LEN,
};
pub(crate) trait WideAes: Copy {
type L: PostXorAes;
type W: Copy;
const BLOCK_PER_VEC: usize;
fn lane(self) -> Self::L;
unsafe fn zero_w(self) -> Self::W;
unsafe fn ld_w(self, ptr: *const u8) -> Self::W;
unsafe fn st_w(self, ptr: *mut u8, v: Self::W);
unsafe fn zext128(self, v: <Self::L as Aes128>::V) -> Self::W;
unsafe fn broadcast(self, v: <Self::L as Aes128>::V) -> Self::W;
unsafe fn xor_w(self, a: Self::W, b: Self::W) -> Self::W;
unsafe fn add_u64_w(self, a: Self::W, b: Self::W) -> Self::W;
unsafe fn aesenc_w(self, a: Self::W, rk: Self::W) -> Self::W;
unsafe fn aesenclast_w(self, a: Self::W, rk: Self::W) -> Self::W;
}
pub(crate) trait Lane256x4: WideAes {
unsafe fn split_pair(self, v: Self::W) -> (<Self::L as Aes128>::V, <Self::L as Aes128>::V);
}
#[cfg(not(all(
target_arch = "x86_64",
target_feature = "avx512f",
target_feature = "vaes",
target_feature = "avx2",
target_feature = "aes",
target_feature = "sse2"
)))]
mod w256 {
use super::*;
#[derive(Clone, Copy)]
#[repr(transparent)]
pub(crate) struct Wide256<B>(pub(in super::super) B);
#[inline(always)]
unsafe fn wide_absorb_swap32<B: Lane256x4, const N: usize>(
b: B,
acc_li: [B::W; N],
ptr: *const u8,
) -> [B::W; N] {
unsafe { from_fn(|i| b.aesenc_w(acc_li[i], b.ld_w(ptr.add((i ^ 1) * 32)))) }
}
impl<B: Lane256x4> MidGroup for Wide256<B> {
type G4 = [B::W; 2];
type G8 = [B::W; 4];
type L = B::L;
#[inline(always)]
fn lane(self) -> Self::L {
self.0.lane()
}
#[inline(always)]
unsafe fn init4<const SEED0: bool>(self, sv: GroupV<Self>) -> Self::G4 {
unsafe { wide_init::<B, 2, SEED0>(self.0, sv, 0) }
}
#[inline(always)]
unsafe fn absorb4_init(self, key_li: Self::G4, ptr: *const u8) -> Self::G4 {
unsafe { wide_absorb_init(self.0, key_li, ptr) }
}
#[inline(always)]
unsafe fn absorb4_linear(self, acc: Self::G4, ptr: *const u8) -> Self::G4 {
let b = self.0;
unsafe { from_fn(|i| b.aesenc_w(acc[i], b.ld_w(ptr.add(i * 32)))) }
}
#[inline(always)]
unsafe fn absorb4_swap32(self, acc: Self::G4, ptr: *const u8) -> Self::G4 {
unsafe { wide_absorb_swap32(self.0, acc, ptr) }
}
#[inline(always)]
unsafe fn merge4<const HI: bool>(
self,
acc: Self::G4,
lk: GroupV<Self>,
) -> (GroupV<Self>, GroupV<Self>) {
let b = self.0;
unsafe {
let l01_lk = b.aesenc_w(acc[0], b.zext128(lk));
let l23 = b.aesenc_w(acc[1], b.zero_w());
wide_finalize_merge4_256_lk::<B, HI>(b, l01_lk, l23, lk)
}
}
#[inline(always)]
unsafe fn init8<const SEED0: bool>(self, sv: GroupV<Self>, lane_base: usize) -> Self::G8 {
unsafe { wide_init::<B, 4, SEED0>(self.0, sv, lane_base) }
}
#[inline(always)]
unsafe fn absorb8_init(self, key_li: Self::G8, ptr: *const u8) -> Self::G8 {
unsafe { wide_absorb_init(self.0, key_li, ptr) }
}
#[inline(always)]
unsafe fn absorb8_pair64(self, acc: Self::G8, p_lo: *const u8, p_hi: *const u8) -> Self::G8 {
unsafe { wide_absorb8_pair64(self.0, acc, p_lo, p_hi) }
}
#[inline(always)]
unsafe fn absorb8_swap32(self, acc: Self::G8, last128: *const u8) -> Self::G8 {
unsafe { wide_absorb_swap32(self.0, acc, last128) }
}
#[inline(always)]
unsafe fn merge8<const HI: bool>(
self,
acc: Self::G8,
lk: GroupV<Self>,
) -> (GroupV<Self>, GroupV<Self>) {
let b = self.0;
unsafe {
let [p01_lk, p23] = wide_fold_pair::<B, 2, 0>(b, [acc[0], acc[1]], [acc[2], acc[3]], lk);
wide_finalize_merge4_256_lk::<B, HI>(b, p01_lk, p23, lk)
}
}
#[inline(always)]
unsafe fn merge16<const HI: bool>(
self,
lo: Self::G8,
hi: Self::G8,
lk: GroupV<Self>,
) -> (GroupV<Self>, GroupV<Self>) {
let b = self.0;
unsafe {
let m_li = wide_fold_pair::<B, 4, 2>(b, lo, hi, lk);
let p01_lk = b.aesenc_w(m_li[0], m_li[2]);
let p23 = b.aesenc_w(m_li[1], m_li[3]);
wide_finalize_merge4_256_lk::<B, HI>(b, p01_lk, p23, lk)
}
}
}
impl<B: Lane256x4> LaneGroup for Wide256<B> {
#[inline(always)]
unsafe fn compress_65_128<const HI: bool, const SEED0: bool>(
self,
ptr: *const u8,
len: usize,
sv: GroupV<Self>,
lk: GroupV<Self>,
) -> (GroupV<Self>, GroupV<Self>) {
let b = self.0;
unsafe {
let key_li = self.init4::<SEED0>(sv);
let h0 = b.ld_w(ptr);
let h1 = b.ld_w(ptr.add(32));
let tail = ptr.add(len - 64);
let t0 = b.ld_w(tail);
let t1 = b.ld_w(tail.add(32));
let acc_li = [
b.aesenc_w(b.xor_w(key_li[0], h0), b.add_u64_w(t1, h1)),
b.aesenc_w(b.xor_w(key_li[1], h1), b.add_u64_w(t0, h0)),
];
self.merge4::<HI>(acc_li, lk)
}
}
#[inline(always)]
unsafe fn compress_33_64<const HI: bool, const SEED0: bool>(
self,
ptr: *const u8,
len: usize,
sv: GroupV<Self>,
lk: GroupV<Self>,
) -> (GroupV<Self>, GroupV<Self>) {
unsafe { wide256_compress_33_64::<B, HI, SEED0>(self.0, ptr, len, sv, lk) }
}
#[inline(always)]
unsafe fn absorb16_swapped(
self,
lo: Self::G8,
hi: Self::G8,
ptr: *const u8,
) -> (Self::G8, Self::G8) {
let b = self.0;
unsafe {
let hi2 = b.aesenc_w(hi[2], b.ld_w(ptr));
let hi3 = b.aesenc_w(hi[3], b.ld_w(ptr.add(32)));
let lo0 = b.aesenc_w(lo[0], b.ld_w(ptr.add(64)));
let lo1 = b.aesenc_w(lo[1], b.ld_w(ptr.add(96)));
let lo2 = b.aesenc_w(lo[2], b.ld_w(ptr.add(128)));
let lo3 = b.aesenc_w(lo[3], b.ld_w(ptr.add(160)));
let hi0 = b.aesenc_w(hi[0], b.ld_w(ptr.add(192)));
let hi1 = b.aesenc_w(hi[1], b.ld_w(ptr.add(224)));
([lo0, lo1, lo2, lo3], [hi0, hi1, hi2, hi3])
}
}
}
}
#[cfg(not(all(
target_arch = "x86_64",
target_feature = "avx512f",
target_feature = "vaes",
target_feature = "avx2",
target_feature = "aes",
target_feature = "sse2"
)))]
pub(crate) use w256::Wide256;
#[inline(always)]
unsafe fn wide_init<W: WideAes, const N: usize, const SEED0: bool>(
ctx: W,
sv: <W::L as Aes128>::V,
lane_base: usize,
) -> [W::W; N] {
let step = W::BLOCK_PER_VEC * BLOCK_LEN;
unsafe {
let lp = lane_ptr::<SEED0>().add(lane_base * BLOCK_LEN);
if SEED0 {
from_fn(|i| ctx.ld_w(lp.add(i * step)))
} else {
let sv_w = ctx.broadcast(sv);
from_fn(|i| ctx.xor_w(sv_w, ctx.ld_w(lp.add(i * step))))
}
}
}
#[inline(always)]
unsafe fn wide_absorb_init<W: WideAes, const N: usize>(
ctx: W,
key_li: [W::W; N],
ptr: *const u8,
) -> [W::W; N] {
let step = W::BLOCK_PER_VEC * BLOCK_LEN;
from_fn(|i| unsafe { ctx.xor_w(key_li[i], ctx.ld_w(ptr.add(i * step))) })
}
#[inline(always)]
unsafe fn wide_absorb8_pair64<W: WideAes, const N: usize>(
ctx: W,
acc_li: [W::W; N],
p_lo: *const u8,
p_hi: *const u8,
) -> [W::W; N] {
let step = W::BLOCK_PER_VEC * BLOCK_LEN;
let half = N / 2;
from_fn(|i| unsafe {
let p = if i < half {
p_lo.add(i * step)
} else {
p_hi.add((i - half) * step)
};
ctx.aesenc_w(acc_li[i], ctx.ld_w(p))
})
}
#[inline(always)]
unsafe fn wide_fold_pair<W: WideAes, const N: usize, const LK_IDX: usize>(
w: W,
a_li: [W::W; N],
b_li: [W::W; N],
lk: <W::L as Aes128>::V,
) -> [W::W; N] {
unsafe {
let z = w.zero_w();
let lk_w = w.zext128(lk);
from_fn(|i| {
let b_rk = if i == LK_IDX { lk_w } else { z };
w.aesenc_w(w.aesenc_w(a_li[i], z), w.aesenc_w(b_li[i], b_rk))
})
}
}
#[inline(always)]
unsafe fn wide_finalize_merge4_256_lk<H: Lane256x4, const HI: bool>(
h: H,
l01_lk: H::W,
l23: H::W,
lk: <H::L as Aes128>::V,
) -> (<H::L as Aes128>::V, <H::L as Aes128>::V) {
let l = h.lane();
unsafe {
let (l0_lk, l1) = h.split_pair(l01_lk);
let (l2, l3) = h.split_pair(l23);
finalize_merge4_lk::<H::L, HI>(l, [l0_lk, l1, l2, l3], lk)
}
}
#[inline(always)]
unsafe fn wide256_compress_33_64<H: Lane256x4, const HI: bool, const SEED0: bool>(
h: H,
ptr: *const u8,
len: usize,
sv: <H::L as Aes128>::V,
lk: <H::L as Aes128>::V,
) -> (<H::L as Aes128>::V, <H::L as Aes128>::V) {
unsafe {
let [k01, k23] = wide_init::<H, 2, SEED0>(h, sv, 0);
let tail = ptr.add(len - 32);
let z = h.zero_w();
let l01_lk = h.aesenc_w(h.xor_w(h.ld_w(ptr), k01), h.zext128(lk));
let l23 = h.aesenc_w(h.xor_w(h.ld_w(tail), k23), z);
wide_finalize_merge4_256_lk::<H, HI>(h, l01_lk, l23, lk)
}
}
#[cfg(all(
target_arch = "x86_64",
any(
feature = "std",
all(
target_feature = "avx512f",
target_feature = "vaes",
target_feature = "avx2",
target_feature = "aes",
target_feature = "sse2"
)
)
))]
mod w512 {
use super::*;
pub(in super::super) trait Lane512x2: WideAes {
type Half: Lane256x4<L = Self::L>;
fn half(self) -> Self::Half;
unsafe fn split_half(
self,
v: Self::W,
) -> (<Self::Half as WideAes>::W, <Self::Half as WideAes>::W);
unsafe fn swap32_in_64(self, v: Self::W) -> Self::W;
}
#[derive(Clone, Copy)]
#[repr(transparent)]
pub(in super::super) struct Wide512<B>(pub(in super::super) B);
#[inline(always)]
unsafe fn wide_finalize_merge4_512_lk<B: Lane512x2, const HI: bool>(
b: B,
v_lk: B::W,
lk: <B::L as Aes128>::V,
) -> (<B::L as Aes128>::V, <B::L as Aes128>::V) {
unsafe {
let (l01_lk, l23) = b.split_half(v_lk);
wide_finalize_merge4_256_lk::<B::Half, HI>(b.half(), l01_lk, l23, lk)
}
}
impl<B: Lane512x2> MidGroup for Wide512<B> {
type G4 = B::W;
type G8 = [B::W; 2];
type L = B::L;
#[inline(always)]
fn lane(self) -> Self::L {
self.0.lane()
}
#[inline(always)]
unsafe fn init4<const SEED0: bool>(self, sv: GroupV<Self>) -> Self::G4 {
unsafe { wide_init::<B, 1, SEED0>(self.0, sv, 0)[0] }
}
#[inline(always)]
unsafe fn absorb4_init(self, key_li: Self::G4, ptr: *const u8) -> Self::G4 {
let b = self.0;
unsafe { b.xor_w(key_li, b.ld_w(ptr)) }
}
#[inline(always)]
unsafe fn absorb4_linear(self, acc: Self::G4, ptr: *const u8) -> Self::G4 {
let b = self.0;
unsafe { b.aesenc_w(acc, b.ld_w(ptr)) }
}
#[inline(always)]
unsafe fn absorb4_swap32(self, acc: Self::G4, ptr: *const u8) -> Self::G4 {
let b = self.0;
unsafe { b.aesenc_w(acc, b.swap32_in_64(b.ld_w(ptr))) }
}
#[inline(always)]
unsafe fn merge4<const HI: bool>(
self,
acc: Self::G4,
lk: GroupV<Self>,
) -> (GroupV<Self>, GroupV<Self>) {
let b = self.0;
unsafe { wide_finalize_merge4_512_lk::<B, HI>(b, b.aesenc_w(acc, b.zext128(lk)), lk) }
}
#[inline(always)]
unsafe fn init8<const SEED0: bool>(self, sv: GroupV<Self>, lane_base: usize) -> Self::G8 {
unsafe { wide_init::<B, 2, SEED0>(self.0, sv, lane_base) }
}
#[inline(always)]
unsafe fn absorb8_init(self, key_li: Self::G8, ptr: *const u8) -> Self::G8 {
unsafe { wide_absorb_init(self.0, key_li, ptr) }
}
#[inline(always)]
unsafe fn absorb8_pair64(self, acc: Self::G8, p_lo: *const u8, p_hi: *const u8) -> Self::G8 {
unsafe { wide_absorb8_pair64(self.0, acc, p_lo, p_hi) }
}
#[inline(always)]
unsafe fn absorb8_swap32(self, acc: Self::G8, last128: *const u8) -> Self::G8 {
let b = self.0;
unsafe { from_fn(|i| b.aesenc_w(acc[i], b.swap32_in_64(b.ld_w(last128.add(i * 64))))) }
}
#[inline(always)]
unsafe fn merge8<const HI: bool>(
self,
acc: Self::G8,
lk: GroupV<Self>,
) -> (GroupV<Self>, GroupV<Self>) {
let b = self.0;
unsafe {
let [v_lk] = wide_fold_pair::<B, 1, 0>(b, [acc[0]], [acc[1]], lk);
wide_finalize_merge4_512_lk::<B, HI>(b, v_lk, lk)
}
}
#[inline(always)]
unsafe fn merge16<const HI: bool>(
self,
lo: Self::G8,
hi: Self::G8,
lk: GroupV<Self>,
) -> (GroupV<Self>, GroupV<Self>) {
let b = self.0;
unsafe {
let [m0, m1_lk] = wide_fold_pair::<B, 2, 1>(b, lo, hi, lk);
wide_finalize_merge4_512_lk::<B, HI>(b, b.aesenc_w(m0, m1_lk), lk)
}
}
}
impl<B: Lane512x2> LaneGroup for Wide512<B> {
#[inline(always)]
unsafe fn compress_65_128<const HI: bool, const SEED0: bool>(
self,
ptr: *const u8,
len: usize,
sv: GroupV<Self>,
lk: GroupV<Self>,
) -> (GroupV<Self>, GroupV<Self>) {
let b = self.0;
unsafe {
let key = self.init4::<SEED0>(sv);
let h = b.ld_w(ptr);
let t = b.ld_w(ptr.add(len - 64));
let acc = b.aesenc_w(b.xor_w(key, h), b.swap32_in_64(b.add_u64_w(t, h)));
self.merge4::<HI>(acc, lk)
}
}
#[inline(always)]
unsafe fn compress_33_64<const HI: bool, const SEED0: bool>(
self,
ptr: *const u8,
len: usize,
sv: GroupV<Self>,
lk: GroupV<Self>,
) -> (GroupV<Self>, GroupV<Self>) {
unsafe { wide256_compress_33_64::<B::Half, HI, SEED0>(self.0.half(), ptr, len, sv, lk) }
}
#[inline(always)]
unsafe fn absorb16_swapped(
self,
lo: Self::G8,
hi: Self::G8,
ptr: *const u8,
) -> (Self::G8, Self::G8) {
let b = self.0;
unsafe {
let hi1 = b.aesenc_w(hi[1], b.ld_w(ptr));
let lo0 = b.aesenc_w(lo[0], b.ld_w(ptr.add(64)));
let lo1 = b.aesenc_w(lo[1], b.ld_w(ptr.add(128)));
let hi0 = b.aesenc_w(hi[0], b.ld_w(ptr.add(192)));
([lo0, lo1], [hi0, hi1])
}
}
}
}
#[cfg(all(
target_arch = "x86_64",
any(
feature = "std",
all(
target_feature = "avx512f",
target_feature = "vaes",
target_feature = "avx2",
target_feature = "aes",
target_feature = "sse2"
)
)
))]
pub(super) use w512::{Lane512x2, Wide512};
#[inline(always)]
pub(super) unsafe fn ctr_4_wide<W: WideAes, const XOR: bool>(
wide: W,
rk_li: [<W::L as Aes128>::V; 4],
blk: &mut u64,
ptr: &mut *mut u8,
rem: &mut usize,
) {
let vec_byte = W::BLOCK_PER_VEC * BLOCK_LEN;
let chunk_byte = 4 * vec_byte;
if *rem < chunk_byte {
return;
}
let lane = wide.lane();
unsafe {
let rk0_w = wide.broadcast(rk_li[0]);
let rk1_w = wide.broadcast(rk_li[1]);
let rk2_w = wide.broadcast(rk_li[2]);
let rk3_w = wide.broadcast(rk_li[3]);
let step_lane = wide.broadcast(lane.ld(CTR_STEP_LI[W::BLOCK_PER_VEC].as_ptr()));
let step_2lane = wide.broadcast(lane.ld(CTR_STEP_LI[2 * W::BLOCK_PER_VEC].as_ptr()));
let step_loop = wide.broadcast(lane.ld(CTR_STEP_LI[4 * W::BLOCK_PER_VEC].as_ptr()));
let c0 = wide.add_u64_w(
wide.broadcast(lane.pair_v(*blk, blk.wrapping_mul(M_CTR))),
wide.ld_w(CTR_STEP_LI.as_ptr().cast()),
);
let c1 = wide.add_u64_w(c0, step_lane);
let mut c_li = [
c0,
c1,
wide.add_u64_w(c0, step_2lane),
wide.add_u64_w(c1, step_2lane),
];
let blk_step = (4 * W::BLOCK_PER_VEC) as u64;
while *rem >= chunk_byte {
let mut k_li: [W::W; 4] = from_fn(|i| wide.xor_w(c_li[i], rk0_w));
for rk in [rk1_w, rk2_w, rk3_w] {
for k in &mut k_li {
*k = wide.aesenc_w(*k, rk);
}
}
for (i, k) in k_li.into_iter().enumerate() {
let ks = wide.aesenclast_w(k, rk0_w);
let p = ptr.add(i * vec_byte);
if XOR {
wide.st_w(p, wide.xor_w(wide.ld_w(p), ks));
} else {
wide.st_w(p, ks);
}
}
for c in &mut c_li {
*c = wide.add_u64_w(*c, step_loop);
}
*ptr = ptr.add(chunk_byte);
*rem -= chunk_byte;
*blk = blk.wrapping_add(blk_step);
}
}
}