use crate::{
backend::{aes128::Aes128, mode::AesMode, tree::finalize_hi},
const_fn::lane_ptr,
state::BLOCK_LEN,
};
pub(crate) type GroupV<G> = <<G as MidGroup>::L as Aes128>::V;
pub(crate) trait MidGroup: Copy {
type L: Aes128;
type G4: Copy;
type G8: Copy;
fn lane(self) -> Self::L;
unsafe fn init4<const SEED0: bool>(self, sv: GroupV<Self>) -> Self::G4;
unsafe fn absorb4_init(self, key_li: Self::G4, ptr: *const u8) -> Self::G4;
unsafe fn absorb4_linear(self, acc: Self::G4, ptr: *const u8) -> Self::G4;
unsafe fn absorb4_swap32(self, acc: Self::G4, ptr: *const u8) -> Self::G4;
unsafe fn merge4<const HI: bool>(
self,
acc: Self::G4,
lk: GroupV<Self>,
) -> (GroupV<Self>, GroupV<Self>);
unsafe fn init8<const SEED0: bool>(self, sv: GroupV<Self>, lane_base: usize) -> Self::G8;
unsafe fn absorb8_init(self, key_li: Self::G8, ptr: *const u8) -> Self::G8;
unsafe fn absorb8_pair64(self, acc: Self::G8, p_lo: *const u8, p_hi: *const u8) -> Self::G8;
unsafe fn absorb8_swap32(self, acc: Self::G8, last128: *const u8) -> Self::G8;
unsafe fn merge8<const HI: bool>(
self,
acc: Self::G8,
lk: GroupV<Self>,
) -> (GroupV<Self>, GroupV<Self>);
unsafe fn merge16<const HI: bool>(
self,
lo: Self::G8,
hi: Self::G8,
lk: GroupV<Self>,
) -> (GroupV<Self>, GroupV<Self>);
}
pub(crate) trait LaneGroup: MidGroup {
#[inline(always)]
unsafe fn compress_17_32<const HI: bool, const SEED0: bool>(
self,
ptr: *const u8,
len: usize,
sv: GroupV<Self>,
lk: GroupV<Self>,
) -> (GroupV<Self>, GroupV<Self>) {
let l = self.lane();
unsafe {
let lp = lane_ptr::<SEED0>();
let r0 = l.ld(lp);
let r1 = l.ld(lp.add(BLOCK_LEN));
let (k0, k1) = if SEED0 {
(r0, r1)
} else {
(l.xor(sv, r0), l.xor(sv, r1))
};
let b0 = l.ld(ptr);
let b1 = l.ld(ptr.add(len - BLOCK_LEN));
let (lo, c) = <Self::L as Aes128>::Mode::compress_17_32_core(l, b0, k0, b1, k1, lk);
finalize_hi::<Self::L, HI>(l, lo, c, lk)
}
}
unsafe fn compress_33_64<const HI: bool, const SEED0: bool>(
self,
ptr: *const u8,
len: usize,
sv: GroupV<Self>,
lk: GroupV<Self>,
) -> (GroupV<Self>, GroupV<Self>);
unsafe fn compress_65_128<const HI: bool, const SEED0: bool>(
self,
ptr: *const u8,
len: usize,
sv: GroupV<Self>,
lk: GroupV<Self>,
) -> (GroupV<Self>, GroupV<Self>);
unsafe fn absorb16_swapped(
self,
lo: Self::G8,
hi: Self::G8,
ptr: *const u8,
) -> (Self::G8, Self::G8);
#[inline(always)]
unsafe fn absorb16_linear(
self,
lo: Self::G8,
hi: Self::G8,
ptr: *const u8,
) -> (Self::G8, Self::G8) {
unsafe {
(
self.absorb8_pair64(lo, ptr, ptr.add(64)),
self.absorb8_pair64(hi, ptr.add(128), ptr.add(192)),
)
}
}
}
pub(crate) trait Lane128x8: Aes128 {}
impl<B: Lane128x8> MidGroup for B {
type G4 = [B::V; 4];
type G8 = [B::V; 8];
type L = B;
#[inline(always)]
fn lane(self) -> B {
self
}
#[inline(always)]
unsafe fn init4<const SEED0: bool>(self, sv: B::V) -> Self::G4 {
unsafe {
let lp = lane_ptr::<SEED0>();
unroll4!(I => {
let key = self.ld(lp.add(I * BLOCK_LEN));
if SEED0 { key } else { self.xor(sv, key) }
})
}
}
#[inline(always)]
unsafe fn absorb4_init(self, key_li: Self::G4, ptr: *const u8) -> Self::G4 {
unsafe { unroll4!(I => self.acc_init_1(key_li[I], self.ld(ptr.add(I * BLOCK_LEN)))) }
}
#[inline(always)]
unsafe fn absorb4_linear(self, acc: Self::G4, ptr: *const u8) -> Self::G4 {
unsafe { unroll4!(I => self.acc_absorb_1(acc[I], self.ld(ptr.add(I * BLOCK_LEN)))) }
}
#[inline(always)]
unsafe fn absorb4_swap32(self, acc: Self::G4, ptr: *const u8) -> Self::G4 {
unsafe { unroll4!(I => self.acc_absorb_1(acc[I], self.ld(ptr.add((I ^ 2) * BLOCK_LEN)))) }
}
#[inline(always)]
unsafe fn merge4<const HI: bool>(self, acc: Self::G4, lk: B::V) -> (B::V, B::V) {
unsafe { B::Mode::merge4_128::<HI>(self, acc, lk) }
}
#[inline(always)]
unsafe fn init8<const SEED0: bool>(self, sv: B::V, lane_base: usize) -> Self::G8 {
unsafe {
let lp = lane_ptr::<SEED0>().add(lane_base * BLOCK_LEN);
unroll8!(I => {
let key = self.ld(lp.add(I * BLOCK_LEN));
if SEED0 { key } else { self.xor(sv, key) }
})
}
}
#[inline(always)]
unsafe fn absorb8_init(self, key_li: Self::G8, ptr: *const u8) -> Self::G8 {
unsafe { unroll8!(I => self.acc_init_1(key_li[I], self.ld(ptr.add(I * BLOCK_LEN)))) }
}
#[inline(always)]
unsafe fn absorb8_pair64(self, acc: Self::G8, p_lo: *const u8, p_hi: *const u8) -> Self::G8 {
unsafe {
unroll8!(I => {
let p = if I < 4 { p_lo.add(I * BLOCK_LEN) } else { p_hi.add((I - 4) * BLOCK_LEN) };
self.acc_absorb_1(acc[I], self.ld(p))
})
}
}
#[inline(always)]
unsafe fn absorb8_swap32(self, acc: Self::G8, last128: *const u8) -> Self::G8 {
unsafe { unroll8!(I => self.acc_absorb_1(acc[I], self.ld(last128.add((I ^ 2) * BLOCK_LEN)))) }
}
#[inline(always)]
unsafe fn merge8<const HI: bool>(self, acc: Self::G8, lk: B::V) -> (B::V, B::V) {
unsafe { B::Mode::merge8_128::<HI>(self, acc, lk) }
}
#[inline(always)]
unsafe fn merge16<const HI: bool>(self, lo: Self::G8, hi: Self::G8, lk: B::V) -> (B::V, B::V) {
unsafe { B::Mode::merge16_128::<HI>(self, lo, hi, lk) }
}
}
#[inline(always)]
fn join4<T: Copy>(a_li: [T; 4], b_li: [T; 4]) -> [T; 8] {
[
a_li[0], a_li[1], a_li[2], a_li[3], b_li[0], b_li[1], b_li[2], b_li[3],
]
}
impl<B: Lane128x8> LaneGroup for B {
#[inline(always)]
unsafe fn compress_65_128<const HI: bool, const SEED0: bool>(
self,
ptr: *const u8,
len: usize,
sv: B::V,
lk: B::V,
) -> (B::V, B::V) {
unsafe {
let key_li = self.init4::<SEED0>(sv);
let head_li = unroll4!(I => self.ld(ptr.add(I * BLOCK_LEN)));
let acc_li = unroll4!(I => self.acc_init_1(key_li[I], head_li[I]));
let tail = ptr.add(len - 64);
let acc_li = unroll4!(I => {
let j = I ^ 2;
self.acc_absorb_1(
acc_li[I],
self.add_u64(self.ld(tail.add(j * BLOCK_LEN)), head_li[j]),
)
});
self.merge4::<HI>(acc_li, lk)
}
}
#[inline(always)]
unsafe fn compress_33_64<const HI: bool, const SEED0: bool>(
self,
ptr: *const u8,
len: usize,
sv: B::V,
lk: B::V,
) -> (B::V, B::V) {
unsafe {
let key_li = self.init4::<SEED0>(sv);
let tail = ptr.add(len - 32);
let b_li = [
self.ld(ptr),
self.ld(ptr.add(BLOCK_LEN)),
self.ld(tail),
self.ld(tail.add(BLOCK_LEN)),
];
B::Mode::compress_33_64_core::<HI>(self, b_li, key_li, lk)
}
}
#[inline(always)]
unsafe fn absorb16_swapped(
self,
lo: Self::G8,
hi: Self::G8,
ptr: *const u8,
) -> (Self::G8, Self::G8) {
unsafe {
let hi_hi_li = unroll4!(I => self.acc_absorb_1(hi[I + 4], self.ld(ptr.add(I * BLOCK_LEN))));
let lo_lo_li = unroll4!(I => self.acc_absorb_1(lo[I], self.ld(ptr.add((I + 4) * BLOCK_LEN))));
let lo_hi_li =
unroll4!(I => self.acc_absorb_1(lo[I + 4], self.ld(ptr.add((I + 8) * BLOCK_LEN))));
let hi_lo_li =
unroll4!(I => self.acc_absorb_1(hi[I], self.ld(ptr.add((I + 12) * BLOCK_LEN))));
(join4(lo_lo_li, lo_hi_li), join4(hi_lo_li, hi_hi_li))
}
}
}