#[cfg(all(
any(target_arch = "x86", target_arch = "x86_64"),
target_endian = "little",
any(feature = "std", all(target_feature = "aes", target_feature = "sse2"))
))]
use crate::backend::mode::PostXorAes;
#[cfg(any(
feature = "cpu",
all(target_arch = "aarch64", target_endian = "little"),
not(all(
any(target_arch = "x86", target_arch = "x86_64"),
target_endian = "little",
target_feature = "aes",
target_feature = "sse2"
))
))]
use crate::backend::mode::PreXorAes;
use crate::{
backend::{
aes128::Aes128,
group::{GroupV, LaneGroup, MidGroup},
},
const_fn::TBL,
state::{BLOCK_LEN, MID_MAX, MID4_MAX, SHORT_MAX, pack_root},
};
#[cold]
#[inline(always)]
pub(crate) const fn cold() {}
#[inline(always)]
unsafe fn len_key<L: Aes128>(ctx: L, len: usize) -> L::V {
unsafe { ctx.dup_u32(len as u32) }
}
#[inline(always)]
pub(crate) unsafe fn len_key_wide<L: Aes128>(ctx: L, len: usize) -> L::V {
let u = len as u64;
unsafe { ctx.dup_u32((u as u32) ^ ((u >> 32) as u32)) }
}
#[inline(always)]
pub(crate) unsafe fn pack_root_v<L: Aes128>(ctx: L, lo: L::V, hi: L::V) -> [u8; 32] {
unsafe { pack_root(ctx.to_state(lo), ctx.to_state(hi)) }
}
#[inline(always)]
pub(crate) unsafe fn expand_seed<L: Aes128, const SEED0: bool>(ctx: L, seed: u64) -> L::V {
unsafe {
if SEED0 {
ctx.ld(TBL.sv0.as_ptr())
} else {
ctx.r1_zero(ctx.seed_v(seed), ctx.ld(TBL.fin_li[6].as_ptr()))
}
}
}
#[inline(always)]
unsafe fn expand_lane_seed<L: Aes128, const SEED0: bool>(ctx: L, seed: u64) -> L::V {
unsafe {
if SEED0 {
ctx.zero()
} else {
expand_seed::<L, false>(ctx, seed)
}
}
}
#[inline(always)]
pub(crate) unsafe fn finalize_hi<L: Aes128, const HI: bool>(
ctx: L,
lo: L::V,
q1: L::V,
lk: L::V,
) -> (L::V, L::V) {
let hi = unsafe {
if HI {
let c1 = ctx.ld(TBL.fin_li[1].as_ptr());
let c2 = ctx.ld(TBL.fin_li[2].as_ptr());
ctx.r3_zero(lo, ctx.xor(q1, c1), lk, c2)
} else {
ctx.zero()
}
};
(lo, hi)
}
#[cfg(all(
any(target_arch = "x86", target_arch = "x86_64"),
target_endian = "little",
any(feature = "std", all(target_feature = "aes", target_feature = "sse2"))
))]
#[inline(always)]
pub(crate) unsafe fn finalize_merge4_lk<L: PostXorAes, const HI: bool>(
ctx: L,
lane_lk_li: [L::V; 4],
lk: L::V,
) -> (L::V, L::V) {
unsafe {
let z = ctx.zero();
let q0 = ctx.post_enclast(lane_lk_li[0], lane_lk_li[2]);
let q1 = ctx.add_u64(ctx.post_enc(lane_lk_li[1], z), lane_lk_li[3]);
let q1_z = ctx.zip_u8(q1);
let s = ctx.post_enc(q0, q1_z);
let u = ctx.post_enc(s, z);
let fin = ctx.post_enc(u, z);
let lo = ctx.add_u64(fin, u);
finalize_hi::<L, HI>(ctx, lo, q1_z, lk)
}
}
#[cfg(any(
feature = "cpu",
all(target_arch = "aarch64", target_endian = "little"),
not(all(
any(target_arch = "x86", target_arch = "x86_64"),
target_endian = "little",
target_feature = "aes",
target_feature = "sse2"
))
))]
#[inline(always)]
pub(crate) unsafe fn finalize_merge4_pre<L: PreXorAes, const HI: bool>(
ctx: L,
lane_li: [L::V; 4],
lk: L::V,
) -> (L::V, L::V) {
unsafe {
let z = ctx.zero();
let q0_a = ctx.pre_fl_zero(lane_li[0], lk);
let q1 = ctx.add_u64(ctx.pre_r1_zero(lane_li[1], z), lane_li[3]);
let q1_z = ctx.zip_u8(q1);
let s_a = ctx.pre_r1_zero(q0_a, lane_li[2]);
let lo = ctx.chain_x01_ff(s_a, q1_z, []);
finalize_hi::<L, HI>(ctx, lo, q1_z, lk)
}
}
#[inline(always)]
pub(crate) unsafe fn compress_0_16_entry<L: Aes128, const HI: bool, const SEED0: bool>(
ctx: L,
data: &[u8],
seed: u64,
) -> (L::V, L::V) {
let len = data.len();
debug_assert!(len <= BLOCK_LEN);
let ptr = data.as_ptr();
unsafe {
let b = ctx.load_short(ptr, len);
let sv = expand_seed::<L, SEED0>(ctx, seed);
let lk = len_key(ctx, len);
let lo = ctx.r3_ff(b, sv, lk);
finalize_hi::<L, HI>(ctx, lo, sv, lk)
}
}
#[inline(always)]
pub(crate) unsafe fn compress_17_128<G: LaneGroup, const HI: bool, const SEED0: bool>(
ctx: G,
data: &[u8],
seed: u64,
) -> (GroupV<G>, GroupV<G>) {
let l = ctx.lane();
let len = data.len();
debug_assert!(len > BLOCK_LEN && len <= SHORT_MAX);
let ptr = data.as_ptr();
unsafe {
let sv = expand_lane_seed::<G::L, SEED0>(l, seed);
let lk = len_key(l, len);
if len <= 32 {
ctx.compress_17_32::<HI, SEED0>(ptr, len, sv, lk)
} else if len <= 64 {
ctx.compress_33_64::<HI, SEED0>(ptr, len, sv, lk)
} else {
ctx.compress_65_128::<HI, SEED0>(ptr, len, sv, lk)
}
}
}
#[cfg(feature = "cpu")]
#[inline(always)]
pub(crate) unsafe fn compress_short<G: LaneGroup, const HI: bool, const SEED0: bool>(
ctx: G,
data: &[u8],
seed: u64,
) -> (GroupV<G>, GroupV<G>) {
if data.len() <= BLOCK_LEN {
unsafe { compress_0_16_entry::<G::L, HI, SEED0>(ctx.lane(), data, seed) }
} else {
unsafe { compress_17_128::<G, HI, SEED0>(ctx, data, seed) }
}
}
#[inline(always)]
pub(crate) unsafe fn compress_mid<G: MidGroup, const HI: bool, const SEED0: bool>(
ctx: G,
data: &[u8],
seed: u64,
) -> (GroupV<G>, GroupV<G>) {
let l = ctx.lane();
let len = data.len();
debug_assert!(len > SHORT_MAX && len <= MID_MAX);
let mut ptr = data.as_ptr();
unsafe {
let sv = expand_lane_seed::<G::L, SEED0>(l, seed);
let lk = len_key(l, len);
if len <= MID4_MAX {
let end = ptr.add(len);
let mut acc = ctx.absorb4_init(ctx.init4::<SEED0>(sv), ptr);
if len <= 192 {
acc = ctx.absorb4_swap32(acc, end.sub(128));
acc = ctx.absorb4_linear(acc, end.sub(64));
} else {
acc = ctx.absorb4_swap32(acc, ptr.add(64));
if len > 256 {
acc = ctx.absorb4_linear(acc, ptr.add(128));
acc = ctx.absorb4_swap32(acc, ptr.add(192));
if len > 384 {
acc = ctx.absorb4_linear(acc, ptr.add(256));
acc = ctx.absorb4_swap32(acc, ptr.add(320));
}
}
acc = ctx.absorb4_linear(acc, end.sub(128));
acc = ctx.absorb4_swap32(acc, end.sub(64));
}
return ctx.merge4::<HI>(acc, lk);
}
let mut acc = ctx.absorb8_init(ctx.init8::<SEED0>(sv, 0), ptr);
ptr = ptr.add(128);
let mut rem = ((len - 1) >> 7) - 1;
while rem >= 2 {
acc = ctx.absorb8_pair64(acc, ptr.add(64), ptr);
let p2 = ptr.add(128);
acc = ctx.absorb8_pair64(acc, p2, p2.add(64));
ptr = ptr.add(256);
rem -= 2;
}
if rem == 1 {
acc = ctx.absorb8_pair64(acc, ptr.add(64), ptr);
}
let last = data.as_ptr().add(len - 128);
acc = ctx.absorb8_swap32(acc, last);
ctx.merge8::<HI>(acc, lk)
}
}
#[inline(always)]
unsafe fn absorb16_pair<G: LaneGroup>(
ctx: G,
lo: G::G8,
hi: G::G8,
ptr: *const u8,
) -> (G::G8, G::G8) {
unsafe {
let (lo, hi) = ctx.absorb16_swapped(lo, hi, ptr);
ctx.absorb16_linear(lo, hi, ptr.add(256))
}
}
#[inline(always)]
pub(crate) unsafe fn compress_bulk<G: LaneGroup, const HI: bool, const SEED0: bool>(
ctx: G,
data: &[u8],
seed: u64,
) -> (GroupV<G>, GroupV<G>) {
const PREFETCH_THRESHOLD: usize = 32 * 1024;
const PREFETCH_DIST: usize = 768;
let l = ctx.lane();
let len = data.len();
debug_assert!(len > MID_MAX);
let mut ptr = data.as_ptr();
unsafe {
let sv = expand_lane_seed::<G::L, SEED0>(l, seed);
let mut lo = ctx.absorb8_init(ctx.init8::<SEED0>(sv, 0), ptr);
let mut hi = ctx.absorb8_init(ctx.init8::<SEED0>(sv, 8), ptr.add(128));
ptr = ptr.add(256);
let mut rem = ((len - 1) >> 8) - 1;
if G::L::HAS_PREFETCH && len > PREFETCH_THRESHOLD {
while rem >= 5 {
l.prefetch_l1(ptr.add(PREFETCH_DIST));
l.prefetch_l1(ptr.add(PREFETCH_DIST + 128));
l.prefetch_l1(ptr.add(PREFETCH_DIST + 256));
l.prefetch_l1(ptr.add(PREFETCH_DIST + 384));
(lo, hi) = absorb16_pair(ctx, lo, hi, ptr);
ptr = ptr.add(512);
rem -= 2;
}
}
while rem >= 2 {
(lo, hi) = absorb16_pair(ctx, lo, hi, ptr);
ptr = ptr.add(512);
rem -= 2;
}
if rem == 1 {
(lo, hi) = ctx.absorb16_swapped(lo, hi, ptr);
}
let last = data.as_ptr().add(len - 256);
lo = ctx.absorb8_pair64(lo, last.add(128), last.add(192));
hi = ctx.absorb8_pair64(hi, last, last.add(64));
let lk = len_key_wide(l, len);
ctx.merge16::<HI>(lo, hi, lk)
}
}
pub(crate) type LongLoFn<V> = unsafe fn(&[u8], u64) -> V;
pub(crate) type LongHiFn<V> = unsafe fn(&[u8], u64) -> (V, V);
#[inline(always)]
pub(crate) unsafe fn compress_long_lo<G: MidGroup, const SEED0: bool>(
ctx: G,
data: &[u8],
seed: u64,
bulk_lo: LongLoFn<GroupV<G>>,
) -> GroupV<G> {
debug_assert!(data.len() > SHORT_MAX);
if data.len() <= MID_MAX {
return unsafe { compress_mid::<G, false, SEED0>(ctx, data, seed).0 };
}
unsafe { bulk_lo(data, seed) }
}
#[inline(always)]
pub(crate) unsafe fn compress_long_hi<G: MidGroup, const SEED0: bool>(
ctx: G,
data: &[u8],
seed: u64,
bulk_hi: LongHiFn<GroupV<G>>,
) -> (GroupV<G>, GroupV<G>) {
debug_assert!(data.len() > SHORT_MAX);
if data.len() <= MID_MAX {
return unsafe { compress_mid::<G, true, SEED0>(ctx, data, seed) };
}
unsafe { bulk_hi(data, seed) }
}