use core::ptr::copy_nonoverlapping;
#[cfg(feature = "cpu")]
use crate::backend::tree::compress_short;
use crate::{
arch::inline_entry::{hash_lo_or, hash256_or},
backend::{
Platform,
aes128::Aes128,
ctr::{CtrKernel, ctr_1x128, split_root},
group::Lane128x8,
mode::{PreXor, PreXorAes},
tree::{compress_bulk, compress_long_hi, compress_long_lo, pack_root_v},
},
const_fn::{SBOX_LI, T0_LI, T1_LI, T2_LI, T3_LI},
state::{BLOCK_LEN, State, rd32_ptr},
};
#[repr(C, align(64))]
struct TTable {
t0_li: [u32; 256],
t1_li: [u32; 256],
t2_li: [u32; 256],
t3_li: [u32; 256],
}
static T_TBL: TTable = TTable {
t0_li: T0_LI,
t1_li: T1_LI,
t2_li: T2_LI,
t3_li: T3_LI,
};
pub(crate) type WordState = [u32; 4];
#[inline(always)]
const fn pair_to_word(lo: u64, hi: u64) -> WordState {
[lo as u32, (lo >> 32) as u32, hi as u32, (hi >> 32) as u32]
}
#[inline(always)]
pub(crate) const fn state_to_word(state: State) -> WordState {
let chunk_li = state.as_chunks::<4>().0;
[
u32::from_le_bytes(chunk_li[0]),
u32::from_le_bytes(chunk_li[1]),
u32::from_le_bytes(chunk_li[2]),
u32::from_le_bytes(chunk_li[3]),
]
}
#[inline(always)]
const fn word_to_u128(w: WordState) -> u128 {
let lo = (w[0] as u64) | ((w[1] as u64) << 32);
let hi = (w[2] as u64) | ((w[3] as u64) << 32);
(lo as u128) | ((hi as u128) << 64)
}
#[inline(always)]
pub(crate) const fn word_to_state(w: WordState) -> State {
word_to_u128(w).to_le_bytes()
}
#[inline(always)]
const fn xor_word(a: WordState, b: WordState) -> WordState {
[a[0] ^ b[0], a[1] ^ b[1], a[2] ^ b[2], a[3] ^ b[3]]
}
#[inline(always)]
fn aes_col_round(
state: WordState,
rk: WordState,
col: impl Fn(u32, u32, u32, u32) -> u32,
) -> WordState {
let [s0, s1, s2, s3] = state;
[
col(s0, s1, s2, s3) ^ rk[0],
col(s1, s2, s3, s0) ^ rk[1],
col(s2, s3, s0, s1) ^ rk[2],
col(s3, s0, s1, s2) ^ rk[3],
]
}
#[inline(always)]
fn aes_enc_word(state: WordState, rk: WordState) -> WordState {
let tbl = &T_TBL;
aes_col_round(state, rk, |c0, c1, c2, c3| {
tbl.t0_li[(c0 & 0xFF) as usize]
^ tbl.t1_li[((c1 >> 8) & 0xFF) as usize]
^ tbl.t2_li[((c2 >> 16) & 0xFF) as usize]
^ tbl.t3_li[(c3 >> 24) as usize]
})
}
#[inline(always)]
fn aes_enclast_word(state: WordState, rk: WordState) -> WordState {
aes_col_round(state, rk, |c0, c1, c2, c3| {
(SBOX_LI[(c0 & 0xFF) as usize] as u32)
| ((SBOX_LI[((c1 >> 8) & 0xFF) as usize] as u32) << 8)
| ((SBOX_LI[((c2 >> 16) & 0xFF) as usize] as u32) << 16)
| ((SBOX_LI[(c3 >> 24) as usize] as u32) << 24)
})
}
#[derive(Clone, Copy)]
pub(crate) struct Soft128;
impl Aes128 for Soft128 {
type V = WordState;
type Mode = PreXor;
#[inline(always)]
unsafe fn zero(self) -> Self::V {
[0; 4]
}
#[inline(always)]
unsafe fn ld(self, ptr: *const u8) -> Self::V {
unsafe {
[
rd32_ptr(ptr, 0),
rd32_ptr(ptr, 4),
rd32_ptr(ptr, 8),
rd32_ptr(ptr, 12),
]
}
}
#[inline(always)]
unsafe fn st(self, ptr: *mut u8, v: Self::V) {
let raw_li = word_to_state(v);
unsafe { copy_nonoverlapping(raw_li.as_ptr(), ptr, BLOCK_LEN) };
}
#[inline(always)]
unsafe fn to_state(self, v: Self::V) -> State {
word_to_state(v)
}
#[inline(always)]
unsafe fn to_u32(self, v: Self::V) -> u32 {
v[0]
}
#[inline(always)]
unsafe fn to_u64(self, v: Self::V) -> u64 {
(v[0] as u64) | ((v[1] as u64) << 32)
}
#[inline(always)]
unsafe fn to_u128(self, v: Self::V) -> u128 {
word_to_u128(v)
}
#[inline(always)]
unsafe fn seed_v(self, seed: u64) -> Self::V {
pair_to_word(seed, seed)
}
#[inline(always)]
unsafe fn pair_v(self, lo: u64, hi: u64) -> Self::V {
pair_to_word(lo, hi)
}
#[inline(always)]
unsafe fn dup_u32(self, x: u32) -> Self::V {
[x; 4]
}
#[inline(always)]
unsafe fn xor(self, a: Self::V, b: Self::V) -> Self::V {
xor_word(a, b)
}
#[inline(always)]
unsafe fn add_u64(self, a: Self::V, b: Self::V) -> Self::V {
let a_lo = (a[0] as u64) | ((a[1] as u64) << 32);
let b_lo = (b[0] as u64) | ((b[1] as u64) << 32);
let a_hi = (a[2] as u64) | ((a[3] as u64) << 32);
let b_hi = (b[2] as u64) | ((b[3] as u64) << 32);
pair_to_word(a_lo.wrapping_add(b_lo), a_hi.wrapping_add(b_hi))
}
#[inline(always)]
unsafe fn zip_u8(self, v: Self::V) -> Self::V {
let s = word_to_state(v);
state_to_word([
s[0], s[8], s[1], s[9], s[2], s[10], s[3], s[11], s[4], s[12], s[5], s[13], s[6], s[14],
s[7], s[15],
])
}
}
impl PreXorAes for Soft128 {
#[inline(always)]
unsafe fn pre_r1_zero(self, a: Self::V, b: Self::V) -> Self::V {
aes_enc_word(xor_word(a, b), [0; 4])
}
#[inline(always)]
unsafe fn pre_fl_zero(self, a: Self::V, b: Self::V) -> Self::V {
aes_enclast_word(xor_word(a, b), [0; 4])
}
#[inline(always)]
unsafe fn pre_enc(self, state: Self::V, rk: Self::V) -> Self::V {
aes_enc_word(state, rk)
}
}
impl Lane128x8 for Soft128 {}
impl CtrKernel for Soft128 {
#[inline]
unsafe fn ctr<const XOR: bool>(self, root: &[u8; 32], blk: u64, buf: &mut [u8]) {
unsafe {
let rk_li = split_root(Soft128, root);
ctr_1x128::<Soft128, XOR>(Soft128, rk_li, blk, buf.as_mut_ptr(), buf.len());
}
}
}
#[inline(never)]
fn bulk_lo_inner<const SEED0: bool>(data: &[u8], seed: u64) -> WordState {
unsafe { compress_bulk::<Soft128, false, SEED0>(Soft128, data, seed).0 }
}
#[inline(never)]
fn bulk_hi_inner<const SEED0: bool>(data: &[u8], seed: u64) -> (WordState, WordState) {
unsafe { compress_bulk::<Soft128, true, SEED0>(Soft128, data, seed) }
}
#[inline(never)]
fn long_lo_inner<const SEED0: bool>(data: &[u8], seed: u64) -> WordState {
unsafe { compress_long_lo::<Soft128, SEED0>(Soft128, data, seed, bulk_lo_inner::<SEED0>) }
}
#[inline(never)]
fn long_hi_inner<const SEED0: bool>(data: &[u8], seed: u64) -> (WordState, WordState) {
unsafe { compress_long_hi::<Soft128, SEED0>(Soft128, data, seed, bulk_hi_inner::<SEED0>) }
}
#[cfg(feature = "cpu")]
#[inline(always)]
pub(crate) fn short_128<const SEED0: bool>(data: &[u8], seed: u64) -> u128 {
unsafe { Soft128.to_u128(compress_short::<Soft128, false, SEED0>(Soft128, data, seed).0) }
}
#[cfg(feature = "cpu")]
#[inline(always)]
pub(crate) fn short_256<const SEED0: bool>(data: &[u8], seed: u64) -> [u8; 32] {
unsafe {
let (lo, hi) = compress_short::<Soft128, true, SEED0>(Soft128, data, seed);
pack_root_v(Soft128, lo, hi)
}
}
#[inline(always)]
fn compress_lo_v<const SEED0: bool>(data: &[u8], seed: u64) -> WordState {
unsafe { hash_lo_or::<Soft128, _, SEED0>(Soft128, data, seed, |_, v| v, long_lo_inner::<SEED0>) }
}
#[cfg_attr(not(debug_assertions), inline(always))]
#[cfg_attr(debug_assertions, inline)]
pub(crate) fn compress_128<const SEED0: bool>(data: &[u8], seed: u64) -> u128 {
unsafe { Soft128.to_u128(compress_lo_v::<SEED0>(data, seed)) }
}
#[cfg_attr(not(debug_assertions), inline(always))]
#[cfg_attr(debug_assertions, inline)]
pub(crate) fn compress_256<const SEED0: bool>(data: &[u8], seed: u64) -> [u8; 32] {
unsafe {
hash256_or::<Soft128, SEED0>(Soft128, data, seed, |d, s| {
let (lo, hi) = long_hi_inner::<SEED0>(d, s);
pack_root_v(Soft128, lo, hi)
})
}
}
impl Platform for Soft128 {
#[inline(always)]
fn hash32<const SEED0: bool>(data: &[u8], seed: u64) -> u32 {
unsafe { Soft128.to_u32(compress_lo_v::<SEED0>(data, seed)) }
}
#[inline(always)]
fn hash64<const SEED0: bool>(data: &[u8], seed: u64) -> u64 {
unsafe { Soft128.to_u64(compress_lo_v::<SEED0>(data, seed)) }
}
#[inline(always)]
fn hash128<const SEED0: bool>(data: &[u8], seed: u64) -> u128 {
compress_128::<SEED0>(data, seed)
}
#[inline(always)]
fn hash256<const SEED0: bool>(data: &[u8], seed: u64) -> [u8; 32] {
compress_256::<SEED0>(data, seed)
}
#[inline(always)]
fn ctr<const XOR: bool>(root: &[u8; 32], blk: u64, buf: &mut [u8]) {
if buf.is_empty() {
return;
}
unsafe { Soft128.ctr::<XOR>(root, blk, buf) };
}
}