#[cfg(any(
all(
target_arch = "aarch64",
target_endian = "little",
target_feature = "aes",
target_feature = "neon"
),
all(
target_arch = "x86_64",
target_endian = "little",
target_feature = "avx512f",
target_feature = "vaes",
target_feature = "avx2",
target_feature = "aes",
target_feature = "sse2"
),
all(
target_arch = "x86",
target_endian = "little",
target_feature = "vaes",
target_feature = "avx2",
target_feature = "aes",
target_feature = "sse2"
),
all(
not(feature = "std"),
any(target_arch = "x86", target_arch = "x86_64"),
target_endian = "little",
target_feature = "aes",
target_feature = "sse2"
)
))]
mod static_platform {
use core::marker::PhantomData;
use crate::backend::Platform;
pub(crate) unsafe trait StaticTier {
fn hash32<const SEED0: bool>(data: &[u8], seed: u64) -> u32;
fn hash64<const SEED0: bool>(data: &[u8], seed: u64) -> u64;
fn hash128<const SEED0: bool>(data: &[u8], seed: u64) -> u128;
fn hash256<const SEED0: bool>(data: &[u8], seed: u64) -> [u8; 32];
fn ctr<const XOR: bool>(root: &[u8; 32], blk: u64, buf: &mut [u8]);
}
pub(crate) struct StaticPlatform<T>(PhantomData<T>);
impl<T: StaticTier> Platform for StaticPlatform<T> {
#[cfg_attr(not(debug_assertions), inline(always))]
#[cfg_attr(debug_assertions, inline)]
fn hash32<const SEED0: bool>(data: &[u8], seed: u64) -> u32 {
T::hash32::<SEED0>(data, seed)
}
#[cfg_attr(not(debug_assertions), inline(always))]
#[cfg_attr(debug_assertions, inline)]
fn hash64<const SEED0: bool>(data: &[u8], seed: u64) -> u64 {
T::hash64::<SEED0>(data, seed)
}
#[cfg_attr(not(debug_assertions), inline(always))]
#[cfg_attr(debug_assertions, inline)]
fn hash128<const SEED0: bool>(data: &[u8], seed: u64) -> u128 {
T::hash128::<SEED0>(data, seed)
}
#[cfg_attr(not(debug_assertions), inline(always))]
#[cfg_attr(debug_assertions, inline)]
fn hash256<const SEED0: bool>(data: &[u8], seed: u64) -> [u8; 32] {
T::hash256::<SEED0>(data, seed)
}
#[inline(always)]
fn ctr<const XOR: bool>(root: &[u8; 32], blk: u64, buf: &mut [u8]) {
if buf.is_empty() {
return;
}
T::ctr::<XOR>(root, blk, buf);
}
}
}
#[cfg(any(
all(
target_arch = "aarch64",
target_endian = "little",
target_feature = "aes",
target_feature = "neon"
),
all(
target_arch = "x86_64",
target_endian = "little",
target_feature = "avx512f",
target_feature = "vaes",
target_feature = "avx2",
target_feature = "aes",
target_feature = "sse2"
),
all(
target_arch = "x86",
target_endian = "little",
target_feature = "vaes",
target_feature = "avx2",
target_feature = "aes",
target_feature = "sse2"
),
all(
not(feature = "std"),
any(target_arch = "x86", target_arch = "x86_64"),
target_endian = "little",
target_feature = "aes",
target_feature = "sse2"
)
))]
pub(crate) use static_platform::{StaticPlatform, StaticTier};
macro_rules! def_hw_tier {
($ctx:expr, $ctr:expr, $vec:ty, $feat_cfg:meta, $is_max:meta, $caller_short:meta, $feat:literal) => {
#[inline(never)]
#[target_feature(enable = $feat)]
unsafe fn bulk_lo_inner<const SEED0: bool>(data: &[u8], seed: u64) -> $vec {
use crate::backend::tree::compress_bulk;
unsafe { compress_bulk::<_, false, SEED0>($ctx, data, seed).0 }
}
#[inline(never)]
#[target_feature(enable = $feat)]
unsafe fn bulk_hi_inner<const SEED0: bool>(data: &[u8], seed: u64) -> ($vec, $vec) {
use crate::backend::tree::compress_bulk;
unsafe { compress_bulk::<_, true, SEED0>($ctx, data, seed) }
}
#[inline(always)]
unsafe fn mid_or_bulk_lo<const SEED0: bool>(data: &[u8], seed: u64) -> $vec {
use crate::backend::tree::compress_long_lo;
unsafe { compress_long_lo::<_, SEED0>($ctx, data, seed, bulk_lo_inner::<SEED0>) }
}
#[inline(always)]
unsafe fn mid_or_bulk_hi<const SEED0: bool>(data: &[u8], seed: u64) -> ($vec, $vec) {
use crate::backend::tree::compress_long_hi;
unsafe { compress_long_hi::<_, SEED0>($ctx, data, seed, bulk_hi_inner::<SEED0>) }
}
#[cfg(any(
$is_max,
all($feat_cfg, not(feature = "std")),
all(feature = "std", not($caller_short))
))]
#[inline(never)]
#[target_feature(enable = $feat)]
unsafe fn long_lo_inner<const SEED0: bool>(data: &[u8], seed: u64) -> $vec {
unsafe { mid_or_bulk_lo::<SEED0>(data, seed) }
}
#[cfg(any(
$is_max,
all($feat_cfg, not(feature = "std")),
all(feature = "std", not($caller_short))
))]
#[inline(never)]
#[target_feature(enable = $feat)]
unsafe fn long_hi_inner<const SEED0: bool>(data: &[u8], seed: u64) -> ($vec, $vec) {
unsafe { mid_or_bulk_hi::<SEED0>(data, seed) }
}
#[cfg(any(
$is_max,
all($feat_cfg, not(feature = "std")),
all(feature = "std", not($caller_short))
))]
#[inline(always)]
unsafe fn hash_lo_v<const SEED0: bool>(data: &[u8], seed: u64) -> $vec {
use crate::arch::inline_entry::hash_lo_or;
unsafe {
hash_lo_or::<_, _, SEED0>(
$ctx,
data,
seed,
|_, v| v,
|d, s| long_lo_inner::<SEED0>(d, s),
)
}
}
#[cfg(any(
$is_max,
all($feat_cfg, not(feature = "std")),
all(feature = "std", not($caller_short))
))]
#[inline(always)]
unsafe fn hash_256<const SEED0: bool>(data: &[u8], seed: u64) -> [u8; 32] {
use crate::{
arch::inline_entry::hash256_or,
backend::{group::MidGroup, tree::pack_root_v},
};
unsafe {
hash256_or::<_, SEED0>($ctx, data, seed, |d, s| {
let (lo, hi) = long_hi_inner::<SEED0>(d, s);
pack_root_v($ctx.lane(), lo, hi)
})
}
}
#[target_feature(enable = $feat)]
pub(crate) unsafe fn ctr<const XOR: bool>(root: &[u8; 32], blk: u64, buf: &mut [u8]) {
use crate::backend::ctr::CtrKernel;
unsafe { $ctr.ctr::<XOR>(root, blk, buf) };
}
#[cfg(any($is_max, all($feat_cfg, not(feature = "std"))))]
mod static_tier {
use super::*;
use crate::{
arch::tier::StaticTier,
backend::{aes128::Aes128, group::MidGroup},
};
pub(crate) struct Tier;
unsafe impl StaticTier for Tier {
#[cfg_attr(not(debug_assertions), inline(always))]
#[cfg_attr(debug_assertions, inline)]
fn hash32<const SEED0: bool>(data: &[u8], seed: u64) -> u32 {
unsafe { $ctx.lane().to_u32(hash_lo_v::<SEED0>(data, seed)) }
}
#[cfg_attr(not(debug_assertions), inline(always))]
#[cfg_attr(debug_assertions, inline)]
fn hash64<const SEED0: bool>(data: &[u8], seed: u64) -> u64 {
unsafe { $ctx.lane().to_u64(hash_lo_v::<SEED0>(data, seed)) }
}
#[cfg_attr(not(debug_assertions), inline(always))]
#[cfg_attr(debug_assertions, inline)]
fn hash128<const SEED0: bool>(data: &[u8], seed: u64) -> u128 {
unsafe { $ctx.lane().to_u128(hash_lo_v::<SEED0>(data, seed)) }
}
#[cfg_attr(not(debug_assertions), inline(always))]
#[cfg_attr(debug_assertions, inline)]
fn hash256<const SEED0: bool>(data: &[u8], seed: u64) -> [u8; 32] {
unsafe { hash_256::<SEED0>(data, seed) }
}
#[inline(always)]
fn ctr<const XOR: bool>(root: &[u8; 32], blk: u64, buf: &mut [u8]) {
unsafe { ctr::<XOR>(root, blk, buf) };
}
}
}
#[cfg(any($is_max, all($feat_cfg, not(feature = "std"))))]
pub(crate) use static_tier::Tier;
#[cfg(all(feature = "std", not($is_max)))]
mod dispatch_tier {
use super::*;
#[cfg($caller_short)]
use crate::backend::tree::pack_root_v;
use crate::backend::{aes128::Aes128, group::MidGroup};
#[inline(always)]
unsafe fn tier_lo_v<const SEED0: bool>(data: &[u8], seed: u64) -> $vec {
#[cfg($caller_short)]
unsafe {
mid_or_bulk_lo::<SEED0>(data, seed)
}
#[cfg(not($caller_short))]
unsafe {
hash_lo_v::<SEED0>(data, seed)
}
}
#[inline(always)]
unsafe fn tier_64<const SEED0: bool>(data: &[u8], seed: u64) -> u64 {
unsafe { $ctx.lane().to_u64(tier_lo_v::<SEED0>(data, seed)) }
}
#[inline(always)]
unsafe fn tier_lo<const SEED0: bool>(data: &[u8], seed: u64) -> u128 {
unsafe { $ctx.lane().to_u128(tier_lo_v::<SEED0>(data, seed)) }
}
#[inline(always)]
unsafe fn tier_hi<const SEED0: bool>(data: &[u8], seed: u64) -> [u8; 32] {
#[cfg($caller_short)]
unsafe {
let (lo, hi) = mid_or_bulk_hi::<SEED0>(data, seed);
pack_root_v($ctx.lane(), lo, hi)
}
#[cfg(not($caller_short))]
unsafe {
hash_256::<SEED0>(data, seed)
}
}
#[target_feature(enable = $feat)]
pub(crate) unsafe fn dispatch_64(data: &[u8], seed: u64) -> u64 {
unsafe { tier_64::<false>(data, seed) }
}
#[target_feature(enable = $feat)]
pub(crate) unsafe fn dispatch_64_0(data: &[u8]) -> u64 {
unsafe { tier_64::<true>(data, 0) }
}
#[target_feature(enable = $feat)]
pub(crate) unsafe fn dispatch_lo(data: &[u8], seed: u64) -> u128 {
unsafe { tier_lo::<false>(data, seed) }
}
#[target_feature(enable = $feat)]
pub(crate) unsafe fn dispatch_lo_0(data: &[u8]) -> u128 {
unsafe { tier_lo::<true>(data, 0) }
}
#[target_feature(enable = $feat)]
pub(crate) unsafe fn dispatch_hi(data: &[u8], seed: u64) -> [u8; 32] {
unsafe { tier_hi::<false>(data, seed) }
}
#[target_feature(enable = $feat)]
pub(crate) unsafe fn dispatch_hi_0(data: &[u8]) -> [u8; 32] {
unsafe { tier_hi::<true>(data, 0) }
}
}
#[cfg(all(feature = "std", not($is_max)))]
pub(crate) use dispatch_tier::{
dispatch_64, dispatch_64_0, dispatch_hi, dispatch_hi_0, dispatch_lo, dispatch_lo_0,
};
};
}
pub(crate) use def_hw_tier;