#[cfg(not(all(
any(target_arch = "x86", target_arch = "x86_64"),
target_feature = "aes",
target_feature = "sse2"
)))]
use crate::backend::Platform;
#[derive(Clone, Copy)]
pub(crate) struct Dispatch {
pub(crate) hash_64: unsafe fn(&[u8], u64) -> u64,
pub(crate) hash_64_0: unsafe fn(&[u8]) -> u64,
pub(crate) hash_lo: unsafe fn(&[u8], u64) -> u128,
pub(crate) hash_lo_0: unsafe fn(&[u8]) -> u128,
pub(crate) hash_hi: unsafe fn(&[u8], u64) -> [u8; 32],
pub(crate) hash_hi_0: unsafe fn(&[u8]) -> [u8; 32],
pub(crate) ctr_fill: unsafe fn(&[u8; 32], u64, &mut [u8]),
pub(crate) ctr_xor: unsafe fn(&[u8; 32], u64, &mut [u8]),
}
impl Dispatch {
#[cfg(not(all(
any(target_arch = "x86", target_arch = "x86_64"),
target_feature = "aes",
target_feature = "sse2"
)))]
pub(crate) const fn for_platform<P: Platform>() -> Self {
unsafe fn p_64<P: Platform>(data: &[u8], seed: u64) -> u64 {
P::hash64::<false>(data, seed)
}
unsafe fn p_64_0<P: Platform>(data: &[u8]) -> u64 {
P::hash64::<true>(data, 0)
}
unsafe fn p_lo<P: Platform>(data: &[u8], seed: u64) -> u128 {
P::hash128::<false>(data, seed)
}
unsafe fn p_lo_0<P: Platform>(data: &[u8]) -> u128 {
P::hash128::<true>(data, 0)
}
unsafe fn p_hi<P: Platform>(data: &[u8], seed: u64) -> [u8; 32] {
P::hash256::<false>(data, seed)
}
unsafe fn p_hi_0<P: Platform>(data: &[u8]) -> [u8; 32] {
P::hash256::<true>(data, 0)
}
unsafe fn p_ctr<P: Platform, const XOR: bool>(root: &[u8; 32], blk: u64, buf: &mut [u8]) {
P::ctr::<XOR>(root, blk, buf);
}
Self {
hash_64: p_64::<P>,
hash_64_0: p_64_0::<P>,
hash_lo: p_lo::<P>,
hash_lo_0: p_lo_0::<P>,
hash_hi: p_hi::<P>,
hash_hi_0: p_hi_0::<P>,
ctr_fill: p_ctr::<P, false>,
ctr_xor: p_ctr::<P, true>,
}
}
#[inline(always)]
pub(crate) unsafe fn lo64<const SEED0: bool>(&self, data: &[u8], seed: u64) -> u64 {
unsafe {
if SEED0 {
(self.hash_64_0)(data)
} else {
(self.hash_64)(data, seed)
}
}
}
#[inline(always)]
pub(crate) unsafe fn lo<const SEED0: bool>(&self, data: &[u8], seed: u64) -> u128 {
unsafe {
if SEED0 {
(self.hash_lo_0)(data)
} else {
(self.hash_lo)(data, seed)
}
}
}
#[inline(always)]
pub(crate) unsafe fn hi<const SEED0: bool>(&self, data: &[u8], seed: u64) -> [u8; 32] {
unsafe {
if SEED0 {
(self.hash_hi_0)(data)
} else {
(self.hash_hi)(data, seed)
}
}
}
#[inline(always)]
pub(crate) fn ctr<const XOR: bool>(&self, root: &[u8; 32], blk: u64, buf: &mut [u8]) {
if buf.is_empty() {
return;
}
let f = if XOR { self.ctr_xor } else { self.ctr_fill };
unsafe { f(root, blk, buf) };
}
}
pub(crate) unsafe trait DispatchSource {
fn active() -> &'static Dispatch;
}
#[cfg(not(all(
any(target_arch = "x86", target_arch = "x86_64"),
target_feature = "aes",
target_feature = "sse2"
)))]
mod runtime_only {
use core::marker::PhantomData;
use super::DispatchSource;
use crate::backend::Platform;
pub(crate) struct RuntimePlatform<S>(PhantomData<S>);
impl<S: DispatchSource> Platform for RuntimePlatform<S> {
#[inline(always)]
fn hash32<const SEED0: bool>(data: &[u8], seed: u64) -> u32 {
Self::hash64::<SEED0>(data, seed) as u32
}
#[inline(always)]
fn hash64<const SEED0: bool>(data: &[u8], seed: u64) -> u64 {
unsafe { S::active().lo64::<SEED0>(data, seed) }
}
#[inline(always)]
fn hash128<const SEED0: bool>(data: &[u8], seed: u64) -> u128 {
unsafe { S::active().lo::<SEED0>(data, seed) }
}
#[inline(always)]
fn hash256<const SEED0: bool>(data: &[u8], seed: u64) -> [u8; 32] {
unsafe { S::active().hi::<SEED0>(data, seed) }
}
#[inline(always)]
fn ctr<const XOR: bool>(root: &[u8; 32], blk: u64, buf: &mut [u8]) {
S::active().ctr::<XOR>(root, blk, buf);
}
}
}
#[cfg(not(all(
any(target_arch = "x86", target_arch = "x86_64"),
target_feature = "aes",
target_feature = "sse2"
)))]
pub(crate) use runtime_only::RuntimePlatform;
#[cfg(all(
any(target_arch = "x86", target_arch = "x86_64"),
target_feature = "aes",
target_feature = "sse2"
))]
mod hybrid {
use core::marker::PhantomData;
use super::DispatchSource;
use crate::{
arch::inline_entry::{hash_lo_or, hash256_or},
backend::{Platform, aes128::Aes128, group::LaneGroup},
};
pub(crate) unsafe trait StaticGroup {
type G: LaneGroup;
fn group() -> Self::G;
}
pub(crate) struct HybridPlatform<S, G>(PhantomData<(S, G)>);
impl<S: DispatchSource, G: StaticGroup> Platform for HybridPlatform<S, G> {
#[cfg_attr(not(debug_assertions), inline(always))]
#[cfg_attr(debug_assertions, inline)]
fn hash32<const SEED0: bool>(data: &[u8], seed: u64) -> u32 {
unsafe {
hash_lo_or::<_, _, SEED0>(
G::group(),
data,
seed,
|l, v| l.to_u32(v),
|d, s| S::active().lo64::<SEED0>(d, s) as u32,
)
}
}
#[cfg_attr(not(debug_assertions), inline(always))]
#[cfg_attr(debug_assertions, inline)]
fn hash64<const SEED0: bool>(data: &[u8], seed: u64) -> u64 {
unsafe {
hash_lo_or::<_, _, SEED0>(
G::group(),
data,
seed,
|l, v| l.to_u64(v),
|d, s| S::active().lo64::<SEED0>(d, s),
)
}
}
#[cfg_attr(not(debug_assertions), inline(always))]
#[cfg_attr(debug_assertions, inline)]
fn hash128<const SEED0: bool>(data: &[u8], seed: u64) -> u128 {
unsafe {
hash_lo_or::<_, _, SEED0>(
G::group(),
data,
seed,
|l, v| l.to_u128(v),
|d, s| S::active().lo::<SEED0>(d, s),
)
}
}
#[cfg_attr(not(debug_assertions), inline(always))]
#[cfg_attr(debug_assertions, inline)]
fn hash256<const SEED0: bool>(data: &[u8], seed: u64) -> [u8; 32] {
unsafe {
hash256_or::<_, SEED0>(G::group(), data, seed, |d, s| S::active().hi::<SEED0>(d, s))
}
}
#[inline(always)]
fn ctr<const XOR: bool>(root: &[u8; 32], blk: u64, buf: &mut [u8]) {
S::active().ctr::<XOR>(root, blk, buf);
}
}
}
#[cfg(all(
any(target_arch = "x86", target_arch = "x86_64"),
target_feature = "aes",
target_feature = "sse2"
))]
pub(crate) use hybrid::{HybridPlatform, StaticGroup};
macro_rules! def_runtime_dispatch {
($name:ident, $len:expr, $tbl:expr, $detect:expr) => {
static ACTIVE_IDX: AtomicUsize = AtomicUsize::new(0);
#[cold]
fn init_idx() -> usize {
let idx = $detect();
ACTIVE_IDX.store(idx, Ordering::Relaxed);
idx
}
unsafe fn init_64(data: &[u8], seed: u64) -> u64 {
unsafe { (DISPATCH_LI.get_unchecked(init_idx()).hash_64)(data, seed) }
}
unsafe fn init_64_0(data: &[u8]) -> u64 {
unsafe { (DISPATCH_LI.get_unchecked(init_idx()).hash_64_0)(data) }
}
unsafe fn init_lo(data: &[u8], seed: u64) -> u128 {
unsafe { (DISPATCH_LI.get_unchecked(init_idx()).hash_lo)(data, seed) }
}
unsafe fn init_lo_0(data: &[u8]) -> u128 {
unsafe { (DISPATCH_LI.get_unchecked(init_idx()).hash_lo_0)(data) }
}
unsafe fn init_hi(data: &[u8], seed: u64) -> [u8; 32] {
unsafe { (DISPATCH_LI.get_unchecked(init_idx()).hash_hi)(data, seed) }
}
unsafe fn init_hi_0(data: &[u8]) -> [u8; 32] {
unsafe { (DISPATCH_LI.get_unchecked(init_idx()).hash_hi_0)(data) }
}
unsafe fn init_ctr_fill(root: &[u8; 32], blk: u64, buf: &mut [u8]) {
unsafe { (DISPATCH_LI.get_unchecked(init_idx()).ctr_fill)(root, blk, buf) };
}
unsafe fn init_ctr_xor(root: &[u8; 32], blk: u64, buf: &mut [u8]) {
unsafe { (DISPATCH_LI.get_unchecked(init_idx()).ctr_xor)(root, blk, buf) };
}
const INIT_DISPATCH: Dispatch = Dispatch {
hash_64: init_64,
hash_64_0: init_64_0,
hash_lo: init_lo,
hash_lo_0: init_lo_0,
hash_hi: init_hi,
hash_hi_0: init_hi_0,
ctr_fill: init_ctr_fill,
ctr_xor: init_ctr_xor,
};
static DISPATCH_LI: [Dispatch; $len] = $tbl(INIT_DISPATCH);
pub(crate) struct $name;
// SAFETY: `$detect` verifies CPU features before publishing the slot index in `ACTIVE_IDX`.
unsafe impl DispatchSource for $name {
#[inline(always)]
fn active() -> &'static Dispatch {
let idx = ACTIVE_IDX.load(Ordering::Relaxed);
unsafe { DISPATCH_LI.get_unchecked(idx) }
}
}
};
}
macro_rules! dispatch_slot {
($($seg:ident)::+) => {
Dispatch {
hash_64: $($seg)::+::dispatch_64,
hash_64_0: $($seg)::+::dispatch_64_0,
hash_lo: $($seg)::+::dispatch_lo,
hash_lo_0: $($seg)::+::dispatch_lo_0,
hash_hi: $($seg)::+::dispatch_hi,
hash_hi_0: $($seg)::+::dispatch_hi_0,
ctr_fill: $($seg)::+::ctr::<false>,
ctr_xor: $($seg)::+::ctr::<true>,
}
};
}
pub(crate) use def_runtime_dispatch;
pub(crate) use dispatch_slot;