ihash 0.1.2

World's fastest high-quality non-cryptographic hash, deterministic cross-platform XOF, and seekable stream cipher
Documentation
#[cfg(not(all(
  any(target_arch = "x86", target_arch = "x86_64"),
  target_feature = "aes",
  target_feature = "sse2"
)))]
use crate::backend::Platform;

/// Function pointer table for branchless, `match`-free runtime dispatch on long inputs and CTR.
#[derive(Clone, Copy)]
pub(crate) struct Dispatch {
  pub(crate) hash_64: unsafe fn(&[u8], u64) -> u64,
  pub(crate) hash_64_0: unsafe fn(&[u8]) -> u64,
  pub(crate) hash_lo: unsafe fn(&[u8], u64) -> u128,
  pub(crate) hash_lo_0: unsafe fn(&[u8]) -> u128,
  pub(crate) hash_hi: unsafe fn(&[u8], u64) -> [u8; 32],
  pub(crate) hash_hi_0: unsafe fn(&[u8]) -> [u8; 32],
  pub(crate) ctr_fill: unsafe fn(&[u8; 32], u64, &mut [u8]),
  pub(crate) ctr_xor: unsafe fn(&[u8; 32], u64, &mut [u8]),
}

impl Dispatch {
  #[cfg(not(all(
    any(target_arch = "x86", target_arch = "x86_64"),
    target_feature = "aes",
    target_feature = "sse2"
  )))]
  pub(crate) const fn for_platform<P: Platform>() -> Self {
    unsafe fn p_64<P: Platform>(data: &[u8], seed: u64) -> u64 {
      P::hash64::<false>(data, seed)
    }

    unsafe fn p_64_0<P: Platform>(data: &[u8]) -> u64 {
      P::hash64::<true>(data, 0)
    }

    unsafe fn p_lo<P: Platform>(data: &[u8], seed: u64) -> u128 {
      P::hash128::<false>(data, seed)
    }

    unsafe fn p_lo_0<P: Platform>(data: &[u8]) -> u128 {
      P::hash128::<true>(data, 0)
    }

    unsafe fn p_hi<P: Platform>(data: &[u8], seed: u64) -> [u8; 32] {
      P::hash256::<false>(data, seed)
    }

    unsafe fn p_hi_0<P: Platform>(data: &[u8]) -> [u8; 32] {
      P::hash256::<true>(data, 0)
    }

    unsafe fn p_ctr<P: Platform, const XOR: bool>(root: &[u8; 32], blk: u64, buf: &mut [u8]) {
      P::ctr::<XOR>(root, blk, buf);
    }

    Self {
      hash_64: p_64::<P>,
      hash_64_0: p_64_0::<P>,
      hash_lo: p_lo::<P>,
      hash_lo_0: p_lo_0::<P>,
      hash_hi: p_hi::<P>,
      hash_hi_0: p_hi_0::<P>,
      ctr_fill: p_ctr::<P, false>,
      ctr_xor: p_ctr::<P, true>,
    }
  }

  #[inline(always)]
  pub(crate) unsafe fn lo64<const SEED0: bool>(&self, data: &[u8], seed: u64) -> u64 {
    // SAFETY: caller ensures data and seed match the active dispatch slot contract.
    unsafe {
      if SEED0 {
        (self.hash_64_0)(data)
      } else {
        (self.hash_64)(data, seed)
      }
    }
  }

  #[inline(always)]
  pub(crate) unsafe fn lo<const SEED0: bool>(&self, data: &[u8], seed: u64) -> u128 {
    // SAFETY: caller ensures data and seed match the active dispatch slot contract.
    unsafe {
      if SEED0 {
        (self.hash_lo_0)(data)
      } else {
        (self.hash_lo)(data, seed)
      }
    }
  }

  #[inline(always)]
  pub(crate) unsafe fn hi<const SEED0: bool>(&self, data: &[u8], seed: u64) -> [u8; 32] {
    // SAFETY: caller ensures data and seed match the active dispatch slot contract.
    unsafe {
      if SEED0 {
        (self.hash_hi_0)(data)
      } else {
        (self.hash_hi)(data, seed)
      }
    }
  }

  #[inline(always)]
  pub(crate) fn ctr<const XOR: bool>(&self, root: &[u8; 32], blk: u64, buf: &mut [u8]) {
    if buf.is_empty() {
      return;
    }
    let f = if XOR { self.ctr_xor } else { self.ctr_fill };
    // SAFETY: dispatch slot matches active CPU features and buf holds valid mutable bytes.
    unsafe { f(root, blk, buf) };
  }
}

/// Lazy atomic runtime slot provider for dynamically dispatched architectures.
///
/// # Safety
///
/// `active()` must only return a [`Dispatch`] slot verified safe for the current CPU.
pub(crate) unsafe trait DispatchSource {
  fn active() -> &'static Dispatch;
}

#[cfg(not(all(
  any(target_arch = "x86", target_arch = "x86_64"),
  target_feature = "aes",
  target_feature = "sse2"
)))]
mod runtime_only {
  use core::marker::PhantomData;

  use super::DispatchSource;
  use crate::backend::Platform;

  /// Bridges a pure runtime [`DispatchSource`] table onto [`Platform`].
  pub(crate) struct RuntimePlatform<S>(PhantomData<S>);

  impl<S: DispatchSource> Platform for RuntimePlatform<S> {
    #[inline(always)]
    fn hash32<const SEED0: bool>(data: &[u8], seed: u64) -> u32 {
      Self::hash64::<SEED0>(data, seed) as u32
    }

    #[inline(always)]
    fn hash64<const SEED0: bool>(data: &[u8], seed: u64) -> u64 {
      // SAFETY: active() resolves to a verified dispatch slot for current CPU features.
      unsafe { S::active().lo64::<SEED0>(data, seed) }
    }

    #[inline(always)]
    fn hash128<const SEED0: bool>(data: &[u8], seed: u64) -> u128 {
      // SAFETY: active() resolves to a verified dispatch slot for current CPU features.
      unsafe { S::active().lo::<SEED0>(data, seed) }
    }

    #[inline(always)]
    fn hash256<const SEED0: bool>(data: &[u8], seed: u64) -> [u8; 32] {
      // SAFETY: active() resolves to a verified dispatch slot for current CPU features.
      unsafe { S::active().hi::<SEED0>(data, seed) }
    }

    #[inline(always)]
    fn ctr<const XOR: bool>(root: &[u8; 32], blk: u64, buf: &mut [u8]) {
      S::active().ctr::<XOR>(root, blk, buf);
    }
  }
}

#[cfg(not(all(
  any(target_arch = "x86", target_arch = "x86_64"),
  target_feature = "aes",
  target_feature = "sse2"
)))]
pub(crate) use runtime_only::RuntimePlatform;

#[cfg(all(
  any(target_arch = "x86", target_arch = "x86_64"),
  target_feature = "aes",
  target_feature = "sse2"
))]
mod hybrid {
  use core::marker::PhantomData;

  use super::DispatchSource;
  use crate::{
    arch::inline_entry::{hash_lo_or, hash256_or},
    backend::{Platform, aes128::Aes128, group::LaneGroup},
  };

  /// Compile-time-guaranteed vector group used to inline `0..=128 B` hashes before runtime dispatch.
  ///
  /// # Safety
  ///
  /// `Self::G` operations must only use target features unconditionally enabled at compile time.
  pub(crate) unsafe trait StaticGroup {
    type G: LaneGroup;
    fn group() -> Self::G;
  }

  /// Combines inline compile-time-safe short hashing (`0..=128 B`) with dynamic [`DispatchSource`] long dispatch.
  pub(crate) struct HybridPlatform<S, G>(PhantomData<(S, G)>);

  impl<S: DispatchSource, G: StaticGroup> Platform for HybridPlatform<S, G> {
    #[cfg_attr(not(debug_assertions), inline(always))]
    #[cfg_attr(debug_assertions, inline)]
    fn hash32<const SEED0: bool>(data: &[u8], seed: u64) -> u32 {
      // SAFETY: StaticGroup matches compile-time target features; DispatchSource resolves valid runtime slot.
      unsafe {
        hash_lo_or::<_, _, SEED0>(
          G::group(),
          data,
          seed,
          |l, v| l.to_u32(v),
          |d, s| S::active().lo64::<SEED0>(d, s) as u32,
        )
      }
    }

    #[cfg_attr(not(debug_assertions), inline(always))]
    #[cfg_attr(debug_assertions, inline)]
    fn hash64<const SEED0: bool>(data: &[u8], seed: u64) -> u64 {
      // SAFETY: StaticGroup matches compile-time target features; DispatchSource resolves valid runtime slot.
      unsafe {
        hash_lo_or::<_, _, SEED0>(
          G::group(),
          data,
          seed,
          |l, v| l.to_u64(v),
          |d, s| S::active().lo64::<SEED0>(d, s),
        )
      }
    }

    #[cfg_attr(not(debug_assertions), inline(always))]
    #[cfg_attr(debug_assertions, inline)]
    fn hash128<const SEED0: bool>(data: &[u8], seed: u64) -> u128 {
      // SAFETY: StaticGroup matches compile-time target features; DispatchSource resolves valid runtime slot.
      unsafe {
        hash_lo_or::<_, _, SEED0>(
          G::group(),
          data,
          seed,
          |l, v| l.to_u128(v),
          |d, s| S::active().lo::<SEED0>(d, s),
        )
      }
    }

    #[cfg_attr(not(debug_assertions), inline(always))]
    #[cfg_attr(debug_assertions, inline)]
    fn hash256<const SEED0: bool>(data: &[u8], seed: u64) -> [u8; 32] {
      // SAFETY: StaticGroup matches compile-time target features; DispatchSource resolves valid runtime slot.
      unsafe {
        hash256_or::<_, SEED0>(G::group(), data, seed, |d, s| S::active().hi::<SEED0>(d, s))
      }
    }

    #[inline(always)]
    fn ctr<const XOR: bool>(root: &[u8; 32], blk: u64, buf: &mut [u8]) {
      S::active().ctr::<XOR>(root, blk, buf);
    }
  }
}

#[cfg(all(
  any(target_arch = "x86", target_arch = "x86_64"),
  target_feature = "aes",
  target_feature = "sse2"
))]
pub(crate) use hybrid::{HybridPlatform, StaticGroup};

macro_rules! def_runtime_dispatch {
  ($name:ident, $len:expr, $tbl:expr, $detect:expr) => {
    static ACTIVE_IDX: AtomicUsize = AtomicUsize::new(0);

    #[cold]
    fn init_idx() -> usize {
      let idx = $detect();
      ACTIVE_IDX.store(idx, Ordering::Relaxed);
      idx
    }

    unsafe fn init_64(data: &[u8], seed: u64) -> u64 {
      // SAFETY: init_idx() returns valid non-zero slot in DISPATCH_LI.
      unsafe { (DISPATCH_LI.get_unchecked(init_idx()).hash_64)(data, seed) }
    }

    unsafe fn init_64_0(data: &[u8]) -> u64 {
      // SAFETY: init_idx() returns valid non-zero slot in DISPATCH_LI.
      unsafe { (DISPATCH_LI.get_unchecked(init_idx()).hash_64_0)(data) }
    }

    unsafe fn init_lo(data: &[u8], seed: u64) -> u128 {
      // SAFETY: init_idx() returns valid non-zero slot in DISPATCH_LI.
      unsafe { (DISPATCH_LI.get_unchecked(init_idx()).hash_lo)(data, seed) }
    }

    unsafe fn init_lo_0(data: &[u8]) -> u128 {
      // SAFETY: init_idx() returns valid non-zero slot in DISPATCH_LI.
      unsafe { (DISPATCH_LI.get_unchecked(init_idx()).hash_lo_0)(data) }
    }

    unsafe fn init_hi(data: &[u8], seed: u64) -> [u8; 32] {
      // SAFETY: init_idx() returns valid non-zero slot in DISPATCH_LI.
      unsafe { (DISPATCH_LI.get_unchecked(init_idx()).hash_hi)(data, seed) }
    }

    unsafe fn init_hi_0(data: &[u8]) -> [u8; 32] {
      // SAFETY: init_idx() returns valid non-zero slot in DISPATCH_LI.
      unsafe { (DISPATCH_LI.get_unchecked(init_idx()).hash_hi_0)(data) }
    }

    unsafe fn init_ctr_fill(root: &[u8; 32], blk: u64, buf: &mut [u8]) {
      // SAFETY: init_idx() returns valid non-zero slot in DISPATCH_LI.
      unsafe { (DISPATCH_LI.get_unchecked(init_idx()).ctr_fill)(root, blk, buf) };
    }

    unsafe fn init_ctr_xor(root: &[u8; 32], blk: u64, buf: &mut [u8]) {
      // SAFETY: init_idx() returns valid non-zero slot in DISPATCH_LI.
      unsafe { (DISPATCH_LI.get_unchecked(init_idx()).ctr_xor)(root, blk, buf) };
    }

    const INIT_DISPATCH: Dispatch = Dispatch {
      hash_64: init_64,
      hash_64_0: init_64_0,
      hash_lo: init_lo,
      hash_lo_0: init_lo_0,
      hash_hi: init_hi,
      hash_hi_0: init_hi_0,
      ctr_fill: init_ctr_fill,
      ctr_xor: init_ctr_xor,
    };

    static DISPATCH_LI: [Dispatch; $len] = $tbl(INIT_DISPATCH);

    pub(crate) struct $name;

    // SAFETY: `$detect` verifies CPU features before publishing the slot index in `ACTIVE_IDX`.
    unsafe impl DispatchSource for $name {
      #[inline(always)]
      fn active() -> &'static Dispatch {
        let idx = ACTIVE_IDX.load(Ordering::Relaxed);
        // SAFETY: ACTIVE_IDX only holds 0 or a valid detected tier index in DISPATCH_LI.
        unsafe { DISPATCH_LI.get_unchecked(idx) }
      }
    }
  };
}

macro_rules! dispatch_slot {
  ($($seg:ident)::+) => {
    Dispatch {
      hash_64: $($seg)::+::dispatch_64,
      hash_64_0: $($seg)::+::dispatch_64_0,
      hash_lo: $($seg)::+::dispatch_lo,
      hash_lo_0: $($seg)::+::dispatch_lo_0,
      hash_hi: $($seg)::+::dispatch_hi,
      hash_hi_0: $($seg)::+::dispatch_hi_0,
      ctr_fill: $($seg)::+::ctr::<false>,
      ctr_xor: $($seg)::+::ctr::<true>,
    }
  };
}

pub(crate) use def_runtime_dispatch;
pub(crate) use dispatch_slot;