ihash 0.1.2

World's fastest high-quality non-cryptographic hash, deterministic cross-platform XOF, and seekable stream cipher
Documentation
#[cfg(any(feature = "std", all(target_feature = "aes", target_feature = "neon")))]
mod hw {
  use core::{
    arch::{
      aarch64::{
        uint8x16_t, vaddq_u64, vcombine_u64, vcreate_u64, vdupq_n_u8, vdupq_n_u32, vdupq_n_u64,
        veorq_u8, vextq_u8, vgetq_lane_u32, vgetq_lane_u64, vld1_u64, vld1q_u8,
        vreinterpretq_u8_u32, vreinterpretq_u8_u64, vreinterpretq_u32_u8, vreinterpretq_u64_u8,
        vst1q_u8, vzip1q_u8,
      },
      asm,
    },
    mem::transmute,
  };

  use crate::{
    arch::{ctr_simd::ctr_8x128, tier::def_hw_tier},
    backend::{
      aes128::Aes128,
      ctr::CtrKernel,
      group::Lane128x8,
      mode::{PreXor, PreXorAes},
    },
    state::State,
  };

  #[inline(always)]
  unsafe fn fmov_u64(x: u64) -> uint8x16_t {
    let v: uint8x16_t;
    // SAFETY: caller ensures neon; `fmov d, x` moves 64-bit GPR to low lane and zeroes high 64 bits.
    unsafe {
      asm!(
        "fmov {v:d}, {x}",
        v = out(vreg) v,
        x = in(reg) x,
        options(pure, nomem, nostack, preserves_flags),
      );
    }
    v
  }

  #[derive(Clone, Copy)]
  pub(super) struct Neon128;

  impl Aes128 for Neon128 {
    type V = uint8x16_t;
    type Mode = PreXor;

    #[inline(always)]
    unsafe fn zero(self) -> Self::V {
      // SAFETY: caller guarantees neon.
      unsafe { vdupq_n_u8(0) }
    }

    #[inline(always)]
    unsafe fn ld(self, ptr: *const u8) -> Self::V {
      // SAFETY: caller ensures neon and >= 16 readable bytes at ptr.
      unsafe { vld1q_u8(ptr) }
    }

    #[inline(always)]
    unsafe fn st(self, ptr: *mut u8, v: Self::V) {
      // SAFETY: caller ensures neon and >= 16 writable bytes at ptr.
      unsafe { vst1q_u8(ptr, v) };
    }

    #[inline(always)]
    unsafe fn to_state(self, v: Self::V) -> State {
      // SAFETY: uint8x16_t and [u8; 16] share identical 16-byte layout on little-endian aarch64.
      unsafe { transmute(v) }
    }

    #[inline(always)]
    unsafe fn to_u32(self, v: Self::V) -> u32 {
      // SAFETY: caller ensures neon.
      unsafe { vgetq_lane_u32::<0>(vreinterpretq_u32_u8(v)) }
    }

    #[inline(always)]
    unsafe fn to_u64(self, v: Self::V) -> u64 {
      // SAFETY: caller ensures neon.
      unsafe { vgetq_lane_u64::<0>(vreinterpretq_u64_u8(v)) }
    }

    #[inline(always)]
    unsafe fn seed_v(self, seed: u64) -> Self::V {
      // SAFETY: caller ensures neon.
      unsafe { vreinterpretq_u8_u64(vdupq_n_u64(seed)) }
    }

    #[inline(always)]
    unsafe fn pair_v(self, lo: u64, hi: u64) -> Self::V {
      // SAFETY: caller ensures neon.
      unsafe { vreinterpretq_u8_u64(vcombine_u64(vcreate_u64(lo), vcreate_u64(hi))) }
    }

    #[inline(always)]
    unsafe fn dup_u32(self, x: u32) -> Self::V {
      // SAFETY: caller ensures neon.
      unsafe { vreinterpretq_u8_u32(vdupq_n_u32(x)) }
    }

    #[inline(always)]
    unsafe fn load_pair_u64(self, p0: *const u8, p1: *const u8) -> Self::V {
      // SAFETY: caller ensures neon and >= 8 readable bytes at p0 and p1.
      unsafe { vreinterpretq_u8_u64(vcombine_u64(vld1_u64(p0.cast()), vld1_u64(p1.cast()))) }
    }

    #[inline(always)]
    unsafe fn load_pair_u32(self, p0: *const u8, p1: *const u8) -> Self::V {
      let lo: uint8x16_t;
      // SAFETY: caller ensures neon and >= 4 readable bytes at p0 and p1; zip1.2s zeroes high 64 bits.
      unsafe {
        asm!(
          "ldr {lo:s}, [{p0}]",
          "ldr {hi:s}, [{p1}]",
          "zip1 {lo:v}.2s, {lo:v}.2s, {hi:v}.2s",
          p0 = in(reg) p0,
          p1 = in(reg) p1,
          lo = out(vreg) lo,
          hi = out(vreg) _,
          options(pure, readonly, nostack, preserves_flags),
        );
      }
      lo
    }

    #[inline(always)]
    unsafe fn u32_v(self, x: u32) -> Self::V {
      // SAFETY: caller ensures neon.
      unsafe { fmov_u64(x as u64) }
    }

    #[inline(always)]
    unsafe fn xor(self, a: Self::V, b: Self::V) -> Self::V {
      // SAFETY: caller ensures neon.
      unsafe { veorq_u8(a, b) }
    }

    #[inline(always)]
    unsafe fn add_u64(self, a: Self::V, b: Self::V) -> Self::V {
      // SAFETY: caller ensures neon.
      unsafe { vreinterpretq_u8_u64(vaddq_u64(vreinterpretq_u64_u8(a), vreinterpretq_u64_u8(b))) }
    }

    #[inline(always)]
    unsafe fn zip_u8(self, v: Self::V) -> Self::V {
      // SAFETY: caller ensures neon.
      unsafe { vzip1q_u8(v, vextq_u8::<8>(v, v)) }
    }
  }

  impl PreXorAes for Neon128 {
    #[inline(always)]
    unsafe fn pre_r1_zero(self, mut a: Self::V, b: Self::V) -> Self::V {
      // SAFETY: caller ensures aes+neon; in-place operand preserves AESE+AESMC macro-op fusion.
      unsafe {
        asm!(
          "aese {a:v}.16b, {b:v}.16b\naesmc {a:v}.16b, {a:v}.16b",
          a = inout(vreg) a,
          b = in(vreg) b,
          options(pure, nomem, nostack, preserves_flags),
        );
      }
      a
    }

    #[inline(always)]
    unsafe fn pre_fl_zero(self, mut a: Self::V, b: Self::V) -> Self::V {
      // SAFETY: caller ensures aes+neon.
      unsafe {
        asm!(
          "aese {a:v}.16b, {b:v}.16b",
          a = inout(vreg) a,
          b = in(vreg) b,
          options(pure, nomem, nostack, preserves_flags),
        );
      }
      a
    }
  }

  impl Lane128x8 for Neon128 {}

  impl CtrKernel for Neon128 {
    #[inline(always)]
    unsafe fn ctr<const XOR: bool>(self, root: &[u8; 32], blk: u64, buf: &mut [u8]) {
      // SAFETY: caller ensures aes+neon and valid buffer.
      unsafe { ctr_8x128::<Neon128, XOR>(Neon128, root, blk, buf) };
    }
  }

  def_hw_tier!(
    Neon128,
    Neon128,
    uint8x16_t,
    all(target_feature = "aes", target_feature = "neon"),
    all(target_feature = "aes", target_feature = "neon"),
    any(),
    "aes,neon"
  );
}

#[cfg(all(target_feature = "aes", target_feature = "neon"))]
use crate::arch::tier::StaticPlatform;

#[cfg(all(target_feature = "aes", target_feature = "neon"))]
pub(crate) type ArmPlatform = StaticPlatform<hw::Tier>;

#[cfg(all(
  feature = "std",
  not(all(target_feature = "aes", target_feature = "neon"))
))]
mod active {
  use core::sync::atomic::{AtomicUsize, Ordering};
  use std::arch::is_aarch64_feature_detected;

  use super::hw;
  use crate::arch::{
    dispatch::{Dispatch, DispatchSource, RuntimePlatform, def_runtime_dispatch, dispatch_slot},
    soft::Soft128,
  };

  fn detect_idx() -> usize {
    if is_aarch64_feature_detected!("aes") && is_aarch64_feature_detected!("neon") {
      2
    } else {
      1
    }
  }

  const fn build_table(init: Dispatch) -> [Dispatch; 3] {
    [
      init,
      Dispatch::for_platform::<Soft128>(),
      dispatch_slot!(hw),
    ]
  }

  def_runtime_dispatch!(ArmDispatch, 3, build_table, detect_idx);

  pub(crate) type ArmPlatform = RuntimePlatform<ArmDispatch>;
}

#[cfg(all(
  feature = "std",
  not(all(target_feature = "aes", target_feature = "neon"))
))]
pub(crate) use active::ArmPlatform;

#[cfg(all(
  not(feature = "std"),
  not(all(target_feature = "aes", target_feature = "neon"))
))]
pub(crate) use crate::arch::soft::Soft128 as ArmPlatform;