ihash 0.1.2

World's fastest high-quality non-cryptographic hash, deterministic cross-platform XOF, and seekable stream cipher
Documentation
#[cfg(target_arch = "x86")]
use core::arch::x86 as arch_simd;
#[cfg(target_arch = "x86_64")]
use core::arch::x86_64 as arch_simd;
use core::mem::transmute;

use arch_simd::{
  __m128i, _MM_HINT_T0, _mm_add_epi64, _mm_aesenc_si128, _mm_aesenclast_si128, _mm_cvtsi32_si128,
  _mm_cvtsi128_si32, _mm_loadl_epi64, _mm_loadu_si128, _mm_prefetch, _mm_set1_epi32,
  _mm_setzero_si128, _mm_storeu_si128, _mm_unpackhi_epi64, _mm_unpacklo_epi8, _mm_unpacklo_epi64,
  _mm_xor_si128,
};

#[cfg(not(all(
  target_feature = "vaes",
  target_feature = "avx2",
  target_feature = "aes",
  target_feature = "sse2"
)))]
use crate::{
  arch::{ctr_simd::ctr_8x128, tier::def_hw_tier},
  backend::ctr::CtrKernel,
};
use crate::{
  backend::{
    aes128::Aes128,
    group::Lane128x8,
    mode::{PostXor, PostXorAes},
  },
  state::State,
};

#[cfg(target_arch = "x86_64")]
mod arch_word {
  use core::arch::x86_64::{
    __m128i, _mm_cvtsi64_si128, _mm_cvtsi128_si64, _mm_set_epi64x, _mm_set1_epi64x,
  };

  #[inline(always)]
  pub(super) unsafe fn to_u64(v: __m128i) -> u64 {
    // SAFETY: caller ensures sse2 on x86_64.
    unsafe { _mm_cvtsi128_si64(v) as u64 }
  }

  #[inline(always)]
  pub(super) unsafe fn seed_v(seed: u64) -> __m128i {
    // SAFETY: caller ensures sse2 on x86_64.
    unsafe { _mm_set1_epi64x(seed as i64) }
  }

  #[inline(always)]
  pub(super) unsafe fn pair_v(lo: u64, hi: u64) -> __m128i {
    // SAFETY: caller ensures sse2 on x86_64.
    unsafe { _mm_set_epi64x(hi as i64, lo as i64) }
  }

  #[inline(always)]
  pub(super) unsafe fn load_pair_u32(p0: *const u8, p1: *const u8) -> __m128i {
    // SAFETY: caller ensures sse2 on x86_64 and >= 4 readable bytes at p0 and p1.
    unsafe {
      let lo = p0.cast::<u32>().read_unaligned() as u64;
      let hi = p1.cast::<u32>().read_unaligned() as u64;
      _mm_cvtsi64_si128((lo | (hi << 32)) as i64)
    }
  }
}

#[cfg(target_arch = "x86")]
mod arch_word {
  use core::{
    arch::x86::{__m128i, _mm_cvtsi32_si128, _mm_set_epi32, _mm_unpacklo_epi32},
    mem::transmute,
  };

  use crate::state::State;

  #[inline(always)]
  pub(super) unsafe fn to_u64(v: __m128i) -> u64 {
    // SAFETY: __m128i and [u8; 16] share identical 16-byte layout on little-endian x86.
    let s: State = unsafe { transmute(v) };
    u64::from_le_bytes(s.as_chunks::<8>().0[0])
  }

  #[inline(always)]
  pub(super) unsafe fn seed_v(seed: u64) -> __m128i {
    let lo = seed as i32;
    let hi = (seed >> 32) as i32;
    // SAFETY: caller ensures sse2.
    unsafe { _mm_set_epi32(hi, lo, hi, lo) }
  }

  #[inline(always)]
  pub(super) unsafe fn pair_v(lo: u64, hi: u64) -> __m128i {
    // SAFETY: caller ensures sse2.
    unsafe { _mm_set_epi32((hi >> 32) as i32, hi as i32, (lo >> 32) as i32, lo as i32) }
  }

  #[inline(always)]
  pub(super) unsafe fn load_pair_u32(p0: *const u8, p1: *const u8) -> __m128i {
    // SAFETY: caller ensures sse2 and >= 4 readable bytes at p0 and p1.
    unsafe {
      let lo = p0.cast::<i32>().read_unaligned();
      let hi = p1.cast::<i32>().read_unaligned();
      _mm_unpacklo_epi32(_mm_cvtsi32_si128(lo), _mm_cvtsi32_si128(hi))
    }
  }
}

#[derive(Clone, Copy)]
pub(crate) struct X86_128;

impl Aes128 for X86_128 {
  type V = __m128i;
  type Mode = PostXor;
  const HAS_PREFETCH: bool = true;

  #[inline(always)]
  unsafe fn zero(self) -> Self::V {
    // SAFETY: caller guarantees sse2.
    unsafe { _mm_setzero_si128() }
  }

  #[inline(always)]
  unsafe fn ld(self, ptr: *const u8) -> Self::V {
    // SAFETY: caller ensures sse2 and >= 16 readable bytes at ptr.
    unsafe { _mm_loadu_si128(ptr.cast()) }
  }

  #[inline(always)]
  unsafe fn st(self, ptr: *mut u8, v: Self::V) {
    // SAFETY: caller ensures sse2 and >= 16 writable bytes at ptr.
    unsafe { _mm_storeu_si128(ptr.cast(), v) };
  }

  #[inline(always)]
  unsafe fn to_state(self, v: Self::V) -> State {
    // SAFETY: __m128i and [u8; 16] share identical 16-byte layout on little-endian x86.
    unsafe { transmute(v) }
  }

  #[inline(always)]
  unsafe fn to_u32(self, v: Self::V) -> u32 {
    // SAFETY: caller ensures sse2.
    unsafe { _mm_cvtsi128_si32(v) as u32 }
  }

  #[inline(always)]
  unsafe fn to_u64(self, v: Self::V) -> u64 {
    // SAFETY: forwarded from caller contract.
    unsafe { arch_word::to_u64(v) }
  }

  #[inline(always)]
  unsafe fn seed_v(self, seed: u64) -> Self::V {
    // SAFETY: forwarded from caller contract.
    unsafe { arch_word::seed_v(seed) }
  }

  #[inline(always)]
  unsafe fn pair_v(self, lo: u64, hi: u64) -> Self::V {
    // SAFETY: forwarded from caller contract.
    unsafe { arch_word::pair_v(lo, hi) }
  }

  #[inline(always)]
  unsafe fn dup_u32(self, x: u32) -> Self::V {
    // SAFETY: caller ensures sse2.
    unsafe { _mm_set1_epi32(x as i32) }
  }

  #[inline(always)]
  unsafe fn xor(self, a: Self::V, b: Self::V) -> Self::V {
    // SAFETY: caller ensures sse2.
    unsafe { _mm_xor_si128(a, b) }
  }

  #[inline(always)]
  unsafe fn add_u64(self, a: Self::V, b: Self::V) -> Self::V {
    // SAFETY: caller ensures sse2.
    unsafe { _mm_add_epi64(a, b) }
  }

  #[inline(always)]
  unsafe fn prefetch_l1(self, ptr: *const u8) {
    // SAFETY: `_mm_prefetch` is a non-faulting hint available on all SSE targets.
    unsafe { _mm_prefetch(ptr.cast::<i8>(), _MM_HINT_T0) };
  }

  #[inline(always)]
  unsafe fn load_pair_u64(self, p0: *const u8, p1: *const u8) -> Self::V {
    // SAFETY: caller ensures sse2 and >= 8 readable bytes at p0 and p1.
    unsafe { _mm_unpacklo_epi64(_mm_loadl_epi64(p0.cast()), _mm_loadl_epi64(p1.cast())) }
  }

  #[inline(always)]
  unsafe fn load_pair_u32(self, p0: *const u8, p1: *const u8) -> Self::V {
    // SAFETY: forwarded from caller contract.
    unsafe { arch_word::load_pair_u32(p0, p1) }
  }

  #[inline(always)]
  unsafe fn u32_v(self, x: u32) -> Self::V {
    // SAFETY: caller ensures sse2.
    unsafe { _mm_cvtsi32_si128(x as i32) }
  }

  #[inline(always)]
  unsafe fn zip_u8(self, v: Self::V) -> Self::V {
    // SAFETY: caller ensures sse2; interleaves low and high 64-bit halves bytewise.
    unsafe { _mm_unpacklo_epi8(v, _mm_unpackhi_epi64(v, v)) }
  }
}

impl PostXorAes for X86_128 {
  #[inline(always)]
  unsafe fn post_enc(self, state: Self::V, rk: Self::V) -> Self::V {
    // SAFETY: caller ensures aes+sse2.
    unsafe { _mm_aesenc_si128(state, rk) }
  }

  #[inline(always)]
  unsafe fn post_enclast(self, state: Self::V, rk: Self::V) -> Self::V {
    // SAFETY: caller ensures aes+sse2.
    unsafe { _mm_aesenclast_si128(state, rk) }
  }
}

impl Lane128x8 for X86_128 {}

#[cfg(not(all(
  target_feature = "vaes",
  target_feature = "avx2",
  target_feature = "aes",
  target_feature = "sse2"
)))]
impl CtrKernel for X86_128 {
  #[inline(always)]
  unsafe fn ctr<const XOR: bool>(self, root: &[u8; 32], blk: u64, buf: &mut [u8]) {
    // SAFETY: caller ensures tier target features and valid buffer.
    unsafe { ctr_8x128::<X86_128, XOR>(X86_128, root, blk, buf) };
  }
}

#[cfg(not(all(
  target_feature = "vaes",
  target_feature = "avx2",
  target_feature = "aes",
  target_feature = "sse2"
)))]
macro_rules! def_128_tier {
  ($mod_name:ident, $feat_cfg:meta, $feat:literal) => {
    pub(super) mod $mod_name {
      use super::*;

      def_hw_tier!(
        X86_128,
        X86_128,
        __m128i,
        $feat_cfg,
        any(),
        all(target_feature = "aes", target_feature = "sse2"),
        $feat
      );
    }
  };
}

#[cfg(all(
  not(all(
    target_feature = "vaes",
    target_feature = "avx2",
    target_feature = "aes",
    target_feature = "sse2"
  )),
  not(all(
    target_feature = "aes",
    target_feature = "avx",
    target_feature = "sse2"
  )),
  any(feature = "std", all(target_feature = "aes", target_feature = "sse2"))
))]
def_128_tier!(
  sse2_aes,
  all(target_feature = "aes", target_feature = "sse2"),
  "aes,sse2"
);

#[cfg(all(
  not(all(
    target_feature = "vaes",
    target_feature = "avx2",
    target_feature = "aes",
    target_feature = "sse2"
  )),
  any(
    feature = "std",
    all(
      target_feature = "aes",
      target_feature = "avx",
      target_feature = "sse2"
    )
  )
))]
def_128_tier!(
  avx_aes,
  all(
    target_feature = "aes",
    target_feature = "avx",
    target_feature = "sse2"
  ),
  "aes,avx,sse2"
);