ihash 0.1.2

World's fastest high-quality non-cryptographic hash, deterministic cross-platform XOF, and seekable stream cipher
Documentation
use crate::{
  backend::{aes128::Aes128, mode::AesMode, tree::finalize_hi},
  const_fn::lane_ptr,
  state::BLOCK_LEN,
};

pub(crate) type GroupV<G> = <<G as MidGroup>::L as Aes128>::V;

/// Medium/long (4/8/16-lane) parallel compression contract shared by hardware and pure-CPU SIMD backends.
pub(crate) trait MidGroup: Copy {
  type L: Aes128;
  type G4: Copy;
  type G8: Copy;

  fn lane(self) -> Self::L;

  unsafe fn init4<const SEED0: bool>(self, sv: GroupV<Self>) -> Self::G4;
  unsafe fn absorb4_init(self, key_li: Self::G4, ptr: *const u8) -> Self::G4;
  unsafe fn absorb4_linear(self, acc: Self::G4, ptr: *const u8) -> Self::G4;
  unsafe fn absorb4_swap32(self, acc: Self::G4, ptr: *const u8) -> Self::G4;
  unsafe fn merge4<const HI: bool>(
    self,
    acc: Self::G4,
    lk: GroupV<Self>,
  ) -> (GroupV<Self>, GroupV<Self>);

  unsafe fn init8<const SEED0: bool>(self, sv: GroupV<Self>, lane_base: usize) -> Self::G8;
  unsafe fn absorb8_init(self, key_li: Self::G8, ptr: *const u8) -> Self::G8;
  unsafe fn absorb8_pair64(self, acc: Self::G8, p_lo: *const u8, p_hi: *const u8) -> Self::G8;
  unsafe fn absorb8_swap32(self, acc: Self::G8, last128: *const u8) -> Self::G8;
  unsafe fn merge8<const HI: bool>(
    self,
    acc: Self::G8,
    lk: GroupV<Self>,
  ) -> (GroupV<Self>, GroupV<Self>);
  unsafe fn merge16<const HI: bool>(
    self,
    lo: Self::G8,
    hi: Self::G8,
    lk: GroupV<Self>,
  ) -> (GroupV<Self>, GroupV<Self>);
}

/// Full hardware 4/8/16-lane compression contract extending [`MidGroup`] with short/bulk striped paths.
pub(crate) trait LaneGroup: MidGroup {
  #[inline(always)]
  unsafe fn compress_17_32<const HI: bool, const SEED0: bool>(
    self,
    ptr: *const u8,
    len: usize,
    sv: GroupV<Self>,
    lk: GroupV<Self>,
  ) -> (GroupV<Self>, GroupV<Self>) {
    let l = self.lane();
    // SAFETY: caller guarantees target features and 17 <= len <= 32 keep both loads strictly in bounds.
    unsafe {
      let lp = lane_ptr::<SEED0>();
      let r0 = l.ld(lp);
      let r1 = l.ld(lp.add(BLOCK_LEN));
      let (k0, k1) = if SEED0 {
        (r0, r1)
      } else {
        (l.xor(sv, r0), l.xor(sv, r1))
      };
      let b0 = l.ld(ptr);
      let b1 = l.ld(ptr.add(len - BLOCK_LEN));
      let (lo, c) = <Self::L as Aes128>::Mode::compress_17_32_core(l, b0, k0, b1, k1, lk);
      finalize_hi::<Self::L, HI>(l, lo, c, lk)
    }
  }

  unsafe fn compress_33_64<const HI: bool, const SEED0: bool>(
    self,
    ptr: *const u8,
    len: usize,
    sv: GroupV<Self>,
    lk: GroupV<Self>,
  ) -> (GroupV<Self>, GroupV<Self>);

  unsafe fn compress_65_128<const HI: bool, const SEED0: bool>(
    self,
    ptr: *const u8,
    len: usize,
    sv: GroupV<Self>,
    lk: GroupV<Self>,
  ) -> (GroupV<Self>, GroupV<Self>);

  unsafe fn absorb16_swapped(
    self,
    lo: Self::G8,
    hi: Self::G8,
    ptr: *const u8,
  ) -> (Self::G8, Self::G8);

  #[inline(always)]
  unsafe fn absorb16_linear(
    self,
    lo: Self::G8,
    hi: Self::G8,
    ptr: *const u8,
  ) -> (Self::G8, Self::G8) {
    // SAFETY: caller guarantees target features and >= 256 readable bytes at ptr.
    unsafe {
      (
        self.absorb8_pair64(lo, ptr, ptr.add(64)),
        self.absorb8_pair64(hi, ptr.add(128), ptr.add(192)),
      )
    }
  }
}

/// Marker trait for 128-bit backends whose 4/8-lane groups are `[V; 4]` / `[V; 8]`.
pub(crate) trait Lane128x8: Aes128 {}

impl<B: Lane128x8> MidGroup for B {
  type G4 = [B::V; 4];
  type G8 = [B::V; 8];
  type L = B;

  #[inline(always)]
  fn lane(self) -> B {
    self
  }

  #[inline(always)]
  unsafe fn init4<const SEED0: bool>(self, sv: B::V) -> Self::G4 {
    // SAFETY: caller guarantees target features and TBL bounds.
    unsafe {
      let lp = lane_ptr::<SEED0>();
      unroll4!(I => {
        let key = self.ld(lp.add(I * BLOCK_LEN));
        if SEED0 { key } else { self.xor(sv, key) }
      })
    }
  }

  #[inline(always)]
  unsafe fn absorb4_init(self, key_li: Self::G4, ptr: *const u8) -> Self::G4 {
    // SAFETY: caller guarantees target features and >= 64 readable bytes at ptr.
    unsafe { unroll4!(I => self.acc_init_1(key_li[I], self.ld(ptr.add(I * BLOCK_LEN)))) }
  }

  #[inline(always)]
  unsafe fn absorb4_linear(self, acc: Self::G4, ptr: *const u8) -> Self::G4 {
    // SAFETY: caller guarantees target features and >= 64 readable bytes at ptr.
    unsafe { unroll4!(I => self.acc_absorb_1(acc[I], self.ld(ptr.add(I * BLOCK_LEN)))) }
  }

  #[inline(always)]
  unsafe fn absorb4_swap32(self, acc: Self::G4, ptr: *const u8) -> Self::G4 {
    // SAFETY: caller guarantees target features and >= 64 readable bytes at ptr.
    unsafe { unroll4!(I => self.acc_absorb_1(acc[I], self.ld(ptr.add((I ^ 2) * BLOCK_LEN)))) }
  }

  #[inline(always)]
  unsafe fn merge4<const HI: bool>(self, acc: Self::G4, lk: B::V) -> (B::V, B::V) {
    // SAFETY: caller guarantees target features and merge4 inputs.
    unsafe { B::Mode::merge4_128::<HI>(self, acc, lk) }
  }

  #[inline(always)]
  unsafe fn init8<const SEED0: bool>(self, sv: B::V, lane_base: usize) -> Self::G8 {
    // SAFETY: caller guarantees target features and lane_base in {0, 8}.
    unsafe {
      let lp = lane_ptr::<SEED0>().add(lane_base * BLOCK_LEN);
      unroll8!(I => {
        let key = self.ld(lp.add(I * BLOCK_LEN));
        if SEED0 { key } else { self.xor(sv, key) }
      })
    }
  }

  #[inline(always)]
  unsafe fn absorb8_init(self, key_li: Self::G8, ptr: *const u8) -> Self::G8 {
    // SAFETY: caller guarantees target features and 128 readable bytes at ptr.
    unsafe { unroll8!(I => self.acc_init_1(key_li[I], self.ld(ptr.add(I * BLOCK_LEN)))) }
  }

  #[inline(always)]
  unsafe fn absorb8_pair64(self, acc: Self::G8, p_lo: *const u8, p_hi: *const u8) -> Self::G8 {
    // SAFETY: caller guarantees target features and >= 64 readable bytes at p_lo and p_hi.
    unsafe {
      unroll8!(I => {
        let p = if I < 4 { p_lo.add(I * BLOCK_LEN) } else { p_hi.add((I - 4) * BLOCK_LEN) };
        self.acc_absorb_1(acc[I], self.ld(p))
      })
    }
  }

  #[inline(always)]
  unsafe fn absorb8_swap32(self, acc: Self::G8, last128: *const u8) -> Self::G8 {
    // SAFETY: caller guarantees target features and 128 readable bytes at last128.
    unsafe { unroll8!(I => self.acc_absorb_1(acc[I], self.ld(last128.add((I ^ 2) * BLOCK_LEN)))) }
  }

  #[inline(always)]
  unsafe fn merge8<const HI: bool>(self, acc: Self::G8, lk: B::V) -> (B::V, B::V) {
    // SAFETY: caller guarantees target features and merge8 inputs.
    unsafe { B::Mode::merge8_128::<HI>(self, acc, lk) }
  }

  #[inline(always)]
  unsafe fn merge16<const HI: bool>(self, lo: Self::G8, hi: Self::G8, lk: B::V) -> (B::V, B::V) {
    // SAFETY: caller guarantees target features.
    unsafe { B::Mode::merge16_128::<HI>(self, lo, hi, lk) }
  }
}

#[inline(always)]
fn join4<T: Copy>(a_li: [T; 4], b_li: [T; 4]) -> [T; 8] {
  [
    a_li[0], a_li[1], a_li[2], a_li[3], b_li[0], b_li[1], b_li[2], b_li[3],
  ]
}

impl<B: Lane128x8> LaneGroup for B {
  #[inline(always)]
  unsafe fn compress_65_128<const HI: bool, const SEED0: bool>(
    self,
    ptr: *const u8,
    len: usize,
    sv: B::V,
    lk: B::V,
  ) -> (B::V, B::V) {
    // SAFETY: caller guarantees target features and 65 <= len <= 128 keep all loads strictly in bounds.
    unsafe {
      let key_li = self.init4::<SEED0>(sv);
      let head_li = unroll4!(I => self.ld(ptr.add(I * BLOCK_LEN)));
      let acc_li = unroll4!(I => self.acc_init_1(key_li[I], head_li[I]));
      let tail = ptr.add(len - 64);
      let acc_li = unroll4!(I => {
        let j = I ^ 2;
        self.acc_absorb_1(
          acc_li[I],
          self.add_u64(self.ld(tail.add(j * BLOCK_LEN)), head_li[j]),
        )
      });
      self.merge4::<HI>(acc_li, lk)
    }
  }

  #[inline(always)]
  unsafe fn compress_33_64<const HI: bool, const SEED0: bool>(
    self,
    ptr: *const u8,
    len: usize,
    sv: B::V,
    lk: B::V,
  ) -> (B::V, B::V) {
    // SAFETY: caller guarantees target features and 33 <= len <= 64 keep all 4 loads strictly in bounds.
    unsafe {
      let key_li = self.init4::<SEED0>(sv);
      let tail = ptr.add(len - 32);
      let b_li = [
        self.ld(ptr),
        self.ld(ptr.add(BLOCK_LEN)),
        self.ld(tail),
        self.ld(tail.add(BLOCK_LEN)),
      ];
      B::Mode::compress_33_64_core::<HI>(self, b_li, key_li, lk)
    }
  }

  #[inline(always)]
  unsafe fn absorb16_swapped(
    self,
    lo: Self::G8,
    hi: Self::G8,
    ptr: *const u8,
  ) -> (Self::G8, Self::G8) {
    // SAFETY: caller guarantees target features and >= 256 readable bytes at ptr.
    unsafe {
      // Load in strictly ascending address order across the 256B stripe before cross-assigning lanes.
      let hi_hi_li = unroll4!(I => self.acc_absorb_1(hi[I + 4], self.ld(ptr.add(I * BLOCK_LEN))));
      let lo_lo_li = unroll4!(I => self.acc_absorb_1(lo[I], self.ld(ptr.add((I + 4) * BLOCK_LEN))));
      let lo_hi_li =
        unroll4!(I => self.acc_absorb_1(lo[I + 4], self.ld(ptr.add((I + 8) * BLOCK_LEN))));
      let hi_lo_li =
        unroll4!(I => self.acc_absorb_1(hi[I], self.ld(ptr.add((I + 12) * BLOCK_LEN))));
      (join4(lo_lo_li, lo_hi_li), join4(hi_lo_li, hi_hi_li))
    }
  }
}