ihash 0.1.2

World's fastest high-quality non-cryptographic hash, deterministic cross-platform XOF, and seekable stream cipher
Documentation
#[cfg(all(
  any(target_arch = "x86", target_arch = "x86_64"),
  target_endian = "little",
  any(feature = "std", all(target_feature = "aes", target_feature = "sse2"))
))]
use crate::backend::mode::PostXorAes;
#[cfg(any(
  feature = "cpu",
  all(target_arch = "aarch64", target_endian = "little"),
  not(all(
    any(target_arch = "x86", target_arch = "x86_64"),
    target_endian = "little",
    target_feature = "aes",
    target_feature = "sse2"
  ))
))]
use crate::backend::mode::PreXorAes;
use crate::{
  backend::{
    aes128::Aes128,
    group::{GroupV, LaneGroup, MidGroup},
  },
  const_fn::TBL,
  state::{BLOCK_LEN, MID_MAX, MID4_MAX, SHORT_MAX, pack_root},
};

/// Branch hint marking cold long-input dispatch paths without compiler intrinsics.
#[cold]
#[inline(always)]
pub(crate) const fn cold() {}

/// Four-column MDS length key (`[len32; 4]`) requiring zero multiplications or table lookups.
#[inline(always)]
unsafe fn len_key<L: Aes128>(ctx: L, len: usize) -> L::V {
  // SAFETY: forwarded from caller contract.
  unsafe { ctx.dup_u32(len as u32) }
}

/// Wide-length variant folding upper 32 bits for bulk inputs (`> 2048 B`).
#[inline(always)]
pub(crate) unsafe fn len_key_wide<L: Aes128>(ctx: L, len: usize) -> L::V {
  let u = len as u64;
  // SAFETY: forwarded from caller contract.
  unsafe { ctx.dup_u32((u as u32) ^ ((u >> 32) as u32)) }
}

/// Packs two 128-bit lane values into the 32-byte little-endian root digest.
#[inline(always)]
pub(crate) unsafe fn pack_root_v<L: Aes128>(ctx: L, lo: L::V, hi: L::V) -> [u8; 32] {
  // SAFETY: forwarded from caller contract.
  unsafe { pack_root(ctx.to_state(lo), ctx.to_state(hi)) }
}

/// Expands `seed` through one AES round so `(key, seed)` pairs cannot cancel linearly.
#[inline(always)]
pub(crate) unsafe fn expand_seed<L: Aes128, const SEED0: bool>(ctx: L, seed: u64) -> L::V {
  // SAFETY: caller satisfies target features and TBL bounds.
  unsafe {
    if SEED0 {
      ctx.ld(TBL.sv0.as_ptr())
    } else {
      ctx.r1_zero(ctx.seed_v(seed), ctx.ld(TBL.fin_li[6].as_ptr()))
    }
  }
}

/// Returns zero for `SEED0` fast paths (already folded into `lane_sv0_li`) or expanded `seed`.
#[inline(always)]
unsafe fn expand_lane_seed<L: Aes128, const SEED0: bool>(ctx: L, seed: u64) -> L::V {
  // SAFETY: forwarded from caller contract.
  unsafe {
    if SEED0 {
      ctx.zero()
    } else {
      expand_seed::<L, false>(ctx, seed)
    }
  }
}

#[inline(always)]
pub(crate) unsafe fn finalize_hi<L: Aes128, const HI: bool>(
  ctx: L,
  lo: L::V,
  q1: L::V,
  lk: L::V,
) -> (L::V, L::V) {
  // SAFETY: caller satisfies target features and TBL bounds.
  let hi = unsafe {
    if HI {
      let c1 = ctx.ld(TBL.fin_li[1].as_ptr());
      let c2 = ctx.ld(TBL.fin_li[2].as_ptr());
      ctx.r3_zero(lo, ctx.xor(q1, c1), lk, c2)
    } else {
      ctx.zero()
    }
  };
  (lo, hi)
}

/// Asymmetric 4-lane AES finalizer when `l0 ^ lk` is already pre-applied (x86 AES-NI/VAES fast path):
/// `q0 = FL(l0_lk) ^ l2`, `q1 = F(l1) +_{u64} l3`, `q1_z = zip_u8(q1)`,
/// `s = F(q0) ^ q1_z`, `u = F(s)`, `lo = F(u) +_{u64} u`.
#[cfg(all(
  any(target_arch = "x86", target_arch = "x86_64"),
  target_endian = "little",
  any(feature = "std", all(target_feature = "aes", target_feature = "sse2"))
))]
#[inline(always)]
pub(crate) unsafe fn finalize_merge4_lk<L: PostXorAes, const HI: bool>(
  ctx: L,
  lane_lk_li: [L::V; 4],
  lk: L::V,
) -> (L::V, L::V) {
  // SAFETY: caller satisfies target features and TBL bounds.
  unsafe {
    let z = ctx.zero();
    let q0 = ctx.post_enclast(lane_lk_li[0], lane_lk_li[2]);
    let q1 = ctx.add_u64(ctx.post_enc(lane_lk_li[1], z), lane_lk_li[3]);
    let q1_z = ctx.zip_u8(q1);
    let s = ctx.post_enc(q0, q1_z);
    let u = ctx.post_enc(s, z);
    let fin = ctx.post_enc(u, z);
    let lo = ctx.add_u64(fin, u);
    finalize_hi::<L, HI>(ctx, lo, q1_z, lk)
  }
}

/// Asymmetric 4-lane AES finalizer for Pre-XOR backends combining final-round `FL`, `u64x2` add, and single `zip_u8`:
/// `q0 = FL(l0 ^ lk) ^ l2`, `q1 = F(l1) +_{u64} l3`, `q1_z = zip_u8(q1)`,
/// `s = F(q0) ^ q1_z`, `u = F(s)`, `lo = F(u) +_{u64} u`.
#[cfg(any(
  feature = "cpu",
  all(target_arch = "aarch64", target_endian = "little"),
  not(all(
    any(target_arch = "x86", target_arch = "x86_64"),
    target_endian = "little",
    target_feature = "aes",
    target_feature = "sse2"
  ))
))]
#[inline(always)]
pub(crate) unsafe fn finalize_merge4_pre<L: PreXorAes, const HI: bool>(
  ctx: L,
  lane_li: [L::V; 4],
  lk: L::V,
) -> (L::V, L::V) {
  // SAFETY: caller satisfies target features and TBL bounds.
  unsafe {
    let z = ctx.zero();
    let q0_a = ctx.pre_fl_zero(lane_li[0], lk);
    let q1 = ctx.add_u64(ctx.pre_r1_zero(lane_li[1], z), lane_li[3]);
    let q1_z = ctx.zip_u8(q1);
    let s_a = ctx.pre_r1_zero(q0_a, lane_li[2]);
    let lo = ctx.chain_x01_ff(s_a, q1_z, []);
    finalize_hi::<L, HI>(ctx, lo, q1_z, lk)
  }
}

/// Top-level inlined entry for `0..=16 B` inputs without computing wide length broadcasts.
#[inline(always)]
pub(crate) unsafe fn compress_0_16_entry<L: Aes128, const HI: bool, const SEED0: bool>(
  ctx: L,
  data: &[u8],
  seed: u64,
) -> (L::V, L::V) {
  let len = data.len();
  debug_assert!(len <= BLOCK_LEN);
  let ptr = data.as_ptr();
  // SAFETY: caller satisfies target features and len <= BLOCK_LEN keeps TBL and slice loads in bounds.
  unsafe {
    let b = ctx.load_short(ptr, len);
    let sv = expand_seed::<L, SEED0>(ctx, seed);
    let lk = len_key(ctx, len);
    let lo = ctx.r3_ff(b, sv, lk);
    finalize_hi::<L, HI>(ctx, lo, sv, lk)
  }
}

/// Inlined AES compression for `17..=128 B` inputs.
#[inline(always)]
pub(crate) unsafe fn compress_17_128<G: LaneGroup, const HI: bool, const SEED0: bool>(
  ctx: G,
  data: &[u8],
  seed: u64,
) -> (GroupV<G>, GroupV<G>) {
  let l = ctx.lane();
  let len = data.len();
  debug_assert!(len > BLOCK_LEN && len <= SHORT_MAX);
  let ptr = data.as_ptr();
  // SAFETY: caller satisfies target features and 17 <= len <= 128 keeps TBL and slice loads in bounds.
  unsafe {
    let sv = expand_lane_seed::<G::L, SEED0>(l, seed);
    let lk = len_key(l, len);
    if len <= 32 {
      ctx.compress_17_32::<HI, SEED0>(ptr, len, sv, lk)
    } else if len <= 64 {
      ctx.compress_33_64::<HI, SEED0>(ptr, len, sv, lk)
    } else {
      ctx.compress_65_128::<HI, SEED0>(ptr, len, sv, lk)
    }
  }
}

/// Inlined AES compression for `0..=128 B` prioritizing `len <= 16` at the top branch.
#[cfg(feature = "cpu")]
#[inline(always)]
pub(crate) unsafe fn compress_short<G: LaneGroup, const HI: bool, const SEED0: bool>(
  ctx: G,
  data: &[u8],
  seed: u64,
) -> (GroupV<G>, GroupV<G>) {
  if data.len() <= BLOCK_LEN {
    // SAFETY: forwarded from caller contract.
    unsafe { compress_0_16_entry::<G::L, HI, SEED0>(ctx.lane(), data, seed) }
  } else {
    // SAFETY: forwarded from caller contract.
    unsafe { compress_17_128::<G, HI, SEED0>(ctx, data, seed) }
  }
}

/// AES compression for medium inputs (`129..=512 B` 4-lane, `513..=2048 B` 8-lane).
#[inline(always)]
pub(crate) unsafe fn compress_mid<G: MidGroup, const HI: bool, const SEED0: bool>(
  ctx: G,
  data: &[u8],
  seed: u64,
) -> (GroupV<G>, GroupV<G>) {
  let l = ctx.lane();
  let len = data.len();
  debug_assert!(len > SHORT_MAX && len <= MID_MAX);
  let mut ptr = data.as_ptr();
  // SAFETY: caller satisfies target features and 129 <= len <= 2048 bounds every slice load.
  unsafe {
    let sv = expand_lane_seed::<G::L, SEED0>(l, seed);
    let lk = len_key(l, len);
    if len <= MID4_MAX {
      let end = ptr.add(len);
      let mut acc = ctx.absorb4_init(ctx.init4::<SEED0>(sv), ptr);
      if len <= 192 {
        acc = ctx.absorb4_swap32(acc, end.sub(128));
        acc = ctx.absorb4_linear(acc, end.sub(64));
      } else {
        acc = ctx.absorb4_swap32(acc, ptr.add(64));
        if len > 256 {
          acc = ctx.absorb4_linear(acc, ptr.add(128));
          acc = ctx.absorb4_swap32(acc, ptr.add(192));
          if len > 384 {
            acc = ctx.absorb4_linear(acc, ptr.add(256));
            acc = ctx.absorb4_swap32(acc, ptr.add(320));
          }
        }
        acc = ctx.absorb4_linear(acc, end.sub(128));
        acc = ctx.absorb4_swap32(acc, end.sub(64));
      }
      return ctx.merge4::<HI>(acc, lk);
    }

    let mut acc = ctx.absorb8_init(ctx.init8::<SEED0>(sv, 0), ptr);
    ptr = ptr.add(128);
    let mut rem = ((len - 1) >> 7) - 1;
    while rem >= 2 {
      acc = ctx.absorb8_pair64(acc, ptr.add(64), ptr);
      let p2 = ptr.add(128);
      acc = ctx.absorb8_pair64(acc, p2, p2.add(64));
      ptr = ptr.add(256);
      rem -= 2;
    }
    if rem == 1 {
      acc = ctx.absorb8_pair64(acc, ptr.add(64), ptr);
    }

    let last = data.as_ptr().add(len - 128);
    acc = ctx.absorb8_swap32(acc, last);
    ctx.merge8::<HI>(acc, lk)
  }
}

#[inline(always)]
unsafe fn absorb16_pair<G: LaneGroup>(
  ctx: G,
  lo: G::G8,
  hi: G::G8,
  ptr: *const u8,
) -> (G::G8, G::G8) {
  // SAFETY: caller satisfies target features and >= 512 readable bytes at ptr.
  unsafe {
    let (lo, hi) = ctx.absorb16_swapped(lo, hi, ptr);
    ctx.absorb16_linear(lo, hi, ptr.add(256))
  }
}

/// 16-lane AES compression for bulk inputs (`> 2048 B`).
#[inline(always)]
pub(crate) unsafe fn compress_bulk<G: LaneGroup, const HI: bool, const SEED0: bool>(
  ctx: G,
  data: &[u8],
  seed: u64,
) -> (GroupV<G>, GroupV<G>) {
  const PREFETCH_THRESHOLD: usize = 32 * 1024;
  const PREFETCH_DIST: usize = 768;

  let l = ctx.lane();
  let len = data.len();
  debug_assert!(len > MID_MAX);
  let mut ptr = data.as_ptr();
  // SAFETY: caller satisfies target features and len > 2048 keeps every 256B stripe in bounds.
  unsafe {
    let sv = expand_lane_seed::<G::L, SEED0>(l, seed);
    let mut lo = ctx.absorb8_init(ctx.init8::<SEED0>(sv, 0), ptr);
    let mut hi = ctx.absorb8_init(ctx.init8::<SEED0>(sv, 8), ptr.add(128));
    ptr = ptr.add(256);

    let mut rem = ((len - 1) >> 8) - 1;
    if G::L::HAS_PREFETCH && len > PREFETCH_THRESHOLD {
      // Loop invariant guarantees at least 5 * 256 + 1 = 1281 readable bytes past ptr,
      // keeping PREFETCH_DIST + 384 (1152) strictly within data bounds.
      while rem >= 5 {
        l.prefetch_l1(ptr.add(PREFETCH_DIST));
        l.prefetch_l1(ptr.add(PREFETCH_DIST + 128));
        l.prefetch_l1(ptr.add(PREFETCH_DIST + 256));
        l.prefetch_l1(ptr.add(PREFETCH_DIST + 384));
        (lo, hi) = absorb16_pair(ctx, lo, hi, ptr);
        ptr = ptr.add(512);
        rem -= 2;
      }
    }
    while rem >= 2 {
      (lo, hi) = absorb16_pair(ctx, lo, hi, ptr);
      ptr = ptr.add(512);
      rem -= 2;
    }
    if rem == 1 {
      (lo, hi) = ctx.absorb16_swapped(lo, hi, ptr);
    }

    let last = data.as_ptr().add(len - 256);
    lo = ctx.absorb8_pair64(lo, last.add(128), last.add(192));
    hi = ctx.absorb8_pair64(hi, last, last.add(64));
    let lk = len_key_wide(l, len);
    ctx.merge16::<HI>(lo, hi, lk)
  }
}

/// Function pointer types for out-of-line `> SHORT_MAX` compression helpers.
pub(crate) type LongLoFn<V> = unsafe fn(&[u8], u64) -> V;
pub(crate) type LongHiFn<V> = unsafe fn(&[u8], u64) -> (V, V);

/// Dispatches `> SHORT_MAX` inputs between 4/8-lane `compress_mid` and cold 16-lane `bulk_lo`.
#[inline(always)]
pub(crate) unsafe fn compress_long_lo<G: MidGroup, const SEED0: bool>(
  ctx: G,
  data: &[u8],
  seed: u64,
  bulk_lo: LongLoFn<GroupV<G>>,
) -> GroupV<G> {
  debug_assert!(data.len() > SHORT_MAX);
  if data.len() <= MID_MAX {
    // SAFETY: forwarded from caller contract with SHORT_MAX < data.len() <= MID_MAX.
    return unsafe { compress_mid::<G, false, SEED0>(ctx, data, seed).0 };
  }
  // SAFETY: forwarded from caller contract with data.len() > MID_MAX.
  unsafe { bulk_lo(data, seed) }
}

/// Dispatches `> SHORT_MAX` 256-bit inputs between `compress_mid` and cold 16-lane `bulk_hi`.
#[inline(always)]
pub(crate) unsafe fn compress_long_hi<G: MidGroup, const SEED0: bool>(
  ctx: G,
  data: &[u8],
  seed: u64,
  bulk_hi: LongHiFn<GroupV<G>>,
) -> (GroupV<G>, GroupV<G>) {
  debug_assert!(data.len() > SHORT_MAX);
  if data.len() <= MID_MAX {
    // SAFETY: forwarded from caller contract with SHORT_MAX < data.len() <= MID_MAX.
    return unsafe { compress_mid::<G, true, SEED0>(ctx, data, seed) };
  }
  // SAFETY: forwarded from caller contract with data.len() > MID_MAX.
  unsafe { bulk_hi(data, seed) }
}