zeppelin_embed/kernels/

mod.rs

//! Runtime-dispatched dot-product and packed-bit Hamming kernels.
//!
//! Callers validate dimensions once at the engine boundary. Every hot kernel
//! therefore uses `debug_assert!` for the equal-length and row-shape contracts
//! instead of repeating release-mode validation in the scan loop. Call
//! [`crate::kernels::initialize`] during engine initialization to apply the optional
//! `ZE_KERNEL=scalar|neon|avx2` override and surface unsupported requests as a
//! typed [`crate::kernels::KernelInitError`]. If no explicit initialization occurs, the first
//! kernel call safely caches the best runtime-detected arm.
//!
//! The scalar implementation is the behavioral oracle. [`crate::kernels::KernelVariant`]
//! exposes every runtime-supported table to property tests, fuzzing, and the
//! future Task 27 frontier harness without moving selection to build time.

mod dispatch;
/// Posting-block decode kernels: bit-unpack and delta prefix sum.
pub mod postings;
mod scalar;
#[cfg(any(test, feature = "test-seams"))]
#[doc(hidden)]
pub mod vector_fault;

#[cfg(target_arch = "x86_64")]
mod avx2;
#[cfg(target_arch = "aarch64")]
mod neon;

/// Largest supported dimension for an i8 dot product.
///
/// The worst product is `(-128) * (-128) = 16_384`; at this dimension the
/// worst sum is `2^30`, which remains representable by [`i32`].
pub const MAX_DOT_I8_DIMENSION: usize = 65_536;

const _: [(); 1] = [(); (MAX_DOT_I8_DIMENSION * (i8::MIN as i16 * i8::MIN as i16) as usize
    <= i32::MAX as usize) as usize];

/// User-selectable runtime kernel families.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum KernelArm {
    /// Portable scalar oracle and fallback.
    Scalar,
    /// AArch64 NEON, using SDOT at runtime when available.
    Neon,
    /// x86-64 AVX2 plus scalar POPCNT.
    Avx2,
}

/// Instruction tiers enumerated for the Task 27 B1 frontier campaign.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum InstructionTier {
    /// Portable scalar instructions.
    Scalar,
    /// Baseline AArch64 widening multiply.
    NeonWiden,
    /// AArch64 FEAT_DotProd SDOT.
    NeonDotprod,
    /// x86-64 AVX2 plus POPCNT.
    Avx2,
    /// AArch64 FEAT_I8MM SMMLA batch tier.
    NeonI8mmReserved,
    /// Reserved, unimplemented FEAT_SME2 campaign tier.
    Sme2Reserved,
}

#[derive(Clone, Copy, Debug, Eq, PartialEq)]
#[cfg_attr(not(any(test, feature = "benchmark-kernels")), allow(dead_code))]
enum KernelBackendTag {
    Scalar,
    #[cfg(target_arch = "aarch64")]
    NeonWiden,
    #[cfg(target_arch = "aarch64")]
    NeonDotprodU4,
    #[cfg(target_arch = "aarch64")]
    NeonI8mm,
    #[cfg(target_arch = "aarch64")]
    NeonDotprodU2,
    #[cfg(target_arch = "aarch64")]
    NeonDotprodU6,
    #[cfg(target_arch = "aarch64")]
    NeonDotprodU8,
    #[cfg(target_arch = "aarch64")]
    NeonDotprodU4Prefetch,
    #[cfg(target_arch = "x86_64")]
    Avx2,
}

/// Stable identity of one concrete runtime kernel table.
///
/// This is test-support evidence rather than a user-selectable production
/// arm. In particular, the experimental DotProd shapes are enumerable for
/// parity campaigns but cannot be selected through `ZE_KERNEL`.
#[cfg(any(test, feature = "benchmark-kernels"))]
#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)]
#[doc(hidden)]
pub enum KernelBackendId {
    /// Portable scalar table.
    Scalar,
    /// Baseline AArch64 widening table.
    NeonWiden,
    /// Shipped four-accumulator AArch64 DotProd table.
    NeonDotprodU4,
    /// AArch64 I8MM batch table.
    NeonI8mm,
    /// Two-accumulator AArch64 DotProd experiment.
    NeonDotprodU2,
    /// Six-accumulator AArch64 DotProd experiment.
    NeonDotprodU6,
    /// Eight-accumulator AArch64 DotProd experiment.
    NeonDotprodU8,
    /// Four-accumulator AArch64 DotProd table with batch prefetch.
    NeonDotprodU4Prefetch,
    /// x86-64 AVX2 table.
    Avx2,
}

#[cfg(any(test, feature = "benchmark-kernels"))]
impl KernelBackendId {
    const fn from_tag(tag: KernelBackendTag) -> Self {
        match tag {
            KernelBackendTag::Scalar => Self::Scalar,
            #[cfg(target_arch = "aarch64")]
            KernelBackendTag::NeonWiden => Self::NeonWiden,
            #[cfg(target_arch = "aarch64")]
            KernelBackendTag::NeonDotprodU4 => Self::NeonDotprodU4,
            #[cfg(target_arch = "aarch64")]
            KernelBackendTag::NeonI8mm => Self::NeonI8mm,
            #[cfg(target_arch = "aarch64")]
            KernelBackendTag::NeonDotprodU2 => Self::NeonDotprodU2,
            #[cfg(target_arch = "aarch64")]
            KernelBackendTag::NeonDotprodU6 => Self::NeonDotprodU6,
            #[cfg(target_arch = "aarch64")]
            KernelBackendTag::NeonDotprodU8 => Self::NeonDotprodU8,
            #[cfg(target_arch = "aarch64")]
            KernelBackendTag::NeonDotprodU4Prefetch => Self::NeonDotprodU4Prefetch,
            #[cfg(target_arch = "x86_64")]
            KernelBackendTag::Avx2 => Self::Avx2,
        }
    }

    /// Returns the permanent canonical evidence label.
    #[must_use]
    pub const fn as_str(self) -> &'static str {
        match self {
            Self::Scalar => "scalar",
            Self::NeonWiden => "neon-widen",
            Self::NeonDotprodU4 => "neon-dotprod-u4",
            Self::NeonI8mm => "neon-i8mm",
            Self::NeonDotprodU2 => "neon-dotprod-u2",
            Self::NeonDotprodU6 => "neon-dotprod-u6",
            Self::NeonDotprodU8 => "neon-dotprod-u8",
            Self::NeonDotprodU4Prefetch => "neon-dotprod-u4-prefetch",
            Self::Avx2 => "avx2",
        }
    }

    #[cfg(any(test, feature = "test-seams"))]
    const fn arm(self) -> KernelArm {
        match self {
            Self::Scalar => KernelArm::Scalar,
            Self::NeonWiden
            | Self::NeonDotprodU4
            | Self::NeonI8mm
            | Self::NeonDotprodU2
            | Self::NeonDotprodU6
            | Self::NeonDotprodU8
            | Self::NeonDotprodU4Prefetch => KernelArm::Neon,
            Self::Avx2 => KernelArm::Avx2,
        }
    }
}

/// Runtime CPU capabilities relevant to current and reserved kernel tiers.
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
pub struct KernelFeatures {
    /// AArch64 Advanced SIMD is executable.
    pub neon: bool,
    /// AArch64 FEAT_DotProd is executable.
    pub dotprod: bool,
    /// AArch64 half-precision conversion/arithmetic is executable.
    pub fp16: bool,
    /// AArch64 FEAT_I8MM is executable.
    pub i8mm: bool,
    /// AArch64 FEAT_SME2 was detected; implementation is deferred to B1.
    pub sme2: bool,
    /// x86-64 AVX2 is executable.
    pub avx2: bool,
    /// x86 POPCNT is executable.
    pub popcnt: bool,
}

/// Typed failures produced while applying a runtime kernel override.
#[derive(Clone, Debug, Eq, PartialEq)]
pub enum KernelInitError {
    /// `ZE_KERNEL` contained a Unicode value outside the supported set.
    UnknownOverride {
        /// Rejected environment value.
        value: String,
    },
    /// `ZE_KERNEL` was not valid Unicode.
    NonUnicodeOverride,
    /// The requested family cannot execute on this CPU/architecture.
    UnsupportedArm {
        /// Requested runtime family.
        requested: KernelArm,
    },
    /// A kernel call selected an arm before a conflicting explicit override.
    AlreadyInitialized {
        /// Arm already cached in the process.
        selected: KernelArm,
        /// Conflicting arm requested by `ZE_KERNEL`.
        requested: KernelArm,
    },
}

impl std::fmt::Display for KernelInitError {
    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        match self {
            Self::UnknownOverride { value } => write!(
                formatter,
                "unknown ZE_KERNEL value {value:?}; expected scalar, neon, or avx2"
            ),
            Self::NonUnicodeOverride => formatter.write_str("ZE_KERNEL is not valid Unicode"),
            Self::UnsupportedArm { requested } => {
                write!(
                    formatter,
                    "forced kernel arm {requested:?} is unsupported on this CPU"
                )
            }
            Self::AlreadyInitialized {
                selected,
                requested,
            } => write!(
                formatter,
                "kernel dispatch already selected {selected:?}; cannot force {requested:?}"
            ),
        }
    }
}

impl std::error::Error for KernelInitError {}

/// Enumerable tuning declarations consumed directly by Task 27-H.
///
/// This is data only. Task 03 intentionally does not implement search,
/// measurement ledgers, roofline calculation, or variant code generation.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct KnobSpace {
    /// Candidate loop-unroll depths.
    pub unroll: &'static [usize],
    /// Candidate independent-accumulator counts.
    pub accumulators: &'static [usize],
    /// Candidate rows grouped per scan block.
    pub rows_per_block: &'static [usize],
    /// Candidate software-prefetch distances in rows; zero disables prefetch.
    pub prefetch_dist: &'static [usize],
    /// Current and reserved instruction tiers.
    pub tier: &'static [InstructionTier],
}

/// Baseline micro-shape selected before the Task 27 B1 campaign.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct BaselineKernelConfig {
    /// Loop-unroll depth.
    pub unroll: usize,
    /// Number of independent accumulators.
    pub accumulators: usize,
    /// Rows handled per row-major batch iteration.
    pub rows_per_block: usize,
    /// Software-prefetch distance in rows.
    pub prefetch_dist: usize,
}

/// Shipped baseline configuration; B1 may replace every value with evidence.
pub const BASELINE_KERNEL_CONFIG: BaselineKernelConfig = BaselineKernelConfig {
    // Retained by tasks/evidence/opt-ledger/B1.md iterations 1–5.
    unroll: 4,
    accumulators: 4,
    rows_per_block: 1,
    prefetch_dist: 0,
};

// Search provenance: tasks/evidence/opt-ledger/B1.md.
const UNROLL_KNOBS: [usize; 4] = [2, 4, 6, 8];
const ACCUMULATOR_KNOBS: [usize; 4] = [2, 4, 6, 8];
const ROW_BLOCK_KNOBS: [usize; 5] = [1, 2, 4, 8, 16];
const PREFETCH_KNOBS: [usize; 5] = [0, 1, 2, 4, 8];
const TIER_KNOBS: [InstructionTier; 6] = [
    InstructionTier::Scalar,
    InstructionTier::NeonWiden,
    InstructionTier::NeonDotprod,
    InstructionTier::Avx2,
    InstructionTier::NeonI8mmReserved,
    InstructionTier::Sme2Reserved,
]; // Search provenance: tasks/evidence/opt-ledger/B1.md.

/// Registered Task 27-H kernel tuning space.
pub const KERNEL_KNOB_SPACE: KnobSpace = KnobSpace {
    unroll: &UNROLL_KNOBS,
    accumulators: &ACCUMULATOR_KNOBS,
    rows_per_block: &ROW_BLOCK_KNOBS,
    prefetch_dist: &PREFETCH_KNOBS,
    tier: &TIER_KNOBS,
};

type DotI8Fn = fn(&[i8], &[i8]) -> i32;
type DotF32Fn = fn(&[f32], &[f32]) -> f32;
type DotF16Fn = fn(&[u16], &[u16]) -> f32;
type HammingU1Fn = fn(&[u8], &[u8]) -> u32;
type DotI8BatchFn = fn(&[i8], &[i8], usize, &mut [i32]);
type HammingU1BatchFn = fn(&[u8], &[u8], usize, &mut [u32]);
type DotPackedFn = fn(&[i8], &[u8]) -> i32;
type DotBit4PreparedFn = fn(&[i8], i32, &[u8]) -> i32;
type DotPackedBatchFn = fn(&[i8], &[u8], usize, &mut [i32]);
type ScoreBit4PreparedBatchFn =
    fn(&[i8], i32, f64, &[u8], usize, &[crate::quant::Bit4Factors], &mut [f32]);
type ScoreBit4PtrsFn = unsafe fn(
    &[i8],
    i32,
    f64,
    &[*const u8; 4],
    usize,
    &[crate::quant::Bit4Factors; 4],
    &mut [f32; 4],
);
#[derive(Clone, Copy)]
struct KernelTable {
    #[cfg_attr(not(any(test, feature = "benchmark-kernels")), allow(dead_code))]
    backend: KernelBackendTag,
    arm: KernelArm,
    tier: InstructionTier,
    dot_i8: DotI8Fn,
    dot_f32: DotF32Fn,
    dot_f16: DotF16Fn,
    hamming_u1: HammingU1Fn,
    dot_i8_batch: DotI8BatchFn,
    hamming_u1_batch: HammingU1BatchFn,
    dot_bit4: DotPackedFn,
    dot_bit4_prepared: DotBit4PreparedFn,
    dot_bit4_batch: DotPackedBatchFn,
    score_bit4_prepared_batch: ScoreBit4PreparedBatchFn,
    score_bit4_ptrs: ScoreBit4PtrsFn,
}

/// One bounds-validated packed four-bit row borrowed from a mapped region.
///
/// `from_mapped_region` validates an in-bounds offset and length; the
/// crate-only `from_validated_bytes` wraps a slice whose persisted range was
/// already validated. Both retain a borrowed slice, so the stored address is
/// readable for that slice's complete length while this handle is alive.
#[derive(Clone, Copy)]
pub struct Bit4Row<'a> {
    bytes: &'a [u8],
}

impl std::fmt::Debug for Bit4Row<'_> {
    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        formatter
            .debug_struct("Bit4Row")
            .field("len", &self.bytes.len())
            .finish()
    }
}

impl<'a> Bit4Row<'a> {
    /// Borrows one row at `offset` when the complete row lies in `region`.
    ///
    /// Returns `None` when `offset + row_bytes` overflows or exceeds the
    /// mapped region. No alignment is required by any installed kernel.
    #[must_use]
    pub fn from_mapped_region(region: &'a [u8], offset: usize, row_bytes: usize) -> Option<Self> {
        let end = offset.checked_add(row_bytes)?;
        region.get(offset..end).map(|bytes| Self { bytes })
    }

    /// Wraps bytes whose complete persisted row range was already validated.
    #[must_use]
    pub(crate) const fn from_validated_bytes(bytes: &'a [u8]) -> Self {
        Self { bytes }
    }

    const fn as_ptr(self) -> *const u8 {
        self.bytes.as_ptr()
    }

    const fn len(self) -> usize {
        self.bytes.len()
    }
}

/// Four equally-sized packed rows whose addresses were validated before scoring.
#[derive(Clone, Copy)]
pub struct Bit4Rows4<'a> {
    pointers: [*const u8; 4],
    row_bytes: usize,
    lifetime: std::marker::PhantomData<&'a [u8]>,
}

impl std::fmt::Debug for Bit4Rows4<'_> {
    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        formatter
            .debug_struct("Bit4Rows4")
            .field("row_bytes", &self.row_bytes)
            .finish()
    }
}

impl<'a> Bit4Rows4<'a> {
    /// Validates four mapped-row handles once for the unchecked gather loop.
    ///
    /// # Errors
    ///
    /// Returns [`GatherShapeError::RowLength`] when any handle was constructed
    /// for a different readable width. No alignment is required.
    pub fn from_rows(rows: [Bit4Row<'a>; 4], row_bytes: usize) -> Result<Self, GatherShapeError> {
        for (index, row) in rows.iter().enumerate() {
            if row.len() != row_bytes {
                return Err(GatherShapeError::RowLength {
                    index,
                    expected: row_bytes,
                    actual: row.len(),
                });
            }
        }
        Ok(Self {
            pointers: rows.map(Bit4Row::as_ptr),
            row_bytes,
            lifetime: std::marker::PhantomData,
        })
    }
}

/// Rejected shape at the safe four-row gather boundary.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum GatherShapeError {
    /// The prepared query exceeds the exact i8 accumulator bound.
    DimensionTooLarge {
        /// Supplied coordinate count.
        actual: usize,
        /// Largest supported coordinate count.
        maximum: usize,
    },
    /// `row_bytes` does not match the packed query dimension.
    RowBytes {
        /// Required packed bytes per row.
        expected: usize,
        /// Supplied packed bytes per row.
        actual: usize,
    },
    /// A validated row handle was constructed for a different row width.
    RowLength {
        /// Zero-based pointer slot.
        index: usize,
        /// Required readable bytes.
        expected: usize,
        /// Bytes proven readable by the row handle.
        actual: usize,
    },
}

impl std::fmt::Display for GatherShapeError {
    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        match self {
            Self::DimensionTooLarge { actual, maximum } => write!(
                formatter,
                "gather dimension {actual} exceeds supported maximum {maximum}"
            ),
            Self::RowBytes { expected, actual } => write!(
                formatter,
                "gather row width mismatch: expected {expected}, got {actual}"
            ),
            Self::RowLength {
                index,
                expected,
                actual,
            } => write!(
                formatter,
                "gather row {index} has {actual} readable bytes, expected {expected}"
            ),
        }
    }
}

impl std::error::Error for GatherShapeError {}

/// Prefetches caller-selected packed rows into L1 for a later gather group.
///
/// Traversal can call this only for degrees where M0 found a benefit (R >= 32),
/// leaving small-degree scoring unchanged. Every validated row handle supplies
/// a live readable address. This is a no-op on non-AArch64 targets.
pub fn prefetch_bit4_rows(rows: &[Bit4Row<'_>]) {
    #[cfg(target_arch = "aarch64")]
    for row in rows.iter().filter(|row| row.len() != 0) {
        let address = row.as_ptr();
        // SAFETY: Bit4Row construction proves address belongs to a live readable
        // region. PRFM is a non-mutating hint and has no alignment requirement.
        unsafe {
            std::arch::asm!(
                "prfm pldl1keep, [{address}]",
                address = in(reg) address,
                options(readonly, nostack)
            );
        }
    }
    #[cfg(not(target_arch = "aarch64"))]
    let _ = rows;
}

/// Prefetches one already-validated four-row gather group.
pub fn prefetch_bit4_row_group(rows: &Bit4Rows4<'_>) {
    #[cfg(target_arch = "aarch64")]
    if rows.row_bytes != 0 {
        for &address in &rows.pointers {
            // SAFETY: Bit4Rows4 construction proves every address belongs to a
            // live region readable for row_bytes. PRFM is a non-mutating hint.
            unsafe {
                std::arch::asm!(
                    "prfm pldl1keep, [{address}]",
                    address = in(reg) address,
                    options(readonly, nostack)
                );
            }
        }
    }
    #[cfg(not(target_arch = "aarch64"))]
    let _ = rows;
}

/// One concrete runtime dispatch table.
///
/// Variants are exposed so the scalar oracle, property suite, fuzzer, and
/// future frontier harness can evaluate every arm independently of the global
/// once-selected default.
#[derive(Clone, Copy)]
pub struct KernelVariant {
    table: KernelTable,
}

impl std::fmt::Debug for KernelVariant {
    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        formatter
            .debug_struct("KernelVariant")
            .field("arm", &self.table.arm)
            .field("tier", &self.table.tier)
            .finish()
    }
}

impl KernelVariant {
    /// Returns the scalar oracle variant.
    #[must_use]
    pub fn scalar() -> Self {
        Self {
            table: scalar::table(),
        }
    }

    /// Returns the once-selected runtime table for a repeated hot loop.
    #[doc(hidden)]
    #[must_use]
    pub fn selected() -> Self {
        Self {
            table: *dispatch::active_table(),
        }
    }

    /// Iterates over every arm executable on the current CPU.
    ///
    /// BL-072: `dispatch::variant_tables()` returns the SMMLA table plus the
    /// u2/u6/u8/prefetch DotProd shape tables, none of which the engine ever
    /// selects on its own. They exist so the scalar oracle, the property
    /// suite, the fuzzer and the frontier harness can evaluate each arm
    /// independently. That is a test and measurement surface, not an engine
    /// surface, so it is gated: a consumer of the shipped staticlib does not
    /// see it.
    #[cfg(any(test, feature = "benchmark-kernels"))]
    pub fn available() -> impl Iterator<Item = Self> {
        dispatch::variant_tables()
            .into_iter()
            .flatten()
            .map(|table| Self { table })
    }

    /// Returns this variant's runtime family.
    #[must_use]
    pub fn arm(self) -> KernelArm {
        self.table.arm
    }

    /// Returns this variant's precise instruction tier.
    #[must_use]
    pub fn tier(self) -> InstructionTier {
        self.table.tier
    }

    /// Returns this concrete table's stable evidence identity.
    #[cfg(any(test, feature = "benchmark-kernels"))]
    #[doc(hidden)]
    #[must_use]
    pub const fn backend_id(self) -> KernelBackendId {
        KernelBackendId::from_tag(self.table.backend)
    }

    /// Computes an i8 dot product through this concrete table.
    #[must_use]
    pub fn dot_i8(self, a: &[i8], b: &[i8]) -> i32 {
        (self.table.dot_i8)(a, b)
    }

    /// Computes an f32 dot product through this concrete table.
    #[must_use]
    pub fn dot_f32(self, a: &[f32], b: &[f32]) -> f32 {
        (self.table.dot_f32)(a, b)
    }

    /// Computes an IEEE-f16-bits dot product through this concrete table.
    #[must_use]
    pub fn dot_f16(self, a: &[u16], b: &[u16]) -> f32 {
        (self.table.dot_f16)(a, b)
    }

    /// Computes packed-byte Hamming distance through this concrete table.
    #[must_use]
    pub fn hamming_u1(self, a: &[u8], b: &[u8]) -> u32 {
        (self.table.hamming_u1)(a, b)
    }

    /// Scores contiguous i8 rows through this concrete table.
    pub fn dot_i8_batch(self, q: &[i8], rows: &[i8], d: usize, out: &mut [i32]) {
        (self.table.dot_i8_batch)(q, rows, d, out);
    }

    /// Scores contiguous packed-bit rows through this concrete table.
    pub fn hamming_u1_batch(self, q: &[u8], rows: &[u8], d_bytes: usize, out: &mut [u32]) {
        (self.table.hamming_u1_batch)(q, rows, d_bytes, out);
    }

    /// Scores a signed-byte query against one packed four-bit row.
    #[must_use]
    pub fn dot_bit4(self, q: &[i8], codes: &[u8]) -> i32 {
        (self.table.dot_bit4)(q, codes)
    }

    /// Scores a block-interleaved four-bit query against one packed row.
    ///
    /// Within each 32-coordinate query block, even coordinates precede odd
    /// coordinates. `query_sum` is the sum of the original signed query codes.
    #[doc(hidden)]
    #[must_use]
    pub fn dot_bit4_prepared(self, q: &[i8], query_sum: i32, codes: &[u8]) -> i32 {
        (self.table.dot_bit4_prepared)(q, query_sum, codes)
    }

    /// Scores a signed-byte query against contiguous packed four-bit rows.
    pub fn dot_bit4_batch(self, q: &[i8], rows: &[u8], d: usize, out: &mut [i32]) {
        (self.table.dot_bit4_batch)(q, rows, d, out);
    }

    /// Scores a prepared four-bit query and row factors through this table.
    #[doc(hidden)]
    pub fn score_bit4_prepared_batch(
        self,
        query: (&[i8], i32, f64),
        rows: &[u8],
        d: usize,
        factors: &[crate::quant::Bit4Factors],
        out: &mut [f32],
    ) {
        let (q, query_sum, query_scale_half) = query;
        (self.table.score_bit4_prepared_batch)(
            q,
            query_sum,
            query_scale_half,
            rows,
            d,
            factors,
            out,
        );
    }

    /// Scores four arbitrary validated packed four-bit rows through this table.
    ///
    /// # Safety
    ///
    /// [`Bit4Rows4::from_rows`] proves that every row address is readable for a
    /// common width and that the mapped regions outlive this call. This method
    /// checks that the width equals `q.len().div_ceil(2)` and that the supported
    /// dimension bound holds. No row alignment is required.
    pub fn score_bit4_ptrs(
        &self,
        query: (&[i8], i32, f64),
        rows: &Bit4Rows4<'_>,
        factors: &[crate::quant::Bit4Factors; 4],
        out: &mut [f32; 4],
    ) -> Result<(), GatherShapeError> {
        score_bit4_ptrs_with_table(&self.table, query, rows, factors, out)
    }
}

fn score_bit4_ptrs_with_table(
    table: &KernelTable,
    query: (&[i8], i32, f64),
    rows: &Bit4Rows4<'_>,
    factors: &[crate::quant::Bit4Factors; 4],
    out: &mut [f32; 4],
) -> Result<(), GatherShapeError> {
    let (q, query_sum, query_scale_half) = query;
    if q.len() > MAX_DOT_I8_DIMENSION {
        return Err(GatherShapeError::DimensionTooLarge {
            actual: q.len(),
            maximum: MAX_DOT_I8_DIMENSION,
        });
    }
    let expected_row_bytes = q.len().div_ceil(2);
    if rows.row_bytes != expected_row_bytes {
        return Err(GatherShapeError::RowBytes {
            expected: expected_row_bytes,
            actual: rows.row_bytes,
        });
    }
    debug_assert!(q.len() <= MAX_DOT_I8_DIMENSION);
    debug_assert_eq!(rows.row_bytes, q.len().div_ceil(2));
    // SAFETY: Bit4Rows4::from_rows verified that every Bit4Row's readable
    // slice is row_bytes long, and their borrow keeps each backing region alive
    // for the dispatched call.
    unsafe {
        (table.score_bit4_ptrs)(
            q,
            query_sum,
            query_scale_half,
            &rows.pointers,
            rows.row_bytes,
            factors,
            out,
        );
    }
    Ok(())
}

/// Applies runtime feature detection and the optional `ZE_KERNEL` override.
///
/// Call this once at engine initialization, before any kernel call. Unknown or
/// unsupported forced arms return a typed error and are never executed.
pub fn initialize() -> Result<KernelArm, KernelInitError> {
    dispatch::initialize()
}

/// Returns the cached arm, selecting the best safe runtime arm if necessary.
#[must_use]
pub fn selected_arm() -> KernelArm {
    dispatch::active_table().arm
}

/// Reports whether a user-selectable arm is executable on this CPU.
#[must_use]
pub fn is_arm_supported(arm: KernelArm) -> bool {
    dispatch::table_for_arm(arm).is_some()
}

/// Returns cached runtime feature detection, including reserved B1 features.
#[must_use]
pub fn detected_features() -> KernelFeatures {
    dispatch::features()
}

/// Streams bytes through four independent wide-load accumulators for the
/// platform-truth bandwidth harness.
///
/// This measurement-only hook returns `None` when an AArch64 NEON
/// implementation is unavailable. It is not part of the distance-kernel API.
#[doc(hidden)]
#[must_use]
pub fn platform_wide_stream_checksum(bytes: &[u8]) -> Option<u64> {
    #[cfg(target_arch = "aarch64")]
    {
        dispatch::features()
            .neon
            .then(|| neon::wide_stream_checksum(bytes))
    }
    #[cfg(not(target_arch = "aarch64"))]
    {
        let _ = bytes;
        None
    }
}

/// Computes the dot product of equal-length signed-byte vectors.
///
/// Callers must pre-validate equal lengths and `a.len() <=
/// MAX_DOT_I8_DIMENSION`; the hot path performs `debug_assert!` checks only.
#[must_use]
pub fn dot_i8(a: &[i8], b: &[i8]) -> i32 {
    let table = dispatch::active_table();
    let result = (table.dot_i8)(a, b);
    #[cfg(any(test, feature = "test-seams"))]
    observe_selected_score(table, vector_fault_kernel_id_dot_i8(), a.len());
    #[cfg(any(test, feature = "test-seams"))]
    vector_fault::observe_selected_result(vector_fault::KernelScoreValue::I32(result));
    result
}

/// Computes the dot product of equal-length f32 vectors.
///
/// Callers must pre-validate equal lengths; the hot path performs a
/// `debug_assert!` check only. SIMD accumulation may reassociate additions.
#[must_use]
pub fn dot_f32(a: &[f32], b: &[f32]) -> f32 {
    let table = dispatch::active_table();
    let result = (table.dot_f32)(a, b);
    #[cfg(any(test, feature = "test-seams"))]
    observe_selected_score(table, vector_fault_kernel_id_dot_f32(), a.len());
    #[cfg(any(test, feature = "test-seams"))]
    vector_fault::observe_selected_result(vector_fault::KernelScoreValue::F32(result.to_bits()));
    result
}

/// Computes an f32 dot product from equal-length IEEE f16 bit vectors.
///
/// Callers must pre-validate equal lengths; the hot path performs a
/// `debug_assert!` check only. Subnormals, infinities, and NaNs follow IEEE
/// conversion and f32 arithmetic behavior.
#[must_use]
pub fn dot_f16(a: &[u16], b: &[u16]) -> f32 {
    let table = dispatch::active_table();
    let result = (table.dot_f16)(a, b);
    #[cfg(any(test, feature = "test-seams"))]
    observe_selected_score(table, vector_fault_kernel_id_dot_f16(), a.len());
    #[cfg(any(test, feature = "test-seams"))]
    vector_fault::observe_selected_result(vector_fault::KernelScoreValue::F32(result.to_bits()));
    result
}

/// Computes Hamming distance over equal-length packed-byte vectors.
///
/// Callers must pre-validate equal lengths; the hot path performs a
/// `debug_assert!` check only. Every bit of every supplied byte is significant,
/// including unused high bits in a caller's final partial logical byte.
#[must_use]
pub fn hamming_u1(a: &[u8], b: &[u8]) -> u32 {
    let table = dispatch::active_table();
    let result = (table.hamming_u1)(a, b);
    #[cfg(any(test, feature = "test-seams"))]
    observe_selected_score(table, vector_fault_kernel_id_hamming(), a.len());
    #[cfg(any(test, feature = "test-seams"))]
    vector_fault::observe_selected_result(vector_fault::KernelScoreValue::U32(result));
    result
}

/// Scores one i8 query against contiguous row-major vectors.
///
/// Callers must pre-validate `q.len() == d`, `rows.len() == d * out.len()`,
/// and `d <= MAX_DOT_I8_DIMENSION`; the hot path uses `debug_assert!` only.
/// A zero dimension deterministically fills `out` with zero.
pub fn dot_i8_batch(q: &[i8], rows: &[i8], d: usize, out: &mut [i32]) {
    let table = dispatch::active_table();
    (table.dot_i8_batch)(q, rows, d, out);
    #[cfg(any(test, feature = "test-seams"))]
    observe_selected_score(
        table,
        vector_fault_kernel_id_dot_i8_batch(),
        d.saturating_mul(out.len()),
    );
    #[cfg(any(test, feature = "test-seams"))]
    vector_fault::observe_selected_result(vector_fault::KernelScoreValue::I32s(out.to_vec()));
}

/// Scores one packed-bit query against contiguous row-major byte vectors.
///
/// Callers must pre-validate `q.len() == d_bytes` and `rows.len() ==
/// d_bytes * out.len()`; the hot path uses `debug_assert!` only. Every bit in
/// the final byte is significant. A zero byte dimension fills `out` with zero.
pub fn hamming_u1_batch(q: &[u8], rows: &[u8], d_bytes: usize, out: &mut [u32]) {
    let table = dispatch::active_table();
    (table.hamming_u1_batch)(q, rows, d_bytes, out);
    #[cfg(any(test, feature = "test-seams"))]
    observe_selected_score(
        table,
        vector_fault_kernel_id_hamming_batch(),
        d_bytes.saturating_mul(out.len()),
    );
    #[cfg(any(test, feature = "test-seams"))]
    vector_fault::observe_selected_result(vector_fault::KernelScoreValue::U32s(out.to_vec()));
}

/// Scores a signed-byte query against one packed four-bit row.
///
/// Two unsigned fields are stored most-significant first in each byte and
/// decoded to the doubled quantization grid `2 * code - 15`. Callers must
/// pre-validate `codes.len() == q.len().div_ceil(2)` and `q.len() <=
/// MAX_DOT_I8_DIMENSION`; an unused trailing field is ignored. The hot path
/// allocates no memory.
#[must_use]
pub fn dot_bit4(q: &[i8], codes: &[u8]) -> i32 {
    let table = dispatch::active_table();
    let result = (table.dot_bit4)(q, codes);
    #[cfg(any(test, feature = "test-seams"))]
    observe_selected_score(table, vector_fault_kernel_id_dot_bit4(), q.len());
    #[cfg(any(test, feature = "test-seams"))]
    vector_fault::observe_selected_result(vector_fault::KernelScoreValue::I32(result));
    result
}

pub(crate) fn dot_bit4_prepared(q: &[i8], query_sum: i32, codes: &[u8]) -> i32 {
    let table = dispatch::active_table();
    let result = (table.dot_bit4_prepared)(q, query_sum, codes);
    #[cfg(any(test, feature = "test-seams"))]
    observe_selected_score(table, vector_fault_kernel_id_dot_bit4_prepared(), q.len());
    #[cfg(any(test, feature = "test-seams"))]
    vector_fault::observe_selected_result(vector_fault::KernelScoreValue::I32(result));
    result
}

pub(crate) fn score_bit4_prepared_batch(
    q: &[i8],
    query_sum: i32,
    query_scale_half: f64,
    rows: &[u8],
    d: usize,
    factors: &[crate::quant::Bit4Factors],
    out: &mut [f32],
) {
    let table = dispatch::active_table();
    (table.score_bit4_prepared_batch)(q, query_sum, query_scale_half, rows, d, factors, out);
    #[cfg(any(test, feature = "test-seams"))]
    observe_selected_score(
        table,
        vector_fault_kernel_id_score_bit4_batch(),
        d.saturating_mul(out.len()),
    );
    #[cfg(any(test, feature = "test-seams"))]
    vector_fault::observe_selected_result(vector_fault::KernelScoreValue::F32s(
        out.iter().map(|value| value.to_bits()).collect(),
    ));
}

#[cfg(any(test, feature = "test-seams"))]
fn observe_selected_score(
    table: &KernelTable,
    kernel: vector_fault::KernelOperationId,
    work_items: usize,
) {
    vector_fault::observe_selected_score(table.backend, kernel, work_items);
}

macro_rules! vector_fault_kernel_id {
    ($name:ident, $variant:ident) => {
        #[cfg(any(test, feature = "test-seams"))]
        const fn $name() -> vector_fault::KernelOperationId {
            vector_fault::KernelOperationId::$variant
        }
    };
}

vector_fault_kernel_id!(vector_fault_kernel_id_dot_i8, DotI8);
vector_fault_kernel_id!(vector_fault_kernel_id_hamming, HammingU1);
vector_fault_kernel_id!(vector_fault_kernel_id_dot_f32, DotF32);
vector_fault_kernel_id!(vector_fault_kernel_id_dot_f16, DotF16);
vector_fault_kernel_id!(vector_fault_kernel_id_dot_i8_batch, DotI8Batch);
vector_fault_kernel_id!(vector_fault_kernel_id_hamming_batch, HammingU1Batch);
vector_fault_kernel_id!(vector_fault_kernel_id_dot_bit4, DotBit4);
vector_fault_kernel_id!(vector_fault_kernel_id_dot_bit4_prepared, DotBit4Prepared);
vector_fault_kernel_id!(vector_fault_kernel_id_dot_bit4_batch, DotBit4Batch);
vector_fault_kernel_id!(
    vector_fault_kernel_id_score_bit4_batch,
    ScoreBit4PreparedBatch
);
vector_fault_kernel_id!(vector_fault_kernel_id_score_bit4_ptrs, ScoreBit4Ptrs);

/// Scores four arbitrary validated packed four-bit rows through runtime dispatch.
///
/// # Safety
///
/// [`Bit4Rows4::from_rows`] proves that every row pointer is readable for one
/// common width and that each backing region outlives the call. This function
/// checks that the width equals `q.len().div_ceil(2)` and that the supported
/// dimension bound holds. No alignment is required.
pub fn score_bit4_ptrs(
    q: &[i8],
    query_sum: i32,
    query_scale_half: f64,
    rows: &Bit4Rows4<'_>,
    factors: &[crate::quant::Bit4Factors; 4],
    out: &mut [f32; 4],
) -> Result<(), GatherShapeError> {
    let table = dispatch::active_table();
    let result =
        score_bit4_ptrs_with_table(table, (q, query_sum, query_scale_half), rows, factors, out);
    #[cfg(any(test, feature = "test-seams"))]
    if result.is_ok() {
        #[cfg(any(test, feature = "test-seams"))]
        observe_selected_score(
            table,
            vector_fault_kernel_id_score_bit4_ptrs(),
            q.len().saturating_mul(out.len()),
        );
        #[cfg(any(test, feature = "test-seams"))]
        vector_fault::observe_selected_result(vector_fault::KernelScoreValue::F32s(
            out.iter().map(|value| value.to_bits()).collect(),
        ));
    }
    result
}

/// Scores one signed-byte query against contiguous packed four-bit rows.
///
/// Callers must pre-validate `q.len() == d` and `rows.len() ==
/// d.div_ceil(2) * out.len()` and `d <= MAX_DOT_I8_DIMENSION`. A zero dimension
/// fills `out` with zero. No scratch storage is allocated.
pub fn dot_bit4_batch(q: &[i8], rows: &[u8], d: usize, out: &mut [i32]) {
    let table = dispatch::active_table();
    (table.dot_bit4_batch)(q, rows, d, out);
    #[cfg(any(test, feature = "test-seams"))]
    observe_selected_score(
        table,
        vector_fault_kernel_id_dot_bit4_batch(),
        d.saturating_mul(out.len()),
    );
    #[cfg(any(test, feature = "test-seams"))]
    vector_fault::observe_selected_result(vector_fault::KernelScoreValue::I32s(out.to_vec()));
}