himada-dispatch 0.1.0

use himada_core::HardwareDNA;

// ── dot ──

#[cfg(all(target_arch = "aarch64", feature = "sve"))]
pub fn dot_f64_sve(a: &[f64], b: &[f64]) -> f64 {
    #[cfg(all(target_arch = "aarch64", feature = "sve"))]
    use std::arch::aarch64::sve::*;
    let n = a.len().min(b.len());
    if n == 0 {
        return 0.0;
    }
    // SAFETY: SVE intrinsics are available because this fn is gated on `feature = "sve"` and `target_arch = "aarch64"`.
    //          Pointer arithmetic is within bounds because `i + vlen <= n` / `i < n` is checked before each access.
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
    let vlen = unsafe { svcntd() as usize };
// SAFETY: SVE intrinsics require aarch64 + sve feature gate; slices valid.
    let pg = unsafe { svptrue_b64() };
    let mut i = 0;
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
    let mut vacc = unsafe { svdup_f64(0.0) };
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
    unsafe {
        while i + vlen <= n {
            let va = svld1_f64(pg, a.as_ptr().add(i));
            let vb = svld1_f64(pg, b.as_ptr().add(i));
            vacc = svmad_f64(va, vb, vacc);
            i += vlen;
        }
    }
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
    let mut total = unsafe { svaddv_f64(pg, vacc) };
    if i < n {
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
        let pg_tail = unsafe { svwhilelt_b64(i as i64, n as i64) };
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
        unsafe {
            let va = svld1_f64(pg_tail, a.as_ptr().add(i));
            let vb = svld1_f64(pg_tail, b.as_ptr().add(i));
            total += svaddv_f64(pg_tail, svmul_f64_x(pg_tail, va, vb));
        }
    }
    total
}

#[cfg(not(all(target_arch = "aarch64", feature = "sve")))]
pub fn dot_f64_sve(a: &[f64], b: &[f64]) -> f64 { crate::kernels::dot_scalar(a, b) }

#[cfg(all(target_arch = "aarch64", feature = "sve"))]
pub fn dot_f64_sve_supported(dna: &HardwareDNA) -> bool {
    dna.cpu.features.iter().any(|f| f == "SVE")
}

#[cfg(not(all(target_arch = "aarch64", feature = "sve")))]
pub fn dot_f64_sve_supported(_: &HardwareDNA) -> bool { false }

#[cfg(all(target_arch = "aarch64", feature = "sve"))]
pub fn dot_f32_sve(a: &[f32], b: &[f32]) -> f32 {
    #[cfg(all(target_arch = "aarch64", feature = "sve"))]
    use std::arch::aarch64::sve::*;
    let n = a.len().min(b.len());
    if n == 0 {
        return 0.0;
    }
    // SAFETY: SVE intrinsics available per cfg gate. Bounds checked by `i + vlen <= n` / `i < n`.
    let vlen = unsafe { svcntw() as usize };
// SAFETY: SVE intrinsics require aarch64 + sve feature gate; slices valid.
    let pg = unsafe { svptrue_b32() };
    let mut i = 0;
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
    let mut vacc = unsafe { svdup_f32(0.0) };
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
    unsafe {
        while i + vlen <= n {
            let va = svld1_f32(pg, a.as_ptr().add(i));
            let vb = svld1_f32(pg, b.as_ptr().add(i));
            vacc = svmad_f32(va, vb, vacc);
            i += vlen;
        }
    }
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
    let mut total = unsafe { svaddv_f32(pg, vacc) };
    if i < n {
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
        let pg_tail = unsafe { svwhilelt_b32(i as i64, n as i64) };
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
        unsafe {
            let va = svld1_f32(pg_tail, a.as_ptr().add(i));
            let vb = svld1_f32(pg_tail, b.as_ptr().add(i));
            total += svaddv_f32(pg_tail, svmul_f32_x(pg_tail, va, vb));
        }
    }
    total
}

#[cfg(not(all(target_arch = "aarch64", feature = "sve")))]
pub fn dot_f32_sve(a: &[f32], b: &[f32]) -> f32 { crate::kernels::dot_f32_scalar(a, b) }

#[cfg(all(target_arch = "aarch64", feature = "sve"))]
pub fn dot_f32_sve_supported(dna: &HardwareDNA) -> bool {
    dna.cpu.features.iter().any(|f| f == "SVE")
}

#[cfg(not(all(target_arch = "aarch64", feature = "sve")))]
pub fn dot_f32_sve_supported(_: &HardwareDNA) -> bool { false }

// ── reduce_sum ──

#[cfg(all(target_arch = "aarch64", feature = "sve"))]
pub fn reduce_sum_f64_sve(a: &[f64]) -> f64 {
    #[cfg(all(target_arch = "aarch64", feature = "sve"))]
    use std::arch::aarch64::sve::*;
    let n = a.len();
    if n == 0 {
        return 0.0;
    }
    // SAFETY: SVE intrinsics available per cfg gate. Bounds checked by `i + vlen <= n` / `i < n`.
    let vlen = unsafe { svcntd() as usize };
// SAFETY: SVE intrinsics require aarch64 + sve feature gate; slices valid.
    let pg = unsafe { svptrue_b64() };
    let mut i = 0;
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
    let mut vacc = unsafe { svdup_f64(0.0) };
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
    unsafe {
        while i + vlen <= n {
            let va = svld1_f64(pg, a.as_ptr().add(i));
            vacc = svadd_f64_x(pg, vacc, va);
            i += vlen;
        }
    }
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
    let mut total = unsafe { svaddv_f64(pg, vacc) };
    if i < n {
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
        let pg_tail = unsafe { svwhilelt_b64(i as i64, n as i64) };
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
        unsafe {
            let va = svld1_f64(pg_tail, a.as_ptr().add(i));
            total += svaddv_f64(pg_tail, va);
        }
    }
    total
}

#[cfg(not(all(target_arch = "aarch64", feature = "sve")))]
pub fn reduce_sum_f64_sve(a: &[f64]) -> f64 { crate::kernels::reduce_sum_scalar(a) }

#[cfg(all(target_arch = "aarch64", feature = "sve"))]
pub fn reduce_sum_f64_sve_supported(dna: &HardwareDNA) -> bool {
    dna.cpu.features.iter().any(|f| f == "SVE")
}

#[cfg(not(all(target_arch = "aarch64", feature = "sve")))]
pub fn reduce_sum_f64_sve_supported(_: &HardwareDNA) -> bool { false }

#[cfg(all(target_arch = "aarch64", feature = "sve"))]
pub fn reduce_sum_f32_sve(a: &[f32]) -> f32 {
    #[cfg(all(target_arch = "aarch64", feature = "sve"))]
    use std::arch::aarch64::sve::*;
    let n = a.len();
    if n == 0 {
        return 0.0;
    }
    // SAFETY: SVE intrinsics available per cfg gate. Bounds checked by loop condition.
    let vlen = unsafe { svcntw() as usize };
// SAFETY: SVE intrinsics require aarch64 + sve feature gate; slices valid.
    let pg = unsafe { svptrue_b32() };
    let mut i = 0;
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
    let mut vacc = unsafe { svdup_f32(0.0) };
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
    unsafe {
        while i + vlen <= n {
            let va = svld1_f32(pg, a.as_ptr().add(i));
            vacc = svadd_f32_x(pg, vacc, va);
            i += vlen;
        }
    }
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
    let mut total = unsafe { svaddv_f32(pg, vacc) };
    if i < n {
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
        let pg_tail = unsafe { svwhilelt_b32(i as i64, n as i64) };
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
        unsafe {
            let va = svld1_f32(pg_tail, a.as_ptr().add(i));
            total += svaddv_f32(pg_tail, va);
        }
    }
    total
}

#[cfg(not(all(target_arch = "aarch64", feature = "sve")))]
pub fn reduce_sum_f32_sve(a: &[f32]) -> f32 { crate::kernels::reduce_sum_f32_scalar(a) }

#[cfg(all(target_arch = "aarch64", feature = "sve"))]
pub fn reduce_sum_f32_sve_supported(dna: &HardwareDNA) -> bool {
    dna.cpu.features.iter().any(|f| f == "SVE")
}

#[cfg(not(all(target_arch = "aarch64", feature = "sve")))]
pub fn reduce_sum_f32_sve_supported(_: &HardwareDNA) -> bool { false }

// ── reduce_max ──

#[cfg(all(target_arch = "aarch64", feature = "sve"))]
pub fn reduce_max_f64_sve(a: &[f64]) -> f64 {
    #[cfg(all(target_arch = "aarch64", feature = "sve"))]
    use std::arch::aarch64::sve::*;
    let n = a.len();
    if n == 0 {
        return f64::NEG_INFINITY;
    }
    // SAFETY: SVE intrinsics available per cfg gate. Bounds checked by loop condition.
    let vlen = unsafe { svcntd() as usize };
// SAFETY: SVE intrinsics require aarch64 + sve feature gate; slices valid.
    let pg = unsafe { svptrue_b64() };
    let mut i = 0;
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
    let mut vmax = unsafe { svdup_f64(f64::NEG_INFINITY) };
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
    unsafe {
        while i + vlen <= n {
            let va = svld1_f64(pg, a.as_ptr().add(i));
            vmax = svmax_f64_x(pg, vmax, va);
            i += vlen;
        }
    }
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
    let mut max = unsafe { svmaxv_f64(pg, vmax) };
    if i < n {
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
        let pg_tail = unsafe { svwhilelt_b64(i as i64, n as i64) };
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
        unsafe {
            let va = svld1_f64(pg_tail, a.as_ptr().add(i));
            max = max.max(svmaxv_f64(pg_tail, va));
        }
    }
    max
}

#[cfg(not(all(target_arch = "aarch64", feature = "sve")))]
pub fn reduce_max_f64_sve(a: &[f64]) -> f64 { crate::kernels::reduce_max_scalar(a) }

#[cfg(all(target_arch = "aarch64", feature = "sve"))]
pub fn reduce_max_f64_sve_supported(dna: &HardwareDNA) -> bool {
    dna.cpu.features.iter().any(|f| f == "SVE")
}

#[cfg(not(all(target_arch = "aarch64", feature = "sve")))]
pub fn reduce_max_f64_sve_supported(_: &HardwareDNA) -> bool { false }

#[cfg(all(target_arch = "aarch64", feature = "sve"))]
pub fn reduce_max_f32_sve(a: &[f32]) -> f32 {
    #[cfg(all(target_arch = "aarch64", feature = "sve"))]
    use std::arch::aarch64::sve::*;
    let n = a.len();
    if n == 0 {
        return f32::NEG_INFINITY;
    }
    // SAFETY: SVE intrinsics available per cfg gate. Bounds checked by loop condition.
    let vlen = unsafe { svcntw() as usize };
// SAFETY: SVE intrinsics require aarch64 + sve feature gate; slices valid.
    let pg = unsafe { svptrue_b32() };
    let mut i = 0;
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
    let mut vmax = unsafe { svdup_f32(f32::NEG_INFINITY) };
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
    unsafe {
        while i + vlen <= n {
            let va = svld1_f32(pg, a.as_ptr().add(i));
            vmax = svmax_f32_x(pg, vmax, va);
            i += vlen;
        }
    }
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
    let mut max = unsafe { svmaxv_f32(pg, vmax) };
    if i < n {
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
        let pg_tail = unsafe { svwhilelt_b32(i as i64, n as i64) };
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
        unsafe {
            let va = svld1_f32(pg_tail, a.as_ptr().add(i));
            max = max.max(svmaxv_f32(pg_tail, va));
        }
    }
    max
}

#[cfg(not(all(target_arch = "aarch64", feature = "sve")))]
pub fn reduce_max_f32_sve(a: &[f32]) -> f32 { crate::kernels::reduce_max_f32_scalar(a) }

#[cfg(all(target_arch = "aarch64", feature = "sve"))]
pub fn reduce_max_f32_sve_supported(dna: &HardwareDNA) -> bool {
    dna.cpu.features.iter().any(|f| f == "SVE")
}

#[cfg(not(all(target_arch = "aarch64", feature = "sve")))]
pub fn reduce_max_f32_sve_supported(_: &HardwareDNA) -> bool { false }

// ── abs_max ──

#[cfg(all(target_arch = "aarch64", feature = "sve"))]
pub fn abs_max_f64_sve(a: &[f64]) -> f64 {
    #[cfg(all(target_arch = "aarch64", feature = "sve"))]
    use std::arch::aarch64::sve::*;
    let n = a.len();
    if n == 0 {
        return 0.0;
    }
    // SAFETY: SVE intrinsics available per cfg gate. Bounds checked by loop condition.
    let vlen = unsafe { svcntd() as usize };
// SAFETY: SVE intrinsics require aarch64 + sve feature gate; slices valid.
    let pg = unsafe { svptrue_b64() };
    let mut i = 0;
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
    let mut vmax = unsafe { svdup_f64(0.0) };
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
    unsafe {
        while i + vlen <= n {
            let va = svld1_f64(pg, a.as_ptr().add(i));
            vmax = svmax_f64_x(pg, vmax, svabs_f64_x(pg, va));
            i += vlen;
        }
    }
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
    let mut max = unsafe { svmaxv_f64(pg, vmax) };
    if i < n {
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
        let pg_tail = unsafe { svwhilelt_b64(i as i64, n as i64) };
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
        unsafe {
            let va = svld1_f64(pg_tail, a.as_ptr().add(i));
            max = max.max(svmaxv_f64(pg_tail, svabs_f64_x(pg_tail, va)));
        }
    }
    max
}

#[cfg(not(all(target_arch = "aarch64", feature = "sve")))]
pub fn abs_max_f64_sve(a: &[f64]) -> f64 { crate::kernels::abs_max_f64_scalar(a) }

#[cfg(all(target_arch = "aarch64", feature = "sve"))]
pub fn abs_max_f64_sve_supported(dna: &HardwareDNA) -> bool {
    dna.cpu.features.iter().any(|f| f == "SVE")
}

#[cfg(not(all(target_arch = "aarch64", feature = "sve")))]
pub fn abs_max_f64_sve_supported(_: &HardwareDNA) -> bool { false }

#[cfg(all(target_arch = "aarch64", feature = "sve"))]
pub fn abs_max_f32_sve(a: &[f32]) -> f32 {
    #[cfg(all(target_arch = "aarch64", feature = "sve"))]
    use std::arch::aarch64::sve::*;
    let n = a.len();
    if n == 0 {
        return 0.0;
    }
    // SAFETY: SVE intrinsics available per cfg gate. Bounds checked by loop condition.
    let vlen = unsafe { svcntw() as usize };
// SAFETY: SVE intrinsics require aarch64 + sve feature gate; slices valid.
    let pg = unsafe { svptrue_b32() };
    let mut i = 0;
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
    let mut vmax = unsafe { svdup_f32(0.0) };
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
    unsafe {
        while i + vlen <= n {
            let va = svld1_f32(pg, a.as_ptr().add(i));
            vmax = svmax_f32_x(pg, vmax, svabs_f32_x(pg, va));
            i += vlen;
        }
    }
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
    let mut max = unsafe { svmaxv_f32(pg, vmax) };
    if i < n {
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
        let pg_tail = unsafe { svwhilelt_b32(i as i64, n as i64) };
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
        unsafe {
            let va = svld1_f32(pg_tail, a.as_ptr().add(i));
            max = max.max(svmaxv_f32(pg_tail, svabs_f32_x(pg_tail, va)));
        }
    }
    max
}

#[cfg(not(all(target_arch = "aarch64", feature = "sve")))]
pub fn abs_max_f32_sve(a: &[f32]) -> f32 { crate::kernels::abs_max_f32_scalar(a) }

#[cfg(all(target_arch = "aarch64", feature = "sve"))]
pub fn abs_max_f32_sve_supported(dna: &HardwareDNA) -> bool {
    dna.cpu.features.iter().any(|f| f == "SVE")
}

#[cfg(not(all(target_arch = "aarch64", feature = "sve")))]
pub fn abs_max_f32_sve_supported(_: &HardwareDNA) -> bool { false }

// ── argmax ──

#[cfg(all(target_arch = "aarch64", feature = "sve"))]
pub fn argmax_f64_sve(a: &[f64]) -> usize {
    #[cfg(all(target_arch = "aarch64", feature = "sve"))]
    use std::arch::aarch64::sve::*;
    let n = a.len();
    if n == 0 {
        return 0;
    }
    // SAFETY: SVE intrinsics available per cfg gate. Bounds checked by loop condition.
    let vlen = unsafe { svcntd() as usize };
// SAFETY: SVE intrinsics require aarch64 + sve feature gate; slices valid.
    let pg = unsafe { svptrue_b64() };
    let mut i = 0;
    let mut best_val = f64::NEG_INFINITY;
    let mut best_idx = 0;
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
    unsafe {
        while i + vlen <= n {
            let va = svld1_f64(pg, a.as_ptr().add(i));
            let mut buf = [0.0f64; 64];
            svst1_f64(pg, buf.as_mut_ptr(), va);
            let ul = vlen.min(n - i);
            for k in 0..ul {
                if buf[k] > best_val {
                    best_val = buf[k];
                    best_idx = i + k;
                }
            }
            i += vlen;
        }
    }
    if i < n {
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
        let pg_tail = unsafe { svwhilelt_b64(i as i64, n as i64) };
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
        unsafe {
            let va = svld1_f64(pg_tail, a.as_ptr().add(i));
            let mut buf = [0.0f64; 64];
            svst1_f64(pg_tail, buf.as_mut_ptr(), va);
            for k in 0..(n - i) {
                if buf[k] > best_val {
                    best_val = buf[k];
                    best_idx = i + k;
                }
            }
        }
    }
    best_idx
}

#[cfg(not(all(target_arch = "aarch64", feature = "sve")))]
pub fn argmax_f64_sve(a: &[f64]) -> usize { crate::kernels::argmax_f64_scalar(a) }

#[cfg(all(target_arch = "aarch64", feature = "sve"))]
pub fn argmax_f64_sve_supported(dna: &HardwareDNA) -> bool {
    dna.cpu.features.iter().any(|f| f == "SVE")
}

#[cfg(not(all(target_arch = "aarch64", feature = "sve")))]
pub fn argmax_f64_sve_supported(_: &HardwareDNA) -> bool { false }

#[cfg(all(target_arch = "aarch64", feature = "sve"))]
pub fn argmax_f32_sve(a: &[f32]) -> usize {
    #[cfg(all(target_arch = "aarch64", feature = "sve"))]
    use std::arch::aarch64::sve::*;
    let n = a.len();
    if n == 0 {
        return 0;
    }
    // SAFETY: SVE intrinsics available per cfg gate. Bounds checked by loop condition.
    let vlen = unsafe { svcntw() as usize };
// SAFETY: SVE intrinsics require aarch64 + sve feature gate; slices valid.
    let pg = unsafe { svptrue_b32() };
    let mut i = 0;
    let mut best_val = f32::NEG_INFINITY;
    let mut best_idx = 0;
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
    unsafe {
        while i + vlen <= n {
            let va = svld1_f32(pg, a.as_ptr().add(i));
            let mut buf = [0.0f32; 64];
            svst1_f32(pg, buf.as_mut_ptr(), va);
            let ul = vlen.min(n - i);
            for k in 0..ul {
                if buf[k] > best_val {
                    best_val = buf[k];
                    best_idx = i + k;
                }
            }
            i += vlen;
        }
    }
    if i < n {
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
        let pg_tail = unsafe { svwhilelt_b32(i as i64, n as i64) };
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
        unsafe {
            let va = svld1_f32(pg_tail, a.as_ptr().add(i));
            let mut buf = [0.0f32; 64];
            svst1_f32(pg_tail, buf.as_mut_ptr(), va);
            for k in 0..(n - i) {
                if buf[k] > best_val {
                    best_val = buf[k];
                    best_idx = i + k;
                }
            }
        }
    }
    best_idx
}

#[cfg(not(all(target_arch = "aarch64", feature = "sve")))]
pub fn argmax_f32_sve(a: &[f32]) -> usize { crate::kernels::argmax_f32_scalar(a) }

#[cfg(all(target_arch = "aarch64", feature = "sve"))]
pub fn argmax_f32_sve_supported(dna: &HardwareDNA) -> bool {
    dna.cpu.features.iter().any(|f| f == "SVE")
}

#[cfg(not(all(target_arch = "aarch64", feature = "sve")))]
pub fn argmax_f32_sve_supported(_: &HardwareDNA) -> bool { false }

// ── hadamard_product (element-wise multiply) ──

#[cfg(all(target_arch = "aarch64", feature = "sve"))]
pub fn hadamard_product_f64_sve(a: &[f64], b: &[f64], c: &mut [f64]) {
    #[cfg(all(target_arch = "aarch64", feature = "sve"))]
    use std::arch::aarch64::sve::*;
    let n = a.len().min(b.len()).min(c.len());
    if n == 0 {
        return;
    }
    // SAFETY: SVE intrinsics available per cfg gate. Bounds checked by loop condition.
    let vlen = unsafe { svcntd() as usize };
// SAFETY: SVE intrinsics require aarch64 + sve feature gate; slices valid.
    let pg = unsafe { svptrue_b64() };
    let mut i = 0;
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
    unsafe {
        while i + vlen <= n {
            let va = svld1_f64(pg, a.as_ptr().add(i));
            let vb = svld1_f64(pg, b.as_ptr().add(i));
            let vc = svmul_f64_x(pg, va, vb);
            svst1_f64(pg, c.as_mut_ptr().add(i), vc);
            i += vlen;
        }
    }
    if i < n {
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
        let pg_tail = unsafe { svwhilelt_b64(i as i64, n as i64) };
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
        unsafe {
            let va = svld1_f64(pg_tail, a.as_ptr().add(i));
            let vb = svld1_f64(pg_tail, b.as_ptr().add(i));
            let vc = svmul_f64_x(pg_tail, va, vb);
            svst1_f64(pg_tail, c.as_mut_ptr().add(i), vc);
        }
    }
}

#[cfg(not(all(target_arch = "aarch64", feature = "sve")))]
pub fn hadamard_product_f64_sve(a: &[f64], b: &[f64], c: &mut [f64]) {
    crate::kernels::hadamard_product_f64_scalar(a, b, c)
}

#[cfg(all(target_arch = "aarch64", feature = "sve"))]
pub fn hadamard_product_f64_sve_supported(dna: &HardwareDNA) -> bool {
    dna.cpu.features.iter().any(|f| f == "SVE")
}

#[cfg(not(all(target_arch = "aarch64", feature = "sve")))]
pub fn hadamard_product_f64_sve_supported(_: &HardwareDNA) -> bool { false }

#[cfg(all(target_arch = "aarch64", feature = "sve"))]
pub fn hadamard_product_f32_sve(a: &[f32], b: &[f32], c: &mut [f32]) {
    #[cfg(all(target_arch = "aarch64", feature = "sve"))]
    use std::arch::aarch64::sve::*;
    let n = a.len().min(b.len()).min(c.len());
    if n == 0 {
        return;
    }
    // SAFETY: SVE intrinsics available per cfg gate. Bounds checked by loop condition.
    let vlen = unsafe { svcntw() as usize };
// SAFETY: SVE intrinsics require aarch64 + sve feature gate; slices valid.
    let pg = unsafe { svptrue_b32() };
    let mut i = 0;
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
    unsafe {
        while i + vlen <= n {
            let va = svld1_f32(pg, a.as_ptr().add(i));
            let vb = svld1_f32(pg, b.as_ptr().add(i));
            let vc = svmul_f32_x(pg, va, vb);
            svst1_f32(pg, c.as_mut_ptr().add(i), vc);
            i += vlen;
        }
    }
    if i < n {
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
        let pg_tail = unsafe { svwhilelt_b32(i as i64, n as i64) };
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
        unsafe {
            let va = svld1_f32(pg_tail, a.as_ptr().add(i));
            let vb = svld1_f32(pg_tail, b.as_ptr().add(i));
            let vc = svmul_f32_x(pg_tail, va, vb);
            svst1_f32(pg_tail, c.as_mut_ptr().add(i), vc);
        }
    }
}

#[cfg(not(all(target_arch = "aarch64", feature = "sve")))]
pub fn hadamard_product_f32_sve(a: &[f32], b: &[f32], c: &mut [f32]) {
    crate::kernels::hadamard_product_f32_scalar(a, b, c)
}

#[cfg(all(target_arch = "aarch64", feature = "sve"))]
pub fn hadamard_product_f32_sve_supported(dna: &HardwareDNA) -> bool {
    dna.cpu.features.iter().any(|f| f == "SVE")
}

#[cfg(not(all(target_arch = "aarch64", feature = "sve")))]
pub fn hadamard_product_f32_sve_supported(_: &HardwareDNA) -> bool { false }

// ── matmul ──

#[cfg(all(target_arch = "aarch64", feature = "sve"))]
pub fn matmul_f64_sve(a: &[f64], b: &[f64], c: &mut [f64], n: usize) {
    #[cfg(all(target_arch = "aarch64", feature = "sve"))]
    use std::arch::aarch64::sve::*;
    if n == 0 {
        return;
    }
    // SAFETY: SVE intrinsics available per cfg gate. All pointer arithmetic is within bounds because
    //          `i * n + j` indices are checked by loop bounds `0..n`.
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
    let vlen = unsafe { svcntd() as usize };
// SAFETY: SVE intrinsics require aarch64 + sve feature gate; slices valid.
    let pg = unsafe { svptrue_b64() };
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
    unsafe {
        for i in 0..n {
            for k in 0..n {
                let aik = a[i * n + k];
                let row_b = k * n;
                let row_c = i * n;
                let vaik = svdup_f64(aik);
                let mut j = 0;
                while j + vlen <= n {
                    let vb = svld1_f64(pg, b.as_ptr().add(row_b + j));
                    let vc = svld1_f64(pg, c.as_ptr().add(row_c + j));
                    let vm = svmul_f64_x(pg, vaik, vb);
                    svst1_f64(pg, c.as_mut_ptr().add(row_c + j), svadd_f64_x(pg, vc, vm));
                    j += vlen;
                }
                if j < n {
                    let pg_tail = svwhilelt_b64(j as i64, n as i64);
                    let vb = svld1_f64(pg_tail, b.as_ptr().add(row_b + j));
                    let vc = svld1_f64(pg_tail, c.as_ptr().add(row_c + j));
                    let vm = svmul_f64_x(pg_tail, vaik, vb);
                    svst1_f64(pg_tail, c.as_mut_ptr().add(row_c + j), svadd_f64_x(pg_tail, vc, vm));
                }
            }
        }
    }
}

#[cfg(not(all(target_arch = "aarch64", feature = "sve")))]
pub fn matmul_f64_sve(a: &[f64], b: &[f64], c: &mut [f64], n: usize) {
    crate::kernels::matmul_scalar(a, b, c, n)
}

#[cfg(all(target_arch = "aarch64", feature = "sve"))]
pub fn matmul_f64_sve_supported(dna: &HardwareDNA) -> bool {
    dna.cpu.features.iter().any(|f| f == "SVE")
}

#[cfg(not(all(target_arch = "aarch64", feature = "sve")))]
pub fn matmul_f64_sve_supported(_: &HardwareDNA) -> bool { false }

#[cfg(all(target_arch = "aarch64", feature = "sve"))]
pub fn matmul_f32_sve(a: &[f32], b: &[f32], c: &mut [f32], n: usize) {
    #[cfg(all(target_arch = "aarch64", feature = "sve"))]
    use std::arch::aarch64::sve::*;
    if n == 0 {
        return;
    }
    // SAFETY: SVE intrinsics available per cfg gate. Loop bounds ensure valid pointer arithmetic.
    let vlen = unsafe { svcntw() as usize };
// SAFETY: SVE intrinsics require aarch64 + sve feature gate; slices valid.
    let pg = unsafe { svptrue_b32() };
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
    unsafe {
        for i in 0..n {
            for k in 0..n {
                let aik = a[i * n + k];
                let row_b = k * n;
                let row_c = i * n;
                let vaik = svdup_f32(aik);
                let mut j = 0;
                while j + vlen <= n {
                    let vb = svld1_f32(pg, b.as_ptr().add(row_b + j));
                    let vc = svld1_f32(pg, c.as_ptr().add(row_c + j));
                    let vm = svmul_f32_x(pg, vaik, vb);
                    svst1_f32(pg, c.as_mut_ptr().add(row_c + j), svadd_f32_x(pg, vc, vm));
                    j += vlen;
                }
                if j < n {
                    let pg_tail = svwhilelt_b32(j as i64, n as i64);
                    let vb = svld1_f32(pg_tail, b.as_ptr().add(row_b + j));
                    let vc = svld1_f32(pg_tail, c.as_ptr().add(row_c + j));
                    let vm = svmul_f32_x(pg_tail, vaik, vb);
                    svst1_f32(pg_tail, c.as_mut_ptr().add(row_c + j), svadd_f32_x(pg_tail, vc, vm));
                }
            }
        }
    }
}

#[cfg(not(all(target_arch = "aarch64", feature = "sve")))]
pub fn matmul_f32_sve(a: &[f32], b: &[f32], c: &mut [f32], n: usize) {
    crate::kernels::matmul_f32_scalar(a, b, c, n)
}

#[cfg(all(target_arch = "aarch64", feature = "sve"))]
pub fn matmul_f32_sve_supported(dna: &HardwareDNA) -> bool {
    dna.cpu.features.iter().any(|f| f == "SVE")
}

#[cfg(not(all(target_arch = "aarch64", feature = "sve")))]
pub fn matmul_f32_sve_supported(_: &HardwareDNA) -> bool { false }

// ── softmax ──

#[cfg(all(target_arch = "aarch64", feature = "sve"))]
pub fn softmax_f64_sve(input: &[f64], output: &mut [f64]) {
    #[cfg(all(target_arch = "aarch64", feature = "sve"))]
    use std::arch::aarch64::sve::*;
    let n = input.len().min(output.len());
    if n == 0 {
        return;
    }
    // SAFETY: SVE intrinsics available per cfg gate. Bounds checked by loop condition.
    let vlen = unsafe { svcntd() as usize };
// SAFETY: SVE intrinsics require aarch64 + sve feature gate; slices valid.
    let pg = unsafe { svptrue_b64() };

    let mut i = 0;
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
    let mut vmax = unsafe { svdup_f64(f64::NEG_INFINITY) };
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
    unsafe {
        while i + vlen <= n {
            let v = svld1_f64(pg, input.as_ptr().add(i));
            vmax = svmax_f64_x(pg, vmax, v);
            i += vlen;
        }
    }
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
    let mut max_val = unsafe { svmaxv_f64(pg, vmax) };
    if i < n {
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
        let pg_tail = unsafe { svwhilelt_b64(i as i64, n as i64) };
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
        unsafe {
            let v = svld1_f64(pg_tail, input.as_ptr().add(i));
            max_val = max_val.max(svmaxv_f64(pg_tail, v));
        }
    }

    let mut i = 0;
    let mut sum = 0.0;
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
    unsafe {
        while i + vlen <= n {
            let v = svld1_f64(pg, input.as_ptr().add(i));
            let mut buf = [0.0f64; 64];
            svst1_f64(pg, buf.as_mut_ptr(), v);
            let mut exp_buf = [0.0f64; 64];
            for k in 0..vlen {
                exp_buf[k] = (buf[k] - max_val).exp();
                sum += exp_buf[k];
            }
            let ve = svld1_f64(pg, exp_buf.as_ptr());
            svst1_f64(pg, output.as_mut_ptr().add(i), ve);
            i += vlen;
        }
    }
    if i < n {
        for j in i..n {
            let e = (input[j] - max_val).exp();
            output[j] = e;
            sum += e;
        }
    }

    let inv = 1.0 / sum;
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
    let vinv = unsafe { svdup_f64(inv) };
    let mut i = 0;
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
    unsafe {
        while i + vlen <= n {
            let v = svld1_f64(pg, output.as_ptr().add(i));
            svst1_f64(pg, output.as_mut_ptr().add(i), svmul_f64_x(pg, v, vinv));
            i += vlen;
        }
    }
    if i < n {
        for v in output[i..n].iter_mut() {
            *v *= inv;
        }
    }
}

#[cfg(not(all(target_arch = "aarch64", feature = "sve")))]
pub fn softmax_f64_sve(input: &[f64], output: &mut [f64]) {
    crate::kernels::softmax_scalar(input, output)
}

#[cfg(all(target_arch = "aarch64", feature = "sve"))]
pub fn softmax_f64_sve_supported(dna: &HardwareDNA) -> bool {
    dna.cpu.features.iter().any(|f| f == "SVE")
}

#[cfg(not(all(target_arch = "aarch64", feature = "sve")))]
pub fn softmax_f64_sve_supported(_: &HardwareDNA) -> bool { false }

#[cfg(all(target_arch = "aarch64", feature = "sve"))]
pub fn softmax_f32_sve(input: &[f32], output: &mut [f32]) {
    #[cfg(all(target_arch = "aarch64", feature = "sve"))]
    use std::arch::aarch64::sve::*;
    let n = input.len().min(output.len());
    if n == 0 {
        return;
    }
    // SAFETY: SVE intrinsics available per cfg gate. Bounds checked by loop condition.
    let vlen = unsafe { svcntw() as usize };
// SAFETY: SVE intrinsics require aarch64 + sve feature gate; slices valid.
    let pg = unsafe { svptrue_b32() };

    let mut i = 0;
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
    let mut vmax = unsafe { svdup_f32(f32::NEG_INFINITY) };
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
    unsafe {
        while i + vlen <= n {
            let v = svld1_f32(pg, input.as_ptr().add(i));
            vmax = svmax_f32_x(pg, vmax, v);
            i += vlen;
        }
    }
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
    let mut max_val = unsafe { svmaxv_f32(pg, vmax) };
    if i < n {
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
        let pg_tail = unsafe { svwhilelt_b32(i as i64, n as i64) };
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
        unsafe {
            let v = svld1_f32(pg_tail, input.as_ptr().add(i));
            max_val = max_val.max(svmaxv_f32(pg_tail, v));
        }
    }

    let mut i = 0;
    let mut sum = 0.0;
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
    unsafe {
        while i + vlen <= n {
            let v = svld1_f32(pg, input.as_ptr().add(i));
            let mut buf = [0.0f32; 64];
            svst1_f32(pg, buf.as_mut_ptr(), v);
            let mut exp_buf = [0.0f32; 64];
            for k in 0..vlen {
                exp_buf[k] = (buf[k] - max_val).exp();
                sum += exp_buf[k];
            }
            let ve = svld1_f32(pg, exp_buf.as_ptr());
            svst1_f32(pg, output.as_mut_ptr().add(i), ve);
            i += vlen;
        }
    }
    if i < n {
        for j in i..n {
            let e = (input[j] - max_val).exp();
            output[j] = e;
            sum += e;
        }
    }

    let inv = 1.0 / sum;
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
    let vinv = unsafe { svdup_f32(inv) };
    let mut i = 0;
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
    unsafe {
        while i + vlen <= n {
            let v = svld1_f32(pg, output.as_ptr().add(i));
            svst1_f32(pg, output.as_mut_ptr().add(i), svmul_f32_x(pg, v, vinv));
            i += vlen;
        }
    }
    if i < n {
        for v in output[i..n].iter_mut() {
            *v *= inv;
        }
    }
}

#[cfg(not(all(target_arch = "aarch64", feature = "sve")))]
pub fn softmax_f32_sve(input: &[f32], output: &mut [f32]) {
    crate::kernels::softmax_f32_scalar(input, output)
}

#[cfg(all(target_arch = "aarch64", feature = "sve"))]
pub fn softmax_f32_sve_supported(dna: &HardwareDNA) -> bool {
    dna.cpu.features.iter().any(|f| f == "SVE")
}

#[cfg(not(all(target_arch = "aarch64", feature = "sve")))]
pub fn softmax_f32_sve_supported(_: &HardwareDNA) -> bool { false }

// ── clamp ──

#[cfg(all(target_arch = "aarch64", feature = "sve"))]
pub fn clamp_f64_sve(a: &[f64], lo: f64, hi: f64, c: &mut [f64]) {
    #[cfg(all(target_arch = "aarch64", feature = "sve"))]
    use std::arch::aarch64::sve::*;
    let n = a.len().min(c.len());
    if n == 0 {
        return;
    }
    // SAFETY: SVE intrinsics available per cfg gate. Bounds checked by loop condition.
    let vlen = unsafe { svcntd() as usize };
// SAFETY: SVE intrinsics require aarch64 + sve feature gate; slices valid.
    let pg = unsafe { svptrue_b64() };
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
    let vlo = unsafe { svdup_f64(lo) };
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
    let vhi = unsafe { svdup_f64(hi) };
    let mut i = 0;
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
    unsafe {
        while i + vlen <= n {
            let va = svld1_f64(pg, a.as_ptr().add(i));
            let clamped = svmin_f64_x(pg, svmax_f64_x(pg, va, vlo), vhi);
            svst1_f64(pg, c.as_mut_ptr().add(i), clamped);
            i += vlen;
        }
    }
    if i < n {
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
        let pg_tail = unsafe { svwhilelt_b64(i as i64, n as i64) };
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
        unsafe {
            let va = svld1_f64(pg_tail, a.as_ptr().add(i));
            let clamped = svmin_f64_x(pg_tail, svmax_f64_x(pg_tail, va, vlo), vhi);
            svst1_f64(pg_tail, c.as_mut_ptr().add(i), clamped);
        }
    }
}

#[cfg(not(all(target_arch = "aarch64", feature = "sve")))]
pub fn clamp_f64_sve(a: &[f64], lo: f64, hi: f64, c: &mut [f64]) {
    crate::kernels::clamp_f64_scalar(a, lo, hi, c)
}

#[cfg(all(target_arch = "aarch64", feature = "sve"))]
pub fn clamp_f64_sve_supported(dna: &HardwareDNA) -> bool {
    dna.cpu.features.iter().any(|f| f == "SVE")
}

#[cfg(not(all(target_arch = "aarch64", feature = "sve")))]
pub fn clamp_f64_sve_supported(_: &HardwareDNA) -> bool { false }

#[cfg(all(target_arch = "aarch64", feature = "sve"))]
pub fn clamp_f32_sve(a: &[f32], lo: f32, hi: f32, c: &mut [f32]) {
    #[cfg(all(target_arch = "aarch64", feature = "sve"))]
    use std::arch::aarch64::sve::*;
    let n = a.len().min(c.len());
    if n == 0 {
        return;
    }
    // SAFETY: SVE intrinsics available per cfg gate. Bounds checked by loop condition.
    let vlen = unsafe { svcntw() as usize };
// SAFETY: SVE intrinsics require aarch64 + sve feature gate; slices valid.
    let pg = unsafe { svptrue_b32() };
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
    let vlo = unsafe { svdup_f32(lo) };
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
    let vhi = unsafe { svdup_f32(hi) };
    let mut i = 0;
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
    unsafe {
        while i + vlen <= n {
            let va = svld1_f32(pg, a.as_ptr().add(i));
            let clamped = svmin_f32_x(pg, svmax_f32_x(pg, va, vlo), vhi);
            svst1_f32(pg, c.as_mut_ptr().add(i), clamped);
            i += vlen;
        }
    }
    if i < n {
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
        let pg_tail = unsafe { svwhilelt_b32(i as i64, n as i64) };
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
        unsafe {
            let va = svld1_f32(pg_tail, a.as_ptr().add(i));
            let clamped = svmin_f32_x(pg_tail, svmax_f32_x(pg_tail, va, vlo), vhi);
            svst1_f32(pg_tail, c.as_mut_ptr().add(i), clamped);
        }
    }
}

#[cfg(not(all(target_arch = "aarch64", feature = "sve")))]
pub fn clamp_f32_sve(a: &[f32], lo: f32, hi: f32, c: &mut [f32]) {
    crate::kernels::clamp_f32_scalar(a, lo, hi, c)
}

#[cfg(all(target_arch = "aarch64", feature = "sve"))]
pub fn clamp_f32_sve_supported(dna: &HardwareDNA) -> bool {
    dna.cpu.features.iter().any(|f| f == "SVE")
}

#[cfg(not(all(target_arch = "aarch64", feature = "sve")))]
pub fn clamp_f32_sve_supported(_: &HardwareDNA) -> bool { false }

// ── memchr ──

#[cfg(all(target_arch = "aarch64", feature = "sve"))]
pub fn memchr_sve(byte: u8, data: &[u8]) -> Option<usize> {
    #[cfg(all(target_arch = "aarch64", feature = "sve"))]
    use std::arch::aarch64::sve::*;
    let n = data.len();
    if n == 0 {
        return None;
    }
    // SAFETY: SVE intrinsics available per cfg gate. Bounds checked by loop condition.
    let vlen = unsafe { svcntb() as usize };
// SAFETY: SVE intrinsics require aarch64 + sve feature gate; slices valid.
    let pg = unsafe { svptrue_b8() };
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
    let vb = unsafe { svdup_u8(byte) };
    let mut i = 0;
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
    unsafe {
        while i + vlen <= n {
            let vd = svld1_u8(pg, data.as_ptr().add(i));
            let cmp = svcmpeq_u8(pg, vd, vb);
            if svptest_any(pg, cmp) {
                let mut buf = [0u8; 256];
                svst1_u8(pg, buf.as_mut_ptr(), vd);
                for k in 0..vlen {
                    if buf[k] == byte {
                        return Some(i + k);
                    }
                }
            }
            i += vlen;
        }
    }
    if i < n {
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
        let pg_tail = unsafe { svwhilelt_b8(i as i64, n as i64) };
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
        unsafe {
            let vd = svld1_u8(pg_tail, data.as_ptr().add(i));
            let mut buf = [0u8; 256];
            svst1_u8(pg_tail, buf.as_mut_ptr(), vd);
            for k in 0..(n - i) {
                if buf[k] == byte {
                    return Some(i + k);
                }
            }
        }
    }
    None
}

#[cfg(not(all(target_arch = "aarch64", feature = "sve")))]
pub fn memchr_sve(byte: u8, data: &[u8]) -> Option<usize> { crate::kernels::memchr_scalar(byte, data) }

#[cfg(all(target_arch = "aarch64", feature = "sve"))]
pub fn memchr_sve_supported(dna: &HardwareDNA) -> bool {
    dna.cpu.features.iter().any(|f| f == "SVE")
}

#[cfg(not(all(target_arch = "aarch64", feature = "sve")))]
pub fn memchr_sve_supported(_: &HardwareDNA) -> bool { false }

// ── cosine_similarity ──

#[cfg(all(target_arch = "aarch64", feature = "sve"))]
pub fn cosine_similarity_f64_sve(a: &[f64], b: &[f64]) -> f64 {
    #[cfg(all(target_arch = "aarch64", feature = "sve"))]
    use std::arch::aarch64::sve::*;
    let n = a.len().min(b.len());
    if n == 0 {
        return 0.0;
    }
    // SAFETY: SVE intrinsics available per cfg gate. Bounds checked by loop condition.
    let vlen = unsafe { svcntd() as usize };
// SAFETY: SVE intrinsics require aarch64 + sve feature gate; slices valid.
    let pg = unsafe { svptrue_b64() };
    let mut i = 0;
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
    let mut vdot = unsafe { svdup_f64(0.0) };
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
    let mut vna = unsafe { svdup_f64(0.0) };
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
    let mut vnb = unsafe { svdup_f64(0.0) };
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
    unsafe {
        while i + vlen <= n {
            let va = svld1_f64(pg, a.as_ptr().add(i));
            let vb = svld1_f64(pg, b.as_ptr().add(i));
            vdot = svmad_f64(va, vb, vdot);
            vna = svmad_f64(va, va, vna);
            vnb = svmad_f64(vb, vb, vnb);
            i += vlen;
        }
    }
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
    let mut sdot = unsafe { svaddv_f64(pg, vdot) };
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
    let mut sna = unsafe { svaddv_f64(pg, vna) };
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
    let mut snb = unsafe { svaddv_f64(pg, vnb) };
    if i < n {
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
        let pg_tail = unsafe { svwhilelt_b64(i as i64, n as i64) };
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
        unsafe {
            let va = svld1_f64(pg_tail, a.as_ptr().add(i));
            let vb = svld1_f64(pg_tail, b.as_ptr().add(i));
            sdot += svaddv_f64(pg_tail, svmul_f64_x(pg_tail, va, vb));
            sna += svaddv_f64(pg_tail, svmul_f64_x(pg_tail, va, va));
            snb += svaddv_f64(pg_tail, svmul_f64_x(pg_tail, vb, vb));
        }
    }
    sdot / (sna.sqrt() * snb.sqrt())
}

#[cfg(not(all(target_arch = "aarch64", feature = "sve")))]
pub fn cosine_similarity_f64_sve(a: &[f64], b: &[f64]) -> f64 {
    crate::kernels::cosine_similarity_f64_scalar(a, b)
}

#[cfg(all(target_arch = "aarch64", feature = "sve"))]
pub fn cosine_similarity_f64_sve_supported(dna: &HardwareDNA) -> bool {
    dna.cpu.features.iter().any(|f| f == "SVE")
}

#[cfg(not(all(target_arch = "aarch64", feature = "sve")))]
pub fn cosine_similarity_f64_sve_supported(_: &HardwareDNA) -> bool { false }

#[cfg(all(target_arch = "aarch64", feature = "sve"))]
pub fn cosine_similarity_f32_sve(a: &[f32], b: &[f32]) -> f32 {
    #[cfg(all(target_arch = "aarch64", feature = "sve"))]
    use std::arch::aarch64::sve::*;
    let n = a.len().min(b.len());
    if n == 0 {
        return 0.0;
    }
    // SAFETY: SVE intrinsics available per cfg gate. Bounds checked by loop condition.
    let vlen = unsafe { svcntw() as usize };
// SAFETY: SVE intrinsics require aarch64 + sve feature gate; slices valid.
    let pg = unsafe { svptrue_b32() };
    let mut i = 0;
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
    let mut vdot = unsafe { svdup_f32(0.0) };
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
    let mut vna = unsafe { svdup_f32(0.0) };
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
    let mut vnb = unsafe { svdup_f32(0.0) };
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
    unsafe {
        while i + vlen <= n {
            let va = svld1_f32(pg, a.as_ptr().add(i));
            let vb = svld1_f32(pg, b.as_ptr().add(i));
            vdot = svmad_f32(va, vb, vdot);
            vna = svmad_f32(va, va, vna);
            vnb = svmad_f32(vb, vb, vnb);
            i += vlen;
        }
    }
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
    let mut sdot = unsafe { svaddv_f32(pg, vdot) };
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
    let mut sna = unsafe { svaddv_f32(pg, vna) };
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
    let mut snb = unsafe { svaddv_f32(pg, vnb) };
    if i < n {
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
        let pg_tail = unsafe { svwhilelt_b32(i as i64, n as i64) };
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
        unsafe {
            let va = svld1_f32(pg_tail, a.as_ptr().add(i));
            let vb = svld1_f32(pg_tail, b.as_ptr().add(i));
            sdot += svaddv_f32(pg_tail, svmul_f32_x(pg_tail, va, vb));
            sna += svaddv_f32(pg_tail, svmul_f32_x(pg_tail, va, va));
            snb += svaddv_f32(pg_tail, svmul_f32_x(pg_tail, vb, vb));
        }
    }
    sdot / (sna.sqrt() * snb.sqrt())
}

#[cfg(not(all(target_arch = "aarch64", feature = "sve")))]
pub fn cosine_similarity_f32_sve(a: &[f32], b: &[f32]) -> f32 {
    crate::kernels::cosine_similarity_f32_scalar(a, b)
}

#[cfg(all(target_arch = "aarch64", feature = "sve"))]
pub fn cosine_similarity_f32_sve_supported(dna: &HardwareDNA) -> bool {
    dna.cpu.features.iter().any(|f| f == "SVE")
}

#[cfg(not(all(target_arch = "aarch64", feature = "sve")))]
pub fn cosine_similarity_f32_sve_supported(_: &HardwareDNA) -> bool { false }

// ── euclidean_distance ──

#[cfg(all(target_arch = "aarch64", feature = "sve"))]
pub fn euclidean_distance_f64_sve(a: &[f64], b: &[f64]) -> f64 {
    #[cfg(all(target_arch = "aarch64", feature = "sve"))]
    use std::arch::aarch64::sve::*;
    let n = a.len().min(b.len());
    if n == 0 {
        return 0.0;
    }
    // SAFETY: SVE intrinsics available per cfg gate. Bounds checked by loop condition.
    let vlen = unsafe { svcntd() as usize };
// SAFETY: SVE intrinsics require aarch64 + sve feature gate; slices valid.
    let pg = unsafe { svptrue_b64() };
    let mut i = 0;
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
    let mut vsum = unsafe { svdup_f64(0.0) };
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
    unsafe {
        while i + vlen <= n {
            let va = svld1_f64(pg, a.as_ptr().add(i));
            let vb = svld1_f64(pg, b.as_ptr().add(i));
            let d = svsub_f64_x(pg, va, vb);
            vsum = svmad_f64(d, d, vsum);
            i += vlen;
        }
    }
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
    let mut total = unsafe { svaddv_f64(pg, vsum) };
    if i < n {
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
        let pg_tail = unsafe { svwhilelt_b64(i as i64, n as i64) };
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
        unsafe {
            let va = svld1_f64(pg_tail, a.as_ptr().add(i));
            let vb = svld1_f64(pg_tail, b.as_ptr().add(i));
            let d = svsub_f64_x(pg_tail, va, vb);
            total += svaddv_f64(pg_tail, svmul_f64_x(pg_tail, d, d));
        }
    }
    total.sqrt()
}

#[cfg(not(all(target_arch = "aarch64", feature = "sve")))]
pub fn euclidean_distance_f64_sve(a: &[f64], b: &[f64]) -> f64 {
    crate::kernels::euclidean_distance_f64_scalar(a, b)
}

#[cfg(all(target_arch = "aarch64", feature = "sve"))]
pub fn euclidean_distance_f64_sve_supported(dna: &HardwareDNA) -> bool {
    dna.cpu.features.iter().any(|f| f == "SVE")
}

#[cfg(not(all(target_arch = "aarch64", feature = "sve")))]
pub fn euclidean_distance_f64_sve_supported(_: &HardwareDNA) -> bool { false }

#[cfg(all(target_arch = "aarch64", feature = "sve"))]
pub fn euclidean_distance_f32_sve(a: &[f32], b: &[f32]) -> f32 {
    #[cfg(all(target_arch = "aarch64", feature = "sve"))]
    use std::arch::aarch64::sve::*;
    let n = a.len().min(b.len());
    if n == 0 {
        return 0.0;
    }
    // SAFETY: SVE intrinsics available per cfg gate. Bounds checked by loop condition.
    let vlen = unsafe { svcntw() as usize };
// SAFETY: SVE intrinsics require aarch64 + sve feature gate; slices valid.
    let pg = unsafe { svptrue_b32() };
    let mut i = 0;
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
    let mut vsum = unsafe { svdup_f32(0.0) };
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
    unsafe {
        while i + vlen <= n {
            let va = svld1_f32(pg, a.as_ptr().add(i));
            let vb = svld1_f32(pg, b.as_ptr().add(i));
            let d = svsub_f32_x(pg, va, vb);
            vsum = svmad_f32(d, d, vsum);
            i += vlen;
        }
    }
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
    let mut total = unsafe { svaddv_f32(pg, vsum) };
    if i < n {
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
        let pg_tail = unsafe { svwhilelt_b32(i as i64, n as i64) };
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
        unsafe {
            let va = svld1_f32(pg_tail, a.as_ptr().add(i));
            let vb = svld1_f32(pg_tail, b.as_ptr().add(i));
            let d = svsub_f32_x(pg_tail, va, vb);
            total += svaddv_f32(pg_tail, svmul_f32_x(pg_tail, d, d));
        }
    }
    total.sqrt()
}

#[cfg(not(all(target_arch = "aarch64", feature = "sve")))]
pub fn euclidean_distance_f32_sve(a: &[f32], b: &[f32]) -> f32 {
    crate::kernels::euclidean_distance_f32_scalar(a, b)
}

#[cfg(all(target_arch = "aarch64", feature = "sve"))]
pub fn euclidean_distance_f32_sve_supported(dna: &HardwareDNA) -> bool {
    dna.cpu.features.iter().any(|f| f == "SVE")
}

#[cfg(not(all(target_arch = "aarch64", feature = "sve")))]
pub fn euclidean_distance_f32_sve_supported(_: &HardwareDNA) -> bool { false }

// ── negate ──

#[cfg(all(target_arch = "aarch64", feature = "sve"))]
pub fn negate_f64_sve(a: &[f64], c: &mut [f64]) {
    #[cfg(all(target_arch = "aarch64", feature = "sve"))]
    use std::arch::aarch64::sve::*;
    let n = a.len().min(c.len());
    if n == 0 {
        return;
    }
    // SAFETY: SVE intrinsics available per cfg gate. Bounds checked by loop condition.
    let vlen = unsafe { svcntd() as usize };
// SAFETY: SVE intrinsics require aarch64 + sve feature gate; slices valid.
    let pg = unsafe { svptrue_b64() };
    let mut i = 0;
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
    unsafe {
        while i + vlen <= n {
            let va = svld1_f64(pg, a.as_ptr().add(i));
            svst1_f64(pg, c.as_mut_ptr().add(i), svneg_f64_x(pg, va));
            i += vlen;
        }
    }
    if i < n {
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
        let pg_tail = unsafe { svwhilelt_b64(i as i64, n as i64) };
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
        unsafe {
            let va = svld1_f64(pg_tail, a.as_ptr().add(i));
            svst1_f64(pg_tail, c.as_mut_ptr().add(i), svneg_f64_x(pg_tail, va));
        }
    }
}

#[cfg(not(all(target_arch = "aarch64", feature = "sve")))]
pub fn negate_f64_sve(a: &[f64], c: &mut [f64]) { crate::kernels::negate_f64_scalar(a, c) }

#[cfg(all(target_arch = "aarch64", feature = "sve"))]
pub fn negate_f64_sve_supported(dna: &HardwareDNA) -> bool {
    dna.cpu.features.iter().any(|f| f == "SVE")
}

#[cfg(not(all(target_arch = "aarch64", feature = "sve")))]
pub fn negate_f64_sve_supported(_: &HardwareDNA) -> bool { false }

#[cfg(all(target_arch = "aarch64", feature = "sve"))]
pub fn negate_f32_sve(a: &[f32], c: &mut [f32]) {
    #[cfg(all(target_arch = "aarch64", feature = "sve"))]
    use std::arch::aarch64::sve::*;
    let n = a.len().min(c.len());
    if n == 0 {
        return;
    }
    // SAFETY: SVE intrinsics available per cfg gate. Bounds checked by loop condition.
    let vlen = unsafe { svcntw() as usize };
// SAFETY: SVE intrinsics require aarch64 + sve feature gate; slices valid.
    let pg = unsafe { svptrue_b32() };
    let mut i = 0;
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
    unsafe {
        while i + vlen <= n {
            let va = svld1_f32(pg, a.as_ptr().add(i));
            svst1_f32(pg, c.as_mut_ptr().add(i), svneg_f32_x(pg, va));
            i += vlen;
        }
    }
    if i < n {
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
        let pg_tail = unsafe { svwhilelt_b32(i as i64, n as i64) };
// SAFETY: SIMD intrinsics; safe under cfg gate with bounds-checked slices.
        unsafe {
            let va = svld1_f32(pg_tail, a.as_ptr().add(i));
            svst1_f32(pg_tail, c.as_mut_ptr().add(i), svneg_f32_x(pg_tail, va));
        }
    }
}

#[cfg(not(all(target_arch = "aarch64", feature = "sve")))]
pub fn negate_f32_sve(a: &[f32], c: &mut [f32]) { crate::kernels::negate_f32_scalar(a, c) }

#[cfg(all(target_arch = "aarch64", feature = "sve"))]
pub fn negate_f32_sve_supported(dna: &HardwareDNA) -> bool {
    dna.cpu.features.iter().any(|f| f == "SVE")
}

#[cfg(not(all(target_arch = "aarch64", feature = "sve")))]
pub fn negate_f32_sve_supported(_: &HardwareDNA) -> bool { false }