hydroplane 0.1.0

//! Hand-written AArch64 NEON backend (4-wide f32, 2-wide f64).
//! NEON is baseline on AArch64, so the [`Neon`] token is always constructible there.
#![allow(unsafe_op_in_unsafe_fn)]
#![cfg(target_arch = "aarch64")]

use core::arch::aarch64::*;

use crate::backend::Backend;

/// AArch64 NEON execution token (NEON is always available on AArch64).
#[derive(Clone, Copy, Debug, Default)]
pub struct Neon;

impl Neon {
    #[inline]
    pub const fn new() -> Self {
        Self
    }
}

impl Backend<f32> for Neon {
    type Vector = float32x4_t;
    type Mask = uint32x4_t;

    #[inline(always)]
    fn lanes(self) -> usize {
        4
    }
    #[inline(always)]
    fn splat(self, v: f32) -> float32x4_t {
        unsafe { vdupq_n_f32(v) }
    }
    #[inline(always)]
    fn load(self, s: &[f32]) -> float32x4_t {
        debug_assert_eq!(s.len(), 4);
        unsafe { vld1q_f32(s.as_ptr()) }
    }
    #[inline(always)]
    fn store(self, v: float32x4_t, s: &mut [f32]) {
        debug_assert_eq!(s.len(), 4);
        unsafe { vst1q_f32(s.as_mut_ptr(), v) }
    }
    #[inline(always)]
    fn add(self, a: float32x4_t, b: float32x4_t) -> float32x4_t {
        unsafe { vaddq_f32(a, b) }
    }
    #[inline(always)]
    fn sub(self, a: float32x4_t, b: float32x4_t) -> float32x4_t {
        unsafe { vsubq_f32(a, b) }
    }
    #[inline(always)]
    fn mul(self, a: float32x4_t, b: float32x4_t) -> float32x4_t {
        unsafe { vmulq_f32(a, b) }
    }
    #[inline(always)]
    fn div(self, a: float32x4_t, b: float32x4_t) -> float32x4_t {
        unsafe { vdivq_f32(a, b) }
    }
    #[inline(always)]
    fn neg(self, a: float32x4_t) -> float32x4_t {
        unsafe { vnegq_f32(a) }
    }
    #[inline(always)]
    fn abs(self, a: float32x4_t) -> float32x4_t {
        unsafe { vabsq_f32(a) }
    }
    #[inline(always)]
    fn fma(self, a: float32x4_t, b: float32x4_t, c: float32x4_t) -> float32x4_t {
        // vfmaq_f32(acc, x, y) = acc + x*y, so this computes a*b + c
        unsafe { vfmaq_f32(c, a, b) }
    }
    #[inline(always)]
    fn madd(self, a: float32x4_t, b: float32x4_t, acc: float32x4_t) -> float32x4_t {
        <Self as Backend<f32>>::fma(self, a, b, acc)
    }
    #[inline(always)]
    fn sqrt(self, a: float32x4_t) -> float32x4_t {
        unsafe { vsqrtq_f32(a) }
    }
    #[inline(always)]
    fn min(self, a: float32x4_t, b: float32x4_t) -> float32x4_t {
        unsafe { vminnmq_f32(a, b) }
    }
    #[inline(always)]
    fn max(self, a: float32x4_t, b: float32x4_t) -> float32x4_t {
        unsafe { vmaxnmq_f32(a, b) }
    }
    #[inline(always)]
    fn le(self, a: float32x4_t, b: float32x4_t) -> uint32x4_t {
        unsafe { vcleq_f32(a, b) }
    }
    #[inline(always)]
    fn lt(self, a: float32x4_t, b: float32x4_t) -> uint32x4_t {
        unsafe { vcltq_f32(a, b) }
    }
    #[inline(always)]
    fn ge(self, a: float32x4_t, b: float32x4_t) -> uint32x4_t {
        unsafe { vcgeq_f32(a, b) }
    }
    #[inline(always)]
    fn gt(self, a: float32x4_t, b: float32x4_t) -> uint32x4_t {
        unsafe { vcgtq_f32(a, b) }
    }
    #[inline(always)]
    fn mask_and(self, a: uint32x4_t, b: uint32x4_t) -> uint32x4_t {
        unsafe { vandq_u32(a, b) }
    }
    #[inline(always)]
    fn mask_or(self, a: uint32x4_t, b: uint32x4_t) -> uint32x4_t {
        unsafe { vorrq_u32(a, b) }
    }
    #[inline(always)]
    fn mask_not(self, a: uint32x4_t) -> uint32x4_t {
        unsafe { vmvnq_u32(a) }
    }
    #[inline(always)]
    fn select(self, m: uint32x4_t, a: float32x4_t, b: float32x4_t) -> float32x4_t {
        // vbslq_f32(mask, a, b) = mask ? a : b
        unsafe { vbslq_f32(m, a, b) }
    }
    #[inline(always)]
    fn any(self, m: uint32x4_t) -> bool {
        unsafe { vmaxvq_u32(m) != 0 }
    }
    #[inline(always)]
    fn all(self, m: uint32x4_t) -> bool {
        unsafe { vminvq_u32(m) == u32::MAX }
    }
    #[inline(always)]
    fn mask_bitmask(self, m: uint32x4_t) -> u32 {
        const WEIGHTS: [u32; 4] = [1, 2, 4, 8];
        unsafe { vaddvq_u32(vandq_u32(m, vld1q_u32(WEIGHTS.as_ptr()))) }
    }
    #[inline(always)]
    fn reduce_sum(self, v: float32x4_t) -> f32 {
        unsafe { vaddvq_f32(v) }
    }
    #[inline(always)]
    fn reduce_min(self, v: float32x4_t) -> f32 {
        unsafe { vminnmvq_f32(v) }
    }
    #[inline(always)]
    fn reduce_max(self, v: float32x4_t) -> f32 {
        unsafe { vmaxnmvq_f32(v) }
    }

    type IVector = uint32x4_t;
    #[inline(always)]
    fn iload(self, s: &[u32]) -> uint32x4_t {
        debug_assert_eq!(s.len(), 4);
        unsafe { vld1q_u32(s.as_ptr()) }
    }
    #[inline(always)]
    fn istore(self, v: uint32x4_t, out: &mut [u32]) {
        debug_assert_eq!(out.len(), 4);
        unsafe { vst1q_u32(out.as_mut_ptr(), v) }
    }
    #[inline(always)]
    fn isplat(self, v: u32) -> uint32x4_t {
        unsafe { vdupq_n_u32(v) }
    }
    #[inline(always)]
    fn iadd(self, a: uint32x4_t, b: uint32x4_t) -> uint32x4_t {
        unsafe { vaddq_u32(a, b) }
    }
    #[inline(always)]
    fn isub(self, a: uint32x4_t, b: uint32x4_t) -> uint32x4_t {
        unsafe { vsubq_u32(a, b) }
    }
    #[inline(always)]
    fn imul(self, a: uint32x4_t, b: uint32x4_t) -> uint32x4_t {
        unsafe { vmulq_u32(a, b) }
    }
    #[inline(always)]
    fn iand(self, a: uint32x4_t, b: uint32x4_t) -> uint32x4_t {
        unsafe { vandq_u32(a, b) }
    }
    #[inline(always)]
    fn ior(self, a: uint32x4_t, b: uint32x4_t) -> uint32x4_t {
        unsafe { vorrq_u32(a, b) }
    }
    #[inline(always)]
    fn ixor(self, a: uint32x4_t, b: uint32x4_t) -> uint32x4_t {
        unsafe { veorq_u32(a, b) }
    }
    #[inline(always)]
    fn inot(self, a: uint32x4_t) -> uint32x4_t {
        unsafe { vmvnq_u32(a) }
    }
    #[inline(always)]
    fn ishl(self, a: uint32x4_t, k: u32) -> uint32x4_t {
        debug_assert!(k < 32);
        unsafe { vshlq_u32(a, vdupq_n_s32(k as i32)) }
    }
    #[inline(always)]
    fn ishr(self, a: uint32x4_t, k: u32) -> uint32x4_t {
        debug_assert!(k < 32);
        unsafe { vshlq_u32(a, vdupq_n_s32(-(k as i32))) }
    }
    #[inline(always)]
    fn ishr_arith(self, a: uint32x4_t, k: u32) -> uint32x4_t {
        debug_assert!(k < 32);
        unsafe {
            vreinterpretq_u32_s32(vshlq_s32(vreinterpretq_s32_u32(a), vdupq_n_s32(-(k as i32))))
        }
    }
    #[inline(always)]
    fn ieq(self, a: uint32x4_t, b: uint32x4_t) -> uint32x4_t {
        unsafe { vceqq_u32(a, b) }
    }
    #[inline(always)]
    fn ilt_u(self, a: uint32x4_t, b: uint32x4_t) -> uint32x4_t {
        unsafe { vcltq_u32(a, b) }
    }
    #[inline(always)]
    fn ilt_s(self, a: uint32x4_t, b: uint32x4_t) -> uint32x4_t {
        unsafe { vcltq_s32(vreinterpretq_s32_u32(a), vreinterpretq_s32_u32(b)) }
    }
    #[inline(always)]
    fn iselect(self, m: uint32x4_t, a: uint32x4_t, b: uint32x4_t) -> uint32x4_t {
        unsafe { vbslq_u32(m, a, b) }
    }
    #[inline(always)]
    fn to_bits(self, v: float32x4_t) -> uint32x4_t {
        unsafe { vreinterpretq_u32_f32(v) }
    }
    #[inline(always)]
    fn from_bits(self, v: uint32x4_t) -> float32x4_t {
        unsafe { vreinterpretq_f32_u32(v) }
    }
}

impl Backend<f64> for Neon {
    type Vector = float64x2_t;
    type Mask = uint64x2_t;

    #[inline(always)]
    fn lanes(self) -> usize {
        2
    }
    #[inline(always)]
    fn splat(self, v: f64) -> float64x2_t {
        unsafe { vdupq_n_f64(v) }
    }
    #[inline(always)]
    fn load(self, s: &[f64]) -> float64x2_t {
        debug_assert_eq!(s.len(), 2);
        unsafe { vld1q_f64(s.as_ptr()) }
    }
    #[inline(always)]
    fn store(self, v: float64x2_t, s: &mut [f64]) {
        debug_assert_eq!(s.len(), 2);
        unsafe { vst1q_f64(s.as_mut_ptr(), v) }
    }
    #[inline(always)]
    fn add(self, a: float64x2_t, b: float64x2_t) -> float64x2_t {
        unsafe { vaddq_f64(a, b) }
    }
    #[inline(always)]
    fn sub(self, a: float64x2_t, b: float64x2_t) -> float64x2_t {
        unsafe { vsubq_f64(a, b) }
    }
    #[inline(always)]
    fn mul(self, a: float64x2_t, b: float64x2_t) -> float64x2_t {
        unsafe { vmulq_f64(a, b) }
    }
    #[inline(always)]
    fn div(self, a: float64x2_t, b: float64x2_t) -> float64x2_t {
        unsafe { vdivq_f64(a, b) }
    }
    #[inline(always)]
    fn neg(self, a: float64x2_t) -> float64x2_t {
        unsafe { vnegq_f64(a) }
    }
    #[inline(always)]
    fn abs(self, a: float64x2_t) -> float64x2_t {
        unsafe { vabsq_f64(a) }
    }
    #[inline(always)]
    fn fma(self, a: float64x2_t, b: float64x2_t, c: float64x2_t) -> float64x2_t {
        unsafe { vfmaq_f64(c, a, b) }
    }
    #[inline(always)]
    fn madd(self, a: float64x2_t, b: float64x2_t, acc: float64x2_t) -> float64x2_t {
        <Self as Backend<f64>>::fma(self, a, b, acc)
    }
    #[inline(always)]
    fn sqrt(self, a: float64x2_t) -> float64x2_t {
        unsafe { vsqrtq_f64(a) }
    }
    #[inline(always)]
    fn min(self, a: float64x2_t, b: float64x2_t) -> float64x2_t {
        unsafe { vminnmq_f64(a, b) }
    }
    #[inline(always)]
    fn max(self, a: float64x2_t, b: float64x2_t) -> float64x2_t {
        unsafe { vmaxnmq_f64(a, b) }
    }
    #[inline(always)]
    fn le(self, a: float64x2_t, b: float64x2_t) -> uint64x2_t {
        unsafe { vcleq_f64(a, b) }
    }
    #[inline(always)]
    fn lt(self, a: float64x2_t, b: float64x2_t) -> uint64x2_t {
        unsafe { vcltq_f64(a, b) }
    }
    #[inline(always)]
    fn ge(self, a: float64x2_t, b: float64x2_t) -> uint64x2_t {
        unsafe { vcgeq_f64(a, b) }
    }
    #[inline(always)]
    fn gt(self, a: float64x2_t, b: float64x2_t) -> uint64x2_t {
        unsafe { vcgtq_f64(a, b) }
    }
    #[inline(always)]
    fn mask_and(self, a: uint64x2_t, b: uint64x2_t) -> uint64x2_t {
        unsafe { vandq_u64(a, b) }
    }
    #[inline(always)]
    fn mask_or(self, a: uint64x2_t, b: uint64x2_t) -> uint64x2_t {
        unsafe { vorrq_u64(a, b) }
    }
    #[inline(always)]
    fn mask_not(self, a: uint64x2_t) -> uint64x2_t {
        unsafe { veorq_u64(a, vdupq_n_u64(u64::MAX)) }
    }
    #[inline(always)]
    fn select(self, m: uint64x2_t, a: float64x2_t, b: float64x2_t) -> float64x2_t {
        unsafe { vbslq_f64(m, a, b) }
    }
    #[inline(always)]
    fn any(self, m: uint64x2_t) -> bool {
        unsafe { (vgetq_lane_u64::<0>(m) | vgetq_lane_u64::<1>(m)) != 0 }
    }
    #[inline(always)]
    fn all(self, m: uint64x2_t) -> bool {
        unsafe { (vgetq_lane_u64::<0>(m) & vgetq_lane_u64::<1>(m)) == u64::MAX }
    }
    #[inline(always)]
    fn mask_bitmask(self, m: uint64x2_t) -> u32 {
        const WEIGHTS: [u64; 2] = [1, 2];
        unsafe { vaddvq_u64(vandq_u64(m, vld1q_u64(WEIGHTS.as_ptr()))) as u32 }
    }
    #[inline(always)]
    fn reduce_sum(self, v: float64x2_t) -> f64 {
        unsafe { vaddvq_f64(v) }
    }
    #[inline(always)]
    fn reduce_min(self, v: float64x2_t) -> f64 {
        unsafe { vgetq_lane_f64::<0>(v).min(vgetq_lane_f64::<1>(v)) }
    }
    #[inline(always)]
    fn reduce_max(self, v: float64x2_t) -> f64 {
        unsafe { vgetq_lane_f64::<0>(v).max(vgetq_lane_f64::<1>(v)) }
    }

    type IVector = [u32; 2];
    #[inline(always)]
    fn iload(self, s: &[u32]) -> [u32; 2] {
        let mut v = [0u32; 2];
        v.copy_from_slice(s);
        v
    }
    #[inline(always)]
    fn istore(self, v: [u32; 2], out: &mut [u32]) {
        out.copy_from_slice(&v);
    }
}

// bf16: 16-bit storage, f32x4 compute (NEON has no native bf16 ALU). Widen on load/splat,
// narrow on store/reduce; conversions live at the memory boundary only.
mod bf16_impl {
    use super::*;
    use half::bf16;

    #[inline(always)]
    unsafe fn load4(s: &[bf16]) -> float32x4_t {
        let t = [s[0].to_f32(), s[1].to_f32(), s[2].to_f32(), s[3].to_f32()];
        vld1q_f32(t.as_ptr())
    }
    #[inline(always)]
    unsafe fn store4(v: float32x4_t, s: &mut [bf16]) {
        let mut t = [0f32; 4];
        vst1q_f32(t.as_mut_ptr(), v);
        for (d, x) in s.iter_mut().zip(t) {
            *d = bf16::from_f32(x);
        }
    }

    impl Backend<bf16> for Neon {
        type Vector = float32x4_t;
        type Mask = uint32x4_t;

        #[inline(always)]
        fn lanes(self) -> usize {
            4
        }
        #[inline(always)]
        fn splat(self, v: bf16) -> float32x4_t {
            unsafe { vdupq_n_f32(v.to_f32()) }
        }
        #[inline(always)]
        fn load(self, s: &[bf16]) -> float32x4_t {
            debug_assert_eq!(s.len(), 4);
            unsafe { load4(s) }
        }
        #[inline(always)]
        fn store(self, v: float32x4_t, s: &mut [bf16]) {
            debug_assert_eq!(s.len(), 4);
            unsafe { store4(v, s) }
        }
        #[inline(always)]
        fn add(self, a: float32x4_t, b: float32x4_t) -> float32x4_t {
            unsafe { vaddq_f32(a, b) }
        }
        #[inline(always)]
        fn sub(self, a: float32x4_t, b: float32x4_t) -> float32x4_t {
            unsafe { vsubq_f32(a, b) }
        }
        #[inline(always)]
        fn mul(self, a: float32x4_t, b: float32x4_t) -> float32x4_t {
            unsafe { vmulq_f32(a, b) }
        }
        #[inline(always)]
        fn div(self, a: float32x4_t, b: float32x4_t) -> float32x4_t {
            unsafe { vdivq_f32(a, b) }
        }
        #[inline(always)]
        fn neg(self, a: float32x4_t) -> float32x4_t {
            unsafe { vnegq_f32(a) }
        }
        #[inline(always)]
        fn abs(self, a: float32x4_t) -> float32x4_t {
            unsafe { vabsq_f32(a) }
        }
        #[inline(always)]
        fn fma(self, a: float32x4_t, b: float32x4_t, c: float32x4_t) -> float32x4_t {
            unsafe { vfmaq_f32(c, a, b) }
        }
            #[inline(always)]
        fn madd(self, a: float32x4_t, b: float32x4_t, acc: float32x4_t) -> float32x4_t {
            <Self as Backend<bf16>>::fma(self, a, b, acc)
        }
    #[inline(always)]
        fn sqrt(self, a: float32x4_t) -> float32x4_t {
            unsafe { vsqrtq_f32(a) }
        }
        #[inline(always)]
        fn min(self, a: float32x4_t, b: float32x4_t) -> float32x4_t {
            unsafe { vminnmq_f32(a, b) }
        }
        #[inline(always)]
        fn max(self, a: float32x4_t, b: float32x4_t) -> float32x4_t {
            unsafe { vmaxnmq_f32(a, b) }
        }
        #[inline(always)]
        fn le(self, a: float32x4_t, b: float32x4_t) -> uint32x4_t {
            unsafe { vcleq_f32(a, b) }
        }
        #[inline(always)]
        fn lt(self, a: float32x4_t, b: float32x4_t) -> uint32x4_t {
            unsafe { vcltq_f32(a, b) }
        }
        #[inline(always)]
        fn ge(self, a: float32x4_t, b: float32x4_t) -> uint32x4_t {
            unsafe { vcgeq_f32(a, b) }
        }
        #[inline(always)]
        fn gt(self, a: float32x4_t, b: float32x4_t) -> uint32x4_t {
            unsafe { vcgtq_f32(a, b) }
        }
        #[inline(always)]
        fn mask_and(self, a: uint32x4_t, b: uint32x4_t) -> uint32x4_t {
            unsafe { vandq_u32(a, b) }
        }
        #[inline(always)]
        fn mask_or(self, a: uint32x4_t, b: uint32x4_t) -> uint32x4_t {
            unsafe { vorrq_u32(a, b) }
        }
        #[inline(always)]
        fn mask_not(self, a: uint32x4_t) -> uint32x4_t {
            unsafe { vmvnq_u32(a) }
        }
        #[inline(always)]
        fn select(self, m: uint32x4_t, a: float32x4_t, b: float32x4_t) -> float32x4_t {
            unsafe { vbslq_f32(m, a, b) }
        }
        #[inline(always)]
        fn any(self, m: uint32x4_t) -> bool {
            unsafe { vmaxvq_u32(m) != 0 }
        }
        #[inline(always)]
        fn all(self, m: uint32x4_t) -> bool {
            unsafe { vminvq_u32(m) == u32::MAX }
        }
        #[inline(always)]
        fn mask_bitmask(self, m: uint32x4_t) -> u32 {
            const WEIGHTS: [u32; 4] = [1, 2, 4, 8];
            unsafe { vaddvq_u32(vandq_u32(m, vld1q_u32(WEIGHTS.as_ptr()))) }
        }
        #[inline(always)]
        fn reduce_sum(self, v: float32x4_t) -> bf16 {
            bf16::from_f32(unsafe { vaddvq_f32(v) })
        }
        #[inline(always)]
        fn reduce_min(self, v: float32x4_t) -> bf16 {
            bf16::from_f32(unsafe { vminnmvq_f32(v) })
        }
        #[inline(always)]
        fn reduce_max(self, v: float32x4_t) -> bf16 {
            bf16::from_f32(unsafe { vmaxnmvq_f32(v) })
        }

        type IVector = [u32; 4];
        #[inline(always)]
        fn iload(self, s: &[u32]) -> [u32; 4] {
            let mut v = [0u32; 4];
            v.copy_from_slice(s);
            v
        }
        #[inline(always)]
        fn istore(self, v: [u32; 4], out: &mut [u32]) {
            out.copy_from_slice(&v);
        }
    }
}

// Native FEAT_FP16: 8 f16 lanes with the full `.h` ALU. Arithmetic, FMA, compares, and
// bit-select are stable intrinsics; the unstable ones are sidestepped: load/store/splat go
// through `uint16x8_t` plus a free `vreinterpretq_f16_u16`, and reductions widen each half to
// `float32x4_t` and use the f32 horizontal ops. Reached only when FEAT_FP16 is detected, so
// the `fp16` target-feature bodies always run on a CPU that has it.
mod f16_impl {
    use super::*;
    use half::f16;

    #[target_feature(enable = "fp16")]
    unsafe fn splat(v: f16) -> float16x8_t {
        vreinterpretq_f16_u16(vdupq_n_u16(v.to_bits()))
    }
    #[target_feature(enable = "fp16")]
    unsafe fn load(p: *const f16) -> float16x8_t {
        vreinterpretq_f16_u16(vld1q_u16(p.cast()))
    }
    #[target_feature(enable = "fp16")]
    unsafe fn store(p: *mut f16, v: float16x8_t) {
        vst1q_u16(p.cast(), vreinterpretq_u16_f16(v));
    }
    #[target_feature(enable = "fp16")]
    unsafe fn add(a: float16x8_t, b: float16x8_t) -> float16x8_t {
        vaddq_f16(a, b)
    }
    #[target_feature(enable = "fp16")]
    unsafe fn sub(a: float16x8_t, b: float16x8_t) -> float16x8_t {
        vsubq_f16(a, b)
    }
    #[target_feature(enable = "fp16")]
    unsafe fn mul(a: float16x8_t, b: float16x8_t) -> float16x8_t {
        vmulq_f16(a, b)
    }
    #[target_feature(enable = "fp16")]
    unsafe fn div(a: float16x8_t, b: float16x8_t) -> float16x8_t {
        vdivq_f16(a, b)
    }
    #[target_feature(enable = "fp16")]
    unsafe fn neg(a: float16x8_t) -> float16x8_t {
        vnegq_f16(a)
    }
    #[target_feature(enable = "fp16")]
    unsafe fn abs(a: float16x8_t) -> float16x8_t {
        vabsq_f16(a)
    }
    #[target_feature(enable = "fp16")]
    unsafe fn fma(a: float16x8_t, b: float16x8_t, c: float16x8_t) -> float16x8_t {
        vfmaq_f16(c, a, b)
    }
    #[target_feature(enable = "fp16")]
    unsafe fn sqrt(a: float16x8_t) -> float16x8_t {
        vsqrtq_f16(a)
    }
    #[target_feature(enable = "fp16")]
    unsafe fn min(a: float16x8_t, b: float16x8_t) -> float16x8_t {
        vminnmq_f16(a, b)
    }
    #[target_feature(enable = "fp16")]
    unsafe fn max(a: float16x8_t, b: float16x8_t) -> float16x8_t {
        vmaxnmq_f16(a, b)
    }
    #[target_feature(enable = "fp16")]
    unsafe fn le(a: float16x8_t, b: float16x8_t) -> uint16x8_t {
        vcleq_f16(a, b)
    }
    #[target_feature(enable = "fp16")]
    unsafe fn lt(a: float16x8_t, b: float16x8_t) -> uint16x8_t {
        vcltq_f16(a, b)
    }
    #[target_feature(enable = "fp16")]
    unsafe fn ge(a: float16x8_t, b: float16x8_t) -> uint16x8_t {
        vcgeq_f16(a, b)
    }
    #[target_feature(enable = "fp16")]
    unsafe fn gt(a: float16x8_t, b: float16x8_t) -> uint16x8_t {
        vcgtq_f16(a, b)
    }
    #[target_feature(enable = "fp16")]
    unsafe fn select(m: uint16x8_t, a: float16x8_t, b: float16x8_t) -> float16x8_t {
        vbslq_f16(m, a, b)
    }
    // The f16 horizontal reductions are unstable; widen each half via the stable `vcvt`s and
    // reduce in f32.
    #[target_feature(enable = "fp16")]
    unsafe fn widen(v: float16x8_t) -> (float32x4_t, float32x4_t) {
        (vcvt_f32_f16(vget_low_f16(v)), vcvt_high_f32_f16(v))
    }

    impl Backend<f16> for Neon {
        type Vector = float16x8_t;
        type Mask = uint16x8_t;

        #[inline(always)]
        fn lanes(self) -> usize {
            8
        }
        #[inline(always)]
        fn splat(self, v: f16) -> float16x8_t {
            unsafe { splat(v) }
        }
        #[inline(always)]
        fn load(self, s: &[f16]) -> float16x8_t {
            debug_assert_eq!(s.len(), 8);
            unsafe { load(s.as_ptr()) }
        }
        #[inline(always)]
        fn store(self, v: float16x8_t, s: &mut [f16]) {
            debug_assert_eq!(s.len(), 8);
            unsafe { store(s.as_mut_ptr(), v) }
        }
        #[inline(always)]
        fn add(self, a: float16x8_t, b: float16x8_t) -> float16x8_t {
            unsafe { add(a, b) }
        }
        #[inline(always)]
        fn sub(self, a: float16x8_t, b: float16x8_t) -> float16x8_t {
            unsafe { sub(a, b) }
        }
        #[inline(always)]
        fn mul(self, a: float16x8_t, b: float16x8_t) -> float16x8_t {
            unsafe { mul(a, b) }
        }
        #[inline(always)]
        fn div(self, a: float16x8_t, b: float16x8_t) -> float16x8_t {
            unsafe { div(a, b) }
        }
        #[inline(always)]
        fn neg(self, a: float16x8_t) -> float16x8_t {
            unsafe { neg(a) }
        }
        #[inline(always)]
        fn abs(self, a: float16x8_t) -> float16x8_t {
            unsafe { abs(a) }
        }
        #[inline(always)]
        fn fma(self, a: float16x8_t, b: float16x8_t, c: float16x8_t) -> float16x8_t {
            unsafe { fma(a, b, c) }
        }
            #[inline(always)]
        fn madd(self, a: float16x8_t, b: float16x8_t, acc: float16x8_t) -> float16x8_t {
            <Self as Backend<f16>>::fma(self, a, b, acc)
        }
    #[inline(always)]
        fn sqrt(self, a: float16x8_t) -> float16x8_t {
            unsafe { sqrt(a) }
        }
        #[inline(always)]
        fn min(self, a: float16x8_t, b: float16x8_t) -> float16x8_t {
            unsafe { min(a, b) }
        }
        #[inline(always)]
        fn max(self, a: float16x8_t, b: float16x8_t) -> float16x8_t {
            unsafe { max(a, b) }
        }
        #[inline(always)]
        fn le(self, a: float16x8_t, b: float16x8_t) -> uint16x8_t {
            unsafe { le(a, b) }
        }
        #[inline(always)]
        fn lt(self, a: float16x8_t, b: float16x8_t) -> uint16x8_t {
            unsafe { lt(a, b) }
        }
        #[inline(always)]
        fn ge(self, a: float16x8_t, b: float16x8_t) -> uint16x8_t {
            unsafe { ge(a, b) }
        }
        #[inline(always)]
        fn gt(self, a: float16x8_t, b: float16x8_t) -> uint16x8_t {
            unsafe { gt(a, b) }
        }
        #[inline(always)]
        fn mask_and(self, a: uint16x8_t, b: uint16x8_t) -> uint16x8_t {
            unsafe { vandq_u16(a, b) }
        }
        #[inline(always)]
        fn mask_or(self, a: uint16x8_t, b: uint16x8_t) -> uint16x8_t {
            unsafe { vorrq_u16(a, b) }
        }
        #[inline(always)]
        fn mask_not(self, a: uint16x8_t) -> uint16x8_t {
            unsafe { vmvnq_u16(a) }
        }
        #[inline(always)]
        fn select(self, m: uint16x8_t, a: float16x8_t, b: float16x8_t) -> float16x8_t {
            unsafe { select(m, a, b) }
        }
        #[inline(always)]
        fn any(self, m: uint16x8_t) -> bool {
            unsafe { vmaxvq_u16(m) != 0 }
        }
        #[inline(always)]
        fn all(self, m: uint16x8_t) -> bool {
            unsafe { vminvq_u16(m) == u16::MAX }
        }
        #[inline(always)]
        fn mask_bitmask(self, m: uint16x8_t) -> u32 {
            const WEIGHTS: [u16; 8] = [1, 2, 4, 8, 16, 32, 64, 128];
            unsafe { vaddvq_u16(vandq_u16(m, vld1q_u16(WEIGHTS.as_ptr()))) as u32 }
        }
        #[inline(always)]
        fn reduce_sum(self, v: float16x8_t) -> f16 {
            unsafe {
                let (lo, hi) = widen(v);
                f16::from_f32(vaddvq_f32(vaddq_f32(lo, hi)))
            }
        }
        #[inline(always)]
        fn reduce_min(self, v: float16x8_t) -> f16 {
            unsafe {
                let (lo, hi) = widen(v);
                f16::from_f32(vminnmvq_f32(vminnmq_f32(lo, hi)))
            }
        }
        #[inline(always)]
        fn reduce_max(self, v: float16x8_t) -> f16 {
            unsafe {
                let (lo, hi) = widen(v);
                f16::from_f32(vmaxnmvq_f32(vmaxnmq_f32(lo, hi)))
            }
        }

        type IVector = [u32; 8];
        #[inline(always)]
        fn iload(self, s: &[u32]) -> [u32; 8] {
            let mut v = [0u32; 8];
            v.copy_from_slice(s);
            v
        }
        #[inline(always)]
        fn istore(self, v: [u32; 8], out: &mut [u32]) {
            out.copy_from_slice(&v);
        }
    }
}

/// The 32-bit integer element backends: same 4-lane geometry and `uint32x4_t` mask convention
/// as the `f32` impl, so float and integer kernels share mask machinery. Arithmetic is wrapping.
macro_rules! neon_int_backend {
    ($t:ty, $vec:ty, $ld:ident, $st:ident, $dup:ident, $add:ident, $sub:ident, $mul:ident,
     $neg:expr, $abs:expr, $min:ident, $max:ident,
     $le:ident, $lt:ident, $ge:ident, $gt:ident, $sel:ident,
     $rsum:ident, $rmin:ident, $rmax:ident,
     $shl:expr, $shr:expr, $and:ident, $orr:ident, $eor:ident, $not:expr,
     $tobits:expr, $frombits:expr) => {
        impl Backend<$t> for Neon {
            type Vector = $vec;
            type Mask = uint32x4_t;

            #[inline(always)]
            fn lanes(self) -> usize {
                4
            }
            #[inline(always)]
            fn splat(self, v: $t) -> $vec {
                unsafe { $dup(v) }
            }
            #[inline(always)]
            fn load(self, s: &[$t]) -> $vec {
                debug_assert_eq!(s.len(), 4);
                unsafe { $ld(s.as_ptr()) }
            }
            #[inline(always)]
            fn store(self, v: $vec, s: &mut [$t]) {
                debug_assert_eq!(s.len(), 4);
                unsafe { $st(s.as_mut_ptr(), v) }
            }
            #[inline(always)]
            fn add(self, a: $vec, b: $vec) -> $vec {
                unsafe { $add(a, b) }
            }
            #[inline(always)]
            fn sub(self, a: $vec, b: $vec) -> $vec {
                unsafe { $sub(a, b) }
            }
            #[inline(always)]
            fn mul(self, a: $vec, b: $vec) -> $vec {
                unsafe { $mul(a, b) }
            }
            #[inline(always)]
            fn neg(self, a: $vec) -> $vec {
                unsafe { ($neg)(a) }
            }
            #[inline(always)]
            #[allow(unused_unsafe)]
            fn abs(self, a: $vec) -> $vec {
                unsafe { ($abs)(a) }
            }
            #[inline(always)]
            fn min(self, a: $vec, b: $vec) -> $vec {
                unsafe { $min(a, b) }
            }
            #[inline(always)]
            fn max(self, a: $vec, b: $vec) -> $vec {
                unsafe { $max(a, b) }
            }
            #[inline(always)]
            fn le(self, a: $vec, b: $vec) -> uint32x4_t {
                unsafe { $le(a, b) }
            }
            #[inline(always)]
            fn lt(self, a: $vec, b: $vec) -> uint32x4_t {
                unsafe { $lt(a, b) }
            }
            #[inline(always)]
            fn ge(self, a: $vec, b: $vec) -> uint32x4_t {
                unsafe { $ge(a, b) }
            }
            #[inline(always)]
            fn gt(self, a: $vec, b: $vec) -> uint32x4_t {
                unsafe { $gt(a, b) }
            }
            #[inline(always)]
            fn mask_and(self, a: uint32x4_t, b: uint32x4_t) -> uint32x4_t {
                unsafe { vandq_u32(a, b) }
            }
            #[inline(always)]
            fn mask_or(self, a: uint32x4_t, b: uint32x4_t) -> uint32x4_t {
                unsafe { vorrq_u32(a, b) }
            }
            #[inline(always)]
            fn mask_not(self, a: uint32x4_t) -> uint32x4_t {
                unsafe { vmvnq_u32(a) }
            }
            #[inline(always)]
            fn select(self, m: uint32x4_t, a: $vec, b: $vec) -> $vec {
                unsafe { $sel(m, a, b) }
            }
            #[inline(always)]
            fn any(self, m: uint32x4_t) -> bool {
                unsafe { vmaxvq_u32(m) != 0 }
            }
            #[inline(always)]
            fn all(self, m: uint32x4_t) -> bool {
                unsafe { vminvq_u32(m) == u32::MAX }
            }
            #[inline(always)]
            fn mask_bitmask(self, m: uint32x4_t) -> u32 {
                const WEIGHTS: [u32; 4] = [1, 2, 4, 8];
                unsafe { vaddvq_u32(vandq_u32(m, vld1q_u32(WEIGHTS.as_ptr()))) }
            }
            #[inline(always)]
            fn reduce_sum(self, v: $vec) -> $t {
                unsafe { $rsum(v) }
            }
            #[inline(always)]
            fn reduce_min(self, v: $vec) -> $t {
                unsafe { $rmin(v) }
            }
            #[inline(always)]
            fn reduce_max(self, v: $vec) -> $t {
                unsafe { $rmax(v) }
            }
            #[inline(always)]
            fn shl(self, a: $vec, k: u32) -> $vec {
                debug_assert!(k < 32);
                unsafe { ($shl)(a, k) }
            }
            #[inline(always)]
            fn shr(self, a: $vec, k: u32) -> $vec {
                debug_assert!(k < 32);
                unsafe { ($shr)(a, k) }
            }
            #[inline(always)]
            fn bit_and(self, a: $vec, b: $vec) -> $vec {
                unsafe { $and(a, b) }
            }
            #[inline(always)]
            fn bit_or(self, a: $vec, b: $vec) -> $vec {
                unsafe { $orr(a, b) }
            }
            #[inline(always)]
            fn bit_xor(self, a: $vec, b: $vec) -> $vec {
                unsafe { $eor(a, b) }
            }
            #[inline(always)]
            fn bit_not(self, a: $vec) -> $vec {
                unsafe { ($not)(a) }
            }

            type IVector = uint32x4_t;
            #[inline(always)]
            fn iload(self, s: &[u32]) -> uint32x4_t {
                debug_assert_eq!(s.len(), 4);
                unsafe { vld1q_u32(s.as_ptr()) }
            }
            #[inline(always)]
            fn istore(self, v: uint32x4_t, out: &mut [u32]) {
                debug_assert_eq!(out.len(), 4);
                unsafe { vst1q_u32(out.as_mut_ptr(), v) }
            }
            #[inline(always)]
            #[allow(unused_unsafe)]
            fn to_bits(self, v: $vec) -> uint32x4_t {
                unsafe { ($tobits)(v) }
            }
            #[inline(always)]
            #[allow(unused_unsafe)]
            fn from_bits(self, v: uint32x4_t) -> $vec {
                unsafe { ($frombits)(v) }
            }
        }
    };
}

neon_int_backend!(
    u32, uint32x4_t, vld1q_u32, vst1q_u32, vdupq_n_u32, vaddq_u32, vsubq_u32, vmulq_u32,
    |a| vreinterpretq_u32_s32(vnegq_s32(vreinterpretq_s32_u32(a))),
    |a| a,
    vminq_u32, vmaxq_u32,
    vcleq_u32, vcltq_u32, vcgeq_u32, vcgtq_u32, vbslq_u32,
    vaddvq_u32, vminvq_u32, vmaxvq_u32,
    |a, k: u32| vshlq_u32(a, vdupq_n_s32(k as i32)),
    |a, k: u32| vshlq_u32(a, vdupq_n_s32(-(k as i32))),
    vandq_u32, vorrq_u32, veorq_u32,
    |a| vmvnq_u32(a),
    |v| v,
    |v| v
);

neon_int_backend!(
    i32, int32x4_t, vld1q_s32, vst1q_s32, vdupq_n_s32, vaddq_s32, vsubq_s32, vmulq_s32,
    |a| vnegq_s32(a),
    |a| vabsq_s32(a),
    vminq_s32, vmaxq_s32,
    vcleq_s32, vcltq_s32, vcgeq_s32, vcgtq_s32, vbslq_s32,
    vaddvq_s32, vminvq_s32, vmaxvq_s32,
    |a, k: u32| vshlq_s32(a, vdupq_n_s32(k as i32)),
    |a, k: u32| vshlq_s32(a, vdupq_n_s32(-(k as i32))),
    vandq_s32, vorrq_s32, veorq_s32,
    |a| vmvnq_s32(a),
    |v| vreinterpretq_u32_s32(v),
    |v| vreinterpretq_s32_u32(v)
);