hydroplane 0.1.0

//! Hand-written WebAssembly SIMD backends over the fixed-width `v128` (4-wide f32, 2-wide f64).
//! [`Simd128`] and [`RelaxedSimd`] differ only in [`fma`](Backend::fma); WASM has no runtime
//! feature detection, so the token is chosen at compile time from `target_feature`.
#![cfg(all(target_arch = "wasm32", target_feature = "simd128"))]
#![allow(unsafe_op_in_unsafe_fn)]

use core::arch::wasm32::*;

use half::{bf16, f16};

use crate::backend::Backend;

#[inline(always)]
fn fma_mul_add_f32(a: v128, b: v128, c: v128) -> v128 {
    f32x4_add(f32x4_mul(a, b), c)
}
#[inline(always)]
fn fma_mul_add_f64(a: v128, b: v128, c: v128) -> v128 {
    f64x2_add(f64x2_mul(a, b), c)
}

#[inline(always)]
unsafe fn load4_bf16(s: &[bf16]) -> v128 {
    let t = [s[0].to_f32(), s[1].to_f32(), s[2].to_f32(), s[3].to_f32()];
    v128_load(t.as_ptr() as *const v128)
}
#[inline(always)]
unsafe fn store4_bf16(v: v128, s: &mut [bf16]) {
    let mut t = [0f32; 4];
    v128_store(t.as_mut_ptr() as *mut v128, v);
    for (d, x) in s.iter_mut().zip(t) {
        *d = bf16::from_f32(x);
    }
}
#[inline(always)]
unsafe fn load4_f16(s: &[f16]) -> v128 {
    let t = [s[0].to_f32(), s[1].to_f32(), s[2].to_f32(), s[3].to_f32()];
    v128_load(t.as_ptr() as *const v128)
}
#[inline(always)]
unsafe fn store4_f16(v: v128, s: &mut [f16]) {
    let mut t = [0f32; 4];
    v128_store(t.as_mut_ptr() as *mut v128, v);
    for (d, x) in s.iter_mut().zip(t) {
        *d = f16::from_f32(x);
    }
}

/// Emits `Backend<f32>`, `Backend<f64>` and the bf16 widen-path `Backend<bf16>` for a WASM token.
/// `$fma32`/`$fma64` are the only point of variation between [`Simd128`] and [`RelaxedSimd`]:
/// `fn(v128, v128, v128) -> v128` computing `a*b + c`.
macro_rules! wasm_backend {
    ($token:ty, $fma32:path, $fma64:path) => {
        impl Backend<f32> for $token {
            type Vector = v128;
            type Mask = v128;

            type IVector = v128;
            #[inline(always)]
            fn iload(self, s: &[u32]) -> v128 {
                debug_assert_eq!(s.len(), 4);
                unsafe { v128_load(s.as_ptr() as *const v128) }
            }
            #[inline(always)]
            fn istore(self, v: v128, out: &mut [u32]) {
                debug_assert_eq!(out.len(), 4);
                unsafe { v128_store(out.as_mut_ptr() as *mut v128, v) }
            }
            #[inline(always)]
            fn isplat(self, v: u32) -> v128 {
                u32x4_splat(v)
            }
            #[inline(always)]
            fn iadd(self, a: v128, b: v128) -> v128 {
                i32x4_add(a, b)
            }
            #[inline(always)]
            fn isub(self, a: v128, b: v128) -> v128 {
                i32x4_sub(a, b)
            }
            #[inline(always)]
            fn imul(self, a: v128, b: v128) -> v128 {
                i32x4_mul(a, b)
            }
            #[inline(always)]
            fn iand(self, a: v128, b: v128) -> v128 {
                v128_and(a, b)
            }
            #[inline(always)]
            fn ior(self, a: v128, b: v128) -> v128 {
                v128_or(a, b)
            }
            #[inline(always)]
            fn ixor(self, a: v128, b: v128) -> v128 {
                v128_xor(a, b)
            }
            #[inline(always)]
            fn inot(self, a: v128) -> v128 {
                v128_not(a)
            }
            #[inline(always)]
            fn ishl(self, a: v128, k: u32) -> v128 {
                debug_assert!(k < 32);
                i32x4_shl(a, k)
            }
            #[inline(always)]
            fn ishr(self, a: v128, k: u32) -> v128 {
                debug_assert!(k < 32);
                u32x4_shr(a, k)
            }
            #[inline(always)]
            fn ishr_arith(self, a: v128, k: u32) -> v128 {
                debug_assert!(k < 32);
                i32x4_shr(a, k)
            }
            #[inline(always)]
            fn ieq(self, a: v128, b: v128) -> v128 {
                i32x4_eq(a, b)
            }
            #[inline(always)]
            fn ilt_u(self, a: v128, b: v128) -> v128 {
                u32x4_lt(a, b)
            }
            #[inline(always)]
            fn ilt_s(self, a: v128, b: v128) -> v128 {
                i32x4_lt(a, b)
            }
            #[inline(always)]
            fn iselect(self, m: v128, a: v128, b: v128) -> v128 {
                v128_bitselect(a, b, m)
            }
            #[inline(always)]
            fn to_bits(self, v: v128) -> v128 {
                v
            }
            #[inline(always)]
            fn from_bits(self, v: v128) -> v128 {
                v
            }

            #[inline(always)]
            fn lanes(self) -> usize {
                4
            }
            #[inline(always)]
            fn splat(self, v: f32) -> v128 {
                f32x4_splat(v)
            }
            #[inline(always)]
            fn load(self, s: &[f32]) -> v128 {
                debug_assert_eq!(s.len(), 4);
                unsafe { v128_load(s.as_ptr() as *const v128) }
            }
            #[inline(always)]
            fn store(self, v: v128, s: &mut [f32]) {
                debug_assert_eq!(s.len(), 4);
                unsafe { v128_store(s.as_mut_ptr() as *mut v128, v) }
            }
            #[inline(always)]
            fn add(self, a: v128, b: v128) -> v128 {
                f32x4_add(a, b)
            }
            #[inline(always)]
            fn sub(self, a: v128, b: v128) -> v128 {
                f32x4_sub(a, b)
            }
            #[inline(always)]
            fn mul(self, a: v128, b: v128) -> v128 {
                f32x4_mul(a, b)
            }
            #[inline(always)]
            fn div(self, a: v128, b: v128) -> v128 {
                f32x4_div(a, b)
            }
            #[inline(always)]
            fn neg(self, a: v128) -> v128 {
                f32x4_neg(a)
            }
            #[inline(always)]
            fn abs(self, a: v128) -> v128 {
                f32x4_abs(a)
            }
            #[inline(always)]
            fn fma(self, a: v128, b: v128, c: v128) -> v128 {
                $fma32(a, b, c)
            }
                    #[inline(always)]
            fn madd(self, a: v128, b: v128, acc: v128) -> v128 {
                <Self as Backend<f32>>::fma(self, a, b, acc)
            }
    #[inline(always)]
            fn sqrt(self, a: v128) -> v128 {
                f32x4_sqrt(a)
            }
            #[inline(always)]
            fn min(self, a: v128, b: v128) -> v128 {
                // IEEE minimumNumber: `pmin` (`b < a ? b : a`) yields `a` when `b` is NaN; the
                // bitselect patches the a-is-NaN case (wasm's `f32x4.min` would propagate NaN).
                v128_bitselect(b, f32x4_pmin(a, b), f32x4_ne(a, a))
            }
            #[inline(always)]
            fn max(self, a: v128, b: v128) -> v128 {
                v128_bitselect(b, f32x4_pmax(a, b), f32x4_ne(a, a))
            }
            #[inline(always)]
            fn le(self, a: v128, b: v128) -> v128 {
                f32x4_le(a, b)
            }
            #[inline(always)]
            fn lt(self, a: v128, b: v128) -> v128 {
                f32x4_lt(a, b)
            }
            #[inline(always)]
            fn ge(self, a: v128, b: v128) -> v128 {
                f32x4_ge(a, b)
            }
            #[inline(always)]
            fn gt(self, a: v128, b: v128) -> v128 {
                f32x4_gt(a, b)
            }
            #[inline(always)]
            fn mask_and(self, a: v128, b: v128) -> v128 {
                v128_and(a, b)
            }
            #[inline(always)]
            fn mask_or(self, a: v128, b: v128) -> v128 {
                v128_or(a, b)
            }
            #[inline(always)]
            fn mask_not(self, a: v128) -> v128 {
                v128_not(a)
            }
            #[inline(always)]
            fn select(self, m: v128, a: v128, b: v128) -> v128 {
                v128_bitselect(a, b, m)
            }
            #[inline(always)]
            fn any(self, m: v128) -> bool {
                v128_any_true(m)
            }
            #[inline(always)]
            fn all(self, m: v128) -> bool {
                i32x4_all_true(m)
            }
            #[inline(always)]
            fn mask_bitmask(self, m: v128) -> u32 {
                i32x4_bitmask(m) as u32
            }
            #[inline(always)]
            fn reduce_sum(self, v: v128) -> f32 {
                f32x4_extract_lane::<0>(v)
                    + f32x4_extract_lane::<1>(v)
                    + f32x4_extract_lane::<2>(v)
                    + f32x4_extract_lane::<3>(v)
            }
            #[inline(always)]
            fn reduce_min(self, v: v128) -> f32 {
                f32x4_extract_lane::<0>(v)
                    .min(f32x4_extract_lane::<1>(v))
                    .min(f32x4_extract_lane::<2>(v))
                    .min(f32x4_extract_lane::<3>(v))
            }
            #[inline(always)]
            fn reduce_max(self, v: v128) -> f32 {
                f32x4_extract_lane::<0>(v)
                    .max(f32x4_extract_lane::<1>(v))
                    .max(f32x4_extract_lane::<2>(v))
                    .max(f32x4_extract_lane::<3>(v))
            }
        }

        impl Backend<f64> for $token {
            type Vector = v128;
            type Mask = v128;

            type IVector = [u32; 2];
            #[inline(always)]
            fn iload(self, s: &[u32]) -> [u32; 2] {
                let mut v = [0u32; 2];
                v.copy_from_slice(s);
                v
            }
            #[inline(always)]
            fn istore(self, v: [u32; 2], out: &mut [u32]) {
                out.copy_from_slice(&v);
            }


            #[inline(always)]
            fn lanes(self) -> usize {
                2
            }
            #[inline(always)]
            fn splat(self, v: f64) -> v128 {
                f64x2_splat(v)
            }
            #[inline(always)]
            fn load(self, s: &[f64]) -> v128 {
                debug_assert_eq!(s.len(), 2);
                unsafe { v128_load(s.as_ptr() as *const v128) }
            }
            #[inline(always)]
            fn store(self, v: v128, s: &mut [f64]) {
                debug_assert_eq!(s.len(), 2);
                unsafe { v128_store(s.as_mut_ptr() as *mut v128, v) }
            }
            #[inline(always)]
            fn add(self, a: v128, b: v128) -> v128 {
                f64x2_add(a, b)
            }
            #[inline(always)]
            fn sub(self, a: v128, b: v128) -> v128 {
                f64x2_sub(a, b)
            }
            #[inline(always)]
            fn mul(self, a: v128, b: v128) -> v128 {
                f64x2_mul(a, b)
            }
            #[inline(always)]
            fn div(self, a: v128, b: v128) -> v128 {
                f64x2_div(a, b)
            }
            #[inline(always)]
            fn neg(self, a: v128) -> v128 {
                f64x2_neg(a)
            }
            #[inline(always)]
            fn abs(self, a: v128) -> v128 {
                f64x2_abs(a)
            }
            #[inline(always)]
            fn fma(self, a: v128, b: v128, c: v128) -> v128 {
                $fma64(a, b, c)
            }
                    #[inline(always)]
            fn madd(self, a: v128, b: v128, acc: v128) -> v128 {
                <Self as Backend<f64>>::fma(self, a, b, acc)
            }
    #[inline(always)]
            fn sqrt(self, a: v128) -> v128 {
                f64x2_sqrt(a)
            }
            #[inline(always)]
            fn min(self, a: v128, b: v128) -> v128 {
                v128_bitselect(b, f64x2_pmin(a, b), f64x2_ne(a, a))
            }
            #[inline(always)]
            fn max(self, a: v128, b: v128) -> v128 {
                v128_bitselect(b, f64x2_pmax(a, b), f64x2_ne(a, a))
            }
            #[inline(always)]
            fn le(self, a: v128, b: v128) -> v128 {
                f64x2_le(a, b)
            }
            #[inline(always)]
            fn lt(self, a: v128, b: v128) -> v128 {
                f64x2_lt(a, b)
            }
            #[inline(always)]
            fn ge(self, a: v128, b: v128) -> v128 {
                f64x2_ge(a, b)
            }
            #[inline(always)]
            fn gt(self, a: v128, b: v128) -> v128 {
                f64x2_gt(a, b)
            }
            #[inline(always)]
            fn mask_and(self, a: v128, b: v128) -> v128 {
                v128_and(a, b)
            }
            #[inline(always)]
            fn mask_or(self, a: v128, b: v128) -> v128 {
                v128_or(a, b)
            }
            #[inline(always)]
            fn mask_not(self, a: v128) -> v128 {
                v128_not(a)
            }
            #[inline(always)]
            fn select(self, m: v128, a: v128, b: v128) -> v128 {
                v128_bitselect(a, b, m)
            }
            #[inline(always)]
            fn any(self, m: v128) -> bool {
                v128_any_true(m)
            }
            #[inline(always)]
            fn all(self, m: v128) -> bool {
                i64x2_all_true(m)
            }
            #[inline(always)]
            fn mask_bitmask(self, m: v128) -> u32 {
                i64x2_bitmask(m) as u32
            }
            #[inline(always)]
            fn reduce_sum(self, v: v128) -> f64 {
                f64x2_extract_lane::<0>(v) + f64x2_extract_lane::<1>(v)
            }
            #[inline(always)]
            fn reduce_min(self, v: v128) -> f64 {
                f64x2_extract_lane::<0>(v).min(f64x2_extract_lane::<1>(v))
            }
            #[inline(always)]
            fn reduce_max(self, v: v128) -> f64 {
                f64x2_extract_lane::<0>(v).max(f64x2_extract_lane::<1>(v))
            }
        }

        // bf16: 16-bit storage, f32x4 compute (WASM has no bf16 ALU). Widen on load/splat,
        // narrow on store/reduce; conversions live at the memory boundary only.
        impl Backend<bf16> for $token {
            type Vector = v128;
            type Mask = v128;

            type IVector = [u32; 4];
            #[inline(always)]
            fn iload(self, s: &[u32]) -> [u32; 4] {
                let mut v = [0u32; 4];
                v.copy_from_slice(s);
                v
            }
            #[inline(always)]
            fn istore(self, v: [u32; 4], out: &mut [u32]) {
                out.copy_from_slice(&v);
            }


            #[inline(always)]
            fn lanes(self) -> usize {
                4
            }
            #[inline(always)]
            fn splat(self, v: bf16) -> v128 {
                f32x4_splat(v.to_f32())
            }
            #[inline(always)]
            fn load(self, s: &[bf16]) -> v128 {
                debug_assert_eq!(s.len(), 4);
                unsafe { load4_bf16(s) }
            }
            #[inline(always)]
            fn store(self, v: v128, s: &mut [bf16]) {
                debug_assert_eq!(s.len(), 4);
                unsafe { store4_bf16(v, s) }
            }
            #[inline(always)]
            fn add(self, a: v128, b: v128) -> v128 {
                f32x4_add(a, b)
            }
            #[inline(always)]
            fn sub(self, a: v128, b: v128) -> v128 {
                f32x4_sub(a, b)
            }
            #[inline(always)]
            fn mul(self, a: v128, b: v128) -> v128 {
                f32x4_mul(a, b)
            }
            #[inline(always)]
            fn div(self, a: v128, b: v128) -> v128 {
                f32x4_div(a, b)
            }
            #[inline(always)]
            fn neg(self, a: v128) -> v128 {
                f32x4_neg(a)
            }
            #[inline(always)]
            fn abs(self, a: v128) -> v128 {
                f32x4_abs(a)
            }
            #[inline(always)]
            fn fma(self, a: v128, b: v128, c: v128) -> v128 {
                $fma32(a, b, c)
            }
                    #[inline(always)]
            fn madd(self, a: v128, b: v128, acc: v128) -> v128 {
                <Self as Backend<bf16>>::fma(self, a, b, acc)
            }
    #[inline(always)]
            fn sqrt(self, a: v128) -> v128 {
                f32x4_sqrt(a)
            }
            #[inline(always)]
            fn min(self, a: v128, b: v128) -> v128 {
                v128_bitselect(b, f32x4_pmin(a, b), f32x4_ne(a, a))
            }
            #[inline(always)]
            fn max(self, a: v128, b: v128) -> v128 {
                v128_bitselect(b, f32x4_pmax(a, b), f32x4_ne(a, a))
            }
            #[inline(always)]
            fn le(self, a: v128, b: v128) -> v128 {
                f32x4_le(a, b)
            }
            #[inline(always)]
            fn lt(self, a: v128, b: v128) -> v128 {
                f32x4_lt(a, b)
            }
            #[inline(always)]
            fn ge(self, a: v128, b: v128) -> v128 {
                f32x4_ge(a, b)
            }
            #[inline(always)]
            fn gt(self, a: v128, b: v128) -> v128 {
                f32x4_gt(a, b)
            }
            #[inline(always)]
            fn mask_and(self, a: v128, b: v128) -> v128 {
                v128_and(a, b)
            }
            #[inline(always)]
            fn mask_or(self, a: v128, b: v128) -> v128 {
                v128_or(a, b)
            }
            #[inline(always)]
            fn mask_not(self, a: v128) -> v128 {
                v128_not(a)
            }
            #[inline(always)]
            fn select(self, m: v128, a: v128, b: v128) -> v128 {
                v128_bitselect(a, b, m)
            }
            #[inline(always)]
            fn any(self, m: v128) -> bool {
                v128_any_true(m)
            }
            #[inline(always)]
            fn all(self, m: v128) -> bool {
                i32x4_all_true(m)
            }
            #[inline(always)]
            fn mask_bitmask(self, m: v128) -> u32 {
                i32x4_bitmask(m) as u32
            }
            #[inline(always)]
            fn reduce_sum(self, v: v128) -> bf16 {
                bf16::from_f32(
                    f32x4_extract_lane::<0>(v)
                        + f32x4_extract_lane::<1>(v)
                        + f32x4_extract_lane::<2>(v)
                        + f32x4_extract_lane::<3>(v),
                )
            }
            #[inline(always)]
            fn reduce_min(self, v: v128) -> bf16 {
                bf16::from_f32(
                    f32x4_extract_lane::<0>(v)
                        .min(f32x4_extract_lane::<1>(v))
                        .min(f32x4_extract_lane::<2>(v))
                        .min(f32x4_extract_lane::<3>(v)),
                )
            }
            #[inline(always)]
            fn reduce_max(self, v: v128) -> bf16 {
                bf16::from_f32(
                    f32x4_extract_lane::<0>(v)
                        .max(f32x4_extract_lane::<1>(v))
                        .max(f32x4_extract_lane::<2>(v))
                        .max(f32x4_extract_lane::<3>(v)),
                )
            }
        }

        // f16: same widen-compute-narrow shape as bf16; only the boundary conversion differs.
        impl Backend<f16> for $token {
            type Vector = v128;
            type Mask = v128;

            type IVector = [u32; 4];
            #[inline(always)]
            fn iload(self, s: &[u32]) -> [u32; 4] {
                let mut v = [0u32; 4];
                v.copy_from_slice(s);
                v
            }
            #[inline(always)]
            fn istore(self, v: [u32; 4], out: &mut [u32]) {
                out.copy_from_slice(&v);
            }


            #[inline(always)]
            fn lanes(self) -> usize {
                4
            }
            #[inline(always)]
            fn splat(self, v: f16) -> v128 {
                f32x4_splat(v.to_f32())
            }
            #[inline(always)]
            fn load(self, s: &[f16]) -> v128 {
                debug_assert_eq!(s.len(), 4);
                unsafe { load4_f16(s) }
            }
            #[inline(always)]
            fn store(self, v: v128, s: &mut [f16]) {
                debug_assert_eq!(s.len(), 4);
                unsafe { store4_f16(v, s) }
            }
            #[inline(always)]
            fn add(self, a: v128, b: v128) -> v128 {
                f32x4_add(a, b)
            }
            #[inline(always)]
            fn sub(self, a: v128, b: v128) -> v128 {
                f32x4_sub(a, b)
            }
            #[inline(always)]
            fn mul(self, a: v128, b: v128) -> v128 {
                f32x4_mul(a, b)
            }
            #[inline(always)]
            fn div(self, a: v128, b: v128) -> v128 {
                f32x4_div(a, b)
            }
            #[inline(always)]
            fn neg(self, a: v128) -> v128 {
                f32x4_neg(a)
            }
            #[inline(always)]
            fn abs(self, a: v128) -> v128 {
                f32x4_abs(a)
            }
            #[inline(always)]
            fn fma(self, a: v128, b: v128, c: v128) -> v128 {
                $fma32(a, b, c)
            }
                    #[inline(always)]
            fn madd(self, a: v128, b: v128, acc: v128) -> v128 {
                <Self as Backend<f16>>::fma(self, a, b, acc)
            }
    #[inline(always)]
            fn sqrt(self, a: v128) -> v128 {
                f32x4_sqrt(a)
            }
            #[inline(always)]
            fn min(self, a: v128, b: v128) -> v128 {
                v128_bitselect(b, f32x4_pmin(a, b), f32x4_ne(a, a))
            }
            #[inline(always)]
            fn max(self, a: v128, b: v128) -> v128 {
                v128_bitselect(b, f32x4_pmax(a, b), f32x4_ne(a, a))
            }
            #[inline(always)]
            fn le(self, a: v128, b: v128) -> v128 {
                f32x4_le(a, b)
            }
            #[inline(always)]
            fn lt(self, a: v128, b: v128) -> v128 {
                f32x4_lt(a, b)
            }
            #[inline(always)]
            fn ge(self, a: v128, b: v128) -> v128 {
                f32x4_ge(a, b)
            }
            #[inline(always)]
            fn gt(self, a: v128, b: v128) -> v128 {
                f32x4_gt(a, b)
            }
            #[inline(always)]
            fn mask_and(self, a: v128, b: v128) -> v128 {
                v128_and(a, b)
            }
            #[inline(always)]
            fn mask_or(self, a: v128, b: v128) -> v128 {
                v128_or(a, b)
            }
            #[inline(always)]
            fn mask_not(self, a: v128) -> v128 {
                v128_not(a)
            }
            #[inline(always)]
            fn select(self, m: v128, a: v128, b: v128) -> v128 {
                v128_bitselect(a, b, m)
            }
            #[inline(always)]
            fn any(self, m: v128) -> bool {
                v128_any_true(m)
            }
            #[inline(always)]
            fn all(self, m: v128) -> bool {
                i32x4_all_true(m)
            }
            #[inline(always)]
            fn mask_bitmask(self, m: v128) -> u32 {
                i32x4_bitmask(m) as u32
            }
            #[inline(always)]
            fn reduce_sum(self, v: v128) -> f16 {
                f16::from_f32(
                    f32x4_extract_lane::<0>(v)
                        + f32x4_extract_lane::<1>(v)
                        + f32x4_extract_lane::<2>(v)
                        + f32x4_extract_lane::<3>(v),
                )
            }
            #[inline(always)]
            fn reduce_min(self, v: v128) -> f16 {
                f16::from_f32(
                    f32x4_extract_lane::<0>(v)
                        .min(f32x4_extract_lane::<1>(v))
                        .min(f32x4_extract_lane::<2>(v))
                        .min(f32x4_extract_lane::<3>(v)),
                )
            }
            #[inline(always)]
            fn reduce_max(self, v: v128) -> f16 {
                f16::from_f32(
                    f32x4_extract_lane::<0>(v)
                        .max(f32x4_extract_lane::<1>(v))
                        .max(f32x4_extract_lane::<2>(v))
                        .max(f32x4_extract_lane::<3>(v)),
                )
            }
        }

        /// The 32-bit integer element backends: 4-lane `v128` with the same all-ones-lane mask
        /// convention as the float impls. Arithmetic is wrapping.
        macro_rules! wasm_int_backend {
            ($t:ty, $le:ident, $lt:ident, $ge:ident, $gt:ident, $min:ident, $max:ident,
             $abs:expr, $shr:ident) => {
                impl Backend<$t> for $token {
                    type Vector = v128;
                    type Mask = v128;

                    #[inline(always)]
                    fn lanes(self) -> usize {
                        4
                    }
                    #[inline(always)]
                    fn splat(self, v: $t) -> v128 {
                        u32x4_splat(v as u32)
                    }
                    #[inline(always)]
                    fn load(self, s: &[$t]) -> v128 {
                        debug_assert_eq!(s.len(), 4);
                        unsafe { v128_load(s.as_ptr() as *const v128) }
                    }
                    #[inline(always)]
                    fn store(self, v: v128, s: &mut [$t]) {
                        debug_assert_eq!(s.len(), 4);
                        unsafe { v128_store(s.as_mut_ptr() as *mut v128, v) }
                    }
                    #[inline(always)]
                    fn add(self, a: v128, b: v128) -> v128 {
                        i32x4_add(a, b)
                    }
                    #[inline(always)]
                    fn sub(self, a: v128, b: v128) -> v128 {
                        i32x4_sub(a, b)
                    }
                    #[inline(always)]
                    fn mul(self, a: v128, b: v128) -> v128 {
                        i32x4_mul(a, b)
                    }
                    #[inline(always)]
                    fn neg(self, a: v128) -> v128 {
                        i32x4_sub(u32x4_splat(0), a)
                    }
                    #[inline(always)]
                    fn abs(self, a: v128) -> v128 {
                        ($abs)(a)
                    }
                    #[inline(always)]
                    fn min(self, a: v128, b: v128) -> v128 {
                        $min(a, b)
                    }
                    #[inline(always)]
                    fn max(self, a: v128, b: v128) -> v128 {
                        $max(a, b)
                    }
                    #[inline(always)]
                    fn le(self, a: v128, b: v128) -> v128 {
                        $le(a, b)
                    }
                    #[inline(always)]
                    fn lt(self, a: v128, b: v128) -> v128 {
                        $lt(a, b)
                    }
                    #[inline(always)]
                    fn ge(self, a: v128, b: v128) -> v128 {
                        $ge(a, b)
                    }
                    #[inline(always)]
                    fn gt(self, a: v128, b: v128) -> v128 {
                        $gt(a, b)
                    }
                    #[inline(always)]
                    fn mask_and(self, a: v128, b: v128) -> v128 {
                        v128_and(a, b)
                    }
                    #[inline(always)]
                    fn mask_or(self, a: v128, b: v128) -> v128 {
                        v128_or(a, b)
                    }
                    #[inline(always)]
                    fn mask_not(self, a: v128) -> v128 {
                        v128_not(a)
                    }
                    #[inline(always)]
                    fn select(self, m: v128, a: v128, b: v128) -> v128 {
                        v128_bitselect(a, b, m)
                    }
                    #[inline(always)]
                    fn any(self, m: v128) -> bool {
                        v128_any_true(m)
                    }
                    #[inline(always)]
                    fn all(self, m: v128) -> bool {
                        i32x4_all_true(m)
                    }
                    #[inline(always)]
                    fn mask_bitmask(self, m: v128) -> u32 {
                        i32x4_bitmask(m) as u32
                    }
                    #[inline(always)]
                    fn reduce_sum(self, v: v128) -> $t {
                        let mut b = [0u32; 4];
                        unsafe { v128_store(b.as_mut_ptr() as *mut v128, v) };
                        b.iter().fold(0 as $t, |acc, &x| acc.wrapping_add(x as $t))
                    }
                    #[inline(always)]
                    fn reduce_min(self, v: v128) -> $t {
                        let mut b = [0u32; 4];
                        unsafe { v128_store(b.as_mut_ptr() as *mut v128, v) };
                        b.iter().map(|&x| x as $t).min().unwrap()
                    }
                    #[inline(always)]
                    fn reduce_max(self, v: v128) -> $t {
                        let mut b = [0u32; 4];
                        unsafe { v128_store(b.as_mut_ptr() as *mut v128, v) };
                        b.iter().map(|&x| x as $t).max().unwrap()
                    }
                    #[inline(always)]
                    fn shl(self, a: v128, k: u32) -> v128 {
                        debug_assert!(k < 32);
                        i32x4_shl(a, k)
                    }
                    #[inline(always)]
                    fn shr(self, a: v128, k: u32) -> v128 {
                        debug_assert!(k < 32);
                        $shr(a, k)
                    }
                    #[inline(always)]
                    fn bit_and(self, a: v128, b: v128) -> v128 {
                        v128_and(a, b)
                    }
                    #[inline(always)]
                    fn bit_or(self, a: v128, b: v128) -> v128 {
                        v128_or(a, b)
                    }
                    #[inline(always)]
                    fn bit_xor(self, a: v128, b: v128) -> v128 {
                        v128_xor(a, b)
                    }
                    #[inline(always)]
                    fn bit_not(self, a: v128) -> v128 {
                        v128_not(a)
                    }

                    type IVector = v128;
                    #[inline(always)]
                    fn iload(self, s: &[u32]) -> v128 {
                        debug_assert_eq!(s.len(), 4);
                        unsafe { v128_load(s.as_ptr() as *const v128) }
                    }
                    #[inline(always)]
                    fn istore(self, v: v128, out: &mut [u32]) {
                        debug_assert_eq!(out.len(), 4);
                        unsafe { v128_store(out.as_mut_ptr() as *mut v128, v) }
                    }
                    #[inline(always)]
                    fn to_bits(self, v: v128) -> v128 {
                        v
                    }
                    #[inline(always)]
                    fn from_bits(self, v: v128) -> v128 {
                        v
                    }
                }
            };
        }

        wasm_int_backend!(u32, u32x4_le, u32x4_lt, u32x4_ge, u32x4_gt, u32x4_min, u32x4_max, |a| a, u32x4_shr);
        wasm_int_backend!(i32, i32x4_le, i32x4_lt, i32x4_ge, i32x4_gt, i32x4_min, i32x4_max, |a| i32x4_abs(a), i32x4_shr);

    };
}

/// Base WASM `simd128` execution token (fixed 128-bit). No fused multiply-add, so [`fma`] lowers
/// to `mul` then `add`. Available whenever the build enables `simd128`.
///
/// [`fma`]: Backend::fma
// A build that also enables `relaxed-simd` dispatches to `RelaxedSimd` instead, leaving this
// token reached only by the in-crate differential tests.
#[allow(dead_code)]
#[derive(Clone, Copy, Debug, Default)]
pub struct Simd128;

impl Simd128 {
    #[allow(dead_code)]
    #[inline]
    pub const fn new() -> Self {
        Self
    }
}

wasm_backend!(Simd128, fma_mul_add_f32, fma_mul_add_f64);

/// WASM relaxed-SIMD execution token. Identical to [`Simd128`] except [`fma`] uses the relaxed
/// `*_relaxed_madd` instructions: fused where the engine supports it, with the non-deterministic
/// rounding the relaxed-SIMD proposal permits. Available only when the build enables
/// `relaxed-simd`.
///
/// [`fma`]: Backend::fma
#[cfg(target_feature = "relaxed-simd")]
#[derive(Clone, Copy, Debug, Default)]
pub struct RelaxedSimd;

#[cfg(target_feature = "relaxed-simd")]
impl RelaxedSimd {
    #[inline]
    pub const fn new() -> Self {
        Self
    }
}

#[cfg(target_feature = "relaxed-simd")]
wasm_backend!(RelaxedSimd, f32x4_relaxed_madd, f64x2_relaxed_madd);