use std::arch::aarch64::*;
#[inline]
#[target_feature(enable = "neon")]
fn update<const FUSED: bool>(dst: &mut [f32; 4], src: &[f32; 4], scale: float32x4_t) {
let dst = dst.as_mut_ptr();
unsafe {
let out = vld1q_f32(dst);
let input = vld1q_f32(src.as_ptr());
let out = if FUSED {
vfmaq_f32(out, scale, input)
} else {
vaddq_f32(out, vmulq_f32(scale, input))
};
vst1q_f32(dst, out);
}
}
#[inline]
#[target_feature(enable = "neon")]
fn add_scaled<const FUSED: bool>(out: &mut [f32], src: &[f32], factor: f32) {
let src = &src[..out.len()];
let scale = vdupq_n_f32(factor);
let (out_blocks, out) = out.as_chunks_mut::<4>();
let (src_blocks, src) = src.as_chunks::<4>();
let (out_pairs, out_blocks) = out_blocks.as_chunks_mut::<2>();
let (src_pairs, src_blocks) = src_blocks.as_chunks::<2>();
for ([dst0, dst1], [src0, src1]) in out_pairs.iter_mut().zip(src_pairs) {
update::<FUSED>(dst0, src0, scale);
update::<FUSED>(dst1, src1, scale);
}
for (dst, src) in out_blocks.iter_mut().zip(src_blocks) {
update::<FUSED>(dst, src, scale);
}
if FUSED {
for (dst, &input) in out.iter_mut().zip(src) {
*dst = factor.mul_add(input, *dst);
}
} else {
crate::simd::scaled_add::add_scaled_scalar(out, src, factor);
}
}
#[target_feature(enable = "neon")]
pub(crate) fn add_scaled_neon(out: &mut [f32], src: &[f32], factor: f32) {
add_scaled::<false>(out, src, factor);
}
#[cfg(feature = "splines")]
#[target_feature(enable = "neon")]
pub(crate) fn add_scaled_fma_neon(out: &mut [f32], src: &[f32], factor: f32) {
add_scaled::<true>(out, src, factor);
}