use crate::fast_divide::{DividerI32, rounded_quotients_scalar};
use std::arch::aarch64::*;
#[inline]
#[target_feature(enable = "neon")]
fn mulhi_s32(m: int32x4_t, magic: int32x4_t) -> int32x4_t {
let lo = vmull_s32(vget_low_s32(m), vget_low_s32(magic));
let hi = vmull_high_s32(m, magic);
vuzp2q_s32(vreinterpretq_s32_s64(lo), vreinterpretq_s32_s64(hi))
}
#[inline]
#[target_feature(enable = "neon")]
fn magic_quotients<const ADD: bool>(v4s: &mut [[i32; 4]], half: i32, divider: DividerI32) {
let half = vdupq_n_s32(half);
let magic = vdupq_n_s32(divider.magic());
let shift = vdupq_n_s32(-(divider.shift() as i32));
for v4 in v4s {
let v = unsafe { vld1q_s32(v4.as_ptr()) };
let m = vaddq_s32(vabsq_s32(v), half);
let mut uq = mulhi_s32(m, magic);
if ADD {
uq = vaddq_s32(uq, m);
}
let q = vshlq_s32(uq, shift);
let q = vsubq_s32(q, vshrq_n_s32::<31>(q));
let out = vbslq_s32(vcltzq_s32(v), vnegq_s32(q), q);
unsafe { vst1q_s32(v4.as_mut_ptr(), out) };
}
}
#[target_feature(enable = "neon")]
pub(crate) fn rounded_quotients_neon(plane: &mut [i32], q: i32) {
debug_assert!(q > 1);
let divider = DividerI32::new(q);
let (v4s, tail) = plane.as_chunks_mut::<4>();
if divider.magic() == 0 {
let half = vdupq_n_s32(q / 2);
let shift = vdupq_n_s32(-(divider.shift() as i32));
for v4 in v4s.iter_mut() {
let v = unsafe { vld1q_s32(v4.as_ptr()) };
let r = vshlq_s32(vaddq_s32(vabsq_s32(v), half), shift);
let out = vbslq_s32(vcltzq_s32(v), vnegq_s32(r), r);
unsafe { vst1q_s32(v4.as_mut_ptr(), out) };
}
} else if divider.adds_numerator() {
magic_quotients::<true>(v4s, q / 2, divider);
} else {
magic_quotients::<false>(v4s, q / 2, divider);
}
rounded_quotients_scalar(tail, q);
}