use crate::fast_divide::{DividerI32, rounded_quotients_scalar};
#[cfg(target_arch = "x86")]
use std::arch::x86::*;
#[cfg(target_arch = "x86_64")]
use std::arch::x86_64::*;
#[inline]
#[target_feature(enable = "sse4.1")]
fn mulhi_epi32(m: __m128i, magic: __m128i) -> __m128i {
let even = _mm_srli_epi64::<32>(_mm_mul_epi32(m, magic));
let odd = _mm_mul_epi32(_mm_srli_epi64::<32>(m), magic);
_mm_blend_epi16::<0xCC>(even, odd)
}
#[inline]
#[target_feature(enable = "sse4.1")]
fn magic_quotients<const ADD: bool>(v4s: &mut [[i32; 4]], half: i32, divider: DividerI32) {
let half = _mm_set1_epi32(half);
let magic = _mm_set1_epi32(divider.magic());
let shift = _mm_cvtsi32_si128(divider.shift() as i32);
for v4 in v4s {
let v = unsafe { _mm_loadu_si128(v4.as_ptr().cast()) };
let sign = _mm_srai_epi32::<31>(v);
let m = _mm_add_epi32(_mm_abs_epi32(v), half);
let mut uq = mulhi_epi32(m, magic);
if ADD {
uq = _mm_add_epi32(uq, m);
}
let q = _mm_sra_epi32(uq, shift);
let q = _mm_sub_epi32(q, _mm_srai_epi32::<31>(q));
let out = _mm_sub_epi32(_mm_xor_si128(q, sign), sign);
unsafe { _mm_storeu_si128(v4.as_mut_ptr().cast(), out) };
}
}
#[target_feature(enable = "sse4.1")]
pub(crate) fn rounded_quotients_sse41(plane: &mut [i32], q: i32) {
debug_assert!(q > 1);
let divider = DividerI32::new(q);
let (v4s, tail) = plane.as_chunks_mut::<4>();
if divider.magic() == 0 {
let half = _mm_set1_epi32(q / 2);
let shift = _mm_cvtsi32_si128(divider.shift() as i32);
for v4 in v4s.iter_mut() {
let v = unsafe { _mm_loadu_si128(v4.as_ptr().cast()) };
let sign = _mm_srai_epi32::<31>(v);
let r = _mm_sra_epi32(_mm_add_epi32(_mm_abs_epi32(v), half), shift);
let out = _mm_sub_epi32(_mm_xor_si128(r, sign), sign);
unsafe { _mm_storeu_si128(v4.as_mut_ptr().cast(), out) };
}
} else if divider.adds_numerator() {
magic_quotients::<true>(v4s, q / 2, divider);
} else {
magic_quotients::<false>(v4s, q / 2, divider);
}
rounded_quotients_scalar(tail, q);
}