#[cfg(target_arch = "x86")]
use core::arch::x86::*;
#[cfg(target_arch = "x86_64")]
use core::arch::x86_64::*;
use crate::aligned::is_aligned;
#[inline]
#[target_feature(enable = "avx512f")]
unsafe fn load_i32x16(p: *const i32, aligned: bool) -> __m512i {
if aligned {
(p as *const __m512i).read() } else {
(p as *const __m512i).read_unaligned() }
}
#[inline]
#[target_feature(enable = "avx512f")]
unsafe fn load_f32x16(p: *const f32, aligned: bool) -> __m512 {
if aligned {
(p as *const __m512).read() } else {
(p as *const __m512).read_unaligned() }
}
#[target_feature(enable = "avx512f")]
pub unsafe fn count_gt_i32(data: &[i32], threshold: i32) -> usize {
let n = data.len();
let p = data.as_ptr();
let aligned = is_aligned(p, 64);
let thr = _mm512_set1_epi32(threshold);
let mut c0 = 0u64;
let mut c1 = 0u64;
let mut c2 = 0u64;
let mut c3 = 0u64;
let mut i = 0usize;
let block = n & !63usize;
while i < block {
let v0 = load_i32x16(p.add(i), aligned);
let v1 = load_i32x16(p.add(i + 16), aligned);
let v2 = load_i32x16(p.add(i + 32), aligned);
let v3 = load_i32x16(p.add(i + 48), aligned);
let m0: __mmask16 = _mm512_cmpgt_epi32_mask(v0, thr);
let m1: __mmask16 = _mm512_cmpgt_epi32_mask(v1, thr);
let m2: __mmask16 = _mm512_cmpgt_epi32_mask(v2, thr);
let m3: __mmask16 = _mm512_cmpgt_epi32_mask(v3, thr);
c0 += (m0 as u32).count_ones() as u64;
c1 += (m1 as u32).count_ones() as u64;
c2 += (m2 as u32).count_ones() as u64;
c3 += (m3 as u32).count_ones() as u64;
i += 64;
}
let mut total = (c0 + c1 + c2 + c3) as usize;
while i < n {
total += (*p.add(i) > threshold) as usize;
i += 1;
}
total
}
#[target_feature(enable = "avx512f")]
pub unsafe fn count_gt_f32(data: &[f32], threshold: f32) -> usize {
let n = data.len();
let p = data.as_ptr();
let aligned = is_aligned(p, 64);
let thr = _mm512_set1_ps(threshold); let mut c0 = 0u64;
let mut c1 = 0u64;
let mut c2 = 0u64;
let mut c3 = 0u64;
let mut i = 0usize;
let block = n & !63usize;
while i < block {
let v0 = load_f32x16(p.add(i), aligned);
let v1 = load_f32x16(p.add(i + 16), aligned);
let v2 = load_f32x16(p.add(i + 32), aligned);
let v3 = load_f32x16(p.add(i + 48), aligned);
let m0 = _mm512_cmp_ps_mask::<_CMP_GT_OQ>(v0, thr);
let m1 = _mm512_cmp_ps_mask::<_CMP_GT_OQ>(v1, thr);
let m2 = _mm512_cmp_ps_mask::<_CMP_GT_OQ>(v2, thr);
let m3 = _mm512_cmp_ps_mask::<_CMP_GT_OQ>(v3, thr);
c0 += (m0 as u32).count_ones() as u64;
c1 += (m1 as u32).count_ones() as u64;
c2 += (m2 as u32).count_ones() as u64;
c3 += (m3 as u32).count_ones() as u64;
i += 64;
}
let mut total = (c0 + c1 + c2 + c3) as usize;
while i < n {
total += (*p.add(i) > threshold) as usize;
i += 1;
}
total
}
#[target_feature(enable = "avx512f")]
pub unsafe fn sum_i32(data: &[i32]) -> i64 {
let n = data.len();
let p = data.as_ptr();
let aligned = is_aligned(p, 64);
let mut a0 = _mm512_setzero_si512(); let mut a1 = _mm512_setzero_si512();
let mut a2 = _mm512_setzero_si512();
let mut a3 = _mm512_setzero_si512();
let mut i = 0usize;
let block = n & !31usize;
while i < block {
let v0 = load_i32x16(p.add(i), aligned);
let v1 = load_i32x16(p.add(i + 16), aligned);
let lo0 = _mm512_cvtepi32_epi64(_mm512_castsi512_si256(v0));
let hi0 = _mm512_cvtepi32_epi64(_mm512_extracti64x4_epi64::<1>(v0));
let lo1 = _mm512_cvtepi32_epi64(_mm512_castsi512_si256(v1));
let hi1 = _mm512_cvtepi32_epi64(_mm512_extracti64x4_epi64::<1>(v1));
a0 = _mm512_add_epi64(a0, lo0);
a1 = _mm512_add_epi64(a1, hi0);
a2 = _mm512_add_epi64(a2, lo1);
a3 = _mm512_add_epi64(a3, hi1);
i += 32;
}
let acc = _mm512_add_epi64(_mm512_add_epi64(a0, a1), _mm512_add_epi64(a2, a3));
let mut total = _mm512_reduce_add_epi64(acc);
while i < n {
total += *p.add(i) as i64;
i += 1;
}
total
}
#[target_feature(enable = "avx512f")]
pub unsafe fn find_i32(data: &[i32], needle: i32) -> Option<usize> {
let n = data.len();
let p = data.as_ptr();
let aligned = is_aligned(p, 64);
let key = _mm512_set1_epi32(needle);
let mut i = 0usize;
let block = n & !63usize;
while i < block {
let v0 = load_i32x16(p.add(i), aligned);
let v1 = load_i32x16(p.add(i + 16), aligned);
let v2 = load_i32x16(p.add(i + 32), aligned);
let v3 = load_i32x16(p.add(i + 48), aligned);
let m0 = _mm512_cmpeq_epi32_mask(v0, key);
let m1 = _mm512_cmpeq_epi32_mask(v1, key);
let m2 = _mm512_cmpeq_epi32_mask(v2, key);
let m3 = _mm512_cmpeq_epi32_mask(v3, key);
let bits = (m0 as u64) | ((m1 as u64) << 16) | ((m2 as u64) << 32) | ((m3 as u64) << 48);
if bits != 0 {
return Some(i + bits.trailing_zeros() as usize);
}
i += 64;
}
while i < n {
if *p.add(i) == needle {
return Some(i);
}
i += 1;
}
None
}
#[target_feature(enable = "avx512f")]
pub unsafe fn filter_into_i32(data: &[i32], threshold: i32, out: &mut [i32]) -> usize {
debug_assert!(data.is_empty() || !out.is_empty());
let n = data.len();
let p = data.as_ptr();
let o = out.as_mut_ptr();
let aligned = is_aligned(p, 64);
let thr = _mm512_set1_epi32(threshold);
let mut w = 0usize;
let mut i = 0usize;
let block = n & !15usize;
while i < block {
let v = load_i32x16(p.add(i), aligned);
let m = _mm512_cmpgt_epi32_mask(v, thr);
_mm512_mask_compressstoreu_epi32(o.add(w), m, v);
w += (m as u32).count_ones() as usize;
i += 16;
}
while i < n {
let x = *p.add(i);
if x > threshold {
*o.add(w) = x;
w += 1;
}
i += 1;
}
w
}