#[cfg(target_arch = "x86")]
use core::arch::x86::*;
#[cfg(target_arch = "x86_64")]
use core::arch::x86_64::*;
use crate::aligned::is_aligned;
#[inline]
#[target_feature(enable = "avx2")]
unsafe fn hsum_epi32(v: __m256i) -> i32 {
let hi = _mm256_extracti128_si256::<1>(v); let lo = _mm256_castsi256_si128(v);
let s = _mm_add_epi32(lo, hi); let s = _mm_add_epi32(s, _mm_shuffle_epi32::<0b01_00_11_10>(s)); let s = _mm_add_epi32(s, _mm_shuffle_epi32::<0b10_11_00_01>(s)); _mm_cvtsi128_si32(s) }
#[inline]
#[target_feature(enable = "avx2")]
unsafe fn hsum_epi64(v: __m256i) -> i64 {
let mut tmp = [0i64; 4];
_mm256_storeu_si256(tmp.as_mut_ptr() as *mut __m256i, v);
tmp[0] + tmp[1] + tmp[2] + tmp[3]
}
#[inline]
#[target_feature(enable = "avx2")]
unsafe fn load_i32x8(p: *const i32, aligned: bool) -> __m256i {
if aligned {
_mm256_load_si256(p as *const __m256i) } else {
_mm256_loadu_si256(p as *const __m256i) }
}
#[inline]
#[target_feature(enable = "avx2")]
unsafe fn load_f32x8(p: *const f32, aligned: bool) -> __m256 {
if aligned {
_mm256_load_ps(p) } else {
_mm256_loadu_ps(p) }
}
#[target_feature(enable = "avx2")]
pub unsafe fn count_gt_i32(data: &[i32], threshold: i32) -> usize {
let n = data.len();
let p = data.as_ptr();
let aligned = is_aligned(p, 32);
let thr = _mm256_set1_epi32(threshold);
let mut a0 = _mm256_setzero_si256();
let mut a1 = _mm256_setzero_si256();
let mut a2 = _mm256_setzero_si256();
let mut a3 = _mm256_setzero_si256();
let mut i = 0usize;
let block = n & !31usize;
while i < block {
let v0 = load_i32x8(p.add(i), aligned);
let v1 = load_i32x8(p.add(i + 8), aligned);
let v2 = load_i32x8(p.add(i + 16), aligned);
let v3 = load_i32x8(p.add(i + 24), aligned);
let m0 = _mm256_cmpgt_epi32(v0, thr);
let m1 = _mm256_cmpgt_epi32(v1, thr);
let m2 = _mm256_cmpgt_epi32(v2, thr);
let m3 = _mm256_cmpgt_epi32(v3, thr);
a0 = _mm256_sub_epi32(a0, m0);
a1 = _mm256_sub_epi32(a1, m1);
a2 = _mm256_sub_epi32(a2, m2);
a3 = _mm256_sub_epi32(a3, m3);
i += 32;
}
let acc = _mm256_add_epi32(_mm256_add_epi32(a0, a1), _mm256_add_epi32(a2, a3));
let mut total = hsum_epi32(acc) as usize;
while i < n {
total += (*p.add(i) > threshold) as usize;
i += 1;
}
total
}
#[target_feature(enable = "avx2")]
pub unsafe fn count_gt_f32(data: &[f32], threshold: f32) -> usize {
let n = data.len();
let p = data.as_ptr();
let aligned = is_aligned(p, 32);
let thr = _mm256_set1_ps(threshold); let mut c0 = 0u32;
let mut c1 = 0u32;
let mut c2 = 0u32;
let mut c3 = 0u32;
let mut i = 0usize;
let block = n & !31usize;
while i < block {
let v0 = load_f32x8(p.add(i), aligned);
let v1 = load_f32x8(p.add(i + 8), aligned);
let v2 = load_f32x8(p.add(i + 16), aligned);
let v3 = load_f32x8(p.add(i + 24), aligned);
let m0 = _mm256_cmp_ps::<_CMP_GT_OQ>(v0, thr);
let m1 = _mm256_cmp_ps::<_CMP_GT_OQ>(v1, thr);
let m2 = _mm256_cmp_ps::<_CMP_GT_OQ>(v2, thr);
let m3 = _mm256_cmp_ps::<_CMP_GT_OQ>(v3, thr);
c0 += (_mm256_movemask_ps(m0) as u32).count_ones();
c1 += (_mm256_movemask_ps(m1) as u32).count_ones();
c2 += (_mm256_movemask_ps(m2) as u32).count_ones();
c3 += (_mm256_movemask_ps(m3) as u32).count_ones();
i += 32;
}
let mut total = (c0 + c1 + c2 + c3) as usize;
while i < n {
total += (*p.add(i) > threshold) as usize;
i += 1;
}
total
}
#[target_feature(enable = "avx2")]
pub unsafe fn sum_i32(data: &[i32]) -> i64 {
let n = data.len();
let p = data.as_ptr();
let aligned = is_aligned(p, 32);
let mut a0 = _mm256_setzero_si256();
let mut a1 = _mm256_setzero_si256();
let mut a2 = _mm256_setzero_si256();
let mut a3 = _mm256_setzero_si256();
let mut i = 0usize;
let block = n & !15usize;
while i < block {
let v0 = load_i32x8(p.add(i), aligned);
let v1 = load_i32x8(p.add(i + 8), aligned);
let lo0 = _mm256_cvtepi32_epi64(_mm256_castsi256_si128(v0));
let hi0 = _mm256_cvtepi32_epi64(_mm256_extracti128_si256::<1>(v0));
let lo1 = _mm256_cvtepi32_epi64(_mm256_castsi256_si128(v1));
let hi1 = _mm256_cvtepi32_epi64(_mm256_extracti128_si256::<1>(v1));
a0 = _mm256_add_epi64(a0, lo0);
a1 = _mm256_add_epi64(a1, hi0);
a2 = _mm256_add_epi64(a2, lo1);
a3 = _mm256_add_epi64(a3, hi1);
i += 16;
}
let acc = _mm256_add_epi64(_mm256_add_epi64(a0, a1), _mm256_add_epi64(a2, a3));
let mut total = hsum_epi64(acc);
while i < n {
total += *p.add(i) as i64;
i += 1;
}
total
}
#[target_feature(enable = "avx2")]
pub unsafe fn find_i32(data: &[i32], needle: i32) -> Option<usize> {
let n = data.len();
let p = data.as_ptr();
let aligned = is_aligned(p, 32);
let key = _mm256_set1_epi32(needle);
let mut i = 0usize;
let block = n & !31usize;
while i < block {
let v0 = load_i32x8(p.add(i), aligned);
let v1 = load_i32x8(p.add(i + 8), aligned);
let v2 = load_i32x8(p.add(i + 16), aligned);
let v3 = load_i32x8(p.add(i + 24), aligned);
let m0 = _mm256_cmpeq_epi32(v0, key);
let m1 = _mm256_cmpeq_epi32(v1, key);
let m2 = _mm256_cmpeq_epi32(v2, key);
let m3 = _mm256_cmpeq_epi32(v3, key);
let any = _mm256_or_si256(_mm256_or_si256(m0, m1), _mm256_or_si256(m2, m3));
if _mm256_testz_si256(any, any) == 0 {
for (k, m) in [m0, m1, m2, m3].into_iter().enumerate() {
let bits = _mm256_movemask_ps(_mm256_castsi256_ps(m)) as u32;
if bits != 0 {
return Some(i + k * 8 + bits.trailing_zeros() as usize);
}
}
}
i += 32;
}
while i < n {
if *p.add(i) == needle {
return Some(i);
}
i += 1;
}
None
}
#[target_feature(enable = "avx2")]
pub unsafe fn filter_into_i32(data: &[i32], threshold: i32, out: &mut [i32]) -> usize {
debug_assert!(data.is_empty() || !out.is_empty());
let n = data.len();
let p = data.as_ptr();
let o = out.as_mut_ptr();
let aligned = is_aligned(p, 32);
let thr = _mm256_set1_epi32(threshold);
let mut w = 0usize;
let mut i = 0usize;
let block = n & !7usize;
while i < block {
let v = load_i32x8(p.add(i), aligned);
let m = _mm256_cmpgt_epi32(v, thr);
let bits = _mm256_movemask_ps(_mm256_castsi256_ps(m)) as usize;
if bits != 0 {
let idx = _mm256_loadu_si256(COMPACT_LUT[bits].as_ptr() as *const __m256i);
let packed = _mm256_permutevar8x32_epi32(v, idx);
_mm256_storeu_si256(o.add(w) as *mut __m256i, packed);
w += (bits as u32).count_ones() as usize;
}
i += 8;
}
while i < n {
let x = *p.add(i);
if x > threshold {
*o.add(w) = x;
w += 1;
}
i += 1;
}
w
}
static COMPACT_LUT: [[u32; 8]; 256] = build_compact_lut();
const fn build_compact_lut() -> [[u32; 8]; 256] {
let mut lut = [[0u32; 8]; 256];
let mut mask = 0usize;
while mask < 256 {
let mut w = 0usize;
let mut lane = 0usize;
while lane < 8 {
if mask & (1 << lane) != 0 {
lut[mask][w] = lane as u32;
w += 1;
}
lane += 1;
}
while w < 8 {
lut[mask][w] = 0;
w += 1;
}
mask += 1;
}
lut
}