simd-vector-engine 0.1.0

Runtime-dispatched SIMD kernels for scanning i32/f32 slices: count, sum, find and stream compaction on AVX-512, AVX2, NEON with a scalar fallback
Documentation
// AVX2, 256-битные YMM, 8 лейнов по 32 бита.
// Вызывать только после is_x86_feature_detected!("avx2").
// Развёртка на 4 вектора: у vpaddd латентность 1, throughput 2-3/такт,
// с одним аккумулятором цикл упирается в зависимость, а не в память.

#[cfg(target_arch = "x86")]
use core::arch::x86::*;
#[cfg(target_arch = "x86_64")]
use core::arch::x86_64::*;

use crate::aligned::is_aligned;

#[inline]
#[target_feature(enable = "avx2")]
unsafe fn hsum_epi32(v: __m256i) -> i32 {
    let hi = _mm256_extracti128_si256::<1>(v); // vextracti128
    let lo = _mm256_castsi256_si128(v);
    let s = _mm_add_epi32(lo, hi); // vpaddd, 8 -> 4
    let s = _mm_add_epi32(s, _mm_shuffle_epi32::<0b01_00_11_10>(s)); // 4 -> 2
    let s = _mm_add_epi32(s, _mm_shuffle_epi32::<0b10_11_00_01>(s)); // 2 -> 1
    _mm_cvtsi128_si32(s) // vmovd
}

#[inline]
#[target_feature(enable = "avx2")]
unsafe fn hsum_epi64(v: __m256i) -> i64 {
    let mut tmp = [0i64; 4];
    _mm256_storeu_si256(tmp.as_mut_ptr() as *mut __m256i, v);
    tmp[0] + tmp[1] + tmp[2] + tmp[3]
}

#[inline]
#[target_feature(enable = "avx2")]
unsafe fn load_i32x8(p: *const i32, aligned: bool) -> __m256i {
    if aligned {
        _mm256_load_si256(p as *const __m256i) // vmovdqa
    } else {
        _mm256_loadu_si256(p as *const __m256i) // vmovdqu
    }
}

#[inline]
#[target_feature(enable = "avx2")]
unsafe fn load_f32x8(p: *const f32, aligned: bool) -> __m256 {
    if aligned {
        _mm256_load_ps(p) // vmovaps
    } else {
        _mm256_loadu_ps(p) // vmovups
    }
}

/// # Safety
///
/// The `avx2` target feature must be available on the current CPU. Use
/// [`crate::cpu::features`] or the safe wrappers in [`crate::engine`] instead of
/// calling this directly.
#[target_feature(enable = "avx2")]
pub unsafe fn count_gt_i32(data: &[i32], threshold: i32) -> usize {
    let n = data.len();
    let p = data.as_ptr();
    let aligned = is_aligned(p, 32);

    let thr = _mm256_set1_epi32(threshold); // vpbroadcastd

    let mut a0 = _mm256_setzero_si256();
    let mut a1 = _mm256_setzero_si256();
    let mut a2 = _mm256_setzero_si256();
    let mut a3 = _mm256_setzero_si256();

    let mut i = 0usize;
    let block = n & !31usize;
    while i < block {
        let v0 = load_i32x8(p.add(i), aligned);
        let v1 = load_i32x8(p.add(i + 8), aligned);
        let v2 = load_i32x8(p.add(i + 16), aligned);
        let v3 = load_i32x8(p.add(i + 24), aligned);

        // vpcmpgtd, маска 0 / -1
        let m0 = _mm256_cmpgt_epi32(v0, thr);
        let m1 = _mm256_cmpgt_epi32(v1, thr);
        let m2 = _mm256_cmpgt_epi32(v2, thr);
        let m3 = _mm256_cmpgt_epi32(v3, thr);

        // vpsubd: acc - (-1) == acc + 1
        a0 = _mm256_sub_epi32(a0, m0);
        a1 = _mm256_sub_epi32(a1, m1);
        a2 = _mm256_sub_epi32(a2, m2);
        a3 = _mm256_sub_epi32(a3, m3);

        i += 32;
    }

    let acc = _mm256_add_epi32(_mm256_add_epi32(a0, a1), _mm256_add_epi32(a2, a3));
    let mut total = hsum_epi32(acc) as usize;

    while i < n {
        total += (*p.add(i) > threshold) as usize;
        i += 1;
    }
    total
}

/// # Safety
///
/// The `avx2` target feature must be available on the current CPU. Use
/// [`crate::cpu::features`] or the safe wrappers in [`crate::engine`] instead of
/// calling this directly.
#[target_feature(enable = "avx2")]
pub unsafe fn count_gt_f32(data: &[f32], threshold: f32) -> usize {
    let n = data.len();
    let p = data.as_ptr();
    let aligned = is_aligned(p, 32);

    let thr = _mm256_set1_ps(threshold); // vbroadcastss
    let mut c0 = 0u32;
    let mut c1 = 0u32;
    let mut c2 = 0u32;
    let mut c3 = 0u32;

    let mut i = 0usize;
    let block = n & !31usize;
    while i < block {
        let v0 = load_f32x8(p.add(i), aligned);
        let v1 = load_f32x8(p.add(i + 8), aligned);
        let v2 = load_f32x8(p.add(i + 16), aligned);
        let v3 = load_f32x8(p.add(i + 24), aligned);

        // vcmpps, _CMP_GT_OQ: NaN даёт false, как `>` в Rust
        let m0 = _mm256_cmp_ps::<_CMP_GT_OQ>(v0, thr);
        let m1 = _mm256_cmp_ps::<_CMP_GT_OQ>(v1, thr);
        let m2 = _mm256_cmp_ps::<_CMP_GT_OQ>(v2, thr);
        let m3 = _mm256_cmp_ps::<_CMP_GT_OQ>(v3, thr);

        // vmovmskps + popcnt
        c0 += (_mm256_movemask_ps(m0) as u32).count_ones();
        c1 += (_mm256_movemask_ps(m1) as u32).count_ones();
        c2 += (_mm256_movemask_ps(m2) as u32).count_ones();
        c3 += (_mm256_movemask_ps(m3) as u32).count_ones();

        i += 32;
    }

    let mut total = (c0 + c1 + c2 + c3) as usize;
    while i < n {
        total += (*p.add(i) > threshold) as usize;
        i += 1;
    }
    total
}

/// # Safety
///
/// The `avx2` target feature must be available on the current CPU. Use
/// [`crate::cpu::features`] or the safe wrappers in [`crate::engine`] instead of
/// calling this directly.
#[target_feature(enable = "avx2")]
pub unsafe fn sum_i32(data: &[i32]) -> i64 {
    let n = data.len();
    let p = data.as_ptr();
    let aligned = is_aligned(p, 32);

    // аккумуляторы в i64: 100M × i32 не влезут в 32 бита
    let mut a0 = _mm256_setzero_si256();
    let mut a1 = _mm256_setzero_si256();
    let mut a2 = _mm256_setzero_si256();
    let mut a3 = _mm256_setzero_si256();

    let mut i = 0usize;
    let block = n & !15usize;
    while i < block {
        let v0 = load_i32x8(p.add(i), aligned);
        let v1 = load_i32x8(p.add(i + 8), aligned);

        // vpmovsxdq: 4 × i32 -> 4 × i64
        let lo0 = _mm256_cvtepi32_epi64(_mm256_castsi256_si128(v0));
        let hi0 = _mm256_cvtepi32_epi64(_mm256_extracti128_si256::<1>(v0));
        let lo1 = _mm256_cvtepi32_epi64(_mm256_castsi256_si128(v1));
        let hi1 = _mm256_cvtepi32_epi64(_mm256_extracti128_si256::<1>(v1));

        // vpaddq
        a0 = _mm256_add_epi64(a0, lo0);
        a1 = _mm256_add_epi64(a1, hi0);
        a2 = _mm256_add_epi64(a2, lo1);
        a3 = _mm256_add_epi64(a3, hi1);

        i += 16;
    }

    let acc = _mm256_add_epi64(_mm256_add_epi64(a0, a1), _mm256_add_epi64(a2, a3));
    let mut total = hsum_epi64(acc);
    while i < n {
        total += *p.add(i) as i64;
        i += 1;
    }
    total
}

/// # Safety
///
/// The `avx2` target feature must be available on the current CPU. Use
/// [`crate::cpu::features`] or the safe wrappers in [`crate::engine`] instead of
/// calling this directly.
#[target_feature(enable = "avx2")]
pub unsafe fn find_i32(data: &[i32], needle: i32) -> Option<usize> {
    let n = data.len();
    let p = data.as_ptr();
    let aligned = is_aligned(p, 32);
    let key = _mm256_set1_epi32(needle);

    let mut i = 0usize;
    let block = n & !31usize;
    while i < block {
        let v0 = load_i32x8(p.add(i), aligned);
        let v1 = load_i32x8(p.add(i + 8), aligned);
        let v2 = load_i32x8(p.add(i + 16), aligned);
        let v3 = load_i32x8(p.add(i + 24), aligned);

        // vpcmpeqd
        let m0 = _mm256_cmpeq_epi32(v0, key);
        let m1 = _mm256_cmpeq_epi32(v1, key);
        let m2 = _mm256_cmpeq_epi32(v2, key);
        let m3 = _mm256_cmpeq_epi32(v3, key);

        let any = _mm256_or_si256(_mm256_or_si256(m0, m1), _mm256_or_si256(m2, m3));

        // vptest: ZF=1 — совпадений нет
        if _mm256_testz_si256(any, any) == 0 {
            for (k, m) in [m0, m1, m2, m3].into_iter().enumerate() {
                let bits = _mm256_movemask_ps(_mm256_castsi256_ps(m)) as u32;
                if bits != 0 {
                    return Some(i + k * 8 + bits.trailing_zeros() as usize);
                }
            }
        }
        i += 32;
    }

    while i < n {
        if *p.add(i) == needle {
            return Some(i);
        }
        i += 1;
    }
    None
}

/// `out` должен вмещать результат плюс 7 элементов запаса (см. parallel::fill_exact).
/// # Safety
///
/// The `avx2` target feature must be available on the current CPU. Use
/// [`crate::cpu::features`] or the safe wrappers in [`crate::engine`] instead of
/// calling this directly.
#[target_feature(enable = "avx2")]
pub unsafe fn filter_into_i32(data: &[i32], threshold: i32, out: &mut [i32]) -> usize {
    debug_assert!(data.is_empty() || !out.is_empty());
    let n = data.len();
    let p = data.as_ptr();
    let o = out.as_mut_ptr();
    let aligned = is_aligned(p, 32);
    let thr = _mm256_set1_epi32(threshold);

    let mut w = 0usize;
    let mut i = 0usize;
    let block = n & !7usize;
    while i < block {
        let v = load_i32x8(p.add(i), aligned);
        let m = _mm256_cmpgt_epi32(v, thr);
        let bits = _mm256_movemask_ps(_mm256_castsi256_ps(m)) as usize;

        if bits != 0 {
            // vpermd собирает отобранные лейны в начало регистра
            let idx = _mm256_loadu_si256(COMPACT_LUT[bits].as_ptr() as *const __m256i);
            let packed = _mm256_permutevar8x32_epi32(v, idx);
            // пишет все 8 лейнов, курсор двигается на popcnt — мусор перезатрётся
            _mm256_storeu_si256(o.add(w) as *mut __m256i, packed);
            w += (bits as u32).count_ones() as usize;
        }
        i += 8;
    }

    while i < n {
        let x = *p.add(i);
        if x > threshold {
            *o.add(w) = x;
            w += 1;
        }
        i += 1;
    }
    w
}

// Порядок лейнов для vpermd по каждой из 256 масок. 8 КБ.
static COMPACT_LUT: [[u32; 8]; 256] = build_compact_lut();

const fn build_compact_lut() -> [[u32; 8]; 256] {
    let mut lut = [[0u32; 8]; 256];
    let mut mask = 0usize;
    while mask < 256 {
        let mut w = 0usize;
        let mut lane = 0usize;
        while lane < 8 {
            if mask & (1 << lane) != 0 {
                lut[mask][w] = lane as u32;
                w += 1;
            }
            lane += 1;
        }
        while w < 8 {
            lut[mask][w] = 0;
            w += 1;
        }
        mask += 1;
    }
    lut
}