simd-vector-engine 0.1.0

Runtime-dispatched SIMD kernels for scanning i32/f32 slices: count, sum, find and stream compaction on AVX-512, AVX2, NEON with a scalar fallback
Documentation
// AVX-512, 512-битные ZMM, 16 лейнов по 32 бита.
// Вызывать только после is_x86_feature_detected!("avx512f").
// Сравнение возвращает __mmask16, а не вектор из 0/-1, поэтому movemask не нужен.

#[cfg(target_arch = "x86")]
use core::arch::x86::*;
#[cfg(target_arch = "x86_64")]
use core::arch::x86_64::*;

use crate::aligned::is_aligned;

#[inline]
#[target_feature(enable = "avx512f")]
unsafe fn load_i32x16(p: *const i32, aligned: bool) -> __m512i {
    if aligned {
        (p as *const __m512i).read() // vmovdqa64
    } else {
        (p as *const __m512i).read_unaligned() // vmovdqu64
    }
}

#[inline]
#[target_feature(enable = "avx512f")]
unsafe fn load_f32x16(p: *const f32, aligned: bool) -> __m512 {
    if aligned {
        (p as *const __m512).read() // vmovaps
    } else {
        (p as *const __m512).read_unaligned() // vmovups
    }
}

// 64 элемента за итерацию = 4 кэш-линии
/// # Safety
///
/// The `avx512f` target feature must be available on the current CPU. Use
/// [`crate::cpu::features`] or the safe wrappers in [`crate::engine`] instead of
/// calling this directly.
#[target_feature(enable = "avx512f")]
pub unsafe fn count_gt_i32(data: &[i32], threshold: i32) -> usize {
    let n = data.len();
    let p = data.as_ptr();
    let aligned = is_aligned(p, 64);

    let thr = _mm512_set1_epi32(threshold); // vpbroadcastd zmm

    let mut c0 = 0u64;
    let mut c1 = 0u64;
    let mut c2 = 0u64;
    let mut c3 = 0u64;

    let mut i = 0usize;
    let block = n & !63usize;
    while i < block {
        let v0 = load_i32x16(p.add(i), aligned);
        let v1 = load_i32x16(p.add(i + 16), aligned);
        let v2 = load_i32x16(p.add(i + 32), aligned);
        let v3 = load_i32x16(p.add(i + 48), aligned);

        // vpcmpd k, zmm, zmm, 6 (imm 6 == _MM_CMPINT_NLE)
        let m0: __mmask16 = _mm512_cmpgt_epi32_mask(v0, thr);
        let m1: __mmask16 = _mm512_cmpgt_epi32_mask(v1, thr);
        let m2: __mmask16 = _mm512_cmpgt_epi32_mask(v2, thr);
        let m3: __mmask16 = _mm512_cmpgt_epi32_mask(v3, thr);

        // kmovw + popcnt, счёт идёт в GPR
        c0 += (m0 as u32).count_ones() as u64;
        c1 += (m1 as u32).count_ones() as u64;
        c2 += (m2 as u32).count_ones() as u64;
        c3 += (m3 as u32).count_ones() as u64;

        i += 64;
    }

    let mut total = (c0 + c1 + c2 + c3) as usize;
    while i < n {
        total += (*p.add(i) > threshold) as usize;
        i += 1;
    }
    total
}

/// # Safety
///
/// The `avx512f` target feature must be available on the current CPU. Use
/// [`crate::cpu::features`] or the safe wrappers in [`crate::engine`] instead of
/// calling this directly.
#[target_feature(enable = "avx512f")]
pub unsafe fn count_gt_f32(data: &[f32], threshold: f32) -> usize {
    let n = data.len();
    let p = data.as_ptr();
    let aligned = is_aligned(p, 64);

    let thr = _mm512_set1_ps(threshold); // vbroadcastss zmm
    let mut c0 = 0u64;
    let mut c1 = 0u64;
    let mut c2 = 0u64;
    let mut c3 = 0u64;

    let mut i = 0usize;
    let block = n & !63usize;
    while i < block {
        let v0 = load_f32x16(p.add(i), aligned);
        let v1 = load_f32x16(p.add(i + 16), aligned);
        let v2 = load_f32x16(p.add(i + 32), aligned);
        let v3 = load_f32x16(p.add(i + 48), aligned);

        // vcmpps k, zmm, zmm, _CMP_GT_OQ — NaN даёт false, как `>` в Rust
        let m0 = _mm512_cmp_ps_mask::<_CMP_GT_OQ>(v0, thr);
        let m1 = _mm512_cmp_ps_mask::<_CMP_GT_OQ>(v1, thr);
        let m2 = _mm512_cmp_ps_mask::<_CMP_GT_OQ>(v2, thr);
        let m3 = _mm512_cmp_ps_mask::<_CMP_GT_OQ>(v3, thr);

        c0 += (m0 as u32).count_ones() as u64;
        c1 += (m1 as u32).count_ones() as u64;
        c2 += (m2 as u32).count_ones() as u64;
        c3 += (m3 as u32).count_ones() as u64;

        i += 64;
    }

    let mut total = (c0 + c1 + c2 + c3) as usize;
    while i < n {
        total += (*p.add(i) > threshold) as usize;
        i += 1;
    }
    total
}

/// # Safety
///
/// The `avx512f` target feature must be available on the current CPU. Use
/// [`crate::cpu::features`] or the safe wrappers in [`crate::engine`] instead of
/// calling this directly.
#[target_feature(enable = "avx512f")]
pub unsafe fn sum_i32(data: &[i32]) -> i64 {
    let n = data.len();
    let p = data.as_ptr();
    let aligned = is_aligned(p, 64);

    let mut a0 = _mm512_setzero_si512(); // 8 × i64
    let mut a1 = _mm512_setzero_si512();
    let mut a2 = _mm512_setzero_si512();
    let mut a3 = _mm512_setzero_si512();

    let mut i = 0usize;
    let block = n & !31usize;
    while i < block {
        let v0 = load_i32x16(p.add(i), aligned);
        let v1 = load_i32x16(p.add(i + 16), aligned);

        // vpmovsxdq: 8 × i32 -> 8 × i64; верхняя половина через vextracti64x4
        let lo0 = _mm512_cvtepi32_epi64(_mm512_castsi512_si256(v0));
        let hi0 = _mm512_cvtepi32_epi64(_mm512_extracti64x4_epi64::<1>(v0));
        let lo1 = _mm512_cvtepi32_epi64(_mm512_castsi512_si256(v1));
        let hi1 = _mm512_cvtepi32_epi64(_mm512_extracti64x4_epi64::<1>(v1));

        // vpaddq zmm
        a0 = _mm512_add_epi64(a0, lo0);
        a1 = _mm512_add_epi64(a1, hi0);
        a2 = _mm512_add_epi64(a2, lo1);
        a3 = _mm512_add_epi64(a3, hi1);

        i += 32;
    }

    let acc = _mm512_add_epi64(_mm512_add_epi64(a0, a1), _mm512_add_epi64(a2, a3));
    // Не одна инструкция, а секвенция shuffle+add, но один раз на весь массив.
    let mut total = _mm512_reduce_add_epi64(acc);

    while i < n {
        total += *p.add(i) as i64;
        i += 1;
    }
    total
}

/// Индекс первого вхождения. 4 маски клеятся в u64 и проверяются одним test.
/// # Safety
///
/// The `avx512f` target feature must be available on the current CPU. Use
/// [`crate::cpu::features`] or the safe wrappers in [`crate::engine`] instead of
/// calling this directly.
#[target_feature(enable = "avx512f")]
pub unsafe fn find_i32(data: &[i32], needle: i32) -> Option<usize> {
    let n = data.len();
    let p = data.as_ptr();
    let aligned = is_aligned(p, 64);
    let key = _mm512_set1_epi32(needle);

    let mut i = 0usize;
    let block = n & !63usize;
    while i < block {
        let v0 = load_i32x16(p.add(i), aligned);
        let v1 = load_i32x16(p.add(i + 16), aligned);
        let v2 = load_i32x16(p.add(i + 32), aligned);
        let v3 = load_i32x16(p.add(i + 48), aligned);

        // vpcmpeqd k, zmm, zmm -> __mmask16
        let m0 = _mm512_cmpeq_epi32_mask(v0, key);
        let m1 = _mm512_cmpeq_epi32_mask(v1, key);
        let m2 = _mm512_cmpeq_epi32_mask(v2, key);
        let m3 = _mm512_cmpeq_epi32_mask(v3, key);

        let bits = (m0 as u64) | ((m1 as u64) << 16) | ((m2 as u64) << 32) | ((m3 as u64) << 48);
        if bits != 0 {
            // tzcnt даёт смещение внутри 64-элементного блока
            return Some(i + bits.trailing_zeros() as usize);
        }
        i += 64;
    }

    while i < n {
        if *p.add(i) == needle {
            return Some(i);
        }
        i += 1;
    }
    None
}

/// Compaction через `vpcompressd`: одна инструкция вместо 8 КБ LUT и vpermd из AVX2.
///
/// Пишет ровно popcnt элементов, без перелива за курсор.
/// Размер `out` обеспечивает вызывающий (см. `crate::parallel`).
/// # Safety
///
/// The `avx512f` target feature must be available on the current CPU. Use
/// [`crate::cpu::features`] or the safe wrappers in [`crate::engine`] instead of
/// calling this directly.
#[target_feature(enable = "avx512f")]
pub unsafe fn filter_into_i32(data: &[i32], threshold: i32, out: &mut [i32]) -> usize {
    debug_assert!(data.is_empty() || !out.is_empty());
    let n = data.len();
    let p = data.as_ptr();
    let o = out.as_mut_ptr();
    let aligned = is_aligned(p, 64);
    let thr = _mm512_set1_epi32(threshold);

    let mut w = 0usize;
    let mut i = 0usize;
    let block = n & !15usize;
    while i < block {
        let v = load_i32x16(p.add(i), aligned);
        let m = _mm512_cmpgt_epi32_mask(v, thr);
        // vpcompressd [mem]{k}, zmm: пишет только отмеченные лейны, плотно
        _mm512_mask_compressstoreu_epi32(o.add(w), m, v);
        w += (m as u32).count_ones() as usize;
        i += 16;
    }

    while i < n {
        let x = *p.add(i);
        if x > threshold {
            *o.add(w) = x;
            w += 1;
        }
        i += 1;
    }
    w
}